def mv_files(source, target, workflow):
"""Move files within workspace."""
absolute_workspace_path = os.path.join(app.config['SHARED_VOLUME_PATH'],
workflow.workspace_path)
absolute_source_path = os.path.join(app.config['SHARED_VOLUME_PATH'],
absolute_workspace_path, source)
absolute_target_path = os.path.join(app.config['SHARED_VOLUME_PATH'],
absolute_workspace_path, target)
if not os.path.exists(absolute_source_path):
message = 'Path {} does not exist'.format(source)
raise REANAWorkflowControllerError(message)
if not absolute_source_path.startswith(absolute_workspace_path):
message = 'Source path is outside user workspace'
raise REANAWorkflowControllerError(message)
if not absolute_source_path.startswith(absolute_workspace_path):
message = 'Target path is outside workspace'
raise REANAWorkflowControllerError(message)
try:
reana_fs = fs.open_fs(absolute_workspace_path)
source_info = reana_fs.getinfo(source)
if source_info.is_dir:
reana_fs.movedir(src_path=source, dst_path=target, create=True)
else:
reana_fs.move(src_path=source, dst_path=target)
reana_fs.close()
except Exception as e:
reana_fs.close()
message = 'Something went wrong:\n {}'.format(e)
raise REANAWorkflowControllerError(message)
def start_workflow(workflow, parameters):
"""Start a workflow."""
def _start_workflow_db(workflow, parameters):
workflow.run_started_at = datetime.now()
workflow.status = WorkflowStatus.running
if parameters:
workflow.input_parameters = parameters.get("input_parameters")
workflow.operational_options = parameters.get("operational_options")
current_db_sessions.add(workflow)
current_db_sessions.commit()
current_db_sessions = Session.object_session(workflow)
kwrm = KubernetesWorkflowRunManager(workflow)
failure_message = (
"Workflow {id_} could not be started because it {verb} " "already {status}."
).format(
id_=workflow.id_,
verb=get_workflow_status_change_verb(workflow.status.name),
status=str(workflow.status.name),
)
if "restart" in parameters.keys():
if parameters["restart"]:
if workflow.status not in [
WorkflowStatus.failed,
WorkflowStatus.finished,
WorkflowStatus.queued,
]:
raise REANAWorkflowControllerError(failure_message)
elif workflow.status not in [WorkflowStatus.created, WorkflowStatus.queued]:
if workflow.status == WorkflowStatus.deleted:
raise REANAWorkflowStatusError(failure_message)
raise REANAWorkflowControllerError(failure_message)
try:
kwrm.start_batch_workflow_run(
overwrite_input_params=parameters.get("input_parameters"),
overwrite_operational_options=parameters.get("operational_options"),
)
_start_workflow_db(workflow, parameters)
except SQLAlchemyError as e:
message = "Database connection failed, please retry."
logging.error(
f"Error while creating {workflow.id_}: {message}\n{e}", exc_info=True
)
# Rollback Kubernetes job creation
kwrm.stop_batch_workflow_run()
logging.error(
f"Stopping Kubernetes jobs associated with workflow " f"{workflow.id_} ..."
)
raise REANAExternalCallError(message)
except ApiException as e:
message = "Kubernetes connection failed, please retry."
logging.error(
f"Error while creating {workflow.id_}: {message}\n{e}", exc_info=True
)
raise REANAExternalCallError(message)
def _delete_workflow_engine_pod(workflow):
"""Delete workflow engine pod."""
try:
jobs = current_k8s_corev1_api_client.list_namespaced_pod(
namespace=REANA_RUNTIME_KUBERNETES_NAMESPACE,
)
for job in jobs.items:
if str(workflow.id_) in job.metadata.name:
workflow_enginge_logs = current_k8s_corev1_api_client.read_namespaced_pod_log(
namespace=job.metadata.namespace,
name=job.metadata.name,
container="workflow-engine",
)
workflow.logs = (workflow.logs or "") + workflow_enginge_logs + "\n"
current_k8s_batchv1_api_client.delete_namespaced_job(
namespace=job.metadata.namespace,
propagation_policy="Background",
name=job.metadata.labels["job-name"],
)
break
except ApiException as e:
raise REANAWorkflowControllerError(
"Workflow engine pod cound not be deleted {}.".format(e)
)
except Exception as e:
logging.error(traceback.format_exc())
logging.error("Unexpected error: {}".format(e))
def _delete_workflow_engine_pod(workflow):
"""Delete workflow engine pod."""
try:
jobs = current_k8s_corev1_api_client.list_namespaced_pod(
namespace='default',
)
for job in jobs.items:
if str(workflow.id_) in job.metadata.name:
workflow_enginge_logs = \
current_k8s_corev1_api_client.read_namespaced_pod_log(
namespace=job.metadata.namespace,
name=job.metadata.name,
container='workflow-engine')
workflow.logs = \
(workflow.logs or '') + workflow_enginge_logs + '\n'
current_k8s_batchv1_api_client.delete_namespaced_job(
namespace='default',
propagation_policy="Background",
name=job.metadata.labels['job-name'])
break
except ApiException as e:
raise REANAWorkflowControllerError(
"Workflow engine pod cound not be deleted {}.".format(e))
except Exception as e:
logging.error(traceback.format_exc())
logging.error("Unexpected error: {}".format(e))
def stop_workflow(workflow):
"""Stop a given workflow."""
if workflow.status == RunStatus.running:
kwrm = KubernetesWorkflowRunManager(workflow)
kwrm.stop_batch_workflow_run()
workflow.status = RunStatus.stopped
Session.add(workflow)
Session.commit()
else:
message = ("Workflow {id_} is not running.").format(id_=workflow.id_)
raise REANAWorkflowControllerError(message)
def _delete_workflow_job(workflow: Workflow) -> None:
job_name = build_unique_component_name("run-batch", workflow.id_)
try:
current_k8s_batchv1_api_client.delete_namespaced_job(
name=job_name,
namespace=REANA_RUNTIME_KUBERNETES_NAMESPACE,
propagation_policy="Background",
)
except ApiException as e:
raise REANAWorkflowControllerError(
f"Workflow engine pod could not be deleted. Error: {e}")
def start_batch_workflow_run(self):
"""Start a batch workflow run."""
workflow_run_name = self._workflow_run_name_generator('batch')
job = self._create_job_spec(workflow_run_name)
try:
current_k8s_batchv1_api_client.create_namespaced_job(
namespace=KubernetesWorkflowRunManager.default_namespace,
body=job)
except ApiException as e:
msg = 'Workflow engine/job controller pod ' \
'creation failed {}'.format(e)
logging.error(msg, exc_info=True)
raise REANAWorkflowControllerError(e)
def stop_workflow(workflow):
"""Stop a given workflow."""
if workflow.status == WorkflowStatus.running:
kwrm = KubernetesWorkflowRunManager(workflow)
workflow.run_stopped_at = datetime.now()
kwrm.stop_batch_workflow_run()
workflow.status = WorkflowStatus.stopped
current_db_sessions = Session.object_session(workflow)
current_db_sessions.add(workflow)
current_db_sessions.commit()
else:
message = ("Workflow {id_} is not running.").format(id_=workflow.id_)
raise REANAWorkflowControllerError(message)
def _delete_workflow_engine_pod(workflow_uuid):
"""Delete workflow engine pod."""
try:
jobs = current_k8s_batchv1_api_client.list_namespaced_job(
namespace='default', )
for job in jobs.items:
if workflow_uuid in job.metadata.name:
current_k8s_batchv1_api_client.delete_namespaced_job(
namespace='default',
propagation_policy="Background",
name=job.metadata.name)
break
except ApiException as e:
raise REANAWorkflowControllerError(
"Workflow engine pod cound not be deleted {}.".format(e))
def _get_workflow_engine_pod_logs(workflow: Workflow) -> str:
try:
pods = current_k8s_corev1_api_client.list_namespaced_pod(
namespace=REANA_RUNTIME_KUBERNETES_NAMESPACE,
label_selector=f"reana-run-batch-workflow-uuid={str(workflow.id_)}",
)
for pod in pods.items:
if str(workflow.id_) in pod.metadata.name:
return current_k8s_corev1_api_client.read_namespaced_pod_log(
namespace=pod.metadata.namespace,
name=pod.metadata.name,
container="workflow-engine",
)
except ApiException as e:
raise REANAWorkflowControllerError(
f"Workflow engine pod logs could not be fetched. Error: {e}")