def _delete_workflow_engine_pod(workflow):
"""Delete workflow engine pod."""
try:
jobs = current_k8s_corev1_api_client.list_namespaced_pod(
namespace=REANA_RUNTIME_KUBERNETES_NAMESPACE,
)
for job in jobs.items:
if str(workflow.id_) in job.metadata.name:
workflow_enginge_logs = current_k8s_corev1_api_client.read_namespaced_pod_log(
namespace=job.metadata.namespace,
name=job.metadata.name,
container="workflow-engine",
)
workflow.logs = (workflow.logs or "") + workflow_enginge_logs + "\n"
current_k8s_batchv1_api_client.delete_namespaced_job(
namespace=job.metadata.namespace,
propagation_policy="Background",
name=job.metadata.labels["job-name"],
)
break
except ApiException as e:
raise REANAWorkflowControllerError(
"Workflow engine pod cound not be deleted {}.".format(e)
)
except Exception as e:
logging.error(traceback.format_exc())
logging.error("Unexpected error: {}".format(e))
def _delete_workflow_engine_pod(workflow):
"""Delete workflow engine pod."""
try:
jobs = current_k8s_corev1_api_client.list_namespaced_pod(
namespace='default',
)
for job in jobs.items:
if str(workflow.id_) in job.metadata.name:
workflow_enginge_logs = \
current_k8s_corev1_api_client.read_namespaced_pod_log(
namespace=job.metadata.namespace,
name=job.metadata.name,
container='workflow-engine')
workflow.logs = \
(workflow.logs or '') + workflow_enginge_logs + '\n'
current_k8s_batchv1_api_client.delete_namespaced_job(
namespace='default',
propagation_policy="Background",
name=job.metadata.labels['job-name'])
break
except ApiException as e:
raise REANAWorkflowControllerError(
"Workflow engine pod cound not be deleted {}.".format(e))
except Exception as e:
logging.error(traceback.format_exc())
logging.error("Unexpected error: {}".format(e))
def stop_batch_workflow_run(self):
"""Stop a batch workflow run along with all its dependent jobs."""
workflow_run_name = self._workflow_run_name_generator("batch")
to_delete = self.get_workflow_running_jobs_as_backend_ids() + [
workflow_run_name
]
error = False
for job in to_delete:
try:
current_k8s_batchv1_api_client.delete_namespaced_job(
job,
REANA_RUNTIME_KUBERNETES_NAMESPACE,
body=V1DeleteOptions(grace_period_seconds=0,
propagation_policy="Background"),
)
except ApiException:
logging.error(
f"Error while trying to stop {self.workflow.id_}"
f": Kubernetes job {job} could not be deleted.",
exc_info=True,
)
error = True
continue
if error:
raise REANAWorkflowStopError(
f"Workflow {self.workflow.id_} could not be stopped.")
def stop_batch_workflow_run(self):
"""Stop a batch workflow run along with all its dependent jobs."""
workflow_run_name = self._workflow_run_name_generator('batch')
to_delete = self.get_workflow_running_jobs_as_backend_ids() + \
[workflow_run_name]
for job in to_delete:
current_k8s_batchv1_api_client.delete_namespaced_job(
job,
KubernetesWorkflowRunManager.default_namespace,
body=V1DeleteOptions(propagation_policy='Background'))
def _delete_workflow_job(workflow: Workflow) -> None:
job_name = build_unique_component_name("run-batch", workflow.id_)
try:
current_k8s_batchv1_api_client.delete_namespaced_job(
name=job_name,
namespace=REANA_RUNTIME_KUBERNETES_NAMESPACE,
propagation_policy="Background",
)
except ApiException as e:
raise REANAWorkflowControllerError(
f"Workflow engine pod could not be deleted. Error: {e}")
def stop_batch_workflow_run(self, workflow_run_jobs=None):
"""Stop a batch workflow run along with all its dependent jobs.
:param workflow_run_jobs: List of active job id's spawned by the
workflow run.
"""
workflow_run_name = self._workflow_run_name_generator('batch')
workflow_run_jobs = workflow_run_jobs or []
to_delete = workflow_run_jobs + [workflow_run_name]
for job in to_delete:
current_k8s_batchv1_api_client.delete_namespaced_job(
job, KubernetesWorkflowRunManager.default_namespace,
V1DeleteOptions(propagation_policy='Background'))
def _delete_workflow_engine_pod(workflow_uuid):
"""Delete workflow engine pod."""
try:
jobs = current_k8s_batchv1_api_client.list_namespaced_job(
namespace='default', )
for job in jobs.items:
if workflow_uuid in job.metadata.name:
current_k8s_batchv1_api_client.delete_namespaced_job(
namespace='default',
propagation_policy="Background",
name=job.metadata.name)
break
except ApiException as e:
raise REANAWorkflowControllerError(
"Workflow engine pod cound not be deleted {}.".format(e))
def stop(backend_job_id, asynchronous=True):
"""Stop Kubernetes job execution.
:param backend_job_id: Kubernetes job id.
:param asynchronous: Whether the function waits for the action to be
performed or does it asynchronously.
"""
try:
propagation_policy = 'Background' if asynchronous else 'Foreground'
delete_options = V1DeleteOptions(
propagation_policy=propagation_policy)
current_k8s_batchv1_api_client.delete_namespaced_job(
backend_job_id, K8S_DEFAULT_NAMESPACE, body=delete_options)
except ApiException as e:
logging.error(
'An error has occurred while connecting to Kubernetes API '
'Server \n {}'.format(e))
raise ComputingBackendSubmissionError(e.reason)
def k8s_delete_job(job, asynchronous=True):
"""Delete Kubernetes job.
:param job: The :class:`kubernetes.client.models.v1_job.V1Job` to be
deleted.
:param asynchronous: Whether the function waits for the action to be
performed or does it asynchronously.
"""
try:
propagation_policy = 'Background' if asynchronous else 'Foreground'
delete_options = V1DeleteOptions(propagation_policy=propagation_policy)
current_k8s_batchv1_api_client.delete_namespaced_job(
job.metadata.name, job.metadata.namespace, delete_options)
except ApiException as e:
logging.error(
'An error has occurred while connecting to Kubernetes API Server'
' \n {}'.format(e))
raise ComputingBackendSubmissionError(e.reason)
def stop_batch_workflow_run(self):
"""Stop a batch workflow run along with all its dependent jobs."""
workflow_run_name = self._workflow_run_name_generator('batch')
to_delete = self.get_workflow_running_jobs_as_backend_ids() + \
[workflow_run_name]
error = False
for job in to_delete:
try:
current_k8s_batchv1_api_client.delete_namespaced_job(
job,
KubernetesWorkflowRunManager.default_namespace,
body=V1DeleteOptions(propagation_policy='Background'))
except ApiException:
logging.error(
f'Error while trying to stop {self.workflow.id_}'
f': Kubernetes job {job} could not be deleted.',
exc_info=True)
error = True
continue
if error:
raise REANAWorkflowStopError(
f'Workflow {self.workflow.id_} could not be stopped.')
