def test_workflow_finish_and_kubernetes_not_available(
in_memory_queue_connection,
sample_serial_workflow_in_db,
consume_queue,
):
"""Test workflow finish with a Kubernetes connection troubles."""
sample_serial_workflow_in_db.status = RunStatus.running
next_status = RunStatus.failed
job_status_consumer = JobStatusConsumer(
connection=in_memory_queue_connection)
workflow_status_publisher = WorkflowStatusPublisher(
connection=in_memory_queue_connection, queue=job_status_consumer.queue)
workflow_status_publisher.publish_workflow_status(
str(sample_serial_workflow_in_db.id_),
next_status.value,
)
k8s_corev1_api_client_mock = Mock()
k8s_corev1_api_client_mock.delete_namespaced_job = Mock(
side_effect=ApiException(reason="Could not delete job.", status=404))
with patch(
"reana_workflow_controller.consumer.current_k8s_corev1_api_client",
k8s_corev1_api_client_mock,
):
consume_queue(job_status_consumer, limit=1)
assert sample_serial_workflow_in_db.status == next_status
def run_cwl_workflow(workflow_uuid,
workflow_workspace,
workflow_json=None,
workflow_parameters=None,
operational_options={}):
"""Run cwl workflow."""
workflow_parameters = parse_str_to_int(workflow_parameters)
log.info(f'running workflow on context: {locals()}')
try:
check_connection_to_job_controller()
publisher = WorkflowStatusPublisher()
rcode = main.main(workflow_uuid, workflow_json, workflow_parameters,
operational_options, workflow_workspace, publisher)
log.info('workflow done')
publisher.publish_workflow_status(workflow_uuid,
rcode_to_workflow_status(rcode))
except Exception as e:
log.error(f'workflow failed: {e}')
publisher.publish_workflow_status(workflow_uuid, 3, message=str(e))
finally:
if publisher:
publisher.close()
else:
log.error(f'Workflow {workflow_uuid} failed but status '
'could not be published.')
def initialize(workflow_uuid, workflow_workspace, operational_options):
"""Initialize engine."""
# configure the logger
logging.basicConfig(level=REANA_LOG_LEVEL, format=REANA_LOG_FORMAT)
# set cache on or off
if not operational_options:
operational_options = {}
if CACHE_ENABLED:
if ("CACHE" not in operational_options
or operational_options.get("CACHE", "").lower() != "off"):
cache_enabled = True
else:
cache_enabled = False
else:
cache_enabled = False
# build workspace path
workflow_workspace = "{0}/{1}".format(SHARED_VOLUME_PATH,
workflow_workspace)
# create a MQ publisher
publisher = WorkflowStatusPublisher()
return workflow_workspace, publisher, cache_enabled
def __new__(cls, instance=None):
"""REANA workflow status publisher object creation."""
if instance:
REANAWorkflowStatusPublisher.__instance = instance
elif REANAWorkflowStatusPublisher.__instance is None:
REANAWorkflowStatusPublisher.__instance = WorkflowStatusPublisher()
return REANAWorkflowStatusPublisher.__instance
def run_workflow_engine_run_command(**kwargs):
"""Click command used to invoke concrete workflow engine adapters."""
workflow_uuid = kwargs.get("workflow_uuid")
workflow_workspace = kwargs.get("workflow_workspace")
def _default_exit_handler(signum, frame):
"""Handle executable exit gracefully."""
if not publisher:
raise Exception(
"Workflow engine graceful exit requires an instance"
"of reana_commons.publisher.WorkflowStatusPublisher")
try:
logging.warning(
"Termination signal {} received. Workflow interrupted ...".
format(signum))
publisher.publish_workflow_status(
workflow_uuid, 3, logs="Workflow exited unexpectedly.")
except Exception:
logging.error(
"Workflow {} could not be stopped gracefully".format(
workflow_uuid), )
try:
signal.signal(signal.SIGTERM, exit_handler
or _default_exit_handler)
publisher = WorkflowStatusPublisher()
rjc_api_client = JobControllerAPIClient("reana-job-controller")
check_connection_to_job_controller()
workflow_engine_run_adapter(publisher, rjc_api_client, **kwargs)
logging.info(
"Workflow {} finished. Files available at {}.".format(
workflow_uuid, workflow_workspace), )
publisher.close()
except Exception as e:
logging.debug(str(e))
if publisher:
publisher.publish_workflow_status(
workflow_uuid,
3,
logs="Workflow exited unexpectedly.\n{e}".format(e=e),
)
else:
logging.error(
"Workflow {} failed but status "
"could not be published causing the workflow to be "
"stuck in running status.".format(workflow_uuid), )
finally:
if publisher:
publisher.close()
def test_workflow_status_publish(ConsumerBaseOnMessageMock,
in_memory_queue_connection, default_queue,
consume_queue):
"""Test WorkflowStatusPublisher."""
consumer = ConsumerBaseOnMessageMock(connection=in_memory_queue_connection,
queue=default_queue)
workflow_status_publisher = WorkflowStatusPublisher(
connection=in_memory_queue_connection,
routing_key=default_queue.routing_key,
exchange=default_queue.exchange.name,
queue=default_queue.name,
)
workflow_id = "test"
status = 1
message = {
"progress": {
"total": {
"total": 1,
"job_ids": []
},
}
}
workflow_status_publisher.publish_workflow_status(workflow_id,
status,
message=message)
workflow_status_publisher.close()
expected = json.dumps({
"workflow_uuid": workflow_id,
"logs": "",
"status": status,
"message": message
})
consume_queue(consumer, limit=1)
consumer.on_message.assert_called_once_with(expected, ANY)
def test_workflow_status_publish(ConsumerBaseOnMessageMock,
in_memory_queue_connection, default_queue,
consume_queue):
"""Test WorkflowStatusPublisher."""
consumer = ConsumerBaseOnMessageMock(connection=in_memory_queue_connection,
queue=default_queue)
workflow_status_publisher = WorkflowStatusPublisher(
connection=in_memory_queue_connection,
routing_key=default_queue.routing_key,
exchange=default_queue.exchange.name,
queue=default_queue.name)
workflow_id = 'test'
status = 1
message = {
'progress': {
'total': {
'total': 1,
'job_ids': []
},
}
}
workflow_status_publisher.publish_workflow_status(workflow_id,
status,
message=message)
workflow_status_publisher.close()
expected = json.dumps({
'workflow_uuid': workflow_id,
'logs': '',
'status': status,
'message': message
})
consume_queue(consumer, limit=1)
consumer.on_message.assert_called_once_with(expected, ANY)
def run_cwl_workflow(workflow_uuid,
workflow_workspace,
workflow_json=None,
workflow_parameters=None,
operational_options={}):
"""Run cwl workflow."""
log.info('running workflow on context: {0}'.format(locals()))
try:
publisher = WorkflowStatusPublisher()
main.main(workflow_uuid, workflow_json, workflow_parameters,
operational_options, workflow_workspace, publisher)
log.info('workflow done')
publisher.publish_workflow_status(workflow_uuid, 2)
except Exception as e:
log.error('workflow failed: {0}'.format(e))
publisher.publish_workflow_status(workflow_uuid, 3, message=str(e))
finally:
if publisher:
publisher.close()
else:
log.error(
'Workflow {workflow_uuid} failed but status '
'could not be published.'.format(workflow_uuid=workflow_uuid))