def __init__(self):
"""Initialize the REANA packtivity backend."""
self.config = packconfig()
self.rjc_api_client = RJC_API_Client("reana-job-controller")
self.jobs_statuses = {}
self._fail_info = ""
def run_workflow_engine_run_command(**kwargs):
"""Click command used to invoke concrete workflow engine adapters."""
workflow_uuid = kwargs.get("workflow_uuid")
workflow_workspace = kwargs.get("workflow_workspace")
def _default_exit_handler(signum, frame):
"""Handle executable exit gracefully."""
if not publisher:
raise Exception(
"Workflow engine graceful exit requires an instance"
"of reana_commons.publisher.WorkflowStatusPublisher")
try:
logging.warning(
"Termination signal {} received. Workflow interrupted ...".
format(signum))
publisher.publish_workflow_status(
workflow_uuid, 3, logs="Workflow exited unexpectedly.")
except Exception:
logging.error(
"Workflow {} could not be stopped gracefully".format(
workflow_uuid), )
try:
signal.signal(signal.SIGTERM, exit_handler
or _default_exit_handler)
publisher = WorkflowStatusPublisher()
rjc_api_client = JobControllerAPIClient("reana-job-controller")
check_connection_to_job_controller()
workflow_engine_run_adapter(publisher, rjc_api_client, **kwargs)
logging.info(
"Workflow {} finished. Files available at {}.".format(
workflow_uuid, workflow_workspace), )
publisher.close()
except Exception as e:
logging.debug(str(e))
if publisher:
publisher.publish_workflow_status(
workflow_uuid,
3,
logs="Workflow exited unexpectedly.\n{e}".format(e=e),
)
else:
logging.error(
"Workflow {} failed but status "
"could not be published causing the workflow to be "
"stuck in running status.".format(workflow_uuid), )
finally:
if publisher:
publisher.close()
class ExternalBackend:
"""REANA yadage external packtivity backend class.
Submits jobs and fetches their statuses from the JobController.
"""
def __init__(self):
"""Initialize the REANA packtivity backend."""
self.config = packconfig()
self.rjc_api_client = RJC_API_Client("reana-job-controller")
self.jobs_statuses = {}
self._fail_info = ""
@staticmethod
def _get_resources(resources: List[Union[Dict, Any]]) -> Dict[str, Any]:
parameters = {}
def set_parameter(resource: Dict[str, Any], key: str) -> None:
if key in resource and resource[key] is not None:
parameters[key] = resource[key]
for item in resources:
if not isinstance(item, dict):
log.info(
"REANA only supports dictionary entries for resources. "
f'"{item}" value is not formatted in such a way and will be ignored.'
)
continue
set_parameter(item, "kerberos")
set_parameter(item, "compute_backend")
set_parameter(item, "kubernetes_uid")
set_parameter(item, "kubernetes_memory_limit")
set_parameter(item, "kubernetes_job_timeout")
set_parameter(item, "unpacked_img")
set_parameter(item, "voms_proxy")
set_parameter(item, "htcondor_max_runtime")
set_parameter(item, "htcondor_accounting_group")
return parameters
def submit( # noqa: C901
self,
spec,
parameters,
state,
metadata # noqa: C901
) -> ReanaExternalProxy: # noqa: C901
"""Submit a yadage packtivity to RJC."""
parameters, state = finalize_inputs(parameters, state)
job = build_job(spec["process"], parameters, state, self.config)
log.debug(f"state context is {state}")
state.ensure()
if "command" in job:
prettified_cmd = wrapped_cmd = job["command"]
elif "script" in job:
prettified_cmd = job["script"]
wrapped_cmd = make_script(job)
image = spec["environment"]["image"]
imagetag = spec["environment"].get("imagetag", "")
if imagetag:
image = f"{image}:{imagetag}"
resources = spec["environment"].get("resources", [])
resources_parameters = self._get_resources(resources)
log.debug(f"would run job {job}")
workflow_uuid = os.getenv("workflow_uuid", "default")
job_request_body = {
"workflow_uuid": workflow_uuid,
"image": image,
"cmd": wrapped_cmd,
"prettified_cmd": prettified_cmd,
"workflow_workspace": os.getenv("workflow_workspace", "default"),
"job_name": metadata["name"],
"cvmfs_mounts": MOUNT_CVMFS,
**resources_parameters,
}
job_submit_response = self.rjc_api_client.submit(**job_request_body)
job_id = job_submit_response.get("job_id")
log.info(f"Submitted job with id: {job_id}")
return ReanaExternalProxy(jobproxy=job_submit_response,
spec=spec,
pardata=parameters,
statedata=state)
def result(self, resultproxy: ReanaExternalProxy):
"""Retrieve the result of a packtivity run by RJC."""
resultproxy.pardata, resultproxy.statedata = finalize_inputs(
resultproxy.pardata, resultproxy.statedata)
return publish(
resultproxy.spec["publisher"],
resultproxy.pardata,
resultproxy.statedata,
self.config,
)
def _get_job_status_from_controller(self, job_id: str) -> str:
response = self.rjc_api_client.check_status(job_id)
return response["status"]
def _refresh_job_status(self, job_id: str) -> None:
self.jobs_statuses[job_id] = self._get_job_status_from_controller(
job_id)
def _should_refresh_job_status(self, job_id: str) -> bool:
if job_id in self.jobs_statuses:
status = self.jobs_statuses[job_id]
return status == JobStatus.started
else:
return True
def _get_state(self, resultproxy: ReanaExternalProxy) -> str:
"""Get the packtivity state."""
job_id = resultproxy.jobproxy["job_id"]
if self._should_refresh_job_status(job_id):
self._refresh_job_status(job_id)
return self.jobs_statuses[job_id]
def ready(self, resultproxy: ReanaExternalProxy) -> bool:
"""Check if a packtivity is finished."""
return self._get_state(resultproxy) != JobStatus.started
def successful(self, resultproxy: ReanaExternalProxy) -> bool:
"""Check if the packtivity was successful."""
return self._get_state(resultproxy) == JobStatus.finished
def fail_info(self, resultproxy):
"""Retrieve the fail info."""
self._fail_info += f"\nraw info: {resultproxy}"
return self._fail_info
def __init__(self):
"""Initialize the REANA packtivity backend."""
self.config = packconfig()
self.rjc_api_client = RJC_API_Client('reana-job-controller')
self._fail_info = None
class ExternalBackend(object):
"""REANA yadage external packtivity backend class."""
def __init__(self):
"""Initialize the REANA packtivity backend."""
self.config = packconfig()
self.rjc_api_client = RJC_API_Client('reana-job-controller')
self._fail_info = None
def submit(self, spec, parameters, state, metadata):
"""Submit a yadage packtivity to RJC."""
parameters, state = finalize_inputs(parameters, state)
job = build_job(spec['process'], parameters, state, self.config)
if 'command' in job:
prettified_cmd = job['command']
wrapped_cmd = make_oneliner(job)
elif 'script' in job:
prettified_cmd = job['script']
wrapped_cmd = make_script(job)
image = spec['environment']['image']
imagetag = spec['environment'].get('imagetag', '')
if imagetag:
image = image + ":" + imagetag
kerberos = None
compute_backend = None
kubernetes_uid = None
resources = spec['environment'].get('resources', None)
if resources:
for item in resources:
if 'kerberos' in item.keys():
kerberos = item['kerberos']
if 'compute_backend' in item.keys():
compute_backend = item['compute_backend']
if 'kubernetes_uid' in item.keys():
kubernetes_uid = item['kubernetes_uid']
log.info('state context is {0}'.format(state))
log.info('would run job {0}'.format(job))
state.ensure()
log.info('submitting!')
workflow_uuid = os.getenv('workflow_uuid', 'default')
job_request_body = {
'workflow_uuid': workflow_uuid,
'image': image,
'cmd': wrapped_cmd,
'prettified_cmd': prettified_cmd,
'workflow_workspace': os.getenv('workflow_workspace', 'default'),
'job_name': metadata['name'],
'cvmfs_mounts': MOUNT_CVMFS,
}
if compute_backend:
job_request_body['compute_backend'] = compute_backend
if kerberos:
job_request_body['kerberos'] = kerberos
if kubernetes_uid:
job_request_body['kubernetes_uid'] = kubernetes_uid
job_id = self.rjc_api_client.submit(**job_request_body)
log.info('submitted job:{0}'.format(job_id))
message = {"job_id": str(job_id)}
workflow_uuid = os.getenv('workflow_uuid', 'default')
status_running = 1
try:
publisher = REANAWorkflowStatusPublisher()
publisher.publish_workflow_status(workflow_uuid,
status_running,
message=message)
except Exception as e:
log.info('Status: workflow - {workflow_uuid} '
'status - {status} message - {message}'.format(
workflow_uuid=workflow_uuid,
status=status_running,
message=message))
log.info('workflow status publish failed: {0}'.format(e))
return ReanaExternalProxy(jobproxy=job_id,
spec=spec,
pardata=parameters,
statedata=state)
def result(self, resultproxy):
"""Retrieve the result of a packtivity run by RJC."""
resultproxy.pardata, resultproxy.statedata \
= finalize_inputs(resultproxy.pardata, resultproxy.statedata)
return publish(resultproxy.spec['publisher'], resultproxy.pardata,
resultproxy.statedata, self.config)
def _get_state(self, resultproxy):
"""Get the packtivity state."""
status_res = self.rjc_api_client.check_status(
resultproxy.jobproxy['job_id'])
return status_res['status']
def ready(self, resultproxy):
"""Check if a packtivity is finished."""
return self._get_state(resultproxy) != 'started'
def successful(self, resultproxy):
"""Check if the packtivity was successful."""
return self._get_state(resultproxy) == 'succeeded'
def fail_info(self, resultproxy):
"""Retrieve the fail info."""
self._fail_info += "\nraw info: {}".format(resultproxy)
return self._fail_info
from reana_commons.api_client import JobControllerAPIClient
from reana_commons.config import (REANA_LOG_LEVEL, REANA_LOG_FORMAT)
from reana_commons.publisher import WorkflowStatusPublisher
from reana_commons.serial import serial_load
from reana_commons.utils import (build_caching_info_message,
build_progress_message)
from .config import SHARED_VOLUME_PATH
from .utils import (build_job_spec, check_cache, copy_workspace_from_cache,
copy_workspace_to_cache, escape_shell_arg, load_json,
poll_job_status, publish_cache_copy,
publish_job_submission, publish_job_success,
publish_workflow_failure, publish_workflow_start,
sanitize_command)
rjc_api_client = JobControllerAPIClient('reana-job-controller')
@click.command()
@click.option('--workflow-uuid',
required=True,
help='UUID of workflow to be run.')
@click.option('--workflow-workspace',
required=True,
help='Name of workspace in which workflow should run.')
@click.option('--workflow-json',
help='JSON representation of workflow object to be run.',
callback=load_json)
@click.option('--workflow-parameters',
help='JSON representation of parameters received by'
' the workflow.',
class ExternalBackend(object):
"""REANA yadage external packtivity backend class."""
def __init__(self):
"""Initialize the REANA packtivity backend."""
self.config = packconfig()
self.rjc_api_client = RJC_API_Client("reana-job-controller")
self._fail_info = None
def submit(self, spec, parameters, state, metadata):
"""Submit a yadage packtivity to RJC."""
parameters, state = finalize_inputs(parameters, state)
job = build_job(spec["process"], parameters, state, self.config)
if "command" in job:
prettified_cmd = job["command"]
wrapped_cmd = make_oneliner(job)
elif "script" in job:
prettified_cmd = job["script"]
wrapped_cmd = make_script(job)
image = spec["environment"]["image"]
imagetag = spec["environment"].get("imagetag", "")
if imagetag:
image = image + ":" + imagetag
kerberos = None
compute_backend = None
kubernetes_uid = None
unpacked_img = None
voms_proxy = None
resources = spec["environment"].get("resources", None)
if resources:
for item in resources:
if "kerberos" in item.keys():
kerberos = item["kerberos"]
if "compute_backend" in item.keys():
compute_backend = item["compute_backend"]
if "kubernetes_uid" in item.keys():
kubernetes_uid = item["kubernetes_uid"]
if "unpacked_img" in item.keys():
unpacked_img = item["unpacked_img"]
if "voms_proxy" in item.keys():
voms_proxy = item["voms_proxy"]
log.info("state context is {0}".format(state))
log.info("would run job {0}".format(job))
state.ensure()
log.info("submitting!")
workflow_uuid = os.getenv("workflow_uuid", "default")
job_request_body = {
"workflow_uuid": workflow_uuid,
"image": image,
"cmd": wrapped_cmd,
"prettified_cmd": prettified_cmd,
"workflow_workspace": os.getenv("workflow_workspace", "default"),
"job_name": metadata["name"],
"cvmfs_mounts": MOUNT_CVMFS,
}
if compute_backend:
job_request_body["compute_backend"] = compute_backend
if kerberos:
job_request_body["kerberos"] = kerberos
if kubernetes_uid:
job_request_body["kubernetes_uid"] = kubernetes_uid
if unpacked_img:
job_request_body["unpacked_img"] = unpacked_img
if voms_proxy:
job_request_body["voms_proxy"] = voms_proxy
job_id = self.rjc_api_client.submit(**job_request_body)
log.info("submitted job:{0}".format(job_id))
message = {"job_id": str(job_id)}
workflow_uuid = os.getenv("workflow_uuid", "default")
status_running = 1
try:
publisher = REANAWorkflowStatusPublisher()
publisher.publish_workflow_status(workflow_uuid,
status_running,
message=message)
except Exception as e:
log.info("Status: workflow - {workflow_uuid} "
"status - {status} message - {message}".format(
workflow_uuid=workflow_uuid,
status=status_running,
message=message))
log.info("workflow status publish failed: {0}".format(e))
return ReanaExternalProxy(jobproxy=job_id,
spec=spec,
pardata=parameters,
statedata=state)
def result(self, resultproxy):
"""Retrieve the result of a packtivity run by RJC."""
resultproxy.pardata, resultproxy.statedata = finalize_inputs(
resultproxy.pardata, resultproxy.statedata)
return publish(
resultproxy.spec["publisher"],
resultproxy.pardata,
resultproxy.statedata,
self.config,
)
def _get_state(self, resultproxy):
"""Get the packtivity state."""
status_res = self.rjc_api_client.check_status(
resultproxy.jobproxy["job_id"])
return status_res["status"]
def ready(self, resultproxy):
"""Check if a packtivity is finished."""
return self._get_state(resultproxy) != "started"
def successful(self, resultproxy):
"""Check if the packtivity was successful."""
return self._get_state(resultproxy) == "succeeded"
def fail_info(self, resultproxy):
"""Retrieve the fail info."""
self._fail_info += "\nraw info: {}".format(resultproxy)
return self._fail_info