def result(self, resultproxy):
"""Retrieve the result of a packtivity run by RJC."""
resultproxy.pardata, resultproxy.statedata \
= finalize_inputs(resultproxy.pardata, resultproxy.statedata)
return publish(resultproxy.spec['publisher'], resultproxy.pardata,
resultproxy.statedata, self.config)
def test_docker_parmounts(tmpdir,basic_localfs_state, docker_env_parmounts):
state = basic_localfs_state
environment = docker_env_parmounts.spec['environment']
parameters, state = finalize_inputs(pdm.create({'outputfile': '{workdir}/hello.txt'}), state)
env = handlers[environment['environment_type']]['default'](environment,parameters,state)
assert env['par_mounts'][0]['mountcontent'] == '"{}"'.format(parameters['outputfile'])
def result(self, resultproxy: ReanaExternalProxy):
"""Retrieve the result of a packtivity run by RJC."""
resultproxy.pardata, resultproxy.statedata = finalize_inputs(
resultproxy.pardata, resultproxy.statedata)
return publish(
resultproxy.spec["publisher"],
resultproxy.pardata,
resultproxy.statedata,
self.config,
)
def test_docker_parmounts(tmpdir, basic_localfs_state, docker_env_parmounts):
state = basic_localfs_state
environment = docker_env_parmounts.spec['environment']
parameters, state = finalize_inputs(
pdm.create({'outputfile': '{workdir}/hello.txt'}), state)
env = handlers[environment['environment_type']]['default'](environment,
parameters,
state)
assert env['par_mounts'][0]['mountcontent'] == '"{}"'.format(
parameters['outputfile'])
def submit( # noqa: C901
self,
spec,
parameters,
state,
metadata # noqa: C901
) -> ReanaExternalProxy: # noqa: C901
"""Submit a yadage packtivity to RJC."""
parameters, state = finalize_inputs(parameters, state)
job = build_job(spec["process"], parameters, state, self.config)
log.debug(f"state context is {state}")
state.ensure()
if "command" in job:
prettified_cmd = wrapped_cmd = job["command"]
elif "script" in job:
prettified_cmd = job["script"]
wrapped_cmd = make_script(job)
image = spec["environment"]["image"]
imagetag = spec["environment"].get("imagetag", "")
if imagetag:
image = f"{image}:{imagetag}"
resources = spec["environment"].get("resources", [])
resources_parameters = self._get_resources(resources)
log.debug(f"would run job {job}")
workflow_uuid = os.getenv("workflow_uuid", "default")
job_request_body = {
"workflow_uuid": workflow_uuid,
"image": image,
"cmd": wrapped_cmd,
"prettified_cmd": prettified_cmd,
"workflow_workspace": os.getenv("workflow_workspace", "default"),
"job_name": metadata["name"],
"cvmfs_mounts": MOUNT_CVMFS,
**resources_parameters,
}
job_submit_response = self.rjc_api_client.submit(**job_request_body)
job_id = job_submit_response.get("job_id")
log.info(f"Submitted job with id: {job_id}")
return ReanaExternalProxy(jobproxy=job_submit_response,
spec=spec,
pardata=parameters,
statedata=state)
def submit(self, spec, parameters, state, metadata):
"""Submit a yadage packtivity to RJC."""
parameters, state = finalize_inputs(parameters, state)
job = build_job(spec['process'], parameters, state, self.config)
if 'command' in job:
prettified_cmd = job['command']
wrapped_cmd = make_oneliner(job)
elif 'script' in job:
prettified_cmd = job['script']
wrapped_cmd = make_script(job)
image = spec['environment']['image']
imagetag = spec['environment'].get('imagetag', '')
if imagetag:
image = image + ":" + imagetag
kerberos = None
compute_backend = None
kubernetes_uid = None
resources = spec['environment'].get('resources', None)
if resources:
for item in resources:
if 'kerberos' in item.keys():
kerberos = item['kerberos']
if 'compute_backend' in item.keys():
compute_backend = item['compute_backend']
if 'kubernetes_uid' in item.keys():
kubernetes_uid = item['kubernetes_uid']
log.info('state context is {0}'.format(state))
log.info('would run job {0}'.format(job))
state.ensure()
log.info('submitting!')
workflow_uuid = os.getenv('workflow_uuid', 'default')
job_request_body = {
'workflow_uuid': workflow_uuid,
'image': image,
'cmd': wrapped_cmd,
'prettified_cmd': prettified_cmd,
'workflow_workspace': os.getenv('workflow_workspace', 'default'),
'job_name': metadata['name'],
'cvmfs_mounts': MOUNT_CVMFS,
}
if compute_backend:
job_request_body['compute_backend'] = compute_backend
if kerberos:
job_request_body['kerberos'] = kerberos
if kubernetes_uid:
job_request_body['kubernetes_uid'] = kubernetes_uid
job_id = self.rjc_api_client.submit(**job_request_body)
log.info('submitted job:{0}'.format(job_id))
message = {"job_id": str(job_id)}
workflow_uuid = os.getenv('workflow_uuid', 'default')
status_running = 1
try:
publisher = REANAWorkflowStatusPublisher()
publisher.publish_workflow_status(workflow_uuid,
status_running,
message=message)
except Exception as e:
log.info('Status: workflow - {workflow_uuid} '
'status - {status} message - {message}'.format(
workflow_uuid=workflow_uuid,
status=status_running,
message=message))
log.info('workflow status publish failed: {0}'.format(e))
return ReanaExternalProxy(jobproxy=job_id,
spec=spec,
pardata=parameters,
statedata=state)
def submit(self, spec, parameters, state, metadata):
"""Submit a yadage packtivity to RJC."""
parameters, state = finalize_inputs(parameters, state)
job = build_job(spec["process"], parameters, state, self.config)
if "command" in job:
prettified_cmd = job["command"]
wrapped_cmd = make_oneliner(job)
elif "script" in job:
prettified_cmd = job["script"]
wrapped_cmd = make_script(job)
image = spec["environment"]["image"]
imagetag = spec["environment"].get("imagetag", "")
if imagetag:
image = image + ":" + imagetag
kerberos = None
compute_backend = None
kubernetes_uid = None
unpacked_img = None
voms_proxy = None
resources = spec["environment"].get("resources", None)
if resources:
for item in resources:
if "kerberos" in item.keys():
kerberos = item["kerberos"]
if "compute_backend" in item.keys():
compute_backend = item["compute_backend"]
if "kubernetes_uid" in item.keys():
kubernetes_uid = item["kubernetes_uid"]
if "unpacked_img" in item.keys():
unpacked_img = item["unpacked_img"]
if "voms_proxy" in item.keys():
voms_proxy = item["voms_proxy"]
log.info("state context is {0}".format(state))
log.info("would run job {0}".format(job))
state.ensure()
log.info("submitting!")
workflow_uuid = os.getenv("workflow_uuid", "default")
job_request_body = {
"workflow_uuid": workflow_uuid,
"image": image,
"cmd": wrapped_cmd,
"prettified_cmd": prettified_cmd,
"workflow_workspace": os.getenv("workflow_workspace", "default"),
"job_name": metadata["name"],
"cvmfs_mounts": MOUNT_CVMFS,
}
if compute_backend:
job_request_body["compute_backend"] = compute_backend
if kerberos:
job_request_body["kerberos"] = kerberos
if kubernetes_uid:
job_request_body["kubernetes_uid"] = kubernetes_uid
if unpacked_img:
job_request_body["unpacked_img"] = unpacked_img
if voms_proxy:
job_request_body["voms_proxy"] = voms_proxy
job_id = self.rjc_api_client.submit(**job_request_body)
log.info("submitted job:{0}".format(job_id))
message = {"job_id": str(job_id)}
workflow_uuid = os.getenv("workflow_uuid", "default")
status_running = 1
try:
publisher = REANAWorkflowStatusPublisher()
publisher.publish_workflow_status(workflow_uuid,
status_running,
message=message)
except Exception as e:
log.info("Status: workflow - {workflow_uuid} "
"status - {status} message - {message}".format(
workflow_uuid=workflow_uuid,
status=status_running,
message=message))
log.info("workflow status publish failed: {0}".format(e))
return ReanaExternalProxy(jobproxy=job_id,
spec=spec,
pardata=parameters,
statedata=state)
def submit(self, spec, parameters, state, metadata): # noqa: C901
"""Submit a yadage packtivity to RJC."""
parameters, state = finalize_inputs(parameters, state)
job = build_job(spec["process"], parameters, state, self.config)
if "command" in job:
prettified_cmd = wrapped_cmd = job["command"]
elif "script" in job:
prettified_cmd = job["script"]
wrapped_cmd = make_script(job)
image = spec["environment"]["image"]
imagetag = spec["environment"].get("imagetag", "")
if imagetag:
image = image + ":" + imagetag
kerberos = None
compute_backend = None
kubernetes_uid = None
kubernetes_memory_limit = None
unpacked_img = None
voms_proxy = None
htcondor_max_runtime = None
htcondor_accounting_group = None
resources = spec["environment"].get("resources", None)
if resources:
for item in resources:
if not isinstance(item, dict):
log.info(
'REANA only supports dictionary entries for resources. "{0}" value is not formatted in such a way and will be ignored.'
.format(item))
continue
if "kerberos" in item.keys():
kerberos = item["kerberos"]
if "compute_backend" in item.keys():
compute_backend = item["compute_backend"]
if "kubernetes_uid" in item.keys():
kubernetes_uid = item["kubernetes_uid"]
if "kubernetes_memory_limit" in item.keys():
kubernetes_memory_limit = item["kubernetes_memory_limit"]
if "unpacked_img" in item.keys():
unpacked_img = item["unpacked_img"]
if "voms_proxy" in item.keys():
voms_proxy = item["voms_proxy"]
if "htcondor_max_runtime" in item.keys():
htcondor_max_runtime = item["htcondor_max_runtime"]
if "htcondor_accounting_group" in item.keys():
htcondor_accounting_group = item[
"htcondor_accounting_group"]
log.info("state context is {0}".format(state))
log.info("would run job {0}".format(job))
state.ensure()
log.info("submitting!")
workflow_uuid = os.getenv("workflow_uuid", "default")
job_request_body = {
"workflow_uuid": workflow_uuid,
"image": image,
"cmd": wrapped_cmd,
"prettified_cmd": prettified_cmd,
"workflow_workspace": os.getenv("workflow_workspace", "default"),
"job_name": metadata["name"],
"cvmfs_mounts": MOUNT_CVMFS,
}
if compute_backend:
job_request_body["compute_backend"] = compute_backend
if kerberos:
job_request_body["kerberos"] = kerberos
if kubernetes_uid:
job_request_body["kubernetes_uid"] = kubernetes_uid
if kubernetes_memory_limit:
job_request_body[
"kubernetes_memory_limit"] = kubernetes_memory_limit
if unpacked_img:
job_request_body["unpacked_img"] = unpacked_img
if voms_proxy:
job_request_body["voms_proxy"] = voms_proxy
if htcondor_max_runtime:
job_request_body["htcondor_max_runtime"] = htcondor_max_runtime
if htcondor_accounting_group:
job_request_body[
"htcondor_accounting_group"] = htcondor_accounting_group
job_id = self.rjc_api_client.submit(**job_request_body)
log.info("submitted job:{0}".format(job_id))
message = {
"progress":
build_progress_message(running={
"total": 1,
"job_ids": [job_id.get("job_id")]
})
}
workflow_uuid = os.getenv("workflow_uuid", "default")
status_running = 1
try:
publisher = REANAWorkflowStatusPublisher()
publisher.publish_workflow_status(workflow_uuid,
status_running,
message=message)
except Exception as e:
log.info("Status: workflow - {workflow_uuid} "
"status - {status} message - {message}".format(
workflow_uuid=workflow_uuid,
status=status_running,
message=message))
log.info("workflow status publish failed: {0}".format(e))
return ReanaExternalProxy(jobproxy=job_id,
spec=spec,
pardata=parameters,
statedata=state)