def result(self, resultproxy): """Retrieve the result of a packtivity run by RJC.""" resultproxy.pardata, resultproxy.statedata \ = finalize_inputs(resultproxy.pardata, resultproxy.statedata) return publish(resultproxy.spec['publisher'], resultproxy.pardata, resultproxy.statedata, self.config)
def test_docker_parmounts(tmpdir,basic_localfs_state, docker_env_parmounts): state = basic_localfs_state environment = docker_env_parmounts.spec['environment'] parameters, state = finalize_inputs(pdm.create({'outputfile': '{workdir}/hello.txt'}), state) env = handlers[environment['environment_type']]['default'](environment,parameters,state) assert env['par_mounts'][0]['mountcontent'] == '"{}"'.format(parameters['outputfile'])
def result(self, resultproxy: ReanaExternalProxy): """Retrieve the result of a packtivity run by RJC.""" resultproxy.pardata, resultproxy.statedata = finalize_inputs( resultproxy.pardata, resultproxy.statedata) return publish( resultproxy.spec["publisher"], resultproxy.pardata, resultproxy.statedata, self.config, )
def test_docker_parmounts(tmpdir, basic_localfs_state, docker_env_parmounts): state = basic_localfs_state environment = docker_env_parmounts.spec['environment'] parameters, state = finalize_inputs( pdm.create({'outputfile': '{workdir}/hello.txt'}), state) env = handlers[environment['environment_type']]['default'](environment, parameters, state) assert env['par_mounts'][0]['mountcontent'] == '"{}"'.format( parameters['outputfile'])
def submit( # noqa: C901 self, spec, parameters, state, metadata # noqa: C901 ) -> ReanaExternalProxy: # noqa: C901 """Submit a yadage packtivity to RJC.""" parameters, state = finalize_inputs(parameters, state) job = build_job(spec["process"], parameters, state, self.config) log.debug(f"state context is {state}") state.ensure() if "command" in job: prettified_cmd = wrapped_cmd = job["command"] elif "script" in job: prettified_cmd = job["script"] wrapped_cmd = make_script(job) image = spec["environment"]["image"] imagetag = spec["environment"].get("imagetag", "") if imagetag: image = f"{image}:{imagetag}" resources = spec["environment"].get("resources", []) resources_parameters = self._get_resources(resources) log.debug(f"would run job {job}") workflow_uuid = os.getenv("workflow_uuid", "default") job_request_body = { "workflow_uuid": workflow_uuid, "image": image, "cmd": wrapped_cmd, "prettified_cmd": prettified_cmd, "workflow_workspace": os.getenv("workflow_workspace", "default"), "job_name": metadata["name"], "cvmfs_mounts": MOUNT_CVMFS, **resources_parameters, } job_submit_response = self.rjc_api_client.submit(**job_request_body) job_id = job_submit_response.get("job_id") log.info(f"Submitted job with id: {job_id}") return ReanaExternalProxy(jobproxy=job_submit_response, spec=spec, pardata=parameters, statedata=state)
def submit(self, spec, parameters, state, metadata): """Submit a yadage packtivity to RJC.""" parameters, state = finalize_inputs(parameters, state) job = build_job(spec['process'], parameters, state, self.config) if 'command' in job: prettified_cmd = job['command'] wrapped_cmd = make_oneliner(job) elif 'script' in job: prettified_cmd = job['script'] wrapped_cmd = make_script(job) image = spec['environment']['image'] imagetag = spec['environment'].get('imagetag', '') if imagetag: image = image + ":" + imagetag kerberos = None compute_backend = None kubernetes_uid = None resources = spec['environment'].get('resources', None) if resources: for item in resources: if 'kerberos' in item.keys(): kerberos = item['kerberos'] if 'compute_backend' in item.keys(): compute_backend = item['compute_backend'] if 'kubernetes_uid' in item.keys(): kubernetes_uid = item['kubernetes_uid'] log.info('state context is {0}'.format(state)) log.info('would run job {0}'.format(job)) state.ensure() log.info('submitting!') workflow_uuid = os.getenv('workflow_uuid', 'default') job_request_body = { 'workflow_uuid': workflow_uuid, 'image': image, 'cmd': wrapped_cmd, 'prettified_cmd': prettified_cmd, 'workflow_workspace': os.getenv('workflow_workspace', 'default'), 'job_name': metadata['name'], 'cvmfs_mounts': MOUNT_CVMFS, } if compute_backend: job_request_body['compute_backend'] = compute_backend if kerberos: job_request_body['kerberos'] = kerberos if kubernetes_uid: job_request_body['kubernetes_uid'] = kubernetes_uid job_id = self.rjc_api_client.submit(**job_request_body) log.info('submitted job:{0}'.format(job_id)) message = {"job_id": str(job_id)} workflow_uuid = os.getenv('workflow_uuid', 'default') status_running = 1 try: publisher = REANAWorkflowStatusPublisher() publisher.publish_workflow_status(workflow_uuid, status_running, message=message) except Exception as e: log.info('Status: workflow - {workflow_uuid} ' 'status - {status} message - {message}'.format( workflow_uuid=workflow_uuid, status=status_running, message=message)) log.info('workflow status publish failed: {0}'.format(e)) return ReanaExternalProxy(jobproxy=job_id, spec=spec, pardata=parameters, statedata=state)
def submit(self, spec, parameters, state, metadata): """Submit a yadage packtivity to RJC.""" parameters, state = finalize_inputs(parameters, state) job = build_job(spec["process"], parameters, state, self.config) if "command" in job: prettified_cmd = job["command"] wrapped_cmd = make_oneliner(job) elif "script" in job: prettified_cmd = job["script"] wrapped_cmd = make_script(job) image = spec["environment"]["image"] imagetag = spec["environment"].get("imagetag", "") if imagetag: image = image + ":" + imagetag kerberos = None compute_backend = None kubernetes_uid = None unpacked_img = None voms_proxy = None resources = spec["environment"].get("resources", None) if resources: for item in resources: if "kerberos" in item.keys(): kerberos = item["kerberos"] if "compute_backend" in item.keys(): compute_backend = item["compute_backend"] if "kubernetes_uid" in item.keys(): kubernetes_uid = item["kubernetes_uid"] if "unpacked_img" in item.keys(): unpacked_img = item["unpacked_img"] if "voms_proxy" in item.keys(): voms_proxy = item["voms_proxy"] log.info("state context is {0}".format(state)) log.info("would run job {0}".format(job)) state.ensure() log.info("submitting!") workflow_uuid = os.getenv("workflow_uuid", "default") job_request_body = { "workflow_uuid": workflow_uuid, "image": image, "cmd": wrapped_cmd, "prettified_cmd": prettified_cmd, "workflow_workspace": os.getenv("workflow_workspace", "default"), "job_name": metadata["name"], "cvmfs_mounts": MOUNT_CVMFS, } if compute_backend: job_request_body["compute_backend"] = compute_backend if kerberos: job_request_body["kerberos"] = kerberos if kubernetes_uid: job_request_body["kubernetes_uid"] = kubernetes_uid if unpacked_img: job_request_body["unpacked_img"] = unpacked_img if voms_proxy: job_request_body["voms_proxy"] = voms_proxy job_id = self.rjc_api_client.submit(**job_request_body) log.info("submitted job:{0}".format(job_id)) message = {"job_id": str(job_id)} workflow_uuid = os.getenv("workflow_uuid", "default") status_running = 1 try: publisher = REANAWorkflowStatusPublisher() publisher.publish_workflow_status(workflow_uuid, status_running, message=message) except Exception as e: log.info("Status: workflow - {workflow_uuid} " "status - {status} message - {message}".format( workflow_uuid=workflow_uuid, status=status_running, message=message)) log.info("workflow status publish failed: {0}".format(e)) return ReanaExternalProxy(jobproxy=job_id, spec=spec, pardata=parameters, statedata=state)
def submit(self, spec, parameters, state, metadata): # noqa: C901 """Submit a yadage packtivity to RJC.""" parameters, state = finalize_inputs(parameters, state) job = build_job(spec["process"], parameters, state, self.config) if "command" in job: prettified_cmd = wrapped_cmd = job["command"] elif "script" in job: prettified_cmd = job["script"] wrapped_cmd = make_script(job) image = spec["environment"]["image"] imagetag = spec["environment"].get("imagetag", "") if imagetag: image = image + ":" + imagetag kerberos = None compute_backend = None kubernetes_uid = None kubernetes_memory_limit = None unpacked_img = None voms_proxy = None htcondor_max_runtime = None htcondor_accounting_group = None resources = spec["environment"].get("resources", None) if resources: for item in resources: if not isinstance(item, dict): log.info( 'REANA only supports dictionary entries for resources. "{0}" value is not formatted in such a way and will be ignored.' .format(item)) continue if "kerberos" in item.keys(): kerberos = item["kerberos"] if "compute_backend" in item.keys(): compute_backend = item["compute_backend"] if "kubernetes_uid" in item.keys(): kubernetes_uid = item["kubernetes_uid"] if "kubernetes_memory_limit" in item.keys(): kubernetes_memory_limit = item["kubernetes_memory_limit"] if "unpacked_img" in item.keys(): unpacked_img = item["unpacked_img"] if "voms_proxy" in item.keys(): voms_proxy = item["voms_proxy"] if "htcondor_max_runtime" in item.keys(): htcondor_max_runtime = item["htcondor_max_runtime"] if "htcondor_accounting_group" in item.keys(): htcondor_accounting_group = item[ "htcondor_accounting_group"] log.info("state context is {0}".format(state)) log.info("would run job {0}".format(job)) state.ensure() log.info("submitting!") workflow_uuid = os.getenv("workflow_uuid", "default") job_request_body = { "workflow_uuid": workflow_uuid, "image": image, "cmd": wrapped_cmd, "prettified_cmd": prettified_cmd, "workflow_workspace": os.getenv("workflow_workspace", "default"), "job_name": metadata["name"], "cvmfs_mounts": MOUNT_CVMFS, } if compute_backend: job_request_body["compute_backend"] = compute_backend if kerberos: job_request_body["kerberos"] = kerberos if kubernetes_uid: job_request_body["kubernetes_uid"] = kubernetes_uid if kubernetes_memory_limit: job_request_body[ "kubernetes_memory_limit"] = kubernetes_memory_limit if unpacked_img: job_request_body["unpacked_img"] = unpacked_img if voms_proxy: job_request_body["voms_proxy"] = voms_proxy if htcondor_max_runtime: job_request_body["htcondor_max_runtime"] = htcondor_max_runtime if htcondor_accounting_group: job_request_body[ "htcondor_accounting_group"] = htcondor_accounting_group job_id = self.rjc_api_client.submit(**job_request_body) log.info("submitted job:{0}".format(job_id)) message = { "progress": build_progress_message(running={ "total": 1, "job_ids": [job_id.get("job_id")] }) } workflow_uuid = os.getenv("workflow_uuid", "default") status_running = 1 try: publisher = REANAWorkflowStatusPublisher() publisher.publish_workflow_status(workflow_uuid, status_running, message=message) except Exception as e: log.info("Status: workflow - {workflow_uuid} " "status - {status} message - {message}".format( workflow_uuid=workflow_uuid, status=status_running, message=message)) log.info("workflow status publish failed: {0}".format(e)) return ReanaExternalProxy(jobproxy=job_id, spec=spec, pardata=parameters, statedata=state)