def stage_files(self):
cwl_job = self.cwl_job()
def stageFunc(resolved_path, target_path):
log.info("resolving %s to %s" % (resolved_path, target_path))
try:
os.symlink(resolved_path, target_path)
except OSError:
pass
if hasattr(cwl_job, "pathmapper"):
process.stageFiles(cwl_job.pathmapper,
stageFunc,
ignoreWritable=True,
symLink=False)
if hasattr(cwl_job, "generatefiles"):
outdir = os.path.join(self._job_directory, "working")
# TODO: Why doesn't cwl_job.generatemapper work?
generate_mapper = pathmapper.PathMapper(
cwl_job.generatefiles["listing"],
outdir,
outdir,
separateDirs=False)
# TODO: figure out what inplace_update should be.
inplace_update = getattr(cwl_job, "inplace_update")
process.stageFiles(generate_mapper,
stageFunc,
ignoreWritable=inplace_update,
symLink=False)
from cwltool import job
job.relink_initialworkdir(generate_mapper,
outdir,
outdir,
inplace_update=inplace_update)
def run(self,
runtimeContext, # type: RuntimeContext
tmpdir_lock=None, # type: Optional[ThreadLock]
): # type: (...) -> None
make_dirs(self.tmpdir, exist_ok=True)
env = self.environment
vars_to_preserve = runtimeContext.preserve_environment
if runtimeContext.preserve_entire_environment:
vars_to_preserve = os.environ
if vars_to_preserve is not None:
for key, value in os.environ.items():
if key in vars_to_preserve and key not in env:
# On Windows, subprocess env can't handle unicode.
env[key] = str(value) if onWindows() else value
env["HOME"] = str(self.outdir) if onWindows() else self.outdir
env["TMPDIR"] = str(self.tmpdir) if onWindows() else self.tmpdir
if "PATH" not in env:
env["PATH"] = str(os.environ["PATH"]) if onWindows() else os.environ["PATH"]
if "SYSTEMROOT" not in env and "SYSTEMROOT" in os.environ:
env["SYSTEMROOT"] = str(os.environ["SYSTEMROOT"]) if onWindows() else os.environ["SYSTEMROOT"]
# stageFiles(self.pathmapper, ignoreWritable=True, symLink=True, secret_store=runtimeContext.secret_store)
if self.generatemapper:
# FIXME: see if this is needed... func doesn't exist anymore in cwltool 2.x
# stageFiles(self.generatemapper, ignoreWritable=self.inplace_update,
# symLink=True, secret_store=runtimeContext.secret_store)
relink_initialworkdir(self.generatemapper, self.outdir,
self.builder.outdir, inplace_update=self.inplace_update)
self.execute([], env, runtimeContext)
def stage_files(self):
cwl_job = self.cwl_job()
def stageFunc(resolved_path, target_path):
log.info("resolving %s to %s" % (resolved_path, target_path))
try:
os.symlink(resolved_path, target_path)
except OSError:
pass
if hasattr(cwl_job, "pathmapper"):
process.stageFiles(cwl_job.pathmapper, stageFunc, ignoreWritable=True, symLink=False)
if hasattr(cwl_job, "generatefiles"):
outdir = os.path.join(self._job_directory, "working")
# TODO: Why doesn't cwl_job.generatemapper work?
generate_mapper = pathmapper.PathMapper(cwl_job.generatefiles["listing"],
outdir, outdir, separateDirs=False)
# TODO: figure out what inplace_update should be.
inplace_update = getattr(cwl_job, "inplace_update")
process.stageFiles(generate_mapper, stageFunc, ignoreWritable=inplace_update, symLink=False)
from cwltool import job
job.relink_initialworkdir(generate_mapper, outdir, outdir, inplace_update=inplace_update)
def execute(self, runtime, env, runtime_context): # noqa: E811
# type: (List[Text], MutableMapping[Text, Text], RuntimeContext) -> None
self.results = self.wps_process.execute(self.builder.job, self.outdir,
self.expected_outputs)
if self.joborder and runtime_context.research_obj:
job_order = self.joborder
assert runtime_context.prov_obj
assert runtime_context.process_run_id
runtime_context.prov_obj.used_artefacts(
job_order, runtime_context.process_run_id, str(self.name))
outputs = {} # type: Dict[Text, Text]
try:
rcode = 0
if self.successCodes:
process_status = "success"
elif self.temporaryFailCodes:
process_status = "temporaryFail"
elif self.permanentFailCodes:
process_status = "permanentFail"
elif rcode == 0:
process_status = "success"
else:
process_status = "permanentFail"
if self.generatefiles["listing"]:
assert self.generatemapper is not None
relink_initialworkdir(self.generatemapper,
self.outdir,
self.builder.outdir,
inplace_update=self.inplace_update)
outputs = self.collect_outputs(self.outdir)
outputs = bytes2str_in_dicts(outputs) # type: ignore
except OSError as exc:
if exc.errno == 2:
if runtime:
LOGGER.exception(u"'%s' not found", runtime[0])
else:
LOGGER.exception(u"'%s' not found", self.command_line[0])
else:
LOGGER.exception("Exception while running job")
process_status = "permanentFail"
except WorkflowException as err:
LOGGER.exception(u"[job %s] Job error:\n%s", self.name, err)
process_status = "permanentFail"
except Exception: # noqa: W0703 # nosec: B110
LOGGER.exception("Exception while running job")
process_status = "permanentFail"
if runtime_context.research_obj and self.prov_obj and \
runtime_context.process_run_id:
# creating entities for the outputs produced by each step (in the provenance document)
self.prov_obj.generate_output_prov(outputs,
runtime_context.process_run_id,
str(self.name))
self.prov_obj.document.wasEndedBy(runtime_context.process_run_id,
None,
self.prov_obj.workflow_run_uri,
now())
if process_status != "success":
LOGGER.warning(u"[job %s] completed %s", self.name, process_status)
else:
LOGGER.info(u"[job %s] completed %s", self.name, process_status)
if LOGGER.isEnabledFor(logging.DEBUG):
LOGGER.debug(u"[job %s] %s", self.name,
json.dumps(outputs, indent=4))
if self.generatemapper and runtime_context.secret_store:
# Delete any runtime-generated files containing secrets.
for _, path_item in self.generatemapper.items():
if path_item.type == "CreateFile":
if runtime_context.secret_store.has_secret(
path_item.resolved):
host_outdir = self.outdir
container_outdir = self.builder.outdir
host_outdir_tgt = path_item.target
if path_item.target.startswith(container_outdir + "/"):
host_outdir_tgt = os.path.join(
host_outdir,
path_item.target[len(container_outdir) + 1:])
os.remove(host_outdir_tgt)
if runtime_context.workflow_eval_lock is None:
raise WorkflowException(
"runtime_context.workflow_eval_lock must not be None")
with runtime_context.workflow_eval_lock:
self.output_callback(outputs, process_status)
if self.stagedir and os.path.exists(self.stagedir):
LOGGER.debug(u"[job %s] Removing input staging directory %s",
self.name, self.stagedir)
shutil.rmtree(self.stagedir, True)
if runtime_context.rm_tmpdir:
LOGGER.debug(u"[job %s] Removing temporary directory %s",
self.name, self.tmpdir)
shutil.rmtree(self.tmpdir, True)
def run(self, runtimeContext): # noqa: C901
"""Run a job."""
self._setup(runtimeContext)
env = self.environment
if not os.path.exists(self.tmpdir):
os.makedirs(self.tmpdir)
vars_to_preserve = runtimeContext.preserve_environment
if runtimeContext.preserve_entire_environment:
vars_to_preserve = os.environ
if vars_to_preserve is not None:
for key, value in os.environ.items():
if key in vars_to_preserve and key not in env:
env[key] = value
env["HOME"] = self.builder.outdir
env["TMPDIR"] = self.tmpdir
if "PATH" not in env:
env["PATH"] = os.environ["PATH"]
if "SYSTEMROOT" not in env and "SYSTEMROOT" in os.environ:
env["SYSTEMROOT"] = os.environ["SYSTEMROOT"]
try:
stage_files(self.pathmapper, ignore_writable=True, symlink=False)
if getattr(self, "generatemapper", ""):
stage_files(
self.generatemapper,
ignore_writable=self.inplace_update,
symlink=False,
)
relink_initialworkdir(
self.generatemapper,
self.outdir,
self.builder.outdir,
inplace_update=self.inplace_update,
)
except OSError:
# cwltool/process.py, line 239, in stage_files
# shutil.copytree(p.resolved, p.target)
pass
self.add_volumes(self.pathmapper)
if getattr(self, "generatemapper", ""):
self.add_volumes(self.generatemapper)
# useful for debugging
log.debug(f"[job {self.name}] self.__dict__ in run() ---------------")
log.debug(pformat(self.__dict__))
task = self.create_task_msg(
runtimeContext.working_dir, runtimeContext.workflow_uuid
)
log.info(f"[job {self.name}] CREATED TASK MSG----------------------")
log.info(pformat(task))
try:
# task_id = job_id received from job-controller
task_id = runtimeContext.pipeline.service.submit(**task)
task_id = str(task_id["job_id"])
running_jobs = {"total": 1, "job_ids": [task_id]}
runtimeContext.publisher.publish_workflow_status(
runtimeContext.workflow_uuid,
1,
message={"progress": {"running": running_jobs,}},
)
log.info(f"[job {self.name}] SUBMITTED TASK --------------------")
log.info(f"[job {self.name}] task id: {task_id} ")
self.task_name_map[self.name] = task_id
operation = runtimeContext.pipeline.service.check_status(task_id)
except Exception as e:
log.error(
f"[job {self.name}] " f"Failed to submit task to job controller:\n{e}"
)
raise WorkflowException(e)
def callback(rcode):
try:
outputs = self.collect_outputs(self.outdir, rcode=rcode)
cleaned_outputs = {}
for k, v in outputs.items():
if isinstance(k, bytes):
k = k.decode("utf8")
if isinstance(v, bytes):
v = v.decode("utf8")
cleaned_outputs[k] = v
self.outputs = cleaned_outputs
self.output_callback(self.outputs, "success")
except WorkflowException as e:
log.error(f"[job {self.name}] workflow job error:\n{e}")
self.output_callback({}, "permanentFail")
except Exception as e:
log.error(f"[job {self.name}] job error:\n{e}")
self.output_callback({}, "permanentFail")
finally:
if self.outputs is not None:
log.info(f"[job {self.name}] OUTPUTS ------------------")
log.info(pformat(self.outputs))
self.cleanup(runtimeContext.rm_tmpdir)
poll = ReanaPipelinePoll(
workflow_uuid=runtimeContext.workflow_uuid,
task_id=self.task_name_map.get(self.name),
jobname=self.name,
service=runtimeContext.pipeline.service,
operation=operation,
callback=callback,
publisher=runtimeContext.publisher,
)
runtimeContext.pipeline.add_thread(poll)
poll.start()
