def executor(self, tool, job_order, **kwargs): final_output = [] final_status = [] def output_callback(out, processStatus): final_status.append(processStatus) final_output.append(out) if "basedir" not in kwargs: raise WorkflowException("Must provide 'basedir' in kwargs") output_dirs = set() if kwargs.get("outdir"): finaloutdir = os.path.abspath(kwargs.get("outdir")) else: finaloutdir = None if kwargs.get("tmp_outdir_prefix"): kwargs["outdir"] = tempfile.mkdtemp( prefix=kwargs["tmp_outdir_prefix"]) else: kwargs["outdir"] = tempfile.mkdtemp() output_dirs.add(kwargs["outdir"]) kwargs["mutation_manager"] = MutationManager() jobReqs = None if "cwl:requirements" in job_order: jobReqs = job_order["cwl:requirements"] elif ("cwl:defaults" in tool.metadata and "cwl:requirements" in tool.metadata["cwl:defaults"]): jobReqs = tool.metadata["cwl:defaults"]["cwl:requirements"] if jobReqs: for req in jobReqs: tool.requirements.append(req) if kwargs.get("default_container"): tool.requirements.insert( 0, { "class": "DockerRequirement", "dockerPull": kwargs["default_container"] }) jobs = tool.job(job_order, output_callback, **kwargs) try: for runnable in jobs: if runnable: builder = kwargs.get("builder", None) if builder is not None: runnable.builder = builder if runnable.outdir: output_dirs.add(runnable.outdir) runnable.run(**kwargs) else: time.sleep(1) except WorkflowException as e: raise e except Exception as e: log.error("Got exception") raise WorkflowException(str(e)) # wait for all processes to finish self.wait() if final_output and final_output[0] and finaloutdir: final_output[0] = relocateOutputs(final_output[0], finaloutdir, output_dirs, kwargs.get("move_outputs"), kwargs["make_fs_access"]("")) if kwargs.get("rm_tmpdir"): cleanIntermediate(output_dirs) if final_output and final_status: return (final_output[0], final_status[0]) else: return (None, "permanentFail")
def executor(self, tool, job_order, runtimeContext, **kwargs): """Executor method.""" final_output = [] final_status = [] def output_callback(out, processStatus): final_status.append(processStatus) final_output.append(out) if not runtimeContext.basedir: raise WorkflowException('`runtimeContext` should contain a ' '`basedir`') output_dirs = set() if runtimeContext.outdir: finaloutdir = os.path.abspath(runtimeContext.outdir) else: finaloutdir = None if runtimeContext.tmp_outdir_prefix: runtimeContext.outdir = tempfile.mkdtemp( prefix=runtimeContext.tmp_outdir_prefix) else: runtimeContext.outdir = tempfile.mkdtemp() output_dirs.add(runtimeContext.outdir) runtimeContext.mutation_manager = MutationManager() jobReqs = None if "cwl:requirements" in job_order: jobReqs = job_order["cwl:requirements"] elif ("cwl:defaults" in tool.metadata and "cwl:requirements" in tool.metadata["cwl:defaults"]): jobReqs = tool.metadata["cwl:defaults"]["cwl:requirements"] if jobReqs: for req in jobReqs: tool.requirements.append(req) if not runtimeContext.default_container: runtimeContext.default_container = 'frolvlad/alpine-bash' runtimeContext.docker_outdir = os.path.join(runtimeContext.working_dir, "cwl/docker_outdir") runtimeContext.docker_tmpdir = os.path.join(runtimeContext.working_dir, "cwl/docker_tmpdir") runtimeContext.docker_stagedir = os.path.join( runtimeContext.working_dir, "cwl/docker_stagedir") jobs = tool.job(job_order, output_callback, runtimeContext) try: for runnable in jobs: if runnable: if runtimeContext.builder: runnable.builder = runtimeContext.builder if runnable.outdir: output_dirs.add(runnable.outdir) runnable.run(runtimeContext) else: # log.error( # "Workflow cannot make any more progress" # ) # break time.sleep(1) except WorkflowException as e: traceback.print_exc() raise e except Exception as e: traceback.print_exc() raise WorkflowException(str(e)) # wait for all processes to finish self.wait() if final_output and final_output[0] and finaloutdir: final_output[0] = relocateOutputs( final_output[0], finaloutdir, output_dirs, runtimeContext.move_outputs, runtimeContext.make_fs_access("")) if runtimeContext.rm_tmpdir: cleanIntermediate(output_dirs) if final_output and final_status: return str(final_output[0]), str(final_status[0]) else: return None, "permanentFail"