def relocate_outputs(workflow, job_data, cwl_args=None): """ Moves or copies filtered outputs to "outputs_folder" depending on "runtime_context.move_outputs" value, however "tmp_folder" is not going to be deleted as it will be done when DAG finishes running. Saves report with relocated outputs as "workflow_report.json" to "outputs_folder". Maps outputs from "workflow" back to normal (from step_id_step_out to workflow output) and filters "job_data" based on them (combining items from "job_data" into a list based on "outputSource" if it was a list). "cwl_args" can be used to update default parameters used for loading and runtime contexts. """ cwl_args = {} if cwl_args is None else cwl_args default_cwl_args = get_default_cwl_args(cwl_args) workflow_tool = fast_cwl_load(workflow=workflow, cwl_args=default_cwl_args) # Filter "job_data" to include only items required by workflow outputs. # Remap keys to the proper workflow outputs IDs (without step id). # If "outputSource" was a list even of len=1, find all correspondent items # from the "job_data" and assign them as list of the same size. job_data_copy = deepcopy(job_data) filtered_job_data = {} for output_id, output_data in get_items(workflow_tool["outputs"]): collected_job_items = [] for source_id, _ in get_items(output_data["outputSource"]): collected_job_items.append(job_data_copy[source_id.replace( "/", "_")]) if isinstance(output_data["outputSource"], list): filtered_job_data[output_id] = collected_job_items else: filtered_job_data[output_id] = collected_job_items[0] runtime_context = RuntimeContext(default_cwl_args) relocated_job_data = relocateOutputs( outputObj=filtered_job_data, destination_path=job_data_copy["outputs_folder"], source_directories=[ job_data_copy["tmp_folder"] ], # need to set it to tmp_folder otherwise it won't be able to delete tmp data if action is "move" action=runtime_context.move_outputs, fs_access=runtime_context.make_fs_access(""), compute_checksum=runtime_context.compute_checksum, path_mapper=runtime_context.path_mapper) # Dump report with relocated outputs workflow_report = os.path.join(job_data_copy["outputs_folder"], "workflow_report.json") dump_json(relocated_job_data, workflow_report) return relocated_job_data, workflow_report
def execute(self, context): post_status(context) self.cwlwf, it_is_workflow = load_cwl( self.dag.default_args["cwl_workflow"], self.dag.default_args) self.cwl_step = [ step for step in self.cwlwf.steps if self.task_id == step.id.split("#")[-1] ][0] if it_is_workflow else self.cwlwf _logger.info('{0}: Running!'.format(self.task_id)) upstream_task_ids = [t.task_id for t in self.upstream_list] + \ ([self.reader_task_id] if self.reader_task_id else []) _logger.debug('{0}: Collecting outputs from: \n{1}'.format( self.task_id, json.dumps(upstream_task_ids, indent=4))) upstream_data = self.xcom_pull(context=context, task_ids=upstream_task_ids) _logger.info('{0}: Upstream data: \n {1}'.format( self.task_id, json.dumps(upstream_data, indent=4))) promises = {} for data in upstream_data: # upstream_data is an array with { promises and outdir } promises = merge(promises, data["promises"]) if "outdir" in data: self.outdir = data["outdir"] _d_args = self.dag.default_args if not self.outdir: self.outdir = _d_args['tmp_folder'] _logger.debug('{0}: Step inputs: {1}'.format( self.task_id, json.dumps(self.cwl_step.tool["inputs"], indent=4))) _logger.debug('{0}: Step outputs: {1}'.format( self.task_id, json.dumps(self.cwl_step.tool["outputs"], indent=4))) jobobj = {} for inp in self.cwl_step.tool["inputs"]: jobobj_id = shortname(inp["id"]).split("/")[-1] source_ids = [] promises_outputs = [] try: source_field = inp["source"] if it_is_workflow else inp.get( "id") source_ids = [shortname(s) for s in source_field] if isinstance( source_field, list) else [shortname(source_field)] promises_outputs = [ promises[source_id] for source_id in source_ids if source_id in promises ] except: _logger.warning( "{0}: Couldn't find source field in step input: {1}". format(self.task_id, json.dumps(inp, indent=4))) _logger.info( '{0}: For input {1} with source_ids: {2} found upstream outputs: \n{3}' .format(self.task_id, jobobj_id, source_ids, promises_outputs)) if len(promises_outputs) > 1: if inp.get("linkMerge", "merge_nested") == "merge_flattened": jobobj[jobobj_id] = flatten(promises_outputs) else: jobobj[jobobj_id] = promises_outputs # Should also check if [None], because in this case we need to take default value elif len(promises_outputs) == 1 and (promises_outputs[0] is not None): jobobj[jobobj_id] = promises_outputs[0] elif "valueFrom" in inp: jobobj[jobobj_id] = None elif "default" in inp: d = copy.copy(inp["default"]) jobobj[jobobj_id] = d else: continue _logger.debug('{0}: Collected job object: \n {1}'.format( self.task_id, json.dumps(jobobj, indent=4))) def _post_scatter_eval(shortio, cwl_step): _value_from = { shortname(i["id"]).split("/")[-1]: i["valueFrom"] for i in cwl_step.tool["inputs"] if "valueFrom" in i } _logger.debug('{0}: Step inputs with valueFrom: \n{1}'.format( self.task_id, json.dumps(_value_from, indent=4))) def value_from_func(k, v): if k in _value_from: return expression.do_eval(_value_from[k], shortio, self.cwlwf.tool.get( "requirements", []), None, None, {}, context=v) else: return v return {k: value_from_func(k, v) for k, v in shortio.items()} job = _post_scatter_eval(jobobj, self.cwl_step) _logger.info('{0}: Final job data: \n {1}'.format( self.task_id, json.dumps(job, indent=4))) _d_args['outdir'] = tempfile.mkdtemp( prefix=os.path.join(self.outdir, "step_tmp")) _d_args['tmpdir_prefix'] = os.path.join(_d_args['outdir'], 'cwl_tmp_') _d_args['tmp_outdir_prefix'] = os.path.join(_d_args['outdir'], 'cwl_outdir_') _d_args["record_container_id"] = True _d_args["cidfile_dir"] = _d_args['outdir'] _d_args["cidfile_prefix"] = self.task_id _logger.debug('{0}: Runtime context: \n {1}'.format(self, _d_args)) executor = SingleJobExecutor() runtimeContext = RuntimeContext(_d_args) runtimeContext.make_fs_access = getdefault( runtimeContext.make_fs_access, StdFsAccess) for inp in self.cwl_step.tool["inputs"]: if inp.get("not_connected"): del job[shortname(inp["id"].split("/")[-1])] _stderr = sys.stderr sys.stderr = sys.__stderr__ (output, status) = executor( self.cwl_step.embedded_tool if it_is_workflow else self.cwl_step, job, runtimeContext, logger=_logger) sys.stderr = _stderr if not output and status == "permanentFail": raise ValueError _logger.debug('{0}: Embedded tool outputs: \n {1}'.format( self.task_id, json.dumps(output, indent=4))) promises = {} for out in self.cwl_step.tool["outputs"]: out_id = shortname(out["id"]) jobout_id = out_id.split("/")[-1] try: promises[out_id] = output[jobout_id] except: continue # Unsetting the Generation from final output object visit_class(promises, ("File", ), MutationManager().unset_generation) data = {"promises": promises, "outdir": self.outdir} _logger.info('{0}: Output: \n {1}'.format(self.task_id, json.dumps(data, indent=4))) return data
def relocate_outputs(workflow, job_data, cwl_args=None, remove_tmp_folder=None): """ Relocates filtered outputs to "outputs_folder" and, by default, removes tmp_folder, unless "remove_tmp_folder" is set to something else. Saves report with relocated outputs as "workflow_report.json" to "outputs_folder". Maps outputs from "workflow" back to normal (from step_id_step_out to workflow output) and filters "job_data" based on them (combining items from "job_data" into a list based on "outputSource" if it was a list). "cwl_args" can be used to update default parameters used for loading and runtime contexts. """ cwl_args = {} if cwl_args is None else cwl_args remove_tmp_folder = True if remove_tmp_folder is None else remove_tmp_folder default_cwl_args = get_default_cwl_args(cwl_args) workflow_tool = fast_cwl_load(workflow=workflow, cwl_args=default_cwl_args) # Filter "job_data" to include only items required by workflow outputs. # Remap keys to the proper workflow outputs IDs (without step id). # If "outputSource" was a list even of len=1, find all correspondent items # from the "job_data" and assign them as list of the same size. job_data_copy = deepcopy(job_data) filtered_job_data = {} for output_id, output_data in get_items(workflow_tool["outputs"]): collected_job_items = [] for source_id, _ in get_items(output_data["outputSource"]): collected_job_items.append(job_data_copy[source_id.replace( "/", "_")]) if isinstance(output_data["outputSource"], list): filtered_job_data[output_id] = collected_job_items else: filtered_job_data[output_id] = collected_job_items[0] # Outputs will be always copied, because source_directories=[] runtime_context = RuntimeContext(default_cwl_args) relocated_job_data = relocateOutputs( outputObj=filtered_job_data, destination_path=job_data_copy["outputs_folder"], source_directories= [], # use it as a placeholder (shouldn't influence anything) action=runtime_context.move_outputs, fs_access=runtime_context.make_fs_access(""), compute_checksum=runtime_context.compute_checksum, path_mapper=runtime_context.path_mapper) # Dump report with relocated outputs workflow_report = os.path.join(job_data_copy["outputs_folder"], "workflow_report.json") dump_json(relocated_job_data, workflow_report) # Clean "tmp_folder" if remove_tmp_folder: shutil.rmtree(job_data_copy["tmp_folder"], ignore_errors=False) return relocated_job_data, workflow_report
def cwlmain( self, argsl=None, # type: List[str] args=None, # type: argparse.Namespace job_order_object=None, # type: MutableMapping[Text, Any] stdin=sys.stdin, # type: IO[Any] stdout=None, # type: Union[TextIO, codecs.StreamWriter] stderr=sys.stderr, # type: IO[Any] versionfunc=versionstring, # type: Callable[[], Text] logger_handler=None, # custom_schema_callback=None, # type: Callable[[], None] executor=None, # type: Callable[..., Tuple[Dict[Text, Any], Text]] loadingContext=None, # type: LoadingContext runtimeContext=None # type: RuntimeContext ): # type: (...) -> int if not stdout: stdout = codecs.getwriter('utf-8')(sys.stdout) _logger.removeHandler(defaultStreamHandler) if logger_handler: stderr_handler = logger_handler else: stderr_handler = logging.StreamHandler(stderr) _logger.addHandler(stderr_handler) try: if args is None: args = arg_parser().parse_args(argsl) if args.workflow and "--outdir" not in argsl: outputPath = args.workflow.split('/')[-1].split('.')[0] setattr( args, "outdir", os.getcwd() + "/" + outputPath + "/" + datetime.datetime.now().strftime('%Y-%m-%d-%H%M')) if runtimeContext is None: runtimeContext = RuntimeContext(vars(args)) else: runtimeContext = runtimeContext.copy() rdflib_logger = logging.getLogger("rdflib.term") rdflib_logger.addHandler(stderr_handler) rdflib_logger.setLevel(logging.ERROR) if args.quiet: _logger.setLevel(logging.WARN) if runtimeContext.debug: _logger.setLevel(logging.DEBUG) rdflib_logger.setLevel(logging.DEBUG) if args.timestamps: formatter = logging.Formatter("[%(asctime)s] %(message)s", "%Y-%m-%d %H:%M:%S") stderr_handler.setFormatter(formatter) # version if args.version: return versionfunc(), 0 else: _logger.info(versionfunc()) if args.print_supported_versions: return "\n".join(supportedCWLversions(args.enable_dev)), 0 if not args.workflow: if os.path.isfile("CWLFile"): setattr(args, "workflow", "CWLFile") else: _logger.error("") _logger.error( "CWL document required, no input file was provided") arg_parser().print_help() return "CWL document required, no input file was provided", 1 if args.relax_path_checks: command_line_tool.ACCEPTLIST_RE = command_line_tool.ACCEPTLIST_EN_RELAXED_RE if args.ga4gh_tool_registries: ga4gh_tool_registries[:] = args.ga4gh_tool_registries if not args.enable_ga4gh_tool_registry: del ga4gh_tool_registries[:] if custom_schema_callback: custom_schema_callback() elif args.enable_ext: res = pkg_resources.resource_stream(__name__, 'extensions.yml') use_custom_schema("v1.0", "http://commonwl.org/cwltool", res.read()) res.close() else: use_standard_schema("v1.0") if loadingContext is None: loadingContext = LoadingContext(vars(args)) else: loadingContext = loadingContext.copy() loadingContext.disable_js_validation = \ args.disable_js_validation or (not args.do_validate) loadingContext.construct_tool_object = getdefault( loadingContext.construct_tool_object, workflow.default_make_tool) loadingContext.resolver = getdefault(loadingContext.resolver, tool_resolver) try: uri, tool_file_uri = resolve_tool_uri( args.workflow, resolver=loadingContext.resolver, fetcher_constructor=loadingContext.fetcher_constructor) except: return "Can't find file " + args.workflow, 0 try_again_msg = "" if args.debug else ", try again with --debug for more information" try: job_order_object, input_basedir, jobloader = load_job_order( args, stdin, loadingContext.fetcher_constructor, loadingContext.overrides_list, tool_file_uri) if args.overrides: loadingContext.overrides_list.extend( load_overrides( file_uri(os.path.abspath(args.overrides)), tool_file_uri)) document_loader, workflowobj, uri = fetch_document( uri, resolver=loadingContext.resolver, fetcher_constructor=loadingContext.fetcher_constructor) if args.print_deps: # printdeps(workflowobj, document_loader, stdout, args.relative_deps, uri) result = returndeps(workflowobj, document_loader, stdout, args.relative_deps, uri) return result, 0 document_loader, avsc_names, processobj, metadata, uri \ = validate_document(document_loader, workflowobj, uri, enable_dev=loadingContext.enable_dev, strict=loadingContext.strict, preprocess_only=(args.print_pre or args.pack), fetcher_constructor=loadingContext.fetcher_constructor, skip_schemas=args.skip_schemas, overrides=loadingContext.overrides_list, do_validate=loadingContext.do_validate) if args.print_pre: # stdout.write(json_dumps(processobj, indent=4)) return json_dumps(processobj, indent=4), 0 loadingContext.overrides_list.extend( metadata.get("cwltool:overrides", [])) tool = make_tool(document_loader, avsc_names, metadata, uri, loadingContext) if args.make_template: yaml.safe_dump(generate_input_template(tool), sys.stdout, default_flow_style=False, indent=4, block_seq_indent=2) return yaml.safe_dump(generate_input_template(tool), indent=4), 0 if args.validate: _logger.info("Tool definition is valid") return "Tool definition is valid", 0 if args.pack: stdout.write( print_pack(document_loader, processobj, uri, metadata)) return print_pack(document_loader, processobj, uri, metadata), 0 if args.print_rdf: stdout.write( printrdf(tool, document_loader.ctx, args.rdf_serializer)) return printrdf(tool, document_loader.ctx, args.rdf_serializer), 0 if args.print_dot: printdot(tool, document_loader.ctx, stdout) return "args.print_dot still not solved", 0 except (validate.ValidationException) as exc: _logger.error(u"Tool definition failed validation:\n%s", exc, exc_info=args.debug) infor = "Tool definition failed validation:\n%s" + exc + args.debug return infor, 1 except (RuntimeError, WorkflowException) as exc: _logger.error(u"Tool definition failed initialization:\n%s", exc, exc_info=args.debug) infor = "Tool definition failed initialization:\n%s" + exc + args.debug return infor, 1 except Exception as exc: _logger.error( u"I'm sorry, I couldn't load this CWL file%s.\nThe error was: %s", try_again_msg, exc if not args.debug else "", exc_info=args.debug) return "I'm sorry, I couldn't load this CWL file", 1 if isinstance(tool, int): return tool, 0 # If on MacOS platform, TMPDIR must be set to be under one of the # shared volumes in Docker for Mac # More info: https://dockstore.org/docs/faq if sys.platform == "darwin": default_mac_path = "/private/tmp/docker_tmp" if runtimeContext.tmp_outdir_prefix == DEFAULT_TMP_PREFIX: runtimeContext.tmp_outdir_prefix = default_mac_path for dirprefix in ("tmpdir_prefix", "tmp_outdir_prefix", "cachedir"): if getattr(runtimeContext, dirprefix) and getattr( runtimeContext, dirprefix) != DEFAULT_TMP_PREFIX: sl = "/" if getattr(runtimeContext, dirprefix).endswith("/") or dirprefix == "cachedir" \ else "" setattr( runtimeContext, dirprefix, os.path.abspath(getattr(runtimeContext, dirprefix)) + sl) if not os.path.exists( os.path.dirname(getattr(runtimeContext, dirprefix))): try: os.makedirs( os.path.dirname( getattr(runtimeContext, dirprefix))) except Exception as e: _logger.error("Failed to create directory: %s", e) infor = "Failed to create directory: %s" + e + "" return infor, 1 if args.cachedir: if args.move_outputs == "move": runtimeContext.move_outputs = "copy" runtimeContext.tmp_outdir_prefix = args.cachedir runtimeContext.secret_store = getdefault( runtimeContext.secret_store, SecretStore()) try: initialized_job_order_object = init_job_order( job_order_object, args, tool, jobloader, stdout, print_input_deps=args.print_input_deps, relative_deps=args.relative_deps, input_basedir=input_basedir, secret_store=runtimeContext.secret_store) except SystemExit as err: return err.code if not executor: if args.parallel: executor = MultithreadedJobExecutor() else: executor = SingleJobExecutor() assert executor is not None if isinstance(initialized_job_order_object, int): return initialized_job_order_object try: runtimeContext.basedir = input_basedir del args.workflow del args.job_order conf_file = getattr(args, "beta_dependency_resolvers_configuration", None) # Text use_conda_dependencies = getattr(args, "beta_conda_dependencies", None) # Text job_script_provider = None # type: Optional[DependenciesConfiguration] if conf_file or use_conda_dependencies: runtimeContext.job_script_provider = DependenciesConfiguration( args) runtimeContext.find_default_container = \ functools.partial(find_default_container, args) runtimeContext.make_fs_access = getdefault( runtimeContext.make_fs_access, StdFsAccess) (out, status) = executor(tool, initialized_job_order_object, runtimeContext, logger=_logger) # This is the workflow output, it needs to be written if out is not None: def loc_to_path(obj): for field in ("path", "nameext", "nameroot", "dirname"): if field in obj: del obj[field] if obj["location"].startswith("file://"): obj["path"] = uri_file_path(obj["location"]) visit_class(out, ("File", "Directory"), loc_to_path) # Unsetting the Generation fron final output object visit_class(out, ("File", ), MutationManager().unset_generation) if isinstance(out, string_types): stdout.write(out) else: stdout.write( json_dumps( out, indent=4, # type: ignore ensure_ascii=False)) stdout.write("\n") if hasattr(stdout, "flush"): stdout.flush() # type: ignore if status != "success": _logger.warning(u"Final process status is %s", status) infor = "Final process status is %s" + status + "" return infor, 1 _logger.info(u"Final process status is %s", status) return out, status except (validate.ValidationException) as exc: _logger.error(u"Input object failed validation:\n%s", exc, exc_info=args.debug) infor = "Input object failed validation:\n%s" + exc + args.debug return infor, 1 except UnsupportedRequirement as exc: _logger.error( u"Workflow or tool uses unsupported feature:\n%s", exc, exc_info=args.debug) infor = "Workflow or tool uses unsupported feature:\n%s" + exc + args.debug return infor, 3 except WorkflowException as exc: _logger.error(u"Workflow error%s:\n%s", try_again_msg, strip_dup_lineno(six.text_type(exc)), exc_info=args.debug) infor = "Workflow error%s:\n%s" + try_again_msg + strip_dup_lineno( six.text_type(exc)) + args.debug return infor, 1 except Exception as exc: _logger.error(u"Unhandled error%s:\n %s", try_again_msg, exc, exc_info=args.debug) infor = "Unhandled error%s:\n %s" + try_again_msg + exc + args.debug return infor, 1 finally: _logger.removeHandler(stderr_handler) _logger.addHandler(defaultStreamHandler)
def execute(self, context): logging.info('Running tool: \n{}'.format( json.dumps(self.cwl_step.tool, indent=4))) collected_outputs = {} for task_outputs in self.xcom_pull( context=context, task_ids=[task.task_id for task in self.upstream_list]): collected_outputs = merge(collected_outputs, task_outputs["outputs"]) logging.debug('Collected outputs:\n{}'.format( json.dumps(collected_outputs, indent=4))) jobobj = {} for inp in self.cwl_step.tool["inputs"]: jobobj_id = shortname(inp["id"]).split("/")[-1] source_ids = [] promises_outputs = [] try: source_ids = [shortname(source) for source in inp["source"]] if isinstance( inp["source"], list) else [shortname(inp["source"])] promises_outputs = [ collected_outputs[source_id] for source_id in source_ids if source_id in collected_outputs ] except Exception as ex: logging.info( "Couldn't find source field in the step input: \n{}". format(json.dumps(inp, indent=4))) logging.info( 'For input {} with sources: \n{} \nfound upstream outputs: \n{}' .format(jobobj_id, source_ids, promises_outputs)) if len(promises_outputs) > 1: if inp.get("linkMerge", "merge_nested") == "merge_flattened": jobobj[jobobj_id] = flatten(promises_outputs) else: jobobj[jobobj_id] = promises_outputs elif len(promises_outputs) == 1 and ( promises_outputs[0] is not None ): # Should also check if [None], because in this case we need to take default value jobobj[jobobj_id] = promises_outputs[0] elif "valueFrom" in inp: jobobj[jobobj_id] = None elif "default" in inp: d = copy.copy(inp["default"]) jobobj[jobobj_id] = d else: continue logging.info('Collected job object: \n{}'.format( json.dumps(jobobj, indent=4))) valueFrom = { shortname(i["id"]).split("/")[-1]: i["valueFrom"] for i in self.cwl_step.tool["inputs"] if "valueFrom" in i } logging.info('Inputs with valueFrom: \n{}'.format( json.dumps(valueFrom, indent=4))) def postScatterEval(shortio): def valueFromFunc(k, v): if k in valueFrom: return cwltool.workflow.expression.do_eval( valueFrom[k], shortio, self.dag.requirements, None, None, {}, context=v) else: return v return {k: valueFromFunc(k, v) for k, v in shortio.items()} job = postScatterEval(jobobj) logging.info( 'Collected job object after valueFrom evaluation: \n{}'.format( json.dumps(job, indent=4))) # maybe need to add here scatter functionality too kwargs = self.dag.default_args tmp_folder = collected_outputs["tmp_folder"] output_folder = collected_outputs["output_folder"] kwargs['outdir'] = tempfile.mkdtemp(dir=tmp_folder, prefix="step_tmp_") kwargs['tmpdir_prefix'] = os.path.join(tmp_folder, "cwl_tmp_") kwargs['tmp_outdir_prefix'] = os.path.join(tmp_folder, "cwl_outdir_tmp_") kwargs['rm_tmpdir'] = False kwargs["basedir"] = os.path.abspath( os.path.dirname(self.dag.default_args["job_data"]["path"])) logger = logging.getLogger("cwltool") sys.stdout = StreamLogWriterUpdated(logger, logging.INFO) sys.stderr = StreamLogWriterUpdated(logger, logging.WARN) executor = cwltool.executors.SingleJobExecutor() runtimeContext = RuntimeContext(kwargs) runtimeContext.make_fs_access = getdefault( runtimeContext.make_fs_access, cwltool.stdfsaccess.StdFsAccess) for inp in self.cwl_step.tool["inputs"]: if inp.get("not_connected"): del job[shortname(inp["id"].split("/")[-1])] (output, status) = executor(self.cwl_step.embedded_tool, job, runtimeContext, logger=logger) if not output and status == "permanentFail": raise ValueError logging.debug('Embedded tool outputs: \n{}'.format( json.dumps(output, indent=4))) promises = {} for out in self.cwl_step.tool["outputs"]: out_id = shortname(out["id"]) jobout_id = out_id.split("/")[-1] try: promises[out_id] = output[jobout_id] except: continue promises["tmp_folder"] = tmp_folder promises["output_folder"] = output_folder data = {"outputs": promises} logging.info('Outputs: \n{}'.format(json.dumps(data, indent=4))) return data