def result( self, job_order: CWLObjectType, jobout: CWLObjectType, runtimeContext: RuntimeContext, ) -> Tuple[Process, CWLObjectType]: try: loadingContext = self.loadingContext.copy() loadingContext.metadata = {} embedded_tool = load_tool( cast(Dict[str, str], jobout["runProcess"])["location"], loadingContext ) except ValidationException as vexc: if runtimeContext.debug: _logger.exception("Validation exception") raise WorkflowException( "Tool definition %s failed validation:\n%s" % (jobout["runProcess"], indent(str(vexc))) ) if "runInputs" in jobout: runinputs = cast(CWLObjectType, jobout["runInputs"]) else: runinputs = copy.deepcopy(job_order) for i in self.embedded_tool.tool["inputs"]: if shortname(i["id"]) in runinputs: del runinputs[shortname(i["id"])] if "id" in runinputs: del runinputs["id"] return embedded_tool, runinputs
def result( self, job_order, # type: Mapping[Text, Any] jobout, # type: Mapping[Text, Any] runtimeContext # type: RuntimeContext ): # type: (...) -> Tuple[Process, MutableMapping[Text, Any]] try: loadingContext = self.loadingContext.copy() loadingContext.metadata = {} embedded_tool = load_tool(jobout["runProcess"]["location"], loadingContext) except validate.ValidationException as vexc: if runtimeContext.debug: _logger.exception("Validation exception") raise WorkflowException( u"Tool definition %s failed validation:\n%s" % (jobout["runProcess"], indent(str(vexc)))) if "runInputs" in jobout: runinputs = cast(MutableMapping[Text, Any], jobout["runInputs"]) else: runinputs = cast(MutableMapping[Text, Any], copy.deepcopy(job_order)) for i in self.embedded_tool.tool["inputs"]: if shortname(i["id"]) in runinputs: del runinputs[shortname(i["id"])] if "id" in runinputs: del runinputs["id"] return embedded_tool, runinputs
def collect_output_ports(self, ports, builder, outdir, compute_checksum=True): # type: (Set[Dict[Text, Any]], Builder, Text, bool) -> Dict[Text, Union[Text, List[Any], Dict[Text, Any]]] ret = {} # type: Dict[Text, Union[Text, List[Any], Dict[Text, Any]]] try: fs_access = builder.make_fs_access(outdir) custom_output = fs_access.join(outdir, "cwl.output.json") if fs_access.exists(custom_output): with fs_access.open(custom_output, "r") as f: ret = json.load(f) if _logger.isEnabledFor(logging.DEBUG): _logger.debug(u"Raw output from %s: %s", custom_output, json.dumps(ret, indent=4)) else: for i, port in enumerate(ports): with SourceLine(ports, i, WorkflowException): fragment = shortname(port["id"]) try: ret[fragment] = self.collect_output( port, builder, outdir, fs_access, compute_checksum=compute_checksum) except Exception as e: _logger.debug( u"Error collecting output for parameter '%s'" % shortname(port["id"]), exc_info=True) raise WorkflowException( u"Error collecting output for parameter '%s':\n%s" % (shortname(port["id"]), indent(u(str(e))))) if ret: adjustDirObjs(ret, trim_listing) adjustFileObjs( ret, cast( Callable[[Any], Any], # known bug in mypy # https://github.com/python/mypy/issues/797 partial(revmap_file, builder, outdir))) adjustFileObjs(ret, remove_path) adjustDirObjs(ret, remove_path) normalizeFilesDirs(ret) if compute_checksum: adjustFileObjs(ret, partial(compute_checksums, fs_access)) validate.validate_ex( self.names.get_name("outputs_record_schema", ""), ret) return ret if ret is not None else {} except validate.ValidationException as e: raise WorkflowException("Error validating output record, " + Text(e) + "\n in " + json.dumps(ret, indent=4))
def load(self, doc, baseuri, loadingOptions, docRoot=None): # type: (Any, Text, LoadingOptions, Optional[Text]) -> Any errors = [] for t in self.alternates: try: return t.load(doc, baseuri, loadingOptions, docRoot=docRoot) except ValidationException as e: errors.append(u"tried %s but\n%s" % (t.__class__.__name__, indent(str(e)))) raise ValidationException(bullets(errors, u"- "))
def collect_output_ports(self, ports, builder, outdir, compute_checksum=True, jobname="", readers=None): # type: (Set[Dict[Text, Any]], Builder, Text, bool, Text, Dict[Text, Any]) -> Dict[Text, Union[Text, List[Any], Dict[Text, Any]]] ret = {} # type: Dict[Text, Union[Text, List[Any], Dict[Text, Any]]] try: fs_access = builder.make_fs_access(outdir) custom_output = fs_access.join(outdir, "cwl.output.json") if fs_access.exists(custom_output): with fs_access.open(custom_output, "r") as f: ret = json.load(f) if _logger.isEnabledFor(logging.DEBUG): _logger.debug(u"Raw output from %s: %s", custom_output, json.dumps(ret, indent=4)) else: for i, port in enumerate(ports): with SourceLine(ports, i, WorkflowException): fragment = shortname(port["id"]) try: ret[fragment] = self.collect_output(port, builder, outdir, fs_access, compute_checksum=compute_checksum) except Exception as e: _logger.debug( u"Error collecting output for parameter '%s'" % shortname(port["id"]), exc_info=True) raise WorkflowException( u"Error collecting output for parameter '%s':\n%s" % (shortname(port["id"]), indent(u(str(e))))) if ret: revmap = partial(revmap_file, builder, outdir) adjustDirObjs(ret, trim_listing) visit_class(ret, ("File", "Directory"), cast(Callable[[Any], Any], revmap)) visit_class(ret, ("File", "Directory"), remove_path) normalizeFilesDirs(ret) visit_class(ret, ("File", "Directory"), partial(check_valid_locations, fs_access)) if compute_checksum: adjustFileObjs(ret, partial(compute_checksums, fs_access)) validate.validate_ex(self.names.get_name("outputs_record_schema", ""), ret, strict=False, logger=_logger_validation_warnings) if ret is not None and builder.mutation_manager is not None: adjustFileObjs(ret, builder.mutation_manager.set_generation) return ret if ret is not None else {} except validate.ValidationException as e: raise WorkflowException("Error validating output record. " + Text(e) + "\n in " + json.dumps(ret, indent=4)) finally: if builder.mutation_manager and readers: for r in readers.values(): builder.mutation_manager.release_reader(jobname, r)
def collect_output_ports(self, ports, builder, outdir, compute_checksum=True, jobname="", readers=None): # type: (Set[Dict[Text, Any]], Builder, Text, bool, Text, Dict[Text, Any]) -> Dict[Text, Union[Text, List[Any], Dict[Text, Any]]] ret = {} # type: Dict[Text, Union[Text, List[Any], Dict[Text, Any]]] try: fs_access = builder.make_fs_access(outdir) custom_output = fs_access.join(outdir, "cwl.output.json") if fs_access.exists(custom_output): with fs_access.open(custom_output, "r") as f: ret = json.load(f) if _logger.isEnabledFor(logging.DEBUG): _logger.debug(u"Raw output from %s: %s", custom_output, json.dumps(ret, indent=4)) else: for i, port in enumerate(ports): with SourceLine(ports, i, WorkflowException): fragment = shortname(port["id"]) try: ret[fragment] = self.collect_output(port, builder, outdir, fs_access, compute_checksum=compute_checksum) except Exception as e: _logger.debug( u"Error collecting output for parameter '%s'" % shortname(port["id"]), exc_info=True) raise WorkflowException( u"Error collecting output for parameter '%s':\n%s" % (shortname(port["id"]), indent(u(str(e))))) if ret: revmap = partial(revmap_file, builder, outdir) adjustDirObjs(ret, trim_listing) visit_class(ret, ("File", "Directory"), cast(Callable[[Any], Any], revmap)) visit_class(ret, ("File", "Directory"), remove_path) normalizeFilesDirs(ret) if builder.mutation_manager: adjustFileObjs(ret, builder.mutation_manager.set_generation) visit_class(ret, ("File", "Directory"), partial(check_valid_locations, fs_access)) if compute_checksum: adjustFileObjs(ret, partial(compute_checksums, fs_access)) validate.validate_ex(self.names.get_name("outputs_record_schema", ""), ret, strict=False, logger=_logger_validation_warnings) return ret if ret is not None else {} except validate.ValidationException as e: raise WorkflowException("Error validating output record. " + Text(e) + "\n in " + json.dumps(ret, indent=4)) finally: if builder.mutation_manager and readers: for r in readers.values(): builder.mutation_manager.release_reader(jobname, r)
def __init__( self, toolpath_object: CommentedMap, loadingContext: LoadingContext, ) -> None: super(ProcessGenerator, self).__init__(toolpath_object, loadingContext) self.loadingContext = loadingContext # type: LoadingContext try: if isinstance(toolpath_object["run"], CommentedMap): self.embedded_tool = loadingContext.construct_tool_object( toolpath_object["run"], loadingContext ) # type: Process else: loadingContext.metadata = {} self.embedded_tool = load_tool(toolpath_object["run"], loadingContext) except ValidationException as vexc: if loadingContext.debug: _logger.exception("Validation exception") raise WorkflowException( "Tool definition %s failed validation:\n%s" % (toolpath_object["run"], indent(str(vexc))) )
def __init__( self, toolpath_object, # type: MutableMapping[Text, Any] loadingContext # type: LoadingContext ): # type: (...) -> None super(ProcessGenerator, self).__init__(toolpath_object, loadingContext) self.loadingContext = loadingContext # type: LoadingContext try: if isinstance(toolpath_object["run"], MutableMapping): self.embedded_tool = loadingContext.construct_tool_object( toolpath_object["run"], loadingContext) # type: Process else: loadingContext.metadata = {} self.embedded_tool = load_tool(toolpath_object["run"], loadingContext) except validate.ValidationException as vexc: if loadingContext.debug: _logger.exception("Validation exception") raise WorkflowException( u"Tool definition %s failed validation:\n%s" % (toolpath_object["run"], indent(str(vexc))))
def collect_output_ports(self, ports, builder, outdir, compute_checksum=True): # type: (Set[Dict[Text, Any]], Builder, Text, bool) -> Dict[Text, Union[Text, List[Any], Dict[Text, Any]]] try: ret = {} # type: Dict[Text, Union[Text, List[Any], Dict[Text, Any]]] fs_access = builder.make_fs_access(outdir) custom_output = fs_access.join(outdir, "cwl.output.json") if fs_access.exists(custom_output): with fs_access.open(custom_output, "r") as f: ret = json.load(f) if _logger.isEnabledFor(logging.DEBUG): _logger.debug(u"Raw output from %s: %s", custom_output, json.dumps(ret, indent=4)) else: for i, port in enumerate(ports): with SourceLine(ports, i, WorkflowException): fragment = shortname(port["id"]) try: ret[fragment] = self.collect_output(port, builder, outdir, fs_access, compute_checksum=compute_checksum) except Exception as e: _logger.debug( u"Error collecting output for parameter '%s'" % shortname(port["id"]), exc_info=True) raise WorkflowException( u"Error collecting output for parameter '%s':\n%s" % (shortname(port["id"]), indent(unicode(e)))) if ret: adjustFileObjs(ret, cast(Callable[[Any], Any], # known bug in mypy # https://github.com/python/mypy/issues/797 partial(revmap_file, builder, outdir))) adjustFileObjs(ret, remove_path) adjustDirObjs(ret, remove_path) normalizeFilesDirs(ret) if compute_checksum: adjustFileObjs(ret, partial(compute_checksums, fs_access)) validate.validate_ex(self.names.get_name("outputs_record_schema", ""), ret) return ret if ret is not None else {} except validate.ValidationException as e: raise WorkflowException("Error validating output record, " + Text(e) + "\n in " + json.dumps(ret, indent=4))
def __init__( self, toolpath_object: CommentedMap, pos: int, loadingContext: LoadingContext, parentworkflowProv: Optional[ProvenanceProfile] = None, ) -> None: """Initialize this WorkflowStep.""" if "id" in toolpath_object: self.id = toolpath_object["id"] else: self.id = "#step" + str(pos) loadingContext = loadingContext.copy() loadingContext.requirements = copy.deepcopy( getdefault(loadingContext.requirements, []) ) assert loadingContext.requirements is not None # nosec loadingContext.requirements.extend(toolpath_object.get("requirements", [])) loadingContext.requirements.extend( cast( List[CWLObjectType], get_overrides( getdefault(loadingContext.overrides_list, []), self.id ).get("requirements", []), ) ) hints = copy.deepcopy(getdefault(loadingContext.hints, [])) hints.extend(toolpath_object.get("hints", [])) loadingContext.hints = hints try: if isinstance(toolpath_object["run"], CommentedMap): self.embedded_tool = loadingContext.construct_tool_object( toolpath_object["run"], loadingContext ) # type: Process else: loadingContext.metadata = {} self.embedded_tool = load_tool(toolpath_object["run"], loadingContext) except ValidationException as vexc: if loadingContext.debug: _logger.exception("Validation exception") raise WorkflowException( "Tool definition %s failed validation:\n%s" % (toolpath_object["run"], indent(str(vexc))) ) from vexc validation_errors = [] self.tool = toolpath_object = copy.deepcopy(toolpath_object) bound = set() for stepfield, toolfield in (("in", "inputs"), ("out", "outputs")): toolpath_object[toolfield] = [] for index, step_entry in enumerate(toolpath_object[stepfield]): if isinstance(step_entry, str): param = CommentedMap() # type: CommentedMap inputid = step_entry else: param = CommentedMap(step_entry.items()) inputid = step_entry["id"] shortinputid = shortname(inputid) found = False for tool_entry in self.embedded_tool.tool[toolfield]: frag = shortname(tool_entry["id"]) if frag == shortinputid: # if the case that the step has a default for a parameter, # we do not want the default of the tool to override it step_default = None if "default" in param and "default" in tool_entry: step_default = param["default"] param.update(tool_entry) param["_tool_entry"] = tool_entry if step_default is not None: param["default"] = step_default found = True bound.add(frag) break if not found: if stepfield == "in": param["type"] = "Any" param["used_by_step"] = used_by_step(self.tool, shortinputid) param["not_connected"] = True else: if isinstance(step_entry, Mapping): step_entry_name = step_entry["id"] else: step_entry_name = step_entry validation_errors.append( SourceLine(self.tool["out"], index).makeError( "Workflow step output '%s' does not correspond to" % shortname(step_entry_name) ) + "\n" + SourceLine(self.embedded_tool.tool, "outputs").makeError( " tool output (expected '%s')" % ( "', '".join( [ shortname(tool_entry["id"]) for tool_entry in self.embedded_tool.tool[ "outputs" ] ] ) ) ) ) param["id"] = inputid param.lc.line = toolpath_object[stepfield].lc.data[index][0] param.lc.col = toolpath_object[stepfield].lc.data[index][1] param.lc.filename = toolpath_object[stepfield].lc.filename toolpath_object[toolfield].append(param) missing_values = [] for _, tool_entry in enumerate(self.embedded_tool.tool["inputs"]): if shortname(tool_entry["id"]) not in bound: if "null" not in tool_entry["type"] and "default" not in tool_entry: missing_values.append(shortname(tool_entry["id"])) if missing_values: validation_errors.append( SourceLine(self.tool, "in").makeError( "Step is missing required parameter%s '%s'" % ( "s" if len(missing_values) > 1 else "", "', '".join(missing_values), ) ) ) if validation_errors: raise ValidationException("\n".join(validation_errors)) super().__init__(toolpath_object, loadingContext) if self.embedded_tool.tool["class"] == "Workflow": (feature, _) = self.get_requirement("SubworkflowFeatureRequirement") if not feature: raise WorkflowException( "Workflow contains embedded workflow but " "SubworkflowFeatureRequirement not in requirements" ) if "scatter" in self.tool: (feature, _) = self.get_requirement("ScatterFeatureRequirement") if not feature: raise WorkflowException( "Workflow contains scatter but ScatterFeatureRequirement " "not in requirements" ) inputparms = copy.deepcopy(self.tool["inputs"]) outputparms = copy.deepcopy(self.tool["outputs"]) scatter = aslist(self.tool["scatter"]) method = self.tool.get("scatterMethod") if method is None and len(scatter) != 1: raise ValidationException( "Must specify scatterMethod when scattering over multiple inputs" ) inp_map = {i["id"]: i for i in inputparms} for inp in scatter: if inp not in inp_map: raise ValidationException( SourceLine(self.tool, "scatter").makeError( "Scatter parameter '%s' does not correspond to " "an input parameter of this step, expecting '%s'" % ( shortname(inp), "', '".join(shortname(k) for k in inp_map.keys()), ) ) ) inp_map[inp]["type"] = {"type": "array", "items": inp_map[inp]["type"]} if self.tool.get("scatterMethod") == "nested_crossproduct": nesting = len(scatter) else: nesting = 1 for _ in range(0, nesting): for oparam in outputparms: oparam["type"] = {"type": "array", "items": oparam["type"]} self.tool["inputs"] = inputparms self.tool["outputs"] = outputparms self.prov_obj = None # type: Optional[ProvenanceProfile] if loadingContext.research_obj is not None: self.prov_obj = parentworkflowProv if self.embedded_tool.tool["class"] == "Workflow": self.parent_wf = self.embedded_tool.parent_wf else: self.parent_wf = self.prov_obj