def result(
self,
job_order: CWLObjectType,
jobout: CWLObjectType,
runtimeContext: RuntimeContext,
) -> Tuple[Process, CWLObjectType]:
try:
loadingContext = self.loadingContext.copy()
loadingContext.metadata = {}
embedded_tool = load_tool(
cast(Dict[str, str], jobout["runProcess"])["location"], loadingContext
)
except ValidationException as vexc:
if runtimeContext.debug:
_logger.exception("Validation exception")
raise WorkflowException(
"Tool definition %s failed validation:\n%s"
% (jobout["runProcess"], indent(str(vexc)))
)
if "runInputs" in jobout:
runinputs = cast(CWLObjectType, jobout["runInputs"])
else:
runinputs = copy.deepcopy(job_order)
for i in self.embedded_tool.tool["inputs"]:
if shortname(i["id"]) in runinputs:
del runinputs[shortname(i["id"])]
if "id" in runinputs:
del runinputs["id"]
return embedded_tool, runinputs
def result(
self,
job_order, # type: Mapping[Text, Any]
jobout, # type: Mapping[Text, Any]
runtimeContext # type: RuntimeContext
): # type: (...) -> Tuple[Process, MutableMapping[Text, Any]]
try:
loadingContext = self.loadingContext.copy()
loadingContext.metadata = {}
embedded_tool = load_tool(jobout["runProcess"]["location"],
loadingContext)
except validate.ValidationException as vexc:
if runtimeContext.debug:
_logger.exception("Validation exception")
raise WorkflowException(
u"Tool definition %s failed validation:\n%s" %
(jobout["runProcess"], indent(str(vexc))))
if "runInputs" in jobout:
runinputs = cast(MutableMapping[Text, Any], jobout["runInputs"])
else:
runinputs = cast(MutableMapping[Text, Any],
copy.deepcopy(job_order))
for i in self.embedded_tool.tool["inputs"]:
if shortname(i["id"]) in runinputs:
del runinputs[shortname(i["id"])]
if "id" in runinputs:
del runinputs["id"]
return embedded_tool, runinputs
def collect_output_ports(self,
ports,
builder,
outdir,
compute_checksum=True):
# type: (Set[Dict[Text, Any]], Builder, Text, bool) -> Dict[Text, Union[Text, List[Any], Dict[Text, Any]]]
ret = {} # type: Dict[Text, Union[Text, List[Any], Dict[Text, Any]]]
try:
fs_access = builder.make_fs_access(outdir)
custom_output = fs_access.join(outdir, "cwl.output.json")
if fs_access.exists(custom_output):
with fs_access.open(custom_output, "r") as f:
ret = json.load(f)
if _logger.isEnabledFor(logging.DEBUG):
_logger.debug(u"Raw output from %s: %s", custom_output,
json.dumps(ret, indent=4))
else:
for i, port in enumerate(ports):
with SourceLine(ports, i, WorkflowException):
fragment = shortname(port["id"])
try:
ret[fragment] = self.collect_output(
port,
builder,
outdir,
fs_access,
compute_checksum=compute_checksum)
except Exception as e:
_logger.debug(
u"Error collecting output for parameter '%s'" %
shortname(port["id"]),
exc_info=True)
raise WorkflowException(
u"Error collecting output for parameter '%s':\n%s"
% (shortname(port["id"]), indent(u(str(e)))))
if ret:
adjustDirObjs(ret, trim_listing)
adjustFileObjs(
ret,
cast(
Callable[[Any], Any], # known bug in mypy
# https://github.com/python/mypy/issues/797
partial(revmap_file, builder, outdir)))
adjustFileObjs(ret, remove_path)
adjustDirObjs(ret, remove_path)
normalizeFilesDirs(ret)
if compute_checksum:
adjustFileObjs(ret, partial(compute_checksums, fs_access))
validate.validate_ex(
self.names.get_name("outputs_record_schema", ""), ret)
return ret if ret is not None else {}
except validate.ValidationException as e:
raise WorkflowException("Error validating output record, " +
Text(e) + "\n in " +
json.dumps(ret, indent=4))
def load(self, doc, baseuri, loadingOptions, docRoot=None):
# type: (Any, Text, LoadingOptions, Optional[Text]) -> Any
errors = []
for t in self.alternates:
try:
return t.load(doc, baseuri, loadingOptions, docRoot=docRoot)
except ValidationException as e:
errors.append(u"tried %s but\n%s" %
(t.__class__.__name__, indent(str(e))))
raise ValidationException(bullets(errors, u"- "))
def collect_output_ports(self, ports, builder, outdir, compute_checksum=True, jobname="", readers=None):
# type: (Set[Dict[Text, Any]], Builder, Text, bool, Text, Dict[Text, Any]) -> Dict[Text, Union[Text, List[Any], Dict[Text, Any]]]
ret = {} # type: Dict[Text, Union[Text, List[Any], Dict[Text, Any]]]
try:
fs_access = builder.make_fs_access(outdir)
custom_output = fs_access.join(outdir, "cwl.output.json")
if fs_access.exists(custom_output):
with fs_access.open(custom_output, "r") as f:
ret = json.load(f)
if _logger.isEnabledFor(logging.DEBUG):
_logger.debug(u"Raw output from %s: %s", custom_output, json.dumps(ret, indent=4))
else:
for i, port in enumerate(ports):
with SourceLine(ports, i, WorkflowException):
fragment = shortname(port["id"])
try:
ret[fragment] = self.collect_output(port, builder, outdir, fs_access,
compute_checksum=compute_checksum)
except Exception as e:
_logger.debug(
u"Error collecting output for parameter '%s'"
% shortname(port["id"]), exc_info=True)
raise WorkflowException(
u"Error collecting output for parameter '%s':\n%s"
% (shortname(port["id"]), indent(u(str(e)))))
if ret:
revmap = partial(revmap_file, builder, outdir)
adjustDirObjs(ret, trim_listing)
visit_class(ret, ("File", "Directory"), cast(Callable[[Any], Any], revmap))
visit_class(ret, ("File", "Directory"), remove_path)
normalizeFilesDirs(ret)
visit_class(ret, ("File", "Directory"), partial(check_valid_locations, fs_access))
if compute_checksum:
adjustFileObjs(ret, partial(compute_checksums, fs_access))
validate.validate_ex(self.names.get_name("outputs_record_schema", ""), ret,
strict=False, logger=_logger_validation_warnings)
if ret is not None and builder.mutation_manager is not None:
adjustFileObjs(ret, builder.mutation_manager.set_generation)
return ret if ret is not None else {}
except validate.ValidationException as e:
raise WorkflowException("Error validating output record. " + Text(e) + "\n in " + json.dumps(ret, indent=4))
finally:
if builder.mutation_manager and readers:
for r in readers.values():
builder.mutation_manager.release_reader(jobname, r)
def collect_output_ports(self, ports, builder, outdir, compute_checksum=True, jobname="", readers=None):
# type: (Set[Dict[Text, Any]], Builder, Text, bool, Text, Dict[Text, Any]) -> Dict[Text, Union[Text, List[Any], Dict[Text, Any]]]
ret = {} # type: Dict[Text, Union[Text, List[Any], Dict[Text, Any]]]
try:
fs_access = builder.make_fs_access(outdir)
custom_output = fs_access.join(outdir, "cwl.output.json")
if fs_access.exists(custom_output):
with fs_access.open(custom_output, "r") as f:
ret = json.load(f)
if _logger.isEnabledFor(logging.DEBUG):
_logger.debug(u"Raw output from %s: %s", custom_output, json.dumps(ret, indent=4))
else:
for i, port in enumerate(ports):
with SourceLine(ports, i, WorkflowException):
fragment = shortname(port["id"])
try:
ret[fragment] = self.collect_output(port, builder, outdir, fs_access,
compute_checksum=compute_checksum)
except Exception as e:
_logger.debug(
u"Error collecting output for parameter '%s'"
% shortname(port["id"]), exc_info=True)
raise WorkflowException(
u"Error collecting output for parameter '%s':\n%s"
% (shortname(port["id"]), indent(u(str(e)))))
if ret:
revmap = partial(revmap_file, builder, outdir)
adjustDirObjs(ret, trim_listing)
visit_class(ret, ("File", "Directory"), cast(Callable[[Any], Any], revmap))
visit_class(ret, ("File", "Directory"), remove_path)
normalizeFilesDirs(ret)
if builder.mutation_manager:
adjustFileObjs(ret, builder.mutation_manager.set_generation)
visit_class(ret, ("File", "Directory"), partial(check_valid_locations, fs_access))
if compute_checksum:
adjustFileObjs(ret, partial(compute_checksums, fs_access))
validate.validate_ex(self.names.get_name("outputs_record_schema", ""), ret,
strict=False, logger=_logger_validation_warnings)
return ret if ret is not None else {}
except validate.ValidationException as e:
raise WorkflowException("Error validating output record. " + Text(e) + "\n in " + json.dumps(ret, indent=4))
finally:
if builder.mutation_manager and readers:
for r in readers.values():
builder.mutation_manager.release_reader(jobname, r)
def __init__(
self, toolpath_object: CommentedMap, loadingContext: LoadingContext,
) -> None:
super(ProcessGenerator, self).__init__(toolpath_object, loadingContext)
self.loadingContext = loadingContext # type: LoadingContext
try:
if isinstance(toolpath_object["run"], CommentedMap):
self.embedded_tool = loadingContext.construct_tool_object(
toolpath_object["run"], loadingContext
) # type: Process
else:
loadingContext.metadata = {}
self.embedded_tool = load_tool(toolpath_object["run"], loadingContext)
except ValidationException as vexc:
if loadingContext.debug:
_logger.exception("Validation exception")
raise WorkflowException(
"Tool definition %s failed validation:\n%s"
% (toolpath_object["run"], indent(str(vexc)))
)
def __init__(
self,
toolpath_object, # type: MutableMapping[Text, Any]
loadingContext # type: LoadingContext
): # type: (...) -> None
super(ProcessGenerator, self).__init__(toolpath_object, loadingContext)
self.loadingContext = loadingContext # type: LoadingContext
try:
if isinstance(toolpath_object["run"], MutableMapping):
self.embedded_tool = loadingContext.construct_tool_object(
toolpath_object["run"], loadingContext) # type: Process
else:
loadingContext.metadata = {}
self.embedded_tool = load_tool(toolpath_object["run"],
loadingContext)
except validate.ValidationException as vexc:
if loadingContext.debug:
_logger.exception("Validation exception")
raise WorkflowException(
u"Tool definition %s failed validation:\n%s" %
(toolpath_object["run"], indent(str(vexc))))
def collect_output_ports(self, ports, builder, outdir, compute_checksum=True):
# type: (Set[Dict[Text, Any]], Builder, Text, bool) -> Dict[Text, Union[Text, List[Any], Dict[Text, Any]]]
try:
ret = {} # type: Dict[Text, Union[Text, List[Any], Dict[Text, Any]]]
fs_access = builder.make_fs_access(outdir)
custom_output = fs_access.join(outdir, "cwl.output.json")
if fs_access.exists(custom_output):
with fs_access.open(custom_output, "r") as f:
ret = json.load(f)
if _logger.isEnabledFor(logging.DEBUG):
_logger.debug(u"Raw output from %s: %s", custom_output, json.dumps(ret, indent=4))
else:
for i, port in enumerate(ports):
with SourceLine(ports, i, WorkflowException):
fragment = shortname(port["id"])
try:
ret[fragment] = self.collect_output(port, builder, outdir, fs_access, compute_checksum=compute_checksum)
except Exception as e:
_logger.debug(
u"Error collecting output for parameter '%s'"
% shortname(port["id"]), exc_info=True)
raise WorkflowException(
u"Error collecting output for parameter '%s':\n%s"
% (shortname(port["id"]), indent(unicode(e))))
if ret:
adjustFileObjs(ret,
cast(Callable[[Any], Any], # known bug in mypy
# https://github.com/python/mypy/issues/797
partial(revmap_file, builder, outdir)))
adjustFileObjs(ret, remove_path)
adjustDirObjs(ret, remove_path)
normalizeFilesDirs(ret)
if compute_checksum:
adjustFileObjs(ret, partial(compute_checksums, fs_access))
validate.validate_ex(self.names.get_name("outputs_record_schema", ""), ret)
return ret if ret is not None else {}
except validate.ValidationException as e:
raise WorkflowException("Error validating output record, " + Text(e) + "\n in " + json.dumps(ret, indent=4))
def __init__(
self,
toolpath_object: CommentedMap,
pos: int,
loadingContext: LoadingContext,
parentworkflowProv: Optional[ProvenanceProfile] = None,
) -> None:
"""Initialize this WorkflowStep."""
if "id" in toolpath_object:
self.id = toolpath_object["id"]
else:
self.id = "#step" + str(pos)
loadingContext = loadingContext.copy()
loadingContext.requirements = copy.deepcopy(
getdefault(loadingContext.requirements, [])
)
assert loadingContext.requirements is not None # nosec
loadingContext.requirements.extend(toolpath_object.get("requirements", []))
loadingContext.requirements.extend(
cast(
List[CWLObjectType],
get_overrides(
getdefault(loadingContext.overrides_list, []), self.id
).get("requirements", []),
)
)
hints = copy.deepcopy(getdefault(loadingContext.hints, []))
hints.extend(toolpath_object.get("hints", []))
loadingContext.hints = hints
try:
if isinstance(toolpath_object["run"], CommentedMap):
self.embedded_tool = loadingContext.construct_tool_object(
toolpath_object["run"], loadingContext
) # type: Process
else:
loadingContext.metadata = {}
self.embedded_tool = load_tool(toolpath_object["run"], loadingContext)
except ValidationException as vexc:
if loadingContext.debug:
_logger.exception("Validation exception")
raise WorkflowException(
"Tool definition %s failed validation:\n%s"
% (toolpath_object["run"], indent(str(vexc)))
) from vexc
validation_errors = []
self.tool = toolpath_object = copy.deepcopy(toolpath_object)
bound = set()
for stepfield, toolfield in (("in", "inputs"), ("out", "outputs")):
toolpath_object[toolfield] = []
for index, step_entry in enumerate(toolpath_object[stepfield]):
if isinstance(step_entry, str):
param = CommentedMap() # type: CommentedMap
inputid = step_entry
else:
param = CommentedMap(step_entry.items())
inputid = step_entry["id"]
shortinputid = shortname(inputid)
found = False
for tool_entry in self.embedded_tool.tool[toolfield]:
frag = shortname(tool_entry["id"])
if frag == shortinputid:
# if the case that the step has a default for a parameter,
# we do not want the default of the tool to override it
step_default = None
if "default" in param and "default" in tool_entry:
step_default = param["default"]
param.update(tool_entry)
param["_tool_entry"] = tool_entry
if step_default is not None:
param["default"] = step_default
found = True
bound.add(frag)
break
if not found:
if stepfield == "in":
param["type"] = "Any"
param["used_by_step"] = used_by_step(self.tool, shortinputid)
param["not_connected"] = True
else:
if isinstance(step_entry, Mapping):
step_entry_name = step_entry["id"]
else:
step_entry_name = step_entry
validation_errors.append(
SourceLine(self.tool["out"], index).makeError(
"Workflow step output '%s' does not correspond to"
% shortname(step_entry_name)
)
+ "\n"
+ SourceLine(self.embedded_tool.tool, "outputs").makeError(
" tool output (expected '%s')"
% (
"', '".join(
[
shortname(tool_entry["id"])
for tool_entry in self.embedded_tool.tool[
"outputs"
]
]
)
)
)
)
param["id"] = inputid
param.lc.line = toolpath_object[stepfield].lc.data[index][0]
param.lc.col = toolpath_object[stepfield].lc.data[index][1]
param.lc.filename = toolpath_object[stepfield].lc.filename
toolpath_object[toolfield].append(param)
missing_values = []
for _, tool_entry in enumerate(self.embedded_tool.tool["inputs"]):
if shortname(tool_entry["id"]) not in bound:
if "null" not in tool_entry["type"] and "default" not in tool_entry:
missing_values.append(shortname(tool_entry["id"]))
if missing_values:
validation_errors.append(
SourceLine(self.tool, "in").makeError(
"Step is missing required parameter%s '%s'"
% (
"s" if len(missing_values) > 1 else "",
"', '".join(missing_values),
)
)
)
if validation_errors:
raise ValidationException("\n".join(validation_errors))
super().__init__(toolpath_object, loadingContext)
if self.embedded_tool.tool["class"] == "Workflow":
(feature, _) = self.get_requirement("SubworkflowFeatureRequirement")
if not feature:
raise WorkflowException(
"Workflow contains embedded workflow but "
"SubworkflowFeatureRequirement not in requirements"
)
if "scatter" in self.tool:
(feature, _) = self.get_requirement("ScatterFeatureRequirement")
if not feature:
raise WorkflowException(
"Workflow contains scatter but ScatterFeatureRequirement "
"not in requirements"
)
inputparms = copy.deepcopy(self.tool["inputs"])
outputparms = copy.deepcopy(self.tool["outputs"])
scatter = aslist(self.tool["scatter"])
method = self.tool.get("scatterMethod")
if method is None and len(scatter) != 1:
raise ValidationException(
"Must specify scatterMethod when scattering over multiple inputs"
)
inp_map = {i["id"]: i for i in inputparms}
for inp in scatter:
if inp not in inp_map:
raise ValidationException(
SourceLine(self.tool, "scatter").makeError(
"Scatter parameter '%s' does not correspond to "
"an input parameter of this step, expecting '%s'"
% (
shortname(inp),
"', '".join(shortname(k) for k in inp_map.keys()),
)
)
)
inp_map[inp]["type"] = {"type": "array", "items": inp_map[inp]["type"]}
if self.tool.get("scatterMethod") == "nested_crossproduct":
nesting = len(scatter)
else:
nesting = 1
for _ in range(0, nesting):
for oparam in outputparms:
oparam["type"] = {"type": "array", "items": oparam["type"]}
self.tool["inputs"] = inputparms
self.tool["outputs"] = outputparms
self.prov_obj = None # type: Optional[ProvenanceProfile]
if loadingContext.research_obj is not None:
self.prov_obj = parentworkflowProv
if self.embedded_tool.tool["class"] == "Workflow":
self.parent_wf = self.embedded_tool.parent_wf
else:
self.parent_wf = self.prov_obj
