def load(self, doc, baseuri, loadingOptions, docRoot=None):
# type: (Any, Text, LoadingOptions, Optional[Text]) -> Any
errors = []
for t in self.alternates:
try:
return t.load(doc, baseuri, loadingOptions, docRoot=docRoot)
except ValidationException as e:
errors.append(u"tried %s but\n%s" %
(t.__class__.__name__, indent(str(e))))
raise ValidationException(bullets(errors, u"- "))
def static_checker(workflow_inputs, workflow_outputs, step_inputs, step_outputs, param_to_step):
# type: (List[Dict[Text, Any]], List[Dict[Text, Any]], List[Dict[Text, Any]], List[Dict[Text, Any]], Dict[Text, Dict[Text, Any]]) -> None
"""Check if all source and sink types of a workflow are compatible before run time.
"""
# source parameters: workflow_inputs and step_outputs
# sink parameters: step_inputs and workflow_outputs
# make a dictionary of source parameters, indexed by the "id" field
src_parms = workflow_inputs + step_outputs
src_dict = {}
for parm in src_parms:
src_dict[parm["id"]] = parm
step_inputs_val = check_all_types(src_dict, step_inputs, "source")
workflow_outputs_val = check_all_types(src_dict, workflow_outputs, "outputSource")
warnings = step_inputs_val["warning"] + workflow_outputs_val["warning"]
exceptions = step_inputs_val["exception"] + workflow_outputs_val["exception"]
warning_msgs = []
exception_msgs = []
for warning in warnings:
src = warning.src
sink = warning.sink
linkMerge = warning.linkMerge
if sink.get("secondaryFiles") and sorted(
sink.get("secondaryFiles", [])) != sorted(src.get("secondaryFiles", [])):
msg1 = "Sink '%s'" % (shortname(sink["id"]))
msg2 = SourceLine(sink.get("_tool_entry", sink), "secondaryFiles").makeError(
"expects secondaryFiles: %s but" % (sink.get("secondaryFiles")))
if "secondaryFiles" in src:
msg3 = SourceLine(src, "secondaryFiles").makeError(
"source '%s' has secondaryFiles %s." % (shortname(src["id"]), src.get("secondaryFiles")))
else:
msg3 = SourceLine(src, "id").makeError(
"source '%s' does not include secondaryFiles." % (shortname(src["id"])))
msg4 = SourceLine(src, "id").makeError("To fix, add secondaryFiles: %s to definition of '%s'." % (sink.get("secondaryFiles"), shortname(src["id"])))
msg = SourceLine(sink).makeError("%s\n%s" % (msg1, bullets([msg2, msg3, msg4], " ")))
elif sink.get("not_connected"):
msg = SourceLine(sink, "type").makeError(
"'%s' is not an input parameter of %s, expected %s"
% (shortname(sink["id"]), param_to_step[sink["id"]]["run"],
", ".join(shortname(s["id"])
for s in param_to_step[sink["id"]]["inputs"]
if not s.get("not_connected"))))
else:
msg = SourceLine(src, "type").makeError(
"Source '%s' of type %s may be incompatible"
% (shortname(src["id"]), json_dumps(src["type"]))) + "\n" + \
SourceLine(sink, "type").makeError(
" with sink '%s' of type %s"
% (shortname(sink["id"]), json_dumps(sink["type"])))
if linkMerge:
msg += "\n" + SourceLine(sink).makeError(" source has linkMerge method %s" % linkMerge)
warning_msgs.append(msg)
for exception in exceptions:
src = exception.src
sink = exception.sink
linkMerge = exception.linkMerge
msg = SourceLine(src, "type").makeError(
"Source '%s' of type %s is incompatible"
% (shortname(src["id"]), json_dumps(src["type"]))) + "\n" + \
SourceLine(sink, "type").makeError(
" with sink '%s' of type %s"
% (shortname(sink["id"]), json_dumps(sink["type"])))
if linkMerge:
msg += "\n" + SourceLine(sink).makeError(" source has linkMerge method %s" % linkMerge)
exception_msgs.append(msg)
for sink in step_inputs:
if ('null' != sink["type"] and 'null' not in sink["type"]
and "source" not in sink and "default" not in sink and "valueFrom" not in sink):
msg = SourceLine(sink).makeError(
"Required parameter '%s' does not have source, default, or valueFrom expression"
% shortname(sink["id"]))
exception_msgs.append(msg)
all_warning_msg = strip_dup_lineno("\n".join(warning_msgs))
all_exception_msg = strip_dup_lineno("\n".join(exception_msgs))
if warnings:
_logger.warning("Workflow checker warning:\n%s", all_warning_msg)
if exceptions:
raise validate.ValidationException(all_exception_msg)
def static_checker(
workflow_inputs: List[CWLObjectType],
workflow_outputs: MutableSequence[CWLObjectType],
step_inputs: MutableSequence[CWLObjectType],
step_outputs: List[CWLObjectType],
param_to_step: Dict[str, CWLObjectType],
) -> None:
"""Check if all source and sink types of a workflow are compatible before run time."""
# source parameters: workflow_inputs and step_outputs
# sink parameters: step_inputs and workflow_outputs
# make a dictionary of source parameters, indexed by the "id" field
src_parms = workflow_inputs + step_outputs
src_dict = {} # type: Dict[str, CWLObjectType]
for parm in src_parms:
src_dict[cast(str, parm["id"])] = parm
step_inputs_val = check_all_types(src_dict, step_inputs, "source",
param_to_step)
workflow_outputs_val = check_all_types(src_dict, workflow_outputs,
"outputSource", param_to_step)
warnings = step_inputs_val["warning"] + workflow_outputs_val["warning"]
exceptions = step_inputs_val["exception"] + workflow_outputs_val[
"exception"]
warning_msgs = []
exception_msgs = []
for warning in warnings:
src = warning.src
sink = warning.sink
linkMerge = warning.linkMerge
sinksf = sorted(p["pattern"] for p in sink.get("secondaryFiles", [])
if p.get("required", True))
srcsf = sorted(p["pattern"] for p in src.get("secondaryFiles", []))
# Every secondaryFile required by the sink, should be declared
# by the source
missing = missing_subset(srcsf, sinksf)
if missing:
msg1 = "Parameter '{}' requires secondaryFiles {} but".format(
shortname(sink["id"]),
missing,
)
msg3 = SourceLine(src, "id").makeError(
"source '%s' does not provide those secondaryFiles." %
(shortname(src["id"])))
msg4 = SourceLine(
src.get("_tool_entry", src), "secondaryFiles"
).makeError(
"To resolve, add missing secondaryFiles patterns to definition of '%s' or"
% (shortname(src["id"])))
msg5 = SourceLine(
sink.get("_tool_entry", sink), "secondaryFiles"
).makeError(
"mark missing secondaryFiles in definition of '%s' as optional."
% shortname(sink["id"]))
msg = SourceLine(sink).makeError("{}\n{}".format(
msg1, bullets([msg3, msg4, msg5], " ")))
elif sink.get("not_connected"):
if not sink.get("used_by_step"):
msg = SourceLine(sink, "type").makeError(
"'%s' is not an input parameter of %s, expected %s" % (
shortname(sink["id"]),
param_to_step[sink["id"]]["run"],
", ".join(
shortname(cast(str, s["id"])) for s in cast(
List[Dict[str, Union[str, bool]]],
param_to_step[sink["id"]]["inputs"],
) if not s.get("not_connected")),
))
else:
msg = ""
else:
msg = (SourceLine(src, "type").makeError(
"Source '%s' of type %s may be incompatible" %
(shortname(src["id"]), json_dumps(src["type"]))) + "\n" +
SourceLine(sink, "type").makeError(
" with sink '%s' of type %s" %
(shortname(sink["id"]), json_dumps(sink["type"]))))
if linkMerge is not None:
msg += "\n" + SourceLine(sink).makeError(
" source has linkMerge method %s" % linkMerge)
if warning.message is not None:
msg += "\n" + SourceLine(sink).makeError(" " + warning.message)
if msg:
warning_msgs.append(msg)
for exception in exceptions:
src = exception.src
sink = exception.sink
linkMerge = exception.linkMerge
extra_message = exception.message
msg = (SourceLine(src, "type").makeError(
"Source '%s' of type %s is incompatible" %
(shortname(src["id"]), json_dumps(src["type"]))) + "\n" +
SourceLine(sink, "type").makeError(
" with sink '%s' of type %s" %
(shortname(sink["id"]), json_dumps(sink["type"]))))
if extra_message is not None:
msg += "\n" + SourceLine(sink).makeError(" " + extra_message)
if linkMerge is not None:
msg += "\n" + SourceLine(sink).makeError(
" source has linkMerge method %s" % linkMerge)
exception_msgs.append(msg)
for sink in step_inputs:
if ("null" != sink["type"] and "null" not in sink["type"]
and "source" not in sink and "default" not in sink
and "valueFrom" not in sink):
msg = SourceLine(sink).makeError(
"Required parameter '%s' does not have source, default, or valueFrom expression"
% shortname(sink["id"]))
exception_msgs.append(msg)
all_warning_msg = strip_dup_lineno("\n".join(warning_msgs))
all_exception_msg = strip_dup_lineno("\n" + "\n".join(exception_msgs))
if all_warning_msg:
_logger.warning("Workflow checker warning:\n%s", all_warning_msg)
if exceptions:
raise ValidationException(all_exception_msg)
def static_checker(workflow_inputs, workflow_outputs, step_inputs,
step_outputs, param_to_step):
# type: (List[Dict[Text, Any]], List[Dict[Text, Any]], List[Dict[Text, Any]], List[Dict[Text, Any]], Dict[Text, Dict[Text, Any]]) -> None
"""Check if all source and sink types of a workflow are compatible before run time."""
# source parameters: workflow_inputs and step_outputs
# sink parameters: step_inputs and workflow_outputs
# make a dictionary of source parameters, indexed by the "id" field
src_parms = workflow_inputs + step_outputs
src_dict = {}
for parm in src_parms:
src_dict[parm["id"]] = parm
step_inputs_val = check_all_types(src_dict, step_inputs, "source")
workflow_outputs_val = check_all_types(src_dict, workflow_outputs,
"outputSource")
warnings = step_inputs_val["warning"] + workflow_outputs_val["warning"]
exceptions = step_inputs_val["exception"] + workflow_outputs_val[
"exception"]
warning_msgs = []
exception_msgs = []
for warning in warnings:
src = warning.src
sink = warning.sink
linkMerge = warning.linkMerge
sinksf = sorted([
p["pattern"] for p in sink.get("secondaryFiles", [])
if p.get("required", True)
])
srcsf = sorted([p["pattern"] for p in src.get("secondaryFiles", [])])
# Every secondaryFile required by the sink, should be declared
# by the source
missing = missing_subset(srcsf, sinksf)
if missing:
msg1 = "Parameter '%s' requires secondaryFiles %s but" % (
shortname(sink["id"]), missing)
msg3 = SourceLine(src, "id").makeError(
"source '%s' does not provide those secondaryFiles." %
(shortname(src["id"])))
msg4 = SourceLine(
src.get("_tool_entry", src), "secondaryFiles"
).makeError(
"To resolve, add missing secondaryFiles patterns to definition of '%s' or"
% (shortname(src["id"])))
msg5 = SourceLine(
sink.get("_tool_entry", sink), "secondaryFiles"
).makeError(
"mark missing secondaryFiles in definition of '%s' as optional."
% shortname(sink["id"]))
msg = SourceLine(sink).makeError(
"%s\n%s" % (msg1, bullets([msg3, msg4, msg5], " ")))
elif sink.get("not_connected"):
msg = SourceLine(sink, "type").makeError(
"'%s' is not an input parameter of %s, expected %s" %
(shortname(
sink["id"]), param_to_step[sink["id"]]["run"], ", ".join(
shortname(s["id"])
for s in param_to_step[sink["id"]]["inputs"]
if not s.get("not_connected"))))
else:
msg = SourceLine(src, "type").makeError(
"Source '%s' of type %s may be incompatible"
% (shortname(src["id"]), json_dumps(src["type"]))) + "\n" + \
SourceLine(sink, "type").makeError(
" with sink '%s' of type %s"
% (shortname(sink["id"]), json_dumps(sink["type"])))
if linkMerge is not None:
msg += "\n" + SourceLine(sink).makeError(
" source has linkMerge method %s" % linkMerge)
warning_msgs.append(msg)
for exception in exceptions:
src = exception.src
sink = exception.sink
linkMerge = exception.linkMerge
msg = SourceLine(src, "type").makeError(
"Source '%s' of type %s is incompatible"
% (shortname(src["id"]), json_dumps(src["type"]))) + "\n" + \
SourceLine(sink, "type").makeError(
" with sink '%s' of type %s"
% (shortname(sink["id"]), json_dumps(sink["type"])))
if linkMerge is not None:
msg += "\n" + SourceLine(sink).makeError(
" source has linkMerge method %s" % linkMerge)
exception_msgs.append(msg)
for sink in step_inputs:
if ('null' != sink["type"] and 'null' not in sink["type"]
and "source" not in sink and "default" not in sink
and "valueFrom" not in sink):
msg = SourceLine(sink).makeError(
"Required parameter '%s' does not have source, default, or valueFrom expression"
% shortname(sink["id"]))
exception_msgs.append(msg)
all_warning_msg = strip_dup_lineno("\n".join(warning_msgs))
all_exception_msg = strip_dup_lineno("\n".join(exception_msgs))
if warnings:
_logger.warning("Workflow checker warning:\n%s", all_warning_msg)
if exceptions:
raise validate.ValidationException(all_exception_msg)
def static_checker(workflow_inputs, workflow_outputs, step_inputs,
step_outputs):
# type: (List[Dict[Text, Any]], List[Dict[Text, Any]], List[Dict[Text, Any]], List[Dict[Text, Any]]) -> None
"""Check if all source and sink types of a workflow are compatible before run time.
"""
# source parameters: workflow_inputs and step_outputs
# sink parameters: step_inputs and workflow_outputs
# make a dictionary of source parameters, indexed by the "id" field
src_parms = workflow_inputs + step_outputs
src_dict = {}
for parm in src_parms:
src_dict[parm["id"]] = parm
step_inputs_val = check_all_types(src_dict, step_inputs, "source")
workflow_outputs_val = check_all_types(src_dict, workflow_outputs,
"outputSource")
warnings = step_inputs_val["warning"] + workflow_outputs_val["warning"]
exceptions = step_inputs_val["exception"] + workflow_outputs_val[
"exception"]
warning_msgs = []
exception_msgs = []
for warning in warnings:
src = warning.src
sink = warning.sink
linkMerge = warning.linkMerge
if sink.get("secondaryFiles") and sorted(sink.get(
"secondaryFiles", [])) != sorted(src.get("secondaryFiles",
[])):
msg1 = "Sink '%s'" % (shortname(sink["id"]))
msg2 = SourceLine(sink.get("_tool_entry", sink),
"secondaryFiles").makeError(
"expects secondaryFiles: %s but" %
(sink.get("secondaryFiles")))
if "secondaryFiles" in src:
msg3 = SourceLine(src, "secondaryFiles").makeError(
"source '%s' has secondaryFiles %s." %
(shortname(src["id"]), src.get("secondaryFiles")))
else:
msg3 = SourceLine(src, "id").makeError(
"source '%s' does not include secondaryFiles." %
(shortname(src["id"])))
msg4 = SourceLine(src, "id").makeError(
"To fix, add secondaryFiles: %s to definition of '%s'." %
(sink.get("secondaryFiles"), shortname(src["id"])))
msg = SourceLine(sink).makeError(
"%s\n%s" % (msg1, bullets([msg2, msg3, msg4], " ")))
else:
msg = SourceLine(src, "type").makeError(
"Source '%s' of type %s may be incompatible"
% (shortname(src["id"]), json.dumps(src["type"]))) + "\n" + \
SourceLine(sink, "type").makeError(
" with sink '%s' of type %s"
% (shortname(sink["id"]), json.dumps(sink["type"])))
if linkMerge:
msg += "\n" + SourceLine(sink).makeError(
" source has linkMerge method %s" % linkMerge)
warning_msgs.append(msg)
for exception in exceptions:
src = exception.src
sink = exception.sink
linkMerge = exception.linkMerge
msg = SourceLine(src, "type").makeError(
"Source '%s' of type %s is incompatible"
% (shortname(src["id"]), json.dumps(src["type"]))) + "\n" + \
SourceLine(sink, "type").makeError(
" with sink '%s' of type %s"
% (shortname(sink["id"]), json.dumps(sink["type"])))
if linkMerge:
msg += "\n" + SourceLine(sink).makeError(
" source has linkMerge method %s" % linkMerge)
exception_msgs.append(msg)
for sink in step_inputs:
if ('null' != sink["type"] and 'null' not in sink["type"]
and "source" not in sink and "default" not in sink
and "valueFrom" not in sink):
msg = SourceLine(sink).makeError(
"Required parameter '%s' does not have source, default, or valueFrom expression"
% shortname(sink["id"]))
exception_msgs.append(msg)
all_warning_msg = strip_dup_lineno("\n".join(warning_msgs))
all_exception_msg = strip_dup_lineno("\n".join(exception_msgs))
if warnings:
_logger.warning("Workflow checker warning:\n%s" % all_warning_msg)
if exceptions:
raise validate.ValidationException(all_exception_msg)