def load(self, doc, baseuri, loadingOptions, docRoot=None): # type: (Any, Text, LoadingOptions, Optional[Text]) -> Any errors = [] for t in self.alternates: try: return t.load(doc, baseuri, loadingOptions, docRoot=docRoot) except ValidationException as e: errors.append(u"tried %s but\n%s" % (t.__class__.__name__, indent(str(e)))) raise ValidationException(bullets(errors, u"- "))
def static_checker(workflow_inputs, workflow_outputs, step_inputs, step_outputs, param_to_step): # type: (List[Dict[Text, Any]], List[Dict[Text, Any]], List[Dict[Text, Any]], List[Dict[Text, Any]], Dict[Text, Dict[Text, Any]]) -> None """Check if all source and sink types of a workflow are compatible before run time. """ # source parameters: workflow_inputs and step_outputs # sink parameters: step_inputs and workflow_outputs # make a dictionary of source parameters, indexed by the "id" field src_parms = workflow_inputs + step_outputs src_dict = {} for parm in src_parms: src_dict[parm["id"]] = parm step_inputs_val = check_all_types(src_dict, step_inputs, "source") workflow_outputs_val = check_all_types(src_dict, workflow_outputs, "outputSource") warnings = step_inputs_val["warning"] + workflow_outputs_val["warning"] exceptions = step_inputs_val["exception"] + workflow_outputs_val["exception"] warning_msgs = [] exception_msgs = [] for warning in warnings: src = warning.src sink = warning.sink linkMerge = warning.linkMerge if sink.get("secondaryFiles") and sorted( sink.get("secondaryFiles", [])) != sorted(src.get("secondaryFiles", [])): msg1 = "Sink '%s'" % (shortname(sink["id"])) msg2 = SourceLine(sink.get("_tool_entry", sink), "secondaryFiles").makeError( "expects secondaryFiles: %s but" % (sink.get("secondaryFiles"))) if "secondaryFiles" in src: msg3 = SourceLine(src, "secondaryFiles").makeError( "source '%s' has secondaryFiles %s." % (shortname(src["id"]), src.get("secondaryFiles"))) else: msg3 = SourceLine(src, "id").makeError( "source '%s' does not include secondaryFiles." % (shortname(src["id"]))) msg4 = SourceLine(src, "id").makeError("To fix, add secondaryFiles: %s to definition of '%s'." % (sink.get("secondaryFiles"), shortname(src["id"]))) msg = SourceLine(sink).makeError("%s\n%s" % (msg1, bullets([msg2, msg3, msg4], " "))) elif sink.get("not_connected"): msg = SourceLine(sink, "type").makeError( "'%s' is not an input parameter of %s, expected %s" % (shortname(sink["id"]), param_to_step[sink["id"]]["run"], ", ".join(shortname(s["id"]) for s in param_to_step[sink["id"]]["inputs"] if not s.get("not_connected")))) else: msg = SourceLine(src, "type").makeError( "Source '%s' of type %s may be incompatible" % (shortname(src["id"]), json_dumps(src["type"]))) + "\n" + \ SourceLine(sink, "type").makeError( " with sink '%s' of type %s" % (shortname(sink["id"]), json_dumps(sink["type"]))) if linkMerge: msg += "\n" + SourceLine(sink).makeError(" source has linkMerge method %s" % linkMerge) warning_msgs.append(msg) for exception in exceptions: src = exception.src sink = exception.sink linkMerge = exception.linkMerge msg = SourceLine(src, "type").makeError( "Source '%s' of type %s is incompatible" % (shortname(src["id"]), json_dumps(src["type"]))) + "\n" + \ SourceLine(sink, "type").makeError( " with sink '%s' of type %s" % (shortname(sink["id"]), json_dumps(sink["type"]))) if linkMerge: msg += "\n" + SourceLine(sink).makeError(" source has linkMerge method %s" % linkMerge) exception_msgs.append(msg) for sink in step_inputs: if ('null' != sink["type"] and 'null' not in sink["type"] and "source" not in sink and "default" not in sink and "valueFrom" not in sink): msg = SourceLine(sink).makeError( "Required parameter '%s' does not have source, default, or valueFrom expression" % shortname(sink["id"])) exception_msgs.append(msg) all_warning_msg = strip_dup_lineno("\n".join(warning_msgs)) all_exception_msg = strip_dup_lineno("\n".join(exception_msgs)) if warnings: _logger.warning("Workflow checker warning:\n%s", all_warning_msg) if exceptions: raise validate.ValidationException(all_exception_msg)
def static_checker( workflow_inputs: List[CWLObjectType], workflow_outputs: MutableSequence[CWLObjectType], step_inputs: MutableSequence[CWLObjectType], step_outputs: List[CWLObjectType], param_to_step: Dict[str, CWLObjectType], ) -> None: """Check if all source and sink types of a workflow are compatible before run time.""" # source parameters: workflow_inputs and step_outputs # sink parameters: step_inputs and workflow_outputs # make a dictionary of source parameters, indexed by the "id" field src_parms = workflow_inputs + step_outputs src_dict = {} # type: Dict[str, CWLObjectType] for parm in src_parms: src_dict[cast(str, parm["id"])] = parm step_inputs_val = check_all_types(src_dict, step_inputs, "source", param_to_step) workflow_outputs_val = check_all_types(src_dict, workflow_outputs, "outputSource", param_to_step) warnings = step_inputs_val["warning"] + workflow_outputs_val["warning"] exceptions = step_inputs_val["exception"] + workflow_outputs_val[ "exception"] warning_msgs = [] exception_msgs = [] for warning in warnings: src = warning.src sink = warning.sink linkMerge = warning.linkMerge sinksf = sorted(p["pattern"] for p in sink.get("secondaryFiles", []) if p.get("required", True)) srcsf = sorted(p["pattern"] for p in src.get("secondaryFiles", [])) # Every secondaryFile required by the sink, should be declared # by the source missing = missing_subset(srcsf, sinksf) if missing: msg1 = "Parameter '{}' requires secondaryFiles {} but".format( shortname(sink["id"]), missing, ) msg3 = SourceLine(src, "id").makeError( "source '%s' does not provide those secondaryFiles." % (shortname(src["id"]))) msg4 = SourceLine( src.get("_tool_entry", src), "secondaryFiles" ).makeError( "To resolve, add missing secondaryFiles patterns to definition of '%s' or" % (shortname(src["id"]))) msg5 = SourceLine( sink.get("_tool_entry", sink), "secondaryFiles" ).makeError( "mark missing secondaryFiles in definition of '%s' as optional." % shortname(sink["id"])) msg = SourceLine(sink).makeError("{}\n{}".format( msg1, bullets([msg3, msg4, msg5], " "))) elif sink.get("not_connected"): if not sink.get("used_by_step"): msg = SourceLine(sink, "type").makeError( "'%s' is not an input parameter of %s, expected %s" % ( shortname(sink["id"]), param_to_step[sink["id"]]["run"], ", ".join( shortname(cast(str, s["id"])) for s in cast( List[Dict[str, Union[str, bool]]], param_to_step[sink["id"]]["inputs"], ) if not s.get("not_connected")), )) else: msg = "" else: msg = (SourceLine(src, "type").makeError( "Source '%s' of type %s may be incompatible" % (shortname(src["id"]), json_dumps(src["type"]))) + "\n" + SourceLine(sink, "type").makeError( " with sink '%s' of type %s" % (shortname(sink["id"]), json_dumps(sink["type"])))) if linkMerge is not None: msg += "\n" + SourceLine(sink).makeError( " source has linkMerge method %s" % linkMerge) if warning.message is not None: msg += "\n" + SourceLine(sink).makeError(" " + warning.message) if msg: warning_msgs.append(msg) for exception in exceptions: src = exception.src sink = exception.sink linkMerge = exception.linkMerge extra_message = exception.message msg = (SourceLine(src, "type").makeError( "Source '%s' of type %s is incompatible" % (shortname(src["id"]), json_dumps(src["type"]))) + "\n" + SourceLine(sink, "type").makeError( " with sink '%s' of type %s" % (shortname(sink["id"]), json_dumps(sink["type"])))) if extra_message is not None: msg += "\n" + SourceLine(sink).makeError(" " + extra_message) if linkMerge is not None: msg += "\n" + SourceLine(sink).makeError( " source has linkMerge method %s" % linkMerge) exception_msgs.append(msg) for sink in step_inputs: if ("null" != sink["type"] and "null" not in sink["type"] and "source" not in sink and "default" not in sink and "valueFrom" not in sink): msg = SourceLine(sink).makeError( "Required parameter '%s' does not have source, default, or valueFrom expression" % shortname(sink["id"])) exception_msgs.append(msg) all_warning_msg = strip_dup_lineno("\n".join(warning_msgs)) all_exception_msg = strip_dup_lineno("\n" + "\n".join(exception_msgs)) if all_warning_msg: _logger.warning("Workflow checker warning:\n%s", all_warning_msg) if exceptions: raise ValidationException(all_exception_msg)
def static_checker(workflow_inputs, workflow_outputs, step_inputs, step_outputs, param_to_step): # type: (List[Dict[Text, Any]], List[Dict[Text, Any]], List[Dict[Text, Any]], List[Dict[Text, Any]], Dict[Text, Dict[Text, Any]]) -> None """Check if all source and sink types of a workflow are compatible before run time.""" # source parameters: workflow_inputs and step_outputs # sink parameters: step_inputs and workflow_outputs # make a dictionary of source parameters, indexed by the "id" field src_parms = workflow_inputs + step_outputs src_dict = {} for parm in src_parms: src_dict[parm["id"]] = parm step_inputs_val = check_all_types(src_dict, step_inputs, "source") workflow_outputs_val = check_all_types(src_dict, workflow_outputs, "outputSource") warnings = step_inputs_val["warning"] + workflow_outputs_val["warning"] exceptions = step_inputs_val["exception"] + workflow_outputs_val[ "exception"] warning_msgs = [] exception_msgs = [] for warning in warnings: src = warning.src sink = warning.sink linkMerge = warning.linkMerge sinksf = sorted([ p["pattern"] for p in sink.get("secondaryFiles", []) if p.get("required", True) ]) srcsf = sorted([p["pattern"] for p in src.get("secondaryFiles", [])]) # Every secondaryFile required by the sink, should be declared # by the source missing = missing_subset(srcsf, sinksf) if missing: msg1 = "Parameter '%s' requires secondaryFiles %s but" % ( shortname(sink["id"]), missing) msg3 = SourceLine(src, "id").makeError( "source '%s' does not provide those secondaryFiles." % (shortname(src["id"]))) msg4 = SourceLine( src.get("_tool_entry", src), "secondaryFiles" ).makeError( "To resolve, add missing secondaryFiles patterns to definition of '%s' or" % (shortname(src["id"]))) msg5 = SourceLine( sink.get("_tool_entry", sink), "secondaryFiles" ).makeError( "mark missing secondaryFiles in definition of '%s' as optional." % shortname(sink["id"])) msg = SourceLine(sink).makeError( "%s\n%s" % (msg1, bullets([msg3, msg4, msg5], " "))) elif sink.get("not_connected"): msg = SourceLine(sink, "type").makeError( "'%s' is not an input parameter of %s, expected %s" % (shortname( sink["id"]), param_to_step[sink["id"]]["run"], ", ".join( shortname(s["id"]) for s in param_to_step[sink["id"]]["inputs"] if not s.get("not_connected")))) else: msg = SourceLine(src, "type").makeError( "Source '%s' of type %s may be incompatible" % (shortname(src["id"]), json_dumps(src["type"]))) + "\n" + \ SourceLine(sink, "type").makeError( " with sink '%s' of type %s" % (shortname(sink["id"]), json_dumps(sink["type"]))) if linkMerge is not None: msg += "\n" + SourceLine(sink).makeError( " source has linkMerge method %s" % linkMerge) warning_msgs.append(msg) for exception in exceptions: src = exception.src sink = exception.sink linkMerge = exception.linkMerge msg = SourceLine(src, "type").makeError( "Source '%s' of type %s is incompatible" % (shortname(src["id"]), json_dumps(src["type"]))) + "\n" + \ SourceLine(sink, "type").makeError( " with sink '%s' of type %s" % (shortname(sink["id"]), json_dumps(sink["type"]))) if linkMerge is not None: msg += "\n" + SourceLine(sink).makeError( " source has linkMerge method %s" % linkMerge) exception_msgs.append(msg) for sink in step_inputs: if ('null' != sink["type"] and 'null' not in sink["type"] and "source" not in sink and "default" not in sink and "valueFrom" not in sink): msg = SourceLine(sink).makeError( "Required parameter '%s' does not have source, default, or valueFrom expression" % shortname(sink["id"])) exception_msgs.append(msg) all_warning_msg = strip_dup_lineno("\n".join(warning_msgs)) all_exception_msg = strip_dup_lineno("\n".join(exception_msgs)) if warnings: _logger.warning("Workflow checker warning:\n%s", all_warning_msg) if exceptions: raise validate.ValidationException(all_exception_msg)
def static_checker(workflow_inputs, workflow_outputs, step_inputs, step_outputs): # type: (List[Dict[Text, Any]], List[Dict[Text, Any]], List[Dict[Text, Any]], List[Dict[Text, Any]]) -> None """Check if all source and sink types of a workflow are compatible before run time. """ # source parameters: workflow_inputs and step_outputs # sink parameters: step_inputs and workflow_outputs # make a dictionary of source parameters, indexed by the "id" field src_parms = workflow_inputs + step_outputs src_dict = {} for parm in src_parms: src_dict[parm["id"]] = parm step_inputs_val = check_all_types(src_dict, step_inputs, "source") workflow_outputs_val = check_all_types(src_dict, workflow_outputs, "outputSource") warnings = step_inputs_val["warning"] + workflow_outputs_val["warning"] exceptions = step_inputs_val["exception"] + workflow_outputs_val[ "exception"] warning_msgs = [] exception_msgs = [] for warning in warnings: src = warning.src sink = warning.sink linkMerge = warning.linkMerge if sink.get("secondaryFiles") and sorted(sink.get( "secondaryFiles", [])) != sorted(src.get("secondaryFiles", [])): msg1 = "Sink '%s'" % (shortname(sink["id"])) msg2 = SourceLine(sink.get("_tool_entry", sink), "secondaryFiles").makeError( "expects secondaryFiles: %s but" % (sink.get("secondaryFiles"))) if "secondaryFiles" in src: msg3 = SourceLine(src, "secondaryFiles").makeError( "source '%s' has secondaryFiles %s." % (shortname(src["id"]), src.get("secondaryFiles"))) else: msg3 = SourceLine(src, "id").makeError( "source '%s' does not include secondaryFiles." % (shortname(src["id"]))) msg4 = SourceLine(src, "id").makeError( "To fix, add secondaryFiles: %s to definition of '%s'." % (sink.get("secondaryFiles"), shortname(src["id"]))) msg = SourceLine(sink).makeError( "%s\n%s" % (msg1, bullets([msg2, msg3, msg4], " "))) else: msg = SourceLine(src, "type").makeError( "Source '%s' of type %s may be incompatible" % (shortname(src["id"]), json.dumps(src["type"]))) + "\n" + \ SourceLine(sink, "type").makeError( " with sink '%s' of type %s" % (shortname(sink["id"]), json.dumps(sink["type"]))) if linkMerge: msg += "\n" + SourceLine(sink).makeError( " source has linkMerge method %s" % linkMerge) warning_msgs.append(msg) for exception in exceptions: src = exception.src sink = exception.sink linkMerge = exception.linkMerge msg = SourceLine(src, "type").makeError( "Source '%s' of type %s is incompatible" % (shortname(src["id"]), json.dumps(src["type"]))) + "\n" + \ SourceLine(sink, "type").makeError( " with sink '%s' of type %s" % (shortname(sink["id"]), json.dumps(sink["type"]))) if linkMerge: msg += "\n" + SourceLine(sink).makeError( " source has linkMerge method %s" % linkMerge) exception_msgs.append(msg) for sink in step_inputs: if ('null' != sink["type"] and 'null' not in sink["type"] and "source" not in sink and "default" not in sink and "valueFrom" not in sink): msg = SourceLine(sink).makeError( "Required parameter '%s' does not have source, default, or valueFrom expression" % shortname(sink["id"])) exception_msgs.append(msg) all_warning_msg = strip_dup_lineno("\n".join(warning_msgs)) all_exception_msg = strip_dup_lineno("\n".join(exception_msgs)) if warnings: _logger.warning("Workflow checker warning:\n%s" % all_warning_msg) if exceptions: raise validate.ValidationException(all_exception_msg)