def test_scoped_id() -> None: ldr = Loader({}) ctx = { "id": "@id", "location": {"@id": "@id", "@type": "@id"}, "bar": "http://example.com/bar", "ex": "http://example.com/", } # type: ContextType ldr.add_context(ctx) ra, _ = ldr.resolve_all( cmap({"id": "foo", "bar": {"id": "baz"}}), "http://example.com" ) assert { "id": "http://example.com/#foo", "bar": {"id": "http://example.com/#foo/baz"}, } == ra g = makerdf(None, ra, ctx) print(g.serialize(format="n3")) ra, _ = ldr.resolve_all( cmap({"location": "foo", "bar": {"location": "baz"}}), "http://example.com", checklinks=False, ) assert { "location": "http://example.com/foo", "bar": {"location": "http://example.com/baz"}, } == ra g = makerdf(None, ra, ctx) print(g.serialize(format="n3")) ra, _ = ldr.resolve_all( cmap({"id": "foo", "bar": {"location": "baz"}}), "http://example.com", checklinks=False, ) assert { "id": "http://example.com/#foo", "bar": {"location": "http://example.com/baz"}, } == ra g = makerdf(None, ra, ctx) print(g.serialize(format="n3")) ra, _ = ldr.resolve_all( cmap({"location": "foo", "bar": {"id": "baz"}}), "http://example.com", checklinks=False, ) assert { "location": "http://example.com/foo", "bar": {"id": "http://example.com/#baz"}, } == ra g = makerdf(None, ra, ctx) print(g.serialize(format="n3"))
def test_secondaryFile_dsl_ref() -> None: ldr = Loader({}) ldr.add_context({"secondaryFiles": {"secondaryFilesDSL": True}}) ra, _ = ldr.resolve_all(cmap({"secondaryFiles": ".foo"}), "") assert {"secondaryFiles": {"pattern": ".foo", "required": None}} == ra ra, _ = ldr.resolve_all(cmap({"secondaryFiles": ".foo?"}), "") assert {"secondaryFiles": {"pattern": ".foo", "required": False}} == ra ra, _ = ldr.resolve_all(cmap({"secondaryFiles": [".foo"]}), "") assert {"secondaryFiles": [{"pattern": ".foo", "required": None}]} == ra ra, _ = ldr.resolve_all(cmap({"secondaryFiles": [".foo?"]}), "") assert {"secondaryFiles": [{"pattern": ".foo", "required": False}]} == ra
def load_job_order( args: argparse.Namespace, stdin: IO[Any], fetcher_constructor: Optional[FetcherCallableType], overrides_list: List[CWLObjectType], tool_file_uri: str, ) -> Tuple[Optional[CWLObjectType], str, Loader]: job_order_object = None job_order_file = None _jobloaderctx = jobloaderctx.copy() loader = Loader(_jobloaderctx, fetcher_constructor=fetcher_constructor) if len(args.job_order) == 1 and args.job_order[0][0] != "-": job_order_file = args.job_order[0] elif len(args.job_order) == 1 and args.job_order[0] == "-": job_order_object = yaml.main.round_trip_load(stdin) job_order_object, _ = loader.resolve_all( job_order_object, file_uri(os.getcwd()) + "/" ) else: job_order_file = None if job_order_object is not None: input_basedir = args.basedir if args.basedir else os.getcwd() elif job_order_file is not None: input_basedir = ( args.basedir if args.basedir else os.path.abspath(os.path.dirname(job_order_file)) ) job_order_object, _ = loader.resolve_ref(job_order_file, checklinks=False) if ( job_order_object is not None and "http://commonwl.org/cwltool#overrides" in job_order_object ): ov_uri = file_uri(job_order_file or input_basedir) overrides_list.extend( resolve_overrides(job_order_object, ov_uri, tool_file_uri) ) del job_order_object["http://commonwl.org/cwltool#overrides"] if job_order_object is None: input_basedir = args.basedir if args.basedir else os.getcwd() if job_order_object is not None and not isinstance( job_order_object, MutableMapping ): _logger.error( "CWL input object at %s is not formatted correctly, it should be a " "JSON/YAML dictionay, not %s.\n" "Raw input object:\n%s", job_order_file or "stdin", type(job_order_object), job_order_object, ) sys.exit(1) return (job_order_object, input_basedir, loader)
def test_idmap() -> None: ldr = Loader({}) ldr.add_context( { "inputs": { "@id": "http://example.com/inputs", "mapSubject": "id", "mapPredicate": "a", }, "outputs": {"@type": "@id", "identity": True}, "id": "@id", } ) ra, _ = ldr.resolve_all( cmap( { "id": "stuff", "inputs": {"zip": 1, "zing": 2}, "outputs": ["out"], "other": {"n": 9}, } ), "http://example2.com/", ) assert isinstance(ra, CommentedMap) assert "http://example2.com/#stuff" == ra["id"] for item in ra["inputs"]: if item["a"] == 2: assert "http://example2.com/#stuff/zing" == item["id"] else: assert "http://example2.com/#stuff/zip" == item["id"] assert ["http://example2.com/#stuff/out"] == ra["outputs"] assert {"n": 9} == ra["other"]
def cwl_dispatch(self, json): try: cwlwf, it_is_workflow = load_cwl(self.dag.default_args["cwl_workflow"], self.dag.default_args) cwl_context = { "outdir": mkdtemp( prefix=os.path.abspath(os.path.join(self.tmp_folder, 'dag_tmp_'))) } _jobloaderctx = jobloaderctx.copy() _jobloaderctx.update(cwlwf.metadata.get("$namespaces", {})) loader = Loader(_jobloaderctx) try: job_order_object = yaml.round_trip_load(io.StringIO(initial_value=dumps(json))) job_order_object, _ = loader.resolve_all(job_order_object, file_uri(os.getcwd()) + "/", checklinks=False) except Exception as e: _logger.error("Job Loader: {}".format(str(e))) job_order_object = init_job_order(job_order_object, None, cwlwf, loader, sys.stdout) cwl_context['promises'] = job_order_object logging.info( '{0}: Final job: \n {1}'.format(self.task_id, dumps(cwl_context, indent=4))) return cwl_context except Exception as e: _logger.info( 'Dispatch Exception {0}: \n {1} {2}'.format(self.task_id, type(e), e)) pass return None
def load_job_order(args, # type: argparse.Namespace stdin, # type: IO[Any] fetcher_constructor, # Fetcher overrides, # type: List[Dict[Text, Any]] tool_file_uri # type: Text ): # type: (...) -> Tuple[Dict[Text, Any], Text, Loader] job_order_object = None _jobloaderctx = jobloaderctx.copy() loader = Loader(_jobloaderctx, fetcher_constructor=fetcher_constructor) # type: ignore if len(args.job_order) == 1 and args.job_order[0][0] != "-": job_order_file = args.job_order[0] elif len(args.job_order) == 1 and args.job_order[0] == "-": job_order_object = yaml.round_trip_load(stdin) job_order_object, _ = loader.resolve_all(job_order_object, file_uri(os.getcwd()) + "/") else: job_order_file = None if job_order_object: input_basedir = args.basedir if args.basedir else os.getcwd() elif job_order_file: input_basedir = args.basedir if args.basedir else os.path.abspath(os.path.dirname(job_order_file)) job_order_object, _ = loader.resolve_ref(job_order_file, checklinks=False) if job_order_object and "http://commonwl.org/cwltool#overrides" in job_order_object: overrides.extend(resolve_overrides(job_order_object, file_uri(job_order_file), tool_file_uri)) del job_order_object["http://commonwl.org/cwltool#overrides"] if not job_order_object: input_basedir = args.basedir if args.basedir else os.getcwd() return (job_order_object, input_basedir, loader)
def test_rdf_datetime() -> None: """Affirm that datetime objects can be serialized in makerdf().""" ldr = Loader({}) ctx: ContextType = { "id": "@id", "location": {"@id": "@id", "@type": "@id"}, "bar": "http://example.com/bar", "ex": "http://example.com/", } ldr.add_context(ctx) ra: CommentedMap = cast( CommentedMap, ldr.resolve_all( cmap( { "id": "foo", "bar": {"id": "baz"}, } ), "http://example.com", )[0], ) ra["s:dateCreated"] = datetime.datetime(2020, 10, 8) g = makerdf(None, ra, ctx) g.serialize(destination=stdout(), format="n3") g2 = makerdf(None, CommentedSeq([ra]), ctx) g2.serialize(destination=stdout(), format="n3")
def resolve_overrides(ov, ov_uri, baseurl): # type: (CommentedMap, Text, Text) -> List[Dict[Text, Any]] ovloader = Loader(overrides_ctx) ret, _ = ovloader.resolve_all(ov, baseurl) if not isinstance(ret, CommentedMap): raise Exception("Expected CommentedMap, got %s" % type(ret)) cwl_docloader = get_schema("v1.0")[0] cwl_docloader.resolve_all(ret, ov_uri) return ret["overrides"]
def resolve_overrides( ov, ov_uri, baseurl): # type: (CommentedMap, Text, Text) -> List[Dict[Text, Any]] ovloader = Loader(overrides_ctx) ret, _ = ovloader.resolve_all(ov, baseurl) if not isinstance(ret, CommentedMap): raise Exception("Expected CommentedMap, got %s" % type(ret)) cwl_docloader = get_schema("v1.0")[0] cwl_docloader.resolve_all(ret, ov_uri) return ret["overrides"]
def resolve_overrides(ov, # Type: CommentedMap ov_uri, # Type: Text baseurl # type: Text ): # type: (...) -> List[Dict[Text, Any]] ovloader = Loader(overrides_ctx) ret, _ = ovloader.resolve_all(ov, baseurl) if not isinstance(ret, CommentedMap): raise Exception("Expected CommentedMap, got %s" % type(ret)) cwl_docloader = get_schema("v1.0")[0] cwl_docloader.resolve_all(ret, ov_uri) return ret["http://commonwl.org/cwltool#overrides"]
def resolve_overrides( ov: IdxResultType, ov_uri: str, baseurl: str, ) -> List[CWLObjectType]: ovloader = Loader(overrides_ctx) ret, _ = ovloader.resolve_all(ov, baseurl) if not isinstance(ret, CommentedMap): raise Exception("Expected CommentedMap, got %s" % type(ret)) cwl_docloader = get_schema("v1.0")[0] cwl_docloader.resolve_all(ret, ov_uri) return cast(List[CWLObjectType], ret["http://commonwl.org/cwltool#overrides"])
def test_blank_node_id() -> None: # Test that blank nodes are passed through and not considered # relative paths. Blank nodes (also called anonymous ids) are # URIs starting with "_:". They are randomly generated # placeholders mainly used internally where an id is needed but # was not given. ldr = Loader({}) ctx = {"id": "@id"} # type: ContextType ldr.add_context(ctx) ra, _ = ldr.resolve_all(cmap({"id": "_:foo"}), "http://example.com") assert {"id": "_:foo"} == ra
def test_typedsl_ref() -> None: ldr = Loader({}) ldr.add_context( { "File": "http://example.com/File", "null": "http://example.com/null", "array": "http://example.com/array", "type": {"@type": "@vocab", "typeDSL": True}, } ) ra, _ = ldr.resolve_all(cmap({"type": "File"}), "") assert {"type": "File"} == ra ra, _ = ldr.resolve_all(cmap({"type": "File?"}), "") assert {"type": ["null", "File"]} == ra ra, _ = ldr.resolve_all(cmap({"type": "File[]"}), "") assert {"type": {"items": "File", "type": "array"}} == ra ra, _ = ldr.resolve_all(cmap({"type": "File[]?"}), "") assert {"type": ["null", {"items": "File", "type": "array"}]} == ra
def load_job_order( args, # type: argparse.Namespace stdin, # type: IO[Any] fetcher_constructor, # Fetcher overrides_list, # type: List[Dict[Text, Any]] tool_file_uri # type: Text ): # type: (...) -> Tuple[Optional[MutableMapping[Text, Any]], Text, Loader] job_order_object = None job_order_file = None _jobloaderctx = jobloaderctx.copy() loader = Loader(_jobloaderctx, fetcher_constructor=fetcher_constructor) # type: ignore if len(args.job_order) == 1 and args.job_order[0][0] != "-": job_order_file = args.job_order[0] elif len(args.job_order) == 1 and args.job_order[0] == "-": job_order_object = yaml.round_trip_load(stdin) job_order_object, _ = loader.resolve_all(job_order_object, file_uri(os.getcwd()) + "/") else: job_order_file = None if job_order_object is not None: input_basedir = args.basedir if args.basedir else os.getcwd() elif job_order_file is not None: input_basedir = args.basedir if args.basedir \ else os.path.abspath(os.path.dirname(job_order_file)) job_order_object, _ = loader.resolve_ref(job_order_file, checklinks=False) if job_order_object is not None and "http://commonwl.org/cwltool#overrides" in job_order_object: ov_uri = file_uri(job_order_file or input_basedir) overrides_list.extend( resolve_overrides(job_order_object, ov_uri, tool_file_uri)) del job_order_object["http://commonwl.org/cwltool#overrides"] if job_order_object is None: input_basedir = args.basedir if args.basedir else os.getcwd() if job_order_object is not None and not isinstance(job_order_object, MutableMapping): _logger.error( 'CWL input object at %s is not formatted correctly, it should be a ' 'JSON/YAML dictionay, not %s.\n' 'Raw input object:\n%s', job_order_file or "stdin", type(job_order_object), job_order_object) sys.exit(1) return (job_order_object, input_basedir, loader)
def test_subscoped_id() -> None: ldr = Loader({}) ctx = { "id": "@id", "bar": {"subscope": "bar"}, } # type: ContextType ldr.add_context(ctx) ra, _ = ldr.resolve_all( cmap({"id": "foo", "bar": {"id": "baz"}}), "http://example.com" ) assert { "id": "http://example.com/#foo", "bar": {"id": "http://example.com/#foo/bar/baz"}, } == ra
def test_mixin() -> None: base_url = file_uri(os.path.join(os.getcwd(), "tests")) ldr = Loader({}) path = get_data("tests/mixin.yml") assert path ra = ldr.resolve_ref(cmap({"$mixin": path, "one": "five"}), base_url=base_url) assert {"id": "four", "one": "five"} == ra[0] ldr = Loader({"id": "@id"}) ra = ldr.resolve_all( cmap([{"id": "a", "m": {"$mixin": path}}, {"id": "b", "m": {"$mixin": path}}]), base_url=base_url, ) assert [ {"id": base_url + "#a", "m": {"id": base_url + "#a/four", "one": "two"}}, {"id": base_url + "#b", "m": {"id": base_url + "#b/four", "one": "two"}}, ] == ra[0]
def load_job_order(args, # type: argparse.Namespace stdin, # type: IO[Any] fetcher_constructor, # Fetcher overrides_list, # type: List[Dict[Text, Any]] tool_file_uri # type: Text ): # type: (...) -> Tuple[Optional[MutableMapping[Text, Any]], Text, Loader] job_order_object = None job_order_file = None _jobloaderctx = jobloaderctx.copy() loader = Loader(_jobloaderctx, fetcher_constructor=fetcher_constructor) # type: ignore if len(args.job_order) == 1 and args.job_order[0][0] != "-": job_order_file = args.job_order[0] elif len(args.job_order) == 1 and args.job_order[0] == "-": job_order_object = yaml.round_trip_load(stdin) job_order_object, _ = loader.resolve_all(job_order_object, file_uri(os.getcwd()) + "/") else: job_order_file = None if job_order_object is not None: input_basedir = args.basedir if args.basedir else os.getcwd() elif job_order_file is not None: input_basedir = args.basedir if args.basedir \ else os.path.abspath(os.path.dirname(job_order_file)) job_order_object, _ = loader.resolve_ref(job_order_file, checklinks=False) if job_order_object is not None and "http://commonwl.org/cwltool#overrides" in job_order_object: ov_uri = file_uri(job_order_file or input_basedir) overrides_list.extend( resolve_overrides(job_order_object, ov_uri, tool_file_uri)) del job_order_object["http://commonwl.org/cwltool#overrides"] if job_order_object is None: input_basedir = args.basedir if args.basedir else os.getcwd() if job_order_object is not None and not isinstance(job_order_object, MutableMapping): _logger.error( 'CWL input object at %s is not formatted correctly, it should be a ' 'JSON/YAML dictionay, not %s.\n' 'Raw input object:\n%s', job_order_file or "stdin", type(job_order_object), job_order_object) sys.exit(1) return (job_order_object, input_basedir, loader)
def test_schemas() -> None: loader = Loader({}) path = get_data("tests/EDAM.owl") assert path ra, _ = loader.resolve_all( cmap( { "$schemas": [file_uri(path)], "$namespaces": {"edam": "http://edamontology.org/"}, "edam:has_format": "edam:format_1915", } ), "", ) assert { "$schemas": [file_uri(path)], "$namespaces": {"edam": "http://edamontology.org/"}, "http://edamontology.org/has_format": "http://edamontology.org/format_1915", } == ra
def validate_document( document_loader, # type: Loader workflowobj, # type: CommentedMap uri, # type: Text enable_dev=False, # type: bool strict=True, # type: bool preprocess_only=False, # type: bool fetcher_constructor=None, # type: FetcherConstructorType skip_schemas=None, # type: bool overrides=None, # type: List[Dict] metadata=None, # type: Optional[Dict] ): # type: (...) -> Tuple[Loader, Names, Union[Dict[Text, Any], List[Dict[Text, Any]]], Dict[Text, Any], Text] """Validate a CWL document.""" if isinstance(workflowobj, list): workflowobj = cmap({"$graph": workflowobj}, fn=uri) if not isinstance(workflowobj, dict): raise ValueError("workflowjobj must be a dict, got '%s': %s" % (type(workflowobj), workflowobj)) jobobj = None if "cwl:tool" in workflowobj: job_loader = default_loader(fetcher_constructor) # type: ignore jobobj, _ = job_loader.resolve_all(workflowobj, uri) uri = urllib.parse.urljoin( uri, workflowobj["https://w3id.org/cwl/cwl#tool"]) del cast(dict, jobobj)["https://w3id.org/cwl/cwl#tool"] if "http://commonwl.org/cwltool#overrides" in jobobj: overrides.extend(resolve_overrides(jobobj, uri, uri)) del jobobj["http://commonwl.org/cwltool#overrides"] workflowobj = fetch_document( uri, fetcher_constructor=fetcher_constructor)[1] fileuri = urllib.parse.urldefrag(uri)[0] if "cwlVersion" not in workflowobj: if metadata and 'cwlVersion' in metadata: workflowobj['cwlVersion'] = metadata['cwlVersion'] else: raise ValidationException( "No cwlVersion found." "Use the following syntax in your CWL document to declare " "the version: cwlVersion: <version>") if not isinstance(workflowobj["cwlVersion"], (str, Text)): raise Exception("'cwlVersion' must be a string, got %s" % type(workflowobj["cwlVersion"])) # strip out version workflowobj["cwlVersion"] = re.sub(r"^(?:cwl:|https://w3id.org/cwl/cwl#)", "", workflowobj["cwlVersion"]) if workflowobj["cwlVersion"] not in list(ALLUPDATES): # print out all the Supported Versions of cwlVersion versions = list(ALLUPDATES) # ALLUPDATES is a dict versions.sort() raise ValidationException( "'cwlVersion' not valid. Supported CWL versions are: \n{}".format( "\n".join(versions))) if workflowobj["cwlVersion"] == "draft-2": workflowobj = cast( CommentedMap, cmap( update._draft2toDraft3dev1(workflowobj, document_loader, uri, update_steps=False))) if "@graph" in workflowobj: workflowobj["$graph"] = workflowobj["@graph"] del workflowobj["@graph"] (sch_document_loader, avsc_names) = \ process.get_schema(workflowobj["cwlVersion"])[:2] if isinstance(avsc_names, Exception): raise avsc_names processobj = None # type: Union[CommentedMap, CommentedSeq, Text] document_loader = Loader(sch_document_loader.ctx, schemagraph=sch_document_loader.graph, idx=document_loader.idx, cache=sch_document_loader.cache, fetcher_constructor=fetcher_constructor, skip_schemas=skip_schemas) _add_blank_ids(workflowobj) workflowobj["id"] = fileuri processobj, new_metadata = document_loader.resolve_all( workflowobj, fileuri) if not isinstance(processobj, (CommentedMap, CommentedSeq)): raise ValidationException("Workflow must be a dict or list.") if not new_metadata: if not isinstance(processobj, dict): raise ValidationException("Draft-2 workflows must be a dict.") new_metadata = cast( CommentedMap, cmap( { "$namespaces": processobj.get("$namespaces", {}), "$schemas": processobj.get("$schemas", []), "cwlVersion": processobj["cwlVersion"] }, fn=fileuri)) _convert_stdstreams_to_files(workflowobj) if preprocess_only: return document_loader, avsc_names, processobj, new_metadata, uri schema.validate_doc(avsc_names, processobj, document_loader, strict) if new_metadata.get("cwlVersion") != update.LATEST: processobj = cast( CommentedMap, cmap( update.update(processobj, document_loader, fileuri, enable_dev, new_metadata))) if jobobj: new_metadata[u"cwl:defaults"] = jobobj if overrides: new_metadata[u"cwltool:overrides"] = overrides return document_loader, avsc_names, processobj, new_metadata, uri
def validate_document( document_loader, # type: Loader workflowobj, # type: CommentedMap uri, # type: Text enable_dev=False, # type: bool strict=True, # type: bool preprocess_only=False, # type: bool fetcher_constructor=None, # type: FetcherConstructorType skip_schemas=None, # type: bool overrides=None, # type: List[Dict] metadata=None, # type: Optional[Dict] do_validate=True): # type: (...) -> Tuple[Loader, schema.Names, Union[Dict[Text, Any], List[Dict[Text, Any]]], Dict[Text, Any], Text] """Validate a CWL document.""" if isinstance(workflowobj, list): workflowobj = cmap({"$graph": workflowobj}, fn=uri) if not isinstance(workflowobj, dict): raise ValueError("workflowjobj must be a dict, got '{}': {}".format( type(workflowobj), workflowobj)) jobobj = None if "cwl:tool" in workflowobj: job_loader = default_loader(fetcher_constructor) # type: ignore jobobj, _ = job_loader.resolve_all(workflowobj, uri, checklinks=do_validate) uri = urllib.parse.urljoin( uri, workflowobj["https://w3id.org/cwl/cwl#tool"]) del cast(dict, jobobj)["https://w3id.org/cwl/cwl#tool"] if "http://commonwl.org/cwltool#overrides" in jobobj: overrides.extend(resolve_overrides(jobobj, uri, uri)) del jobobj["http://commonwl.org/cwltool#overrides"] workflowobj = fetch_document( uri, fetcher_constructor=fetcher_constructor)[1] def var_spool_cwl_detector( obj, # type: Union[Mapping, Iterable, Text] item=None, # type: Optional[Any] obj_key=None, # type: Optional[Any] ): # type: (...)->None """ Detects any textual reference to /var/spool/cwl. """ if isinstance(obj, string_types): if "var/spool/cwl" in obj: message = SourceLine( item=item, key=obj_key, raise_type=Text, include_traceback=_logger.isEnabledFor( logging.DEBUG)).makeError( "Non-portable reference to /var/spool/cwl found: " "'{}'.\n Replace with /var/spool/cwl/ with " "$(runtime.outdir).".format(obj)) if not strict: _logger.warning(message) else: raise ValidationException(message) else: return elif isinstance(obj, Mapping): for key, value in iteritems(obj): var_spool_cwl_detector(value, obj, key) elif isinstance(obj, Iterable): for element in obj: var_spool_cwl_detector(element, obj, None) var_spool_cwl_detector(workflowobj) fileuri = urllib.parse.urldefrag(uri)[0] if "cwlVersion" not in workflowobj: if metadata and 'cwlVersion' in metadata: workflowobj['cwlVersion'] = metadata['cwlVersion'] else: raise ValidationException( "No cwlVersion found. " "Use the following syntax in your CWL document to declare " "the version: cwlVersion: <version>.\n" "Note: if this is a CWL draft-2 (pre v1.0) document then it " "will need to be upgraded first.") if not isinstance(workflowobj["cwlVersion"], (str, Text)): with SourceLine(workflowobj, "cwlVersion", ValidationException): raise ValidationException("'cwlVersion' must be a string, " "got {}".format( type(workflowobj["cwlVersion"]))) # strip out version workflowobj["cwlVersion"] = re.sub(r"^(?:cwl:|https://w3id.org/cwl/cwl#)", "", workflowobj["cwlVersion"]) if workflowobj["cwlVersion"] not in list(ALLUPDATES): # print out all the Supported Versions of cwlVersion versions = [] for version in list(ALLUPDATES): if "dev" in version: version += " (with --enable-dev flag only)" versions.append(version) versions.sort() raise ValidationException( "The CWL reference runner no longer supports pre CWL v1.0 " "documents. Supported versions are: " "\n{}".format("\n".join(versions))) (sch_document_loader, avsc_names) = \ process.get_schema(workflowobj["cwlVersion"])[:2] if isinstance(avsc_names, Exception): raise avsc_names processobj = None # type: Union[CommentedMap, CommentedSeq, Text] document_loader = Loader(sch_document_loader.ctx, schemagraph=sch_document_loader.graph, idx=document_loader.idx, cache=sch_document_loader.cache, fetcher_constructor=fetcher_constructor, skip_schemas=skip_schemas) _add_blank_ids(workflowobj) workflowobj["id"] = fileuri processobj, new_metadata = document_loader.resolve_all( workflowobj, fileuri, checklinks=do_validate) if not isinstance(processobj, (CommentedMap, CommentedSeq)): raise ValidationException("Workflow must be a dict or list.") if not new_metadata: new_metadata = cast( CommentedMap, cmap( { "$namespaces": processobj.get("$namespaces", {}), "$schemas": processobj.get("$schemas", []), "cwlVersion": processobj["cwlVersion"] }, fn=fileuri)) _convert_stdstreams_to_files(workflowobj) if preprocess_only: return document_loader, avsc_names, processobj, new_metadata, uri if do_validate: schema.validate_doc(avsc_names, processobj, document_loader, strict) if new_metadata.get("cwlVersion") != update.LATEST: processobj = cast( CommentedMap, cmap( update.update(processobj, document_loader, fileuri, enable_dev, new_metadata))) if jobobj: new_metadata[u"cwl:defaults"] = jobobj if overrides: new_metadata[u"cwltool:overrides"] = overrides return document_loader, avsc_names, processobj, new_metadata, uri
def resolve_and_validate_document( loadingContext, workflowobj, uri, preprocess_only=False, # type: bool skip_schemas=None, # type: bool ): # type: (...) -> Tuple[LoadingContext, Text] """Validate a CWL document.""" loadingContext = loadingContext.copy() if not isinstance(workflowobj, MutableMapping): raise ValueError("workflowjobj must be a dict, got '{}': {}".format( type(workflowobj), workflowobj)) jobobj = None if "cwl:tool" in workflowobj: jobobj, _ = loadingContext.loader.resolve_all( workflowobj, uri, checklinks=loadingContext.do_validate) uri = urllib.parse.urljoin( uri, workflowobj["https://w3id.org/cwl/cwl#tool"]) del cast(dict, jobobj)["https://w3id.org/cwl/cwl#tool"] workflowobj = fetch_document(uri, loadingContext)[1] fileuri = urllib.parse.urldefrag(uri)[0] cwlVersion = workflowobj.get("cwlVersion") if not cwlVersion: fileobj = fetch_document(fileuri, loadingContext)[1] cwlVersion = fileobj.get("cwlVersion") if not cwlVersion: raise ValidationException( "No cwlVersion found. " "Use the following syntax in your CWL document to declare " "the version: cwlVersion: <version>.\n" "Note: if this is a CWL draft-2 (pre v1.0) document then it " "will need to be upgraded first.") if not isinstance(cwlVersion, string_types): with SourceLine(workflowobj, "cwlVersion", ValidationException): raise ValidationException("'cwlVersion' must be a string, " "got {}".format(type(cwlVersion))) # strip out version cwlVersion = re.sub(r"^(?:cwl:|https://w3id.org/cwl/cwl#)", "", cwlVersion) if cwlVersion not in list(ALLUPDATES): # print out all the Supported Versions of cwlVersion versions = [] for version in list(ALLUPDATES): if "dev" in version: version += " (with --enable-dev flag only)" versions.append(version) versions.sort() raise ValidationException( "The CWL reference runner no longer supports pre CWL v1.0 " "documents. Supported versions are: " "\n{}".format("\n".join(versions))) if isinstance(jobobj, CommentedMap ) and "http://commonwl.org/cwltool#overrides" in jobobj: loadingContext.overrides_list.extend( resolve_overrides(jobobj, uri, uri)) del jobobj["http://commonwl.org/cwltool#overrides"] if isinstance(jobobj, CommentedMap ) and "https://w3id.org/cwl/cwl#requirements" in jobobj: if cwlVersion not in ("v1.1.0-dev1", ): raise ValidationException( "`cwl:requirements` in the input object is not part of CWL " "v1.0. You can adjust to use `cwltool:overrides` instead; or you " "can set the cwlVersion to v1.1.0-dev1 or greater and re-run with " "--enable-dev.") loadingContext.overrides_list.append({ "overrideTarget": uri, "requirements": jobobj["https://w3id.org/cwl/cwl#requirements"] }) del jobobj["https://w3id.org/cwl/cwl#requirements"] (sch_document_loader, avsc_names) = \ process.get_schema(cwlVersion)[:2] if isinstance(avsc_names, Exception): raise avsc_names processobj = None # type: Union[CommentedMap, CommentedSeq, Text, None] document_loader = Loader( sch_document_loader.ctx, schemagraph=sch_document_loader.graph, idx=loadingContext.loader.idx, cache=sch_document_loader.cache, fetcher_constructor=loadingContext.fetcher_constructor, skip_schemas=skip_schemas) if cwlVersion == "v1.0": _add_blank_ids(workflowobj) workflowobj["id"] = fileuri processobj, metadata = document_loader.resolve_all( workflowobj, fileuri, checklinks=loadingContext.do_validate) if not isinstance(processobj, (CommentedMap, CommentedSeq)): raise ValidationException( "Workflow must be a CommentedMap or CommentedSeq.") if not isinstance(metadata, CommentedMap): raise ValidationException("metadata must be a CommentedMap, was %s" % type(metadata)) _convert_stdstreams_to_files(workflowobj) if preprocess_only: return loadingContext, uri if loadingContext.do_validate: schema.validate_doc(avsc_names, processobj, document_loader, loadingContext.strict) if loadingContext.do_update: processobj = cast( CommentedMap, cmap( update.update(processobj, document_loader, fileuri, loadingContext.enable_dev, metadata))) if isinstance(processobj, MutableMapping): document_loader.idx[processobj["id"]] = processobj elif isinstance(processobj, MutableSequence): document_loader.idx[metadata["id"]] = metadata for po in processobj: document_loader.idx[po["id"]] = po if jobobj is not None: loadingContext.jobdefaults = jobobj loadingContext.loader = document_loader loadingContext.avsc_names = avsc_names loadingContext.metadata = metadata return loadingContext, uri
def resolve_and_validate_document(loadingContext, workflowobj, uri, preprocess_only=False, # type: bool skip_schemas=None, # type: bool ): # type: (...) -> Tuple[LoadingContext, Text] """Validate a CWL document.""" loadingContext = loadingContext.copy() if not isinstance(workflowobj, MutableMapping): raise ValueError("workflowjobj must be a dict, got '{}': {}".format( type(workflowobj), workflowobj)) jobobj = None if "cwl:tool" in workflowobj: jobobj, _ = loadingContext.loader.resolve_all(workflowobj, uri, checklinks=loadingContext.do_validate) uri = urllib.parse.urljoin(uri, workflowobj["https://w3id.org/cwl/cwl#tool"]) del cast(dict, jobobj)["https://w3id.org/cwl/cwl#tool"] workflowobj = fetch_document(uri, loadingContext)[1] fileuri = urllib.parse.urldefrag(uri)[0] cwlVersion = workflowobj.get("cwlVersion") if not cwlVersion: fileobj = fetch_document(fileuri, loadingContext)[1] cwlVersion = fileobj.get("cwlVersion") if not cwlVersion: raise ValidationException( "No cwlVersion found. " "Use the following syntax in your CWL document to declare " "the version: cwlVersion: <version>.\n" "Note: if this is a CWL draft-2 (pre v1.0) document then it " "will need to be upgraded first.") if not isinstance(cwlVersion, string_types): with SourceLine(workflowobj, "cwlVersion", ValidationException): raise ValidationException("'cwlVersion' must be a string, " "got {}".format( type(cwlVersion))) # strip out version cwlVersion = re.sub( r"^(?:cwl:|https://w3id.org/cwl/cwl#)", "", cwlVersion) if cwlVersion not in list(ALLUPDATES): # print out all the Supported Versions of cwlVersion versions = [] for version in list(ALLUPDATES): if "dev" in version: version += " (with --enable-dev flag only)" versions.append(version) versions.sort() raise ValidationException( "The CWL reference runner no longer supports pre CWL v1.0 " "documents. Supported versions are: " "\n{}".format("\n".join(versions))) if isinstance(jobobj, CommentedMap) and "http://commonwl.org/cwltool#overrides" in jobobj: loadingContext.overrides_list.extend(resolve_overrides(jobobj, uri, uri)) del jobobj["http://commonwl.org/cwltool#overrides"] if isinstance(jobobj, CommentedMap) and "https://w3id.org/cwl/cwl#requirements" in jobobj: if cwlVersion not in ("v1.1.0-dev1",): raise ValidationException( "`cwl:requirements` in the input object is not part of CWL " "v1.0. You can adjust to use `cwltool:overrides` instead; or you " "can set the cwlVersion to v1.1.0-dev1 or greater and re-run with " "--enable-dev.") loadingContext.overrides_list.append({"overrideTarget": uri, "requirements": jobobj["https://w3id.org/cwl/cwl#requirements"]}) del jobobj["https://w3id.org/cwl/cwl#requirements"] (sch_document_loader, avsc_names) = \ process.get_schema(cwlVersion)[:2] if isinstance(avsc_names, Exception): raise avsc_names processobj = None # type: Union[CommentedMap, CommentedSeq, Text, None] document_loader = Loader(sch_document_loader.ctx, schemagraph=sch_document_loader.graph, idx=loadingContext.loader.idx, cache=sch_document_loader.cache, fetcher_constructor=loadingContext.fetcher_constructor, skip_schemas=skip_schemas) if cwlVersion == "v1.0": _add_blank_ids(workflowobj) workflowobj["id"] = fileuri processobj, metadata = document_loader.resolve_all( workflowobj, fileuri, checklinks=loadingContext.do_validate) if not isinstance(processobj, (CommentedMap, CommentedSeq)): raise ValidationException("Workflow must be a CommentedMap or CommentedSeq.") if not isinstance(metadata, CommentedMap): raise ValidationException("metadata must be a CommentedMap, was %s" % type(metadata)) _convert_stdstreams_to_files(workflowobj) if preprocess_only: return loadingContext, uri if loadingContext.do_validate: schema.validate_doc(avsc_names, processobj, document_loader, loadingContext.strict) if loadingContext.do_update: processobj = cast(CommentedMap, cmap(update.update( processobj, document_loader, fileuri, loadingContext.enable_dev, metadata))) if isinstance(processobj, MutableMapping): document_loader.idx[processobj["id"]] = processobj elif isinstance(processobj, MutableSequence): document_loader.idx[metadata["id"]] = metadata for po in processobj: document_loader.idx[po["id"]] = po if jobobj is not None: loadingContext.jobdefaults = jobobj loadingContext.loader = document_loader loadingContext.avsc_names = avsc_names loadingContext.metadata = metadata return loadingContext, uri
def validate_document( document_loader, # type: Loader workflowobj, # type: CommentedMap uri, # type: Text enable_dev=False, # type: bool strict=True, # type: bool preprocess_only=False, # type: bool fetcher_constructor=None # type: Callable[[Dict[unicode, unicode], requests.sessions.Session], Fetcher] ): # type: (...) -> Tuple[Loader, Names, Union[Dict[Text, Any], List[Dict[Text, Any]]], Dict[Text, Any], Text] """Validate a CWL document.""" if isinstance(workflowobj, list): workflowobj = {"$graph": workflowobj} if not isinstance(workflowobj, dict): raise ValueError("workflowjobj must be a dict, got '%s': %s" % (type(workflowobj), workflowobj)) jobobj = None if "cwl:tool" in workflowobj: jobobj, _ = document_loader.resolve_all(workflowobj, uri) uri = urllib.parse.urljoin( uri, workflowobj["https://w3id.org/cwl/cwl#tool"]) del cast(dict, jobobj)["https://w3id.org/cwl/cwl#tool"] workflowobj = fetch_document( uri, fetcher_constructor=fetcher_constructor)[1] fileuri = urllib.parse.urldefrag(uri)[0] if "cwlVersion" in workflowobj: if not isinstance(workflowobj["cwlVersion"], (str, Text)): raise Exception("'cwlVersion' must be a string, got %s" % type(workflowobj["cwlVersion"])) workflowobj["cwlVersion"] = re.sub( r"^(?:cwl:|https://w3id.org/cwl/cwl#)", "", workflowobj["cwlVersion"]) else: _logger.warn("No cwlVersion found, treating this file as draft-2.") workflowobj["cwlVersion"] = "draft-2" if workflowobj["cwlVersion"] == "draft-2": workflowobj = cast( CommentedMap, cmap( update._draft2toDraft3dev1(workflowobj, document_loader, uri, update_steps=False))) if "@graph" in workflowobj: workflowobj["$graph"] = workflowobj["@graph"] del workflowobj["@graph"] (sch_document_loader, avsc_names) = \ process.get_schema(workflowobj["cwlVersion"])[:2] if isinstance(avsc_names, Exception): raise avsc_names processobj = None # type: Union[CommentedMap, CommentedSeq, unicode] document_loader = Loader(sch_document_loader.ctx, schemagraph=sch_document_loader.graph, idx=document_loader.idx, cache=sch_document_loader.cache, fetcher_constructor=fetcher_constructor) _add_blank_ids(workflowobj) workflowobj["id"] = fileuri processobj, metadata = document_loader.resolve_all(workflowobj, fileuri) if not isinstance(processobj, (CommentedMap, CommentedSeq)): raise ValidationException("Workflow must be a dict or list.") if not metadata: if not isinstance(processobj, dict): raise ValidationException("Draft-2 workflows must be a dict.") metadata = cast( CommentedMap, cmap( { "$namespaces": processobj.get("$namespaces", {}), "$schemas": processobj.get("$schemas", []), "cwlVersion": processobj["cwlVersion"] }, fn=fileuri)) _convert_stdstreams_to_files(workflowobj) if preprocess_only: return document_loader, avsc_names, processobj, metadata, uri schema.validate_doc(avsc_names, processobj, document_loader, strict) if metadata.get("cwlVersion") != update.LATEST: processobj = cast( CommentedMap, cmap( update.update(processobj, document_loader, fileuri, enable_dev, metadata))) if jobobj: metadata[u"cwl:defaults"] = jobobj return document_loader, avsc_names, processobj, metadata, uri
def test_scoped_ref() -> None: ldr = Loader({}) ldr.add_context({ "scatter": { "@type": "@id", "refScope": 0 }, "source": { "@type": "@id", "refScope": 2 }, "in": { "mapSubject": "id", "mapPredicate": "source" }, "out": { "@type": "@id", "identity": True }, "inputs": { "mapSubject": "id", "mapPredicate": "type" }, "outputs": { "mapSubject": "id" }, "steps": { "mapSubject": "id" }, "id": "@id", }) ra, _ = ldr.resolve_all( cmap({ "inputs": { "inp": "string", "inp2": "string" }, "outputs": { "out": { "type": "string", "source": "step2/out" } }, "steps": { "step1": { "in": { "inp": "inp", "inp2": "#inp2", "inp3": ["inp", "inp2"] }, "out": ["out"], "scatter": "inp", }, "step2": { "in": { "inp": "step1/out" }, "scatter": "inp", "out": ["out"], }, }, }), "http://example2.com/", ) assert { "inputs": [ { "id": "http://example2.com/#inp", "type": "string" }, { "id": "http://example2.com/#inp2", "type": "string" }, ], "outputs": [{ "id": "http://example2.com/#out", "type": "string", "source": "http://example2.com/#step2/out", }], "steps": [ { "id": "http://example2.com/#step1", "scatter": "http://example2.com/#step1/inp", "in": [ { "id": "http://example2.com/#step1/inp", "source": "http://example2.com/#inp", }, { "id": "http://example2.com/#step1/inp2", "source": "http://example2.com/#inp2", }, { "id": "http://example2.com/#step1/inp3", "source": [ "http://example2.com/#inp", "http://example2.com/#inp2", ], }, ], "out": ["http://example2.com/#step1/out"], }, { "id": "http://example2.com/#step2", "scatter": "http://example2.com/#step2/inp", "in": [{ "id": "http://example2.com/#step2/inp", "source": "http://example2.com/#step1/out", }], "out": ["http://example2.com/#step2/out"], }, ], } == ra
def validate_document( document_loader, # type: Loader workflowobj, # type: CommentedMap uri, # type: Text overrides, # type: List[Dict] metadata, # type: Dict[Text, Any] enable_dev=False, # type: bool strict=True, # type: bool preprocess_only=False, # type: bool fetcher_constructor=None, # type: FetcherConstructorType skip_schemas=None, # type: bool do_validate=True # type: bool ): # type: (...) -> Tuple[Loader, schema.Names, Union[Dict[Text, Any], List[Dict[Text, Any]]], Dict[Text, Any], Text] """Validate a CWL document.""" if isinstance(workflowobj, MutableSequence): workflowobj = cmap({"$graph": workflowobj}, fn=uri) if not isinstance(workflowobj, MutableMapping): raise ValueError("workflowjobj must be a dict, got '{}': {}".format( type(workflowobj), workflowobj)) jobobj = None if "cwl:tool" in workflowobj: job_loader = default_loader(fetcher_constructor) # type: ignore jobobj, _ = job_loader.resolve_all(workflowobj, uri, checklinks=do_validate) uri = urllib.parse.urljoin( uri, workflowobj["https://w3id.org/cwl/cwl#tool"]) del cast(dict, jobobj)["https://w3id.org/cwl/cwl#tool"] if isinstance(jobobj, CommentedMap ) and "http://commonwl.org/cwltool#overrides" in jobobj: overrides.extend(resolve_overrides(jobobj, uri, uri)) del jobobj["http://commonwl.org/cwltool#overrides"] workflowobj = fetch_document( uri, fetcher_constructor=fetcher_constructor)[1] fileuri = urllib.parse.urldefrag(uri)[0] if "cwlVersion" not in workflowobj: if 'cwlVersion' in metadata: workflowobj['cwlVersion'] = metadata['cwlVersion'] else: raise ValidationException( "No cwlVersion found. " "Use the following syntax in your CWL document to declare " "the version: cwlVersion: <version>.\n" "Note: if this is a CWL draft-2 (pre v1.0) document then it " "will need to be upgraded first.") if not isinstance(workflowobj["cwlVersion"], string_types): with SourceLine(workflowobj, "cwlVersion", ValidationException): raise ValidationException("'cwlVersion' must be a string, " "got {}".format( type(workflowobj["cwlVersion"]))) # strip out version workflowobj["cwlVersion"] = re.sub(r"^(?:cwl:|https://w3id.org/cwl/cwl#)", "", workflowobj["cwlVersion"]) if workflowobj["cwlVersion"] not in list(ALLUPDATES): # print out all the Supported Versions of cwlVersion versions = [] for version in list(ALLUPDATES): if "dev" in version: version += " (with --enable-dev flag only)" versions.append(version) versions.sort() raise ValidationException( "The CWL reference runner no longer supports pre CWL v1.0 " "documents. Supported versions are: " "\n{}".format("\n".join(versions))) (sch_document_loader, avsc_names) = \ process.get_schema(workflowobj["cwlVersion"])[:2] if isinstance(avsc_names, Exception): raise avsc_names processobj = None # type: Union[CommentedMap, CommentedSeq, Text, None] document_loader = Loader(sch_document_loader.ctx, schemagraph=sch_document_loader.graph, idx=document_loader.idx, cache=sch_document_loader.cache, fetcher_constructor=fetcher_constructor, skip_schemas=skip_schemas) _add_blank_ids(workflowobj) workflowobj["id"] = fileuri processobj, new_metadata = document_loader.resolve_all( workflowobj, fileuri, checklinks=do_validate) if not isinstance(processobj, (CommentedMap, CommentedSeq)): raise ValidationException("Workflow must be a dict or list.") if not new_metadata and isinstance(processobj, CommentedMap): new_metadata = cast( CommentedMap, cmap( { "$namespaces": processobj.get("$namespaces", {}), "$schemas": processobj.get("$schemas", []), "cwlVersion": processobj["cwlVersion"] }, fn=fileuri)) _convert_stdstreams_to_files(workflowobj) if preprocess_only: return document_loader, avsc_names, processobj, new_metadata, uri if do_validate: schema.validate_doc(avsc_names, processobj, document_loader, strict) if new_metadata.get("cwlVersion") != update.LATEST: processobj = cast( CommentedMap, cmap( update.update(processobj, document_loader, fileuri, enable_dev, new_metadata))) if jobobj is not None: new_metadata[u"cwl:defaults"] = jobobj if overrides: new_metadata[u"cwltool:overrides"] = overrides return document_loader, avsc_names, processobj, new_metadata, uri
def main(args=None, executor=single_job_executor, makeTool=workflow.defaultMakeTool, parser=None, stdin=sys.stdin, stdout=sys.stdout, stderr=sys.stderr): _logger.removeHandler(defaultStreamHandler) _logger.addHandler(logging.StreamHandler(stderr)) if args is None: args = sys.argv[1:] if parser is None: parser = arg_parser() args = parser.parse_args(args) if args.quiet: _logger.setLevel(logging.WARN) if args.debug: _logger.setLevel(logging.DEBUG) pkg = pkg_resources.require("cwltool") if pkg: if args.version: print "%s %s" % (sys.argv[0], pkg[0].version) return 0 else: _logger.info("%s %s", sys.argv[0], pkg[0].version) if not args.workflow: parser.print_help() _logger.error("") _logger.error("CWL document required") return 1 t = load_tool(args.workflow, args.update, args.strict, makeTool, args.debug, args.print_pre) if type(t) == int: return t if args.print_rdf: printrdf(args.workflow, processobj, ctx, args.rdf_serializer) return 0 if args.print_dot: printdot(args.workflow, processobj, ctx, args.rdf_serializer) return 0 if args.tmp_outdir_prefix != 'tmp': # Use user defined temp directory (if it exists) args.tmp_outdir_prefix = os.path.abspath(args.tmp_outdir_prefix) if not os.path.exists(args.tmp_outdir_prefix): _logger.error("Intermediate output directory prefix doesn't exist, reverting to default") return 1 if args.tmpdir_prefix != 'tmp': # Use user defined prefix (if the folder exists) args.tmpdir_prefix = os.path.abspath(args.tmpdir_prefix) if not os.path.exists(args.tmpdir_prefix): _logger.error("Temporary directory prefix doesn't exist.") return 1 job_order_object = None if args.conformance_test: loader = Loader({}) else: loader = Loader({"id": "@id", "path": {"@type": "@id"}}) if len(args.job_order) == 1 and args.job_order[0][0] != "-": job_order_file = args.job_order[0] elif len(args.job_order) == 1 and args.job_order[0] == "-": job_order_object = yaml.load(stdin) job_order_object, _ = loader.resolve_all(job_order_object, "") else: job_order_file = None if job_order_object: input_basedir = args.basedir if args.basedir else os.getcwd() elif job_order_file: input_basedir = args.basedir if args.basedir else os.path.abspath(os.path.dirname(job_order_file)) try: job_order_object, _ = loader.resolve_ref(job_order_file) except Exception as e: _logger.error(e, exc_info=(e if args.debug else False)) return 1 toolparser = None else: input_basedir = args.basedir if args.basedir else os.getcwd() namemap = {} toolparser = generate_parser(argparse.ArgumentParser(prog=args.workflow), t, namemap) if toolparser: if args.tool_help: toolparser.print_help() return 0 cmd_line = vars(toolparser.parse_args(args.job_order)) if cmd_line["job_order"]: try: input_basedir = args.basedir if args.basedir else os.path.abspath(os.path.dirname(cmd_line["job_order"])) job_order_object = loader.resolve_ref(cmd_line["job_order"]) except Exception as e: _logger.error(e, exc_info=(e if args.debug else False)) return 1 else: job_order_object = {} job_order_object.update({namemap[k]: v for k,v in cmd_line.items()}) _logger.debug("Parsed job order from command line: %s", job_order_object) else: job_order_object = None if not job_order_object: parser.print_help() if toolparser: print "\nOptions for %s " % args.workflow toolparser.print_help() _logger.error("") _logger.error("Input object required") return 1 try: out = executor(t, job_order_object, input_basedir, args, conformance_test=args.conformance_test, dry_run=args.dry_run, outdir=args.outdir, tmp_outdir_prefix=args.tmp_outdir_prefix, use_container=args.use_container, preserve_environment=args.preserve_environment, pull_image=args.enable_pull, rm_container=args.rm_container, tmpdir_prefix=args.tmpdir_prefix, rm_tmpdir=args.rm_tmpdir, makeTool=makeTool, move_outputs=args.move_outputs ) # This is the workflow output, it needs to be written stdout.write(json.dumps(out, indent=4)) stdout.flush() except (validate.ValidationException) as e: _logger.error("Input object failed validation:\n%s", e, exc_info=(e if args.debug else False)) return 1 except workflow.WorkflowException as e: _logger.error("Workflow error:\n %s", e, exc_info=(e if args.debug else False)) return 1 return 0
def validate_document(document_loader, # type: Loader workflowobj, # type: CommentedMap uri, # type: Text overrides, # type: List[Dict] metadata, # type: Dict[Text, Any] enable_dev=False, # type: bool strict=True, # type: bool preprocess_only=False, # type: bool fetcher_constructor=None, # type: FetcherConstructorType skip_schemas=None, # type: bool do_validate=True # type: bool ): # type: (...) -> Tuple[Loader, schema.Names, Union[Dict[Text, Any], List[Dict[Text, Any]]], Dict[Text, Any], Text] """Validate a CWL document.""" if isinstance(workflowobj, MutableSequence): workflowobj = cmap({ "$graph": workflowobj }, fn=uri) if not isinstance(workflowobj, MutableMapping): raise ValueError("workflowjobj must be a dict, got '{}': {}".format( type(workflowobj), workflowobj)) jobobj = None if "cwl:tool" in workflowobj: job_loader = default_loader(fetcher_constructor) # type: ignore jobobj, _ = job_loader.resolve_all(workflowobj, uri, checklinks=do_validate) uri = urllib.parse.urljoin(uri, workflowobj["https://w3id.org/cwl/cwl#tool"]) del cast(dict, jobobj)["https://w3id.org/cwl/cwl#tool"] if isinstance(jobobj, CommentedMap) and "http://commonwl.org/cwltool#overrides" in jobobj: overrides.extend(resolve_overrides(jobobj, uri, uri)) del jobobj["http://commonwl.org/cwltool#overrides"] workflowobj = fetch_document(uri, fetcher_constructor=fetcher_constructor)[1] fileuri = urllib.parse.urldefrag(uri)[0] if "cwlVersion" not in workflowobj: if 'cwlVersion' in metadata: workflowobj['cwlVersion'] = metadata['cwlVersion'] else: raise ValidationException( "No cwlVersion found. " "Use the following syntax in your CWL document to declare " "the version: cwlVersion: <version>.\n" "Note: if this is a CWL draft-2 (pre v1.0) document then it " "will need to be upgraded first.") if not isinstance(workflowobj["cwlVersion"], string_types): with SourceLine(workflowobj, "cwlVersion", ValidationException): raise ValidationException("'cwlVersion' must be a string, " "got {}".format( type(workflowobj["cwlVersion"]))) # strip out version workflowobj["cwlVersion"] = re.sub( r"^(?:cwl:|https://w3id.org/cwl/cwl#)", "", workflowobj["cwlVersion"]) if workflowobj["cwlVersion"] not in list(ALLUPDATES): # print out all the Supported Versions of cwlVersion versions = [] for version in list(ALLUPDATES): if "dev" in version: version += " (with --enable-dev flag only)" versions.append(version) versions.sort() raise ValidationException( "The CWL reference runner no longer supports pre CWL v1.0 " "documents. Supported versions are: " "\n{}".format("\n".join(versions))) (sch_document_loader, avsc_names) = \ process.get_schema(workflowobj["cwlVersion"])[:2] if isinstance(avsc_names, Exception): raise avsc_names processobj = None # type: Union[CommentedMap, CommentedSeq, Text, None] document_loader = Loader(sch_document_loader.ctx, schemagraph=sch_document_loader.graph, idx=document_loader.idx, cache=sch_document_loader.cache, fetcher_constructor=fetcher_constructor, skip_schemas=skip_schemas) _add_blank_ids(workflowobj) workflowobj["id"] = fileuri processobj, new_metadata = document_loader.resolve_all( workflowobj, fileuri, checklinks=do_validate) if not isinstance(processobj, (CommentedMap, CommentedSeq)): raise ValidationException("Workflow must be a dict or list.") if not new_metadata and isinstance(processobj, CommentedMap): new_metadata = cast(CommentedMap, cmap( {"$namespaces": processobj.get("$namespaces", {}), "$schemas": processobj.get("$schemas", []), "cwlVersion": processobj["cwlVersion"]}, fn=fileuri)) _convert_stdstreams_to_files(workflowobj) if preprocess_only: return document_loader, avsc_names, processobj, new_metadata, uri if do_validate: schema.validate_doc(avsc_names, processobj, document_loader, strict) if new_metadata.get("cwlVersion") != update.LATEST: processobj = cast(CommentedMap, cmap(update.update( processobj, document_loader, fileuri, enable_dev, new_metadata))) if jobobj is not None: new_metadata[u"cwl:defaults"] = jobobj if overrides: new_metadata[u"cwltool:overrides"] = overrides return document_loader, avsc_names, processobj, new_metadata, uri
def load_job_order(args, t, parser): job_order_object = None if args.conformance_test: loader = Loader({}) else: jobloaderctx = {"path": {"@type": "@id"}, "format": {"@type": "@id"}} jobloaderctx.update(t.metadata.get("$namespaces", {})) loader = Loader(jobloaderctx) if len(args.job_order) == 1 and args.job_order[0][0] != "-": job_order_file = args.job_order[0] elif len(args.job_order) == 1 and args.job_order[0] == "-": job_order_object = yaml.load(stdin) job_order_object, _ = loader.resolve_all(job_order_object, "") else: job_order_file = None if job_order_object: input_basedir = args.basedir if args.basedir else os.getcwd() elif job_order_file: input_basedir = args.basedir if args.basedir else os.path.abspath(os.path.dirname(job_order_file)) try: job_order_object, _ = loader.resolve_ref(job_order_file) except Exception as e: _logger.error(e, exc_info=(e if args.debug else False)) return 1 toolparser = None else: input_basedir = args.basedir if args.basedir else os.getcwd() namemap = {} toolparser = generate_parser(argparse.ArgumentParser(prog=args.workflow), t, namemap) if toolparser: if args.tool_help: toolparser.print_help() return 0 cmd_line = vars(toolparser.parse_args(args.job_order)) if cmd_line["job_order"]: try: input_basedir = args.basedir if args.basedir else os.path.abspath(os.path.dirname(cmd_line["job_order"])) job_order_object = loader.resolve_ref(cmd_line["job_order"]) except Exception as e: _logger.error(e, exc_info=(e if args.debug else False)) return 1 else: job_order_object = {} job_order_object.update({namemap[k]: v for k,v in cmd_line.items()}) _logger.debug("Parsed job order from command line: %s", job_order_object) else: job_order_object = None for inp in t.tool["inputs"]: if "default" in inp and (not job_order_object or shortname(inp["id"]) not in job_order_object): if not job_order_object: job_order_object = {} job_order_object[shortname(inp["id"])] = inp["default"] if not job_order_object and len(t.tool["inputs"]) > 0: parser.print_help() if toolparser: print "\nOptions for %s " % args.workflow toolparser.print_help() _logger.error("") _logger.error("Input object required") return 1 return (job_order_object, input_basedir)
def validate_document(document_loader, # type: Loader workflowobj, # type: CommentedMap uri, # type: Text enable_dev=False, # type: bool strict=True, # type: bool preprocess_only=False, # type: bool fetcher_constructor=None # type: Callable[[Dict[unicode, unicode], requests.sessions.Session], Fetcher] ): # type: (...) -> Tuple[Loader, Names, Union[Dict[Text, Any], List[Dict[Text, Any]]], Dict[Text, Any], Text] """Validate a CWL document.""" if isinstance(workflowobj, list): workflowobj = { "$graph": workflowobj } if not isinstance(workflowobj, dict): raise ValueError("workflowjobj must be a dict") jobobj = None if "cwl:tool" in workflowobj: jobobj, _ = document_loader.resolve_all(workflowobj, uri) uri = urlparse.urljoin(uri, workflowobj["https://w3id.org/cwl/cwl#tool"]) del cast(dict, jobobj)["https://w3id.org/cwl/cwl#tool"] workflowobj = fetch_document(uri, fetcher_constructor=fetcher_constructor)[1] fileuri = urlparse.urldefrag(uri)[0] if "cwlVersion" in workflowobj: if not isinstance(workflowobj["cwlVersion"], (str, Text)): raise Exception("'cwlVersion' must be a string, got %s" % type(workflowobj["cwlVersion"])) workflowobj["cwlVersion"] = re.sub( r"^(?:cwl:|https://w3id.org/cwl/cwl#)", "", workflowobj["cwlVersion"]) else: _logger.warn("No cwlVersion found, treating this file as draft-2.") workflowobj["cwlVersion"] = "draft-2" if workflowobj["cwlVersion"] == "draft-2": workflowobj = cast(CommentedMap, cmap(update._draft2toDraft3dev1( workflowobj, document_loader, uri, update_steps=False))) if "@graph" in workflowobj: workflowobj["$graph"] = workflowobj["@graph"] del workflowobj["@graph"] (sch_document_loader, avsc_names) = \ process.get_schema(workflowobj["cwlVersion"])[:2] if isinstance(avsc_names, Exception): raise avsc_names processobj = None # type: Union[CommentedMap, CommentedSeq, unicode] document_loader = Loader(sch_document_loader.ctx, schemagraph=sch_document_loader.graph, idx=document_loader.idx, cache=sch_document_loader.cache, fetcher_constructor=fetcher_constructor) workflowobj["id"] = fileuri processobj, metadata = document_loader.resolve_all(workflowobj, fileuri) if not isinstance(processobj, (CommentedMap, CommentedSeq)): raise ValidationException("Workflow must be a dict or list.") if not metadata: if not isinstance(processobj, dict): raise ValidationException("Draft-2 workflows must be a dict.") metadata = cast(CommentedMap, cmap({"$namespaces": processobj.get("$namespaces", {}), "$schemas": processobj.get("$schemas", []), "cwlVersion": processobj["cwlVersion"]}, fn=fileuri)) _convert_stdstreams_to_files(workflowobj) if preprocess_only: return document_loader, avsc_names, processobj, metadata, uri schema.validate_doc(avsc_names, processobj, document_loader, strict) if metadata.get("cwlVersion") != update.LATEST: processobj = cast(CommentedMap, cmap(update.update( processobj, document_loader, fileuri, enable_dev, metadata))) if jobobj: metadata[u"cwl:defaults"] = jobobj return document_loader, avsc_names, processobj, metadata, uri
def load_job_order(args, t, stdin, print_input_deps=False, relative_deps=False, stdout=sys.stdout, make_fs_access=None, fetcher_constructor=None): # type: (argparse.Namespace, Process, IO[Any], bool, bool, IO[Any], Callable[[Text], StdFsAccess], Callable[[Dict[Text, Text], requests.sessions.Session], Fetcher]) -> Union[int, Tuple[Dict[Text, Any], Text]] job_order_object = None _jobloaderctx = jobloaderctx.copy() _jobloaderctx.update(t.metadata.get("$namespaces", {})) loader = Loader(_jobloaderctx, fetcher_constructor=fetcher_constructor) # type: ignore if len(args.job_order) == 1 and args.job_order[0][0] != "-": job_order_file = args.job_order[0] elif len(args.job_order) == 1 and args.job_order[0] == "-": job_order_object = yaml.round_trip_load(stdin) job_order_object, _ = loader.resolve_all(job_order_object, file_uri(os.getcwd()) + "/") else: job_order_file = None if job_order_object: input_basedir = args.basedir if args.basedir else os.getcwd() elif job_order_file: input_basedir = args.basedir if args.basedir else os.path.abspath( os.path.dirname(job_order_file)) try: job_order_object, _ = loader.resolve_ref(job_order_file, checklinks=False) except Exception as e: _logger.error(Text(e), exc_info=args.debug) return 1 toolparser = None else: input_basedir = args.basedir if args.basedir else os.getcwd() namemap = {} # type: Dict[Text, Text] records = [] # type: List[Text] toolparser = generate_parser( argparse.ArgumentParser(prog=args.workflow), t, namemap, records) if toolparser: if args.tool_help: toolparser.print_help() return 0 cmd_line = vars(toolparser.parse_args(args.job_order)) for record_name in records: record = {} record_items = { k: v for k, v in six.iteritems(cmd_line) if k.startswith(record_name) } for key, value in six.iteritems(record_items): record[key[len(record_name) + 1:]] = value del cmd_line[key] cmd_line[str(record_name)] = record if cmd_line["job_order"]: try: input_basedir = args.basedir if args.basedir else os.path.abspath( os.path.dirname(cmd_line["job_order"])) job_order_object = loader.resolve_ref( cmd_line["job_order"]) except Exception as e: _logger.error(Text(e), exc_info=args.debug) return 1 else: job_order_object = {"id": args.workflow} del cmd_line["job_order"] job_order_object.update( {namemap[k]: v for k, v in cmd_line.items()}) if _logger.isEnabledFor(logging.DEBUG): _logger.debug(u"Parsed job order from command line: %s", json.dumps(job_order_object, indent=4)) else: job_order_object = None for inp in t.tool["inputs"]: if "default" in inp and (not job_order_object or shortname(inp["id"]) not in job_order_object): if not job_order_object: job_order_object = {} job_order_object[shortname(inp["id"])] = inp["default"] if not job_order_object and len(t.tool["inputs"]) > 0: if toolparser: print(u"\nOptions for {} ".format(args.workflow)) toolparser.print_help() _logger.error("") _logger.error("Input object required, use --help for details") return 1 if print_input_deps: printdeps(job_order_object, loader, stdout, relative_deps, "", basedir=file_uri(input_basedir + "/")) return 0 def pathToLoc(p): if "location" not in p and "path" in p: p["location"] = p["path"] del p["path"] visit_class(job_order_object, ("File", "Directory"), pathToLoc) adjustDirObjs(job_order_object, trim_listing) normalizeFilesDirs(job_order_object) if "cwl:tool" in job_order_object: del job_order_object["cwl:tool"] if "id" in job_order_object: del job_order_object["id"] return (job_order_object, input_basedir)
def load_job_order(args, t, parser, stdin, print_input_deps=False, relative_deps=False, stdout=sys.stdout): job_order_object = None if args.conformance_test: loader = Loader({}) else: jobloaderctx = { "path": { "@type": "@id" }, "format": { "@type": "@id" }, "id": "@id" } jobloaderctx.update(t.metadata.get("$namespaces", {})) loader = Loader(jobloaderctx) if len(args.job_order) == 1 and args.job_order[0][0] != "-": job_order_file = args.job_order[0] elif len(args.job_order) == 1 and args.job_order[0] == "-": job_order_object = yaml.load(stdin) job_order_object, _ = loader.resolve_all(job_order_object, "") else: job_order_file = None if job_order_object: input_basedir = args.basedir if args.basedir else os.getcwd() elif job_order_file: input_basedir = args.basedir if args.basedir else os.path.abspath( os.path.dirname(job_order_file)) try: job_order_object, _ = loader.resolve_ref(job_order_file) except Exception as e: _logger.error(e, exc_info=(e if args.debug else False)) return 1 toolparser = None else: input_basedir = args.basedir if args.basedir else os.getcwd() namemap = {} toolparser = generate_parser( argparse.ArgumentParser(prog=args.workflow), t, namemap) if toolparser: if args.tool_help: toolparser.print_help() return 0 cmd_line = vars(toolparser.parse_args(args.job_order)) if cmd_line["job_order"]: try: input_basedir = args.basedir if args.basedir else os.path.abspath( os.path.dirname(cmd_line["job_order"])) job_order_object = loader.resolve_ref( cmd_line["job_order"]) except Exception as e: _logger.error(e, exc_info=(e if args.debug else False)) return 1 else: job_order_object = {"id": args.workflow} job_order_object.update( {namemap[k]: v for k, v in cmd_line.items()}) _logger.debug("Parsed job order from command line: %s", json.dumps(job_order_object, indent=4)) else: job_order_object = None for inp in t.tool["inputs"]: if "default" in inp and (not job_order_object or shortname(inp["id"]) not in job_order_object): if not job_order_object: job_order_object = {} job_order_object[shortname(inp["id"])] = inp["default"] if not job_order_object and len(t.tool["inputs"]) > 0: parser.print_help() if toolparser: print "\nOptions for %s " % args.workflow toolparser.print_help() _logger.error("") _logger.error("Input object required") return 1 if print_input_deps: printdeps(job_order_object, loader, stdout, relative_deps, basedir="file://%s/" % input_basedir) return 0 return (job_order_object, input_basedir)
def load_job_order(args, t, parser, stdin, print_input_deps=False, relative_deps=False, stdout=sys.stdout): # type: (argparse.Namespace, Process, argparse.ArgumentParser, IO[Any], bool, bool, IO[Any]) -> Union[int,Tuple[Dict[str,Any],str]] job_order_object = None if args.conformance_test: loader = Loader({}) else: jobloaderctx = { "path": {"@type": "@id"}, "format": {"@type": "@id"}, "id": "@id"} jobloaderctx.update(t.metadata.get("$namespaces", {})) loader = Loader(jobloaderctx) if len(args.job_order) == 1 and args.job_order[0][0] != "-": job_order_file = args.job_order[0] elif len(args.job_order) == 1 and args.job_order[0] == "-": job_order_object = yaml.load(stdin) job_order_object, _ = loader.resolve_all(job_order_object, "") else: job_order_file = None if job_order_object: input_basedir = args.basedir if args.basedir else os.getcwd() elif job_order_file: input_basedir = args.basedir if args.basedir else os.path.abspath(os.path.dirname(job_order_file)) try: job_order_object, _ = loader.resolve_ref(job_order_file) except Exception as e: _logger.error(str(e), exc_info=(e if args.debug else False)) return 1 toolparser = None else: input_basedir = args.basedir if args.basedir else os.getcwd() namemap = {} # type: Dict[str,str] toolparser = generate_parser(argparse.ArgumentParser(prog=args.workflow), t, namemap) if toolparser: if args.tool_help: toolparser.print_help() return 0 cmd_line = vars(toolparser.parse_args(args.job_order)) if cmd_line["job_order"]: try: input_basedir = args.basedir if args.basedir else os.path.abspath(os.path.dirname(cmd_line["job_order"])) job_order_object = loader.resolve_ref(cmd_line["job_order"]) except Exception as e: _logger.error(str(e), exc_info=(e if args.debug else False)) return 1 else: job_order_object = {"id": args.workflow} job_order_object.update({namemap[k]: v for k,v in cmd_line.items()}) _logger.debug(u"Parsed job order from command line: %s", json.dumps(job_order_object, indent=4)) else: job_order_object = None for inp in t.tool["inputs"]: if "default" in inp and (not job_order_object or shortname(inp["id"]) not in job_order_object): if not job_order_object: job_order_object = {} job_order_object[shortname(inp["id"])] = inp["default"] if not job_order_object and len(t.tool["inputs"]) > 0: parser.print_help() if toolparser: print u"\nOptions for %s " % args.workflow toolparser.print_help() _logger.error("") _logger.error("Input object required") return 1 if print_input_deps: printdeps(job_order_object, loader, stdout, relative_deps, basedir=u"file://%s/" % input_basedir) return 0 if "cwl:tool" in job_order_object: del job_order_object["cwl:tool"] if "id" in job_order_object: del job_order_object["id"] return (job_order_object, input_basedir)
def load_job_order(args, t, stdin, print_input_deps=False, relative_deps=False, stdout=sys.stdout): # type: (argparse.Namespace, Process, IO[Any], bool, bool, IO[Any]) -> Union[int,Tuple[Dict[str,Any],str]] job_order_object = None if args.conformance_test: loader = Loader({}) else: jobloaderctx = { u"path": { u"@type": u"@id" }, u"location": { u"@type": u"@id" }, u"format": { u"@type": u"@id" }, u"id": u"@id" } jobloaderctx.update(t.metadata.get("$namespaces", {})) loader = Loader(jobloaderctx) if len(args.job_order) == 1 and args.job_order[0][0] != "-": job_order_file = args.job_order[0] elif len(args.job_order) == 1 and args.job_order[0] == "-": job_order_object = yaml.load(stdin) job_order_object, _ = loader.resolve_all(job_order_object, "") else: job_order_file = None if job_order_object: input_basedir = args.basedir if args.basedir else os.getcwd() elif job_order_file: input_basedir = args.basedir if args.basedir else os.path.abspath( os.path.dirname(job_order_file)) try: job_order_object, _ = loader.resolve_ref(job_order_file, checklinks=False) except Exception as e: _logger.error(str(e), exc_info=(e if args.debug else False)) return 1 toolparser = None else: input_basedir = args.basedir if args.basedir else os.getcwd() namemap = {} # type: Dict[unicode, unicode] toolparser = generate_parser( argparse.ArgumentParser(prog=args.workflow), t, namemap) if toolparser: if args.tool_help: toolparser.print_help() return 0 cmd_line = vars(toolparser.parse_args(args.job_order)) if cmd_line["job_order"]: try: input_basedir = args.basedir if args.basedir else os.path.abspath( os.path.dirname(cmd_line["job_order"])) job_order_object = loader.resolve_ref( cmd_line["job_order"]) except Exception as e: _logger.error(str(e), exc_info=(e if args.debug else False)) return 1 else: job_order_object = {"id": args.workflow} job_order_object.update( {namemap[k]: v for k, v in cmd_line.items()}) _logger.debug(u"Parsed job order from command line: %s", json.dumps(job_order_object, indent=4)) else: job_order_object = None for inp in t.tool["inputs"]: if "default" in inp and (not job_order_object or shortname(inp["id"]) not in job_order_object): if not job_order_object: job_order_object = {} job_order_object[shortname(inp["id"])] = inp["default"] if not job_order_object and len(t.tool["inputs"]) > 0: if toolparser: print u"\nOptions for %s " % args.workflow toolparser.print_help() _logger.error("") _logger.error("Input object required, use --help for details") return 1 if print_input_deps: printdeps(job_order_object, loader, stdout, relative_deps, "", basedir=u"file://%s/" % input_basedir) return 0 def pathToLoc(p): if "location" not in p and "path" in p: p["location"] = p["path"] del p["path"] adjustDirObjs(job_order_object, pathToLoc) adjustFileObjs(job_order_object, pathToLoc) normalizeFilesDirs(job_order_object) adjustDirObjs( job_order_object, cast(Callable[..., Any], functools.partial(getListing, StdFsAccess(input_basedir)))) if "cwl:tool" in job_order_object: del job_order_object["cwl:tool"] if "id" in job_order_object: del job_order_object["id"] return (job_order_object, input_basedir)
def load_job_order(args, t, stdin, print_input_deps=False, relative_deps=False, stdout=sys.stdout, make_fs_access=None, fetcher_constructor=None): # type: (argparse.Namespace, Process, IO[Any], bool, bool, IO[Any], Callable[[Text], StdFsAccess], Callable[[Dict[unicode, unicode], requests.sessions.Session], Fetcher]) -> Union[int, Tuple[Dict[Text, Any], Text]] job_order_object = None _jobloaderctx = jobloaderctx.copy() _jobloaderctx.update(t.metadata.get("$namespaces", {})) loader = Loader(_jobloaderctx, fetcher_constructor=fetcher_constructor) if len(args.job_order) == 1 and args.job_order[0][0] != "-": job_order_file = args.job_order[0] elif len(args.job_order) == 1 and args.job_order[0] == "-": job_order_object = yaml.round_trip_load(stdin) # type: ignore job_order_object, _ = loader.resolve_all(job_order_object, file_uri(os.getcwd()) + "/") else: job_order_file = None if job_order_object: input_basedir = args.basedir if args.basedir else os.getcwd() elif job_order_file: input_basedir = args.basedir if args.basedir else os.path.abspath(os.path.dirname(job_order_file)) try: job_order_object, _ = loader.resolve_ref(job_order_file, checklinks=False) except Exception as e: _logger.error(Text(e), exc_info=args.debug) return 1 toolparser = None else: input_basedir = args.basedir if args.basedir else os.getcwd() namemap = {} # type: Dict[Text, Text] records = [] # type: List[Text] toolparser = generate_parser( argparse.ArgumentParser(prog=args.workflow), t, namemap, records) if toolparser: if args.tool_help: toolparser.print_help() return 0 cmd_line = vars(toolparser.parse_args(args.job_order)) for record_name in records: record = {} record_items = { k: v for k, v in cmd_line.iteritems() if k.startswith(record_name)} for key, value in record_items.iteritems(): record[key[len(record_name) + 1:]] = value del cmd_line[key] cmd_line[str(record_name)] = record if cmd_line["job_order"]: try: input_basedir = args.basedir if args.basedir else os.path.abspath( os.path.dirname(cmd_line["job_order"])) job_order_object = loader.resolve_ref(cmd_line["job_order"]) except Exception as e: _logger.error(Text(e), exc_info=args.debug) return 1 else: job_order_object = {"id": args.workflow} del cmd_line["job_order"] job_order_object.update({namemap[k]: v for k, v in cmd_line.items()}) if _logger.isEnabledFor(logging.DEBUG): _logger.debug(u"Parsed job order from command line: %s", json.dumps(job_order_object, indent=4)) else: job_order_object = None for inp in t.tool["inputs"]: if "default" in inp and (not job_order_object or shortname(inp["id"]) not in job_order_object): if not job_order_object: job_order_object = {} job_order_object[shortname(inp["id"])] = inp["default"] if not job_order_object and len(t.tool["inputs"]) > 0: if toolparser: print(u"\nOptions for {} ".format(args.workflow)) toolparser.print_help() _logger.error("") _logger.error("Input object required, use --help for details") return 1 if print_input_deps: printdeps(job_order_object, loader, stdout, relative_deps, "", basedir=file_uri(input_basedir + "/")) return 0 def pathToLoc(p): if "location" not in p and "path" in p: p["location"] = p["path"] del p["path"] visit_class(job_order_object, ("File", "Directory"), pathToLoc) adjustDirObjs(job_order_object, trim_listing) normalizeFilesDirs(job_order_object) if "cwl:tool" in job_order_object: del job_order_object["cwl:tool"] if "id" in job_order_object: del job_order_object["id"] return (job_order_object, input_basedir)
def resolve_and_validate_document( loadingContext, # type: LoadingContext workflowobj, # type: Union[CommentedMap, CommentedSeq] uri, # type: str preprocess_only=False, # type: bool skip_schemas=None, # type: Optional[bool] ): # type: (...) -> Tuple[LoadingContext, str] """Validate a CWL document.""" if not loadingContext.loader: raise ValueError("loadingContext must have a loader.") else: loader = loadingContext.loader loadingContext = loadingContext.copy() if not isinstance(workflowobj, MutableMapping): raise ValueError("workflowjobj must be a dict, got '{}': {}".format( type(workflowobj), workflowobj)) jobobj = None if "cwl:tool" in workflowobj: jobobj, _ = loader.resolve_all(workflowobj, uri) uri = urllib.parse.urljoin( uri, workflowobj["https://w3id.org/cwl/cwl#tool"]) del cast(Dict[str, Any], jobobj)["https://w3id.org/cwl/cwl#tool"] workflowobj = fetch_document(uri, loadingContext)[1] fileuri = urllib.parse.urldefrag(uri)[0] cwlVersion = loadingContext.metadata.get("cwlVersion") if not cwlVersion: cwlVersion = workflowobj.get("cwlVersion") if not cwlVersion and fileuri != uri: # The tool we're loading is a fragment of a bigger file. Get # the document root element and look for cwlVersion there. metadata = fetch_document(fileuri, loadingContext)[1] # type: Dict[str, Any] cwlVersion = metadata.get("cwlVersion") if not cwlVersion: raise ValidationException( "No cwlVersion found. " "Use the following syntax in your CWL document to declare " "the version: cwlVersion: <version>.\n" "Note: if this is a CWL draft-2 (pre v1.0) document then it " "will need to be upgraded first.") if not isinstance(cwlVersion, str): with SourceLine(workflowobj, "cwlVersion", ValidationException): raise ValidationException("'cwlVersion' must be a string, " "got {}".format(type(cwlVersion))) # strip out version cwlVersion = re.sub(r"^(?:cwl:|https://w3id.org/cwl/cwl#)", "", cwlVersion) if cwlVersion not in list(ALLUPDATES): # print out all the Supported Versions of cwlVersion versions = [] for version in list(ALLUPDATES): if "dev" in version: version += " (with --enable-dev flag only)" versions.append(version) versions.sort() raise ValidationException( "The CWL reference runner no longer supports pre CWL v1.0 " "documents. Supported versions are: " "\n{}".format("\n".join(versions))) if (isinstance(jobobj, CommentedMap) and "http://commonwl.org/cwltool#overrides" in jobobj): loadingContext.overrides_list.extend( resolve_overrides(jobobj, uri, uri)) del jobobj["http://commonwl.org/cwltool#overrides"] if (isinstance(jobobj, CommentedMap) and "https://w3id.org/cwl/cwl#requirements" in jobobj): if cwlVersion not in ("v1.1.0-dev1", "v1.1"): raise ValidationException( "`cwl:requirements` in the input object is not part of CWL " "v1.0. You can adjust to use `cwltool:overrides` instead; or you " "can set the cwlVersion to v1.1 or greater.") loadingContext.overrides_list.append({ "overrideTarget": uri, "requirements": jobobj["https://w3id.org/cwl/cwl#requirements"], }) del jobobj["https://w3id.org/cwl/cwl#requirements"] (sch_document_loader, avsc_names) = process.get_schema(cwlVersion)[:2] if isinstance(avsc_names, Exception): raise avsc_names processobj = None # type: Union[CommentedMap, CommentedSeq, str, None] document_loader = Loader( sch_document_loader.ctx, schemagraph=sch_document_loader.graph, idx=loader.idx, cache=sch_document_loader.cache, fetcher_constructor=loadingContext.fetcher_constructor, skip_schemas=skip_schemas, doc_cache=loadingContext.doc_cache, ) if cwlVersion == "v1.0": _add_blank_ids(workflowobj) processobj, metadata = document_loader.resolve_all(workflowobj, fileuri) if loadingContext.metadata: metadata = loadingContext.metadata if not isinstance(processobj, (CommentedMap, CommentedSeq)): raise ValidationException( "Workflow must be a CommentedMap or CommentedSeq.") if not isinstance(metadata, CommentedMap): raise ValidationException("metadata must be a CommentedMap, was %s" % type(metadata)) if isinstance(processobj, CommentedMap): uri = processobj["id"] _convert_stdstreams_to_files(workflowobj) if preprocess_only: return loadingContext, uri if loadingContext.do_validate: schema.validate_doc(avsc_names, processobj, document_loader, loadingContext.strict) # None means default behavior (do update) if loadingContext.do_update in (True, None): if "cwlVersion" not in metadata: metadata["cwlVersion"] = cwlVersion processobj = update.update(processobj, document_loader, fileuri, loadingContext.enable_dev, metadata) document_loader.idx[processobj["id"]] = processobj def update_index(pr: CommentedMap) -> None: if "id" in pr: document_loader.idx[pr["id"]] = pr visit_class(processobj, ("CommandLineTool", "Workflow", "ExpressionTool"), update_index) if isinstance(jobobj, CommentedMap): loadingContext.jobdefaults = jobobj loadingContext.loader = document_loader loadingContext.avsc_names = avsc_names loadingContext.metadata = metadata return loadingContext, uri
def load_job_order(args, t, stdin, print_input_deps=False, relative_deps=False, stdout=sys.stdout, make_fs_access=None): # type: (argparse.Namespace, Process, IO[Any], bool, bool, IO[Any], Type[StdFsAccess]) -> Union[int, Tuple[Dict[Text, Any], Text]] job_order_object = None if args.conformance_test: loader = Loader({}) else: jobloaderctx = { u"path": {u"@type": u"@id"}, u"location": {u"@type": u"@id"}, u"format": {u"@type": u"@id"}, u"id": u"@id"} jobloaderctx.update(t.metadata.get("$namespaces", {})) loader = Loader(jobloaderctx) if len(args.job_order) == 1 and args.job_order[0][0] != "-": job_order_file = args.job_order[0] elif len(args.job_order) == 1 and args.job_order[0] == "-": job_order_object = yaml.load(stdin) job_order_object, _ = loader.resolve_all(job_order_object, "") else: job_order_file = None if job_order_object: input_basedir = args.basedir if args.basedir else os.getcwd() elif job_order_file: input_basedir = args.basedir if args.basedir else os.path.abspath(os.path.dirname(job_order_file)) try: job_order_object, _ = loader.resolve_ref(job_order_file, checklinks=False) except Exception as e: _logger.error(Text(e), exc_info=args.debug) return 1 toolparser = None else: input_basedir = args.basedir if args.basedir else os.getcwd() namemap = {} # type: Dict[Text, Text] toolparser = generate_parser(argparse.ArgumentParser(prog=args.workflow), t, namemap) if toolparser: if args.tool_help: toolparser.print_help() return 0 cmd_line = vars(toolparser.parse_args(args.job_order)) if cmd_line["job_order"]: try: input_basedir = args.basedir if args.basedir else os.path.abspath(os.path.dirname(cmd_line["job_order"])) job_order_object = loader.resolve_ref(cmd_line["job_order"]) except Exception as e: _logger.error(Text(e), exc_info=args.debug) return 1 else: job_order_object = {"id": args.workflow} job_order_object.update({namemap[k]: v for k,v in cmd_line.items()}) _logger.debug(u"Parsed job order from command line: %s", json.dumps(job_order_object, indent=4)) else: job_order_object = None for inp in t.tool["inputs"]: if "default" in inp and (not job_order_object or shortname(inp["id"]) not in job_order_object): if not job_order_object: job_order_object = {} job_order_object[shortname(inp["id"])] = inp["default"] if not job_order_object and len(t.tool["inputs"]) > 0: if toolparser: print u"\nOptions for %s " % args.workflow toolparser.print_help() _logger.error("") _logger.error("Input object required, use --help for details") return 1 if print_input_deps: printdeps(job_order_object, loader, stdout, relative_deps, "", basedir=u"file://%s/" % input_basedir) return 0 def pathToLoc(p): if "location" not in p and "path" in p: p["location"] = p["path"] del p["path"] adjustDirObjs(job_order_object, pathToLoc) adjustFileObjs(job_order_object, pathToLoc) normalizeFilesDirs(job_order_object) adjustDirObjs(job_order_object, cast(Callable[..., Any], functools.partial(getListing, make_fs_access(input_basedir)))) if "cwl:tool" in job_order_object: del job_order_object["cwl:tool"] if "id" in job_order_object: del job_order_object["id"] return (job_order_object, input_basedir)
def validate_document(document_loader, # type: Loader workflowobj, # type: CommentedMap uri, # type: Text enable_dev=False, # type: bool strict=True, # type: bool preprocess_only=False, # type: bool fetcher_constructor=None, # type: FetcherConstructorType skip_schemas=None, # type: bool overrides=None, # type: List[Dict] metadata=None, # type: Optional[Dict] ): # type: (...) -> Tuple[Loader, Names, Union[Dict[Text, Any], List[Dict[Text, Any]]], Dict[Text, Any], Text] """Validate a CWL document.""" if isinstance(workflowobj, list): workflowobj = cmap({ "$graph": workflowobj }, fn=uri) if not isinstance(workflowobj, dict): raise ValueError("workflowjobj must be a dict, got '%s': %s" % (type(workflowobj), workflowobj)) jobobj = None if "cwl:tool" in workflowobj: job_loader = default_loader(fetcher_constructor) # type: ignore jobobj, _ = job_loader.resolve_all(workflowobj, uri) uri = urllib.parse.urljoin(uri, workflowobj["https://w3id.org/cwl/cwl#tool"]) del cast(dict, jobobj)["https://w3id.org/cwl/cwl#tool"] if "http://commonwl.org/cwltool#overrides" in jobobj: overrides.extend(resolve_overrides(jobobj, uri, uri)) del jobobj["http://commonwl.org/cwltool#overrides"] workflowobj = fetch_document(uri, fetcher_constructor=fetcher_constructor)[1] fileuri = urllib.parse.urldefrag(uri)[0] if "cwlVersion" not in workflowobj: if metadata and 'cwlVersion' in metadata: workflowobj['cwlVersion'] = metadata['cwlVersion'] else: raise ValidationException("No cwlVersion found." "Use the following syntax in your CWL document to declare " "the version: cwlVersion: <version>") if not isinstance(workflowobj["cwlVersion"], (str, Text)): raise Exception("'cwlVersion' must be a string, got %s" % type(workflowobj["cwlVersion"])) # strip out version workflowobj["cwlVersion"] = re.sub( r"^(?:cwl:|https://w3id.org/cwl/cwl#)", "", workflowobj["cwlVersion"]) if workflowobj["cwlVersion"] not in list(ALLUPDATES): # print out all the Supported Versions of cwlVersion versions = list(ALLUPDATES) # ALLUPDATES is a dict versions.sort() raise ValidationException("'cwlVersion' not valid. Supported CWL versions are: \n{}".format("\n".join(versions))) if workflowobj["cwlVersion"] == "draft-2": workflowobj = cast(CommentedMap, cmap(update._draft2toDraft3dev1( workflowobj, document_loader, uri, update_steps=False))) if "@graph" in workflowobj: workflowobj["$graph"] = workflowobj["@graph"] del workflowobj["@graph"] (sch_document_loader, avsc_names) = \ process.get_schema(workflowobj["cwlVersion"])[:2] if isinstance(avsc_names, Exception): raise avsc_names processobj = None # type: Union[CommentedMap, CommentedSeq, Text] document_loader = Loader(sch_document_loader.ctx, schemagraph=sch_document_loader.graph, idx=document_loader.idx, cache=sch_document_loader.cache, fetcher_constructor=fetcher_constructor, skip_schemas=skip_schemas) _add_blank_ids(workflowobj) workflowobj["id"] = fileuri processobj, new_metadata = document_loader.resolve_all(workflowobj, fileuri) if not isinstance(processobj, (CommentedMap, CommentedSeq)): raise ValidationException("Workflow must be a dict or list.") if not new_metadata: if not isinstance(processobj, dict): raise ValidationException("Draft-2 workflows must be a dict.") new_metadata = cast(CommentedMap, cmap( {"$namespaces": processobj.get("$namespaces", {}), "$schemas": processobj.get("$schemas", []), "cwlVersion": processobj["cwlVersion"]}, fn=fileuri)) _convert_stdstreams_to_files(workflowobj) if preprocess_only: return document_loader, avsc_names, processobj, new_metadata, uri schema.validate_doc(avsc_names, processobj, document_loader, strict) if new_metadata.get("cwlVersion") != update.LATEST: processobj = cast(CommentedMap, cmap(update.update( processobj, document_loader, fileuri, enable_dev, new_metadata))) if jobobj: new_metadata[u"cwl:defaults"] = jobobj if overrides: new_metadata[u"cwltool:overrides"] = overrides return document_loader, avsc_names, processobj, new_metadata, uri