def validate_document(document_loader, workflowobj, uri, enable_dev=False, strict=True, preprocess_only=False): # type: (Loader, Dict[unicode, Any], unicode, bool, bool, bool) -> Tuple[Loader, Names, Any, Dict[str, str], unicode] """Validate a CWL document.""" jobobj = None if "cwl:tool" in workflowobj: jobobj = workflowobj uri = urlparse.urljoin(uri, jobobj["cwl:tool"]) del jobobj["cwl:tool"] workflowobj = fetch_document(uri)[1] if isinstance(workflowobj, list): workflowobj = { "$graph": workflowobj } fileuri = urlparse.urldefrag(uri)[0] if "cwlVersion" in workflowobj: workflowobj["cwlVersion"] = re.sub( r"^(?:cwl:|https://w3id.org/cwl/cwl#)", "", workflowobj["cwlVersion"]) else: workflowobj["cwlVersion"] = "draft-2" if workflowobj["cwlVersion"] == "draft-2": workflowobj = update._draft2toDraft3dev1( workflowobj, document_loader, uri, update_steps=False) if "@graph" in workflowobj: workflowobj["$graph"] = workflowobj["@graph"] del workflowobj["@graph"] (document_loader, avsc_names) = \ process.get_schema(workflowobj["cwlVersion"])[:2] if isinstance(avsc_names, Exception): raise avsc_names workflowobj["id"] = fileuri processobj, metadata = document_loader.resolve_all(workflowobj, fileuri) if preprocess_only: return document_loader, avsc_names, processobj, metadata, uri document_loader.validate_links(processobj) schema.validate_doc(avsc_names, processobj, document_loader, strict) if not metadata: metadata = {"$namespaces": processobj.get("$namespaces", {}), "$schemas": processobj.get("$schemas", []), "cwlVersion": processobj["cwlVersion"]} if metadata.get("cwlVersion") != update.LATEST: processobj = update.update( processobj, document_loader, fileuri, enable_dev, metadata) if jobobj: metadata["cwl:defaults"] = jobobj return document_loader, avsc_names, processobj, metadata, uri
def main(): parser = argparse.ArgumentParser() parser.add_argument("swagger") parser.add_argument("annotations") parser.add_argument("url") parser.add_argument("--dockstore-fixup", action="store_true", default=False) parser.add_argument("--print-rdf", action="store_true", default=False) parser.add_argument("--serve", action="store_true", default=False) parser.add_argument("--fuseki-path", type=str, default=".") args = parser.parse_args() with open(args.annotations) as f2: annotations = yaml.load(f2) with open(args.swagger) as f: sld = swg2salad.swg2salad(yaml.load(f), annotations) sld["$base"] = "http://ga4gh.org/schemas/tool-registry-schemas" sld["name"] = "file://" + os.path.realpath(args.swagger) document_loader, avsc_names, schema_metadata, metaschema_loader = load_schema( sld) txt = document_loader.fetch_text( urlparse.urljoin("file://" + os.getcwd() + "/", args.url)) r = yaml.load(txt) if args.dockstore_fixup: dockstore_fixup(r) validate_doc(avsc_names, r, document_loader, True) sys.stderr.write("API returned valid response\n") toolreg = Namespace("http://ga4gh.org/schemas/tool-registry-schemas#") td = Namespace( "http://ga4gh.org/schemas/tool-registry-schemas#ToolDescriptor/") if args.print_rdf or args.serve: g = jsonld_context.makerdf(args.url, r, document_loader.ctx) for s, _, o in g.triples((None, td["type"], Literal("CWL"))): for _, _, d in g.triples((s, toolreg["descriptor"], None)): expand_cwl(d, unicode(s), g) if args.print_rdf: print(g.serialize(format="turtle")) if args.serve: t = tempfile.NamedTemporaryFile(suffix=".ttl") g.serialize(t, format="turtle") t.flush() subprocess.check_call( ["./fuseki-server", "--file=" + t.name, "/tools"], cwd=args.fuseki_path)
def main(): parser = argparse.ArgumentParser() parser.add_argument("swagger") parser.add_argument("annotations") parser.add_argument("url") parser.add_argument("--print-rdf", action="store_true", default=False) parser.add_argument("--serve", action="store_true", default=False) parser.add_argument("--fuseki-path", type=str, default=".") args = parser.parse_args() warnings.simplefilter('ignore', yaml.error.UnsafeLoaderWarning) with open(args.annotations) as f2: annotations = yaml.load(f2) with open(args.swagger) as f: sld = swg2salad.swg2salad(yaml.load(f), annotations) sld["$base"] = "http://ga4gh.org/schemas/tool-registry-schemas" sld["name"] = "file://" + os.path.realpath(args.swagger) document_loader, avsc_names, schema_metadata, metaschema_loader = load_schema( cmap(sld)) txt = document_loader.fetch_text( urlparse.urljoin( "file://" + os.getcwd() + "/", args.url)) r = yaml.load(txt) validate_doc(avsc_names, r, document_loader, True) sys.stderr.write("API returned valid response\n") toolreg = Namespace("http://ga4gh.org/schemas/tool-registry-schemas#") td = Namespace( "http://ga4gh.org/schemas/tool-registry-schemas#ToolDescriptor/") if args.print_rdf or args.serve: g = jsonld_context.makerdf(args.url, r, document_loader.ctx) for s, _, o in g.triples((None, td["type"], Literal("CWL"))): for _, _, d in g.triples((s, toolreg["descriptor"], None)): expand_cwl(d, unicode(s), g) if args.print_rdf: print(g.serialize(format="turtle")) if args.serve: t = tempfile.NamedTemporaryFile(suffix=".ttl") g.serialize(t, format="turtle") t.flush() subprocess.check_call( ["./fuseki-server", "--file=" + t.name, "/tools"], cwd=args.fuseki_path)
def _schema_salad_validate(self, schema_path, document_path): ''' Adapted from schema_salad main(). :param schema_path: :param document_path: :return: ''' strict_foreign_properties = False strict = True metaschema_names, metaschema_doc, metaschema_loader = get_metaschema() schema_uri = str(schema_path) if not (urlparse(schema_uri)[0] and urlparse(schema_uri)[0] in ['http', 'https', 'file']): schema_uri = file_uri(schema_uri) schema_raw_doc = metaschema_loader.fetch(schema_uri) schema_doc, schema_metadata = metaschema_loader.resolve_all( schema_raw_doc, schema_uri) # Validate schema against metaschema validate_doc(metaschema_names, schema_doc, metaschema_loader, True) # Get the json-ld context and RDFS representation from the schema metactx = collect_namespaces(schema_metadata) if "$base" in schema_metadata: metactx["@base"] = schema_metadata["$base"] (schema_ctx, rdfs) = salad_to_jsonld_context(schema_doc, metactx) # Create the loader that will be used to load the target document. document_loader = Loader(schema_ctx, skip_schemas=False) # Make the Avro validation that will be used to validate the target # document avsc_obj = make_avro(schema_doc, document_loader) avsc_names = make_avro_schema_from_avro(avsc_obj) # Load target document and resolve refs uri = str(document_path) document, doc_metadata = document_loader.resolve_ref( uri, strict_foreign_properties=strict_foreign_properties, checklinks=False ) # This is what's getting us around file link checking. validate_doc(avsc_names, document, document_loader, strict=strict, strict_foreign_properties=strict_foreign_properties) return
def resolve_and_validate_document( loadingContext, workflowobj, uri, preprocess_only=False, # type: bool skip_schemas=None, # type: bool ): # type: (...) -> Tuple[LoadingContext, Text] """Validate a CWL document.""" loadingContext = loadingContext.copy() if not isinstance(workflowobj, MutableMapping): raise ValueError("workflowjobj must be a dict, got '{}': {}".format( type(workflowobj), workflowobj)) jobobj = None if "cwl:tool" in workflowobj: jobobj, _ = loadingContext.loader.resolve_all( workflowobj, uri, checklinks=loadingContext.do_validate) uri = urllib.parse.urljoin( uri, workflowobj["https://w3id.org/cwl/cwl#tool"]) del cast(dict, jobobj)["https://w3id.org/cwl/cwl#tool"] workflowobj = fetch_document(uri, loadingContext)[1] fileuri = urllib.parse.urldefrag(uri)[0] cwlVersion = workflowobj.get("cwlVersion") if not cwlVersion: fileobj = fetch_document(fileuri, loadingContext)[1] cwlVersion = fileobj.get("cwlVersion") if not cwlVersion: raise ValidationException( "No cwlVersion found. " "Use the following syntax in your CWL document to declare " "the version: cwlVersion: <version>.\n" "Note: if this is a CWL draft-2 (pre v1.0) document then it " "will need to be upgraded first.") if not isinstance(cwlVersion, string_types): with SourceLine(workflowobj, "cwlVersion", ValidationException): raise ValidationException("'cwlVersion' must be a string, " "got {}".format(type(cwlVersion))) # strip out version cwlVersion = re.sub(r"^(?:cwl:|https://w3id.org/cwl/cwl#)", "", cwlVersion) if cwlVersion not in list(ALLUPDATES): # print out all the Supported Versions of cwlVersion versions = [] for version in list(ALLUPDATES): if "dev" in version: version += " (with --enable-dev flag only)" versions.append(version) versions.sort() raise ValidationException( "The CWL reference runner no longer supports pre CWL v1.0 " "documents. Supported versions are: " "\n{}".format("\n".join(versions))) if isinstance(jobobj, CommentedMap ) and "http://commonwl.org/cwltool#overrides" in jobobj: loadingContext.overrides_list.extend( resolve_overrides(jobobj, uri, uri)) del jobobj["http://commonwl.org/cwltool#overrides"] if isinstance(jobobj, CommentedMap ) and "https://w3id.org/cwl/cwl#requirements" in jobobj: if cwlVersion not in ("v1.1.0-dev1", ): raise ValidationException( "`cwl:requirements` in the input object is not part of CWL " "v1.0. You can adjust to use `cwltool:overrides` instead; or you " "can set the cwlVersion to v1.1.0-dev1 or greater and re-run with " "--enable-dev.") loadingContext.overrides_list.append({ "overrideTarget": uri, "requirements": jobobj["https://w3id.org/cwl/cwl#requirements"] }) del jobobj["https://w3id.org/cwl/cwl#requirements"] (sch_document_loader, avsc_names) = \ process.get_schema(cwlVersion)[:2] if isinstance(avsc_names, Exception): raise avsc_names processobj = None # type: Union[CommentedMap, CommentedSeq, Text, None] document_loader = Loader( sch_document_loader.ctx, schemagraph=sch_document_loader.graph, idx=loadingContext.loader.idx, cache=sch_document_loader.cache, fetcher_constructor=loadingContext.fetcher_constructor, skip_schemas=skip_schemas) if cwlVersion == "v1.0": _add_blank_ids(workflowobj) workflowobj["id"] = fileuri processobj, metadata = document_loader.resolve_all( workflowobj, fileuri, checklinks=loadingContext.do_validate) if not isinstance(processobj, (CommentedMap, CommentedSeq)): raise ValidationException( "Workflow must be a CommentedMap or CommentedSeq.") if not isinstance(metadata, CommentedMap): raise ValidationException("metadata must be a CommentedMap, was %s" % type(metadata)) _convert_stdstreams_to_files(workflowobj) if preprocess_only: return loadingContext, uri if loadingContext.do_validate: schema.validate_doc(avsc_names, processobj, document_loader, loadingContext.strict) if loadingContext.do_update: processobj = cast( CommentedMap, cmap( update.update(processobj, document_loader, fileuri, loadingContext.enable_dev, metadata))) if isinstance(processobj, MutableMapping): document_loader.idx[processobj["id"]] = processobj elif isinstance(processobj, MutableSequence): document_loader.idx[metadata["id"]] = metadata for po in processobj: document_loader.idx[po["id"]] = po if jobobj is not None: loadingContext.jobdefaults = jobobj loadingContext.loader = document_loader loadingContext.avsc_names = avsc_names loadingContext.metadata = metadata return loadingContext, uri
def validate_document( document_loader, # type: Loader workflowobj, # type: CommentedMap uri, # type: Text enable_dev=False, # type: bool strict=True, # type: bool preprocess_only=False, # type: bool fetcher_constructor=None, # type: FetcherConstructorType skip_schemas=None, # type: bool overrides=None, # type: List[Dict] metadata=None, # type: Optional[Dict] ): # type: (...) -> Tuple[Loader, Names, Union[Dict[Text, Any], List[Dict[Text, Any]]], Dict[Text, Any], Text] """Validate a CWL document.""" if isinstance(workflowobj, list): workflowobj = cmap({"$graph": workflowobj}, fn=uri) if not isinstance(workflowobj, dict): raise ValueError("workflowjobj must be a dict, got '%s': %s" % (type(workflowobj), workflowobj)) jobobj = None if "cwl:tool" in workflowobj: job_loader = default_loader(fetcher_constructor) # type: ignore jobobj, _ = job_loader.resolve_all(workflowobj, uri) uri = urllib.parse.urljoin( uri, workflowobj["https://w3id.org/cwl/cwl#tool"]) del cast(dict, jobobj)["https://w3id.org/cwl/cwl#tool"] if "http://commonwl.org/cwltool#overrides" in jobobj: overrides.extend(resolve_overrides(jobobj, uri, uri)) del jobobj["http://commonwl.org/cwltool#overrides"] workflowobj = fetch_document( uri, fetcher_constructor=fetcher_constructor)[1] fileuri = urllib.parse.urldefrag(uri)[0] if "cwlVersion" not in workflowobj: if metadata and 'cwlVersion' in metadata: workflowobj['cwlVersion'] = metadata['cwlVersion'] else: raise ValidationException( "No cwlVersion found." "Use the following syntax in your CWL document to declare " "the version: cwlVersion: <version>") if not isinstance(workflowobj["cwlVersion"], (str, Text)): raise Exception("'cwlVersion' must be a string, got %s" % type(workflowobj["cwlVersion"])) # strip out version workflowobj["cwlVersion"] = re.sub(r"^(?:cwl:|https://w3id.org/cwl/cwl#)", "", workflowobj["cwlVersion"]) if workflowobj["cwlVersion"] not in list(ALLUPDATES): # print out all the Supported Versions of cwlVersion versions = list(ALLUPDATES) # ALLUPDATES is a dict versions.sort() raise ValidationException( "'cwlVersion' not valid. Supported CWL versions are: \n{}".format( "\n".join(versions))) if workflowobj["cwlVersion"] == "draft-2": workflowobj = cast( CommentedMap, cmap( update._draft2toDraft3dev1(workflowobj, document_loader, uri, update_steps=False))) if "@graph" in workflowobj: workflowobj["$graph"] = workflowobj["@graph"] del workflowobj["@graph"] (sch_document_loader, avsc_names) = \ process.get_schema(workflowobj["cwlVersion"])[:2] if isinstance(avsc_names, Exception): raise avsc_names processobj = None # type: Union[CommentedMap, CommentedSeq, Text] document_loader = Loader(sch_document_loader.ctx, schemagraph=sch_document_loader.graph, idx=document_loader.idx, cache=sch_document_loader.cache, fetcher_constructor=fetcher_constructor, skip_schemas=skip_schemas) _add_blank_ids(workflowobj) workflowobj["id"] = fileuri processobj, new_metadata = document_loader.resolve_all( workflowobj, fileuri) if not isinstance(processobj, (CommentedMap, CommentedSeq)): raise ValidationException("Workflow must be a dict or list.") if not new_metadata: if not isinstance(processobj, dict): raise ValidationException("Draft-2 workflows must be a dict.") new_metadata = cast( CommentedMap, cmap( { "$namespaces": processobj.get("$namespaces", {}), "$schemas": processobj.get("$schemas", []), "cwlVersion": processobj["cwlVersion"] }, fn=fileuri)) _convert_stdstreams_to_files(workflowobj) if preprocess_only: return document_loader, avsc_names, processobj, new_metadata, uri schema.validate_doc(avsc_names, processobj, document_loader, strict) if new_metadata.get("cwlVersion") != update.LATEST: processobj = cast( CommentedMap, cmap( update.update(processobj, document_loader, fileuri, enable_dev, new_metadata))) if jobobj: new_metadata[u"cwl:defaults"] = jobobj if overrides: new_metadata[u"cwltool:overrides"] = overrides return document_loader, avsc_names, processobj, new_metadata, uri
def validate_document(document_loader, # type: Loader workflowobj, # type: CommentedMap uri, # type: Text enable_dev=False, # type: bool strict=True, # type: bool preprocess_only=False, # type: bool fetcher_constructor=None, # type: FetcherConstructorType skip_schemas=None, # type: bool overrides=None, # type: List[Dict] metadata=None, # type: Optional[Dict] ): # type: (...) -> Tuple[Loader, Names, Union[Dict[Text, Any], List[Dict[Text, Any]]], Dict[Text, Any], Text] """Validate a CWL document.""" if isinstance(workflowobj, list): workflowobj = cmap({ "$graph": workflowobj }, fn=uri) if not isinstance(workflowobj, dict): raise ValueError("workflowjobj must be a dict, got '%s': %s" % (type(workflowobj), workflowobj)) jobobj = None if "cwl:tool" in workflowobj: job_loader = default_loader(fetcher_constructor) # type: ignore jobobj, _ = job_loader.resolve_all(workflowobj, uri) uri = urllib.parse.urljoin(uri, workflowobj["https://w3id.org/cwl/cwl#tool"]) del cast(dict, jobobj)["https://w3id.org/cwl/cwl#tool"] if "http://commonwl.org/cwltool#overrides" in jobobj: overrides.extend(resolve_overrides(jobobj, uri, uri)) del jobobj["http://commonwl.org/cwltool#overrides"] workflowobj = fetch_document(uri, fetcher_constructor=fetcher_constructor)[1] fileuri = urllib.parse.urldefrag(uri)[0] if "cwlVersion" not in workflowobj: if metadata and 'cwlVersion' in metadata: workflowobj['cwlVersion'] = metadata['cwlVersion'] else: raise ValidationException("No cwlVersion found." "Use the following syntax in your CWL document to declare " "the version: cwlVersion: <version>") if not isinstance(workflowobj["cwlVersion"], (str, Text)): raise Exception("'cwlVersion' must be a string, got %s" % type(workflowobj["cwlVersion"])) # strip out version workflowobj["cwlVersion"] = re.sub( r"^(?:cwl:|https://w3id.org/cwl/cwl#)", "", workflowobj["cwlVersion"]) if workflowobj["cwlVersion"] not in list(ALLUPDATES): # print out all the Supported Versions of cwlVersion versions = list(ALLUPDATES) # ALLUPDATES is a dict versions.sort() raise ValidationException("'cwlVersion' not valid. Supported CWL versions are: \n{}".format("\n".join(versions))) if workflowobj["cwlVersion"] == "draft-2": workflowobj = cast(CommentedMap, cmap(update._draft2toDraft3dev1( workflowobj, document_loader, uri, update_steps=False))) if "@graph" in workflowobj: workflowobj["$graph"] = workflowobj["@graph"] del workflowobj["@graph"] (sch_document_loader, avsc_names) = \ process.get_schema(workflowobj["cwlVersion"])[:2] if isinstance(avsc_names, Exception): raise avsc_names processobj = None # type: Union[CommentedMap, CommentedSeq, Text] document_loader = Loader(sch_document_loader.ctx, schemagraph=sch_document_loader.graph, idx=document_loader.idx, cache=sch_document_loader.cache, fetcher_constructor=fetcher_constructor, skip_schemas=skip_schemas) _add_blank_ids(workflowobj) workflowobj["id"] = fileuri processobj, new_metadata = document_loader.resolve_all(workflowobj, fileuri) if not isinstance(processobj, (CommentedMap, CommentedSeq)): raise ValidationException("Workflow must be a dict or list.") if not new_metadata: if not isinstance(processobj, dict): raise ValidationException("Draft-2 workflows must be a dict.") new_metadata = cast(CommentedMap, cmap( {"$namespaces": processobj.get("$namespaces", {}), "$schemas": processobj.get("$schemas", []), "cwlVersion": processobj["cwlVersion"]}, fn=fileuri)) _convert_stdstreams_to_files(workflowobj) if preprocess_only: return document_loader, avsc_names, processobj, new_metadata, uri schema.validate_doc(avsc_names, processobj, document_loader, strict) if new_metadata.get("cwlVersion") != update.LATEST: processobj = cast(CommentedMap, cmap(update.update( processobj, document_loader, fileuri, enable_dev, new_metadata))) if jobobj: new_metadata[u"cwl:defaults"] = jobobj if overrides: new_metadata[u"cwltool:overrides"] = overrides return document_loader, avsc_names, processobj, new_metadata, uri
def validate_document( document_loader, # type: Loader workflowobj, # type: CommentedMap uri, # type: Text enable_dev=False, # type: bool strict=True, # type: bool preprocess_only=False, # type: bool fetcher_constructor=None, # type: FetcherConstructorType skip_schemas=None, # type: bool overrides=None, # type: List[Dict] metadata=None, # type: Optional[Dict] do_validate=True): # type: (...) -> Tuple[Loader, schema.Names, Union[Dict[Text, Any], List[Dict[Text, Any]]], Dict[Text, Any], Text] """Validate a CWL document.""" if isinstance(workflowobj, list): workflowobj = cmap({"$graph": workflowobj}, fn=uri) if not isinstance(workflowobj, dict): raise ValueError("workflowjobj must be a dict, got '{}': {}".format( type(workflowobj), workflowobj)) jobobj = None if "cwl:tool" in workflowobj: job_loader = default_loader(fetcher_constructor) # type: ignore jobobj, _ = job_loader.resolve_all(workflowobj, uri, checklinks=do_validate) uri = urllib.parse.urljoin( uri, workflowobj["https://w3id.org/cwl/cwl#tool"]) del cast(dict, jobobj)["https://w3id.org/cwl/cwl#tool"] if "http://commonwl.org/cwltool#overrides" in jobobj: overrides.extend(resolve_overrides(jobobj, uri, uri)) del jobobj["http://commonwl.org/cwltool#overrides"] workflowobj = fetch_document( uri, fetcher_constructor=fetcher_constructor)[1] def var_spool_cwl_detector( obj, # type: Union[Mapping, Iterable, Text] item=None, # type: Optional[Any] obj_key=None, # type: Optional[Any] ): # type: (...)->None """ Detects any textual reference to /var/spool/cwl. """ if isinstance(obj, string_types): if "var/spool/cwl" in obj: message = SourceLine( item=item, key=obj_key, raise_type=Text, include_traceback=_logger.isEnabledFor( logging.DEBUG)).makeError( "Non-portable reference to /var/spool/cwl found: " "'{}'.\n Replace with /var/spool/cwl/ with " "$(runtime.outdir).".format(obj)) if not strict: _logger.warning(message) else: raise ValidationException(message) else: return elif isinstance(obj, Mapping): for key, value in iteritems(obj): var_spool_cwl_detector(value, obj, key) elif isinstance(obj, Iterable): for element in obj: var_spool_cwl_detector(element, obj, None) var_spool_cwl_detector(workflowobj) fileuri = urllib.parse.urldefrag(uri)[0] if "cwlVersion" not in workflowobj: if metadata and 'cwlVersion' in metadata: workflowobj['cwlVersion'] = metadata['cwlVersion'] else: raise ValidationException( "No cwlVersion found. " "Use the following syntax in your CWL document to declare " "the version: cwlVersion: <version>.\n" "Note: if this is a CWL draft-2 (pre v1.0) document then it " "will need to be upgraded first.") if not isinstance(workflowobj["cwlVersion"], (str, Text)): with SourceLine(workflowobj, "cwlVersion", ValidationException): raise ValidationException("'cwlVersion' must be a string, " "got {}".format( type(workflowobj["cwlVersion"]))) # strip out version workflowobj["cwlVersion"] = re.sub(r"^(?:cwl:|https://w3id.org/cwl/cwl#)", "", workflowobj["cwlVersion"]) if workflowobj["cwlVersion"] not in list(ALLUPDATES): # print out all the Supported Versions of cwlVersion versions = [] for version in list(ALLUPDATES): if "dev" in version: version += " (with --enable-dev flag only)" versions.append(version) versions.sort() raise ValidationException( "The CWL reference runner no longer supports pre CWL v1.0 " "documents. Supported versions are: " "\n{}".format("\n".join(versions))) (sch_document_loader, avsc_names) = \ process.get_schema(workflowobj["cwlVersion"])[:2] if isinstance(avsc_names, Exception): raise avsc_names processobj = None # type: Union[CommentedMap, CommentedSeq, Text] document_loader = Loader(sch_document_loader.ctx, schemagraph=sch_document_loader.graph, idx=document_loader.idx, cache=sch_document_loader.cache, fetcher_constructor=fetcher_constructor, skip_schemas=skip_schemas) _add_blank_ids(workflowobj) workflowobj["id"] = fileuri processobj, new_metadata = document_loader.resolve_all( workflowobj, fileuri, checklinks=do_validate) if not isinstance(processobj, (CommentedMap, CommentedSeq)): raise ValidationException("Workflow must be a dict or list.") if not new_metadata: new_metadata = cast( CommentedMap, cmap( { "$namespaces": processobj.get("$namespaces", {}), "$schemas": processobj.get("$schemas", []), "cwlVersion": processobj["cwlVersion"] }, fn=fileuri)) _convert_stdstreams_to_files(workflowobj) if preprocess_only: return document_loader, avsc_names, processobj, new_metadata, uri if do_validate: schema.validate_doc(avsc_names, processobj, document_loader, strict) if new_metadata.get("cwlVersion") != update.LATEST: processobj = cast( CommentedMap, cmap( update.update(processobj, document_loader, fileuri, enable_dev, new_metadata))) if jobobj: new_metadata[u"cwl:defaults"] = jobobj if overrides: new_metadata[u"cwltool:overrides"] = overrides return document_loader, avsc_names, processobj, new_metadata, uri
def validate_document( document_loader, # type: Loader workflowobj, # type: CommentedMap uri, # type: Text enable_dev=False, # type: bool strict=True, # type: bool preprocess_only=False, # type: bool fetcher_constructor=None # type: Callable[[Dict[unicode, unicode], requests.sessions.Session], Fetcher] ): # type: (...) -> Tuple[Loader, Names, Union[Dict[Text, Any], List[Dict[Text, Any]]], Dict[Text, Any], Text] """Validate a CWL document.""" if isinstance(workflowobj, list): workflowobj = {"$graph": workflowobj} if not isinstance(workflowobj, dict): raise ValueError("workflowjobj must be a dict, got '%s': %s" % (type(workflowobj), workflowobj)) jobobj = None if "cwl:tool" in workflowobj: jobobj, _ = document_loader.resolve_all(workflowobj, uri) uri = urllib.parse.urljoin( uri, workflowobj["https://w3id.org/cwl/cwl#tool"]) del cast(dict, jobobj)["https://w3id.org/cwl/cwl#tool"] workflowobj = fetch_document( uri, fetcher_constructor=fetcher_constructor)[1] fileuri = urllib.parse.urldefrag(uri)[0] if "cwlVersion" in workflowobj: if not isinstance(workflowobj["cwlVersion"], (str, Text)): raise Exception("'cwlVersion' must be a string, got %s" % type(workflowobj["cwlVersion"])) workflowobj["cwlVersion"] = re.sub( r"^(?:cwl:|https://w3id.org/cwl/cwl#)", "", workflowobj["cwlVersion"]) else: _logger.warn("No cwlVersion found, treating this file as draft-2.") workflowobj["cwlVersion"] = "draft-2" if workflowobj["cwlVersion"] == "draft-2": workflowobj = cast( CommentedMap, cmap( update._draft2toDraft3dev1(workflowobj, document_loader, uri, update_steps=False))) if "@graph" in workflowobj: workflowobj["$graph"] = workflowobj["@graph"] del workflowobj["@graph"] (sch_document_loader, avsc_names) = \ process.get_schema(workflowobj["cwlVersion"])[:2] if isinstance(avsc_names, Exception): raise avsc_names processobj = None # type: Union[CommentedMap, CommentedSeq, unicode] document_loader = Loader(sch_document_loader.ctx, schemagraph=sch_document_loader.graph, idx=document_loader.idx, cache=sch_document_loader.cache, fetcher_constructor=fetcher_constructor) _add_blank_ids(workflowobj) workflowobj["id"] = fileuri processobj, metadata = document_loader.resolve_all(workflowobj, fileuri) if not isinstance(processobj, (CommentedMap, CommentedSeq)): raise ValidationException("Workflow must be a dict or list.") if not metadata: if not isinstance(processobj, dict): raise ValidationException("Draft-2 workflows must be a dict.") metadata = cast( CommentedMap, cmap( { "$namespaces": processobj.get("$namespaces", {}), "$schemas": processobj.get("$schemas", []), "cwlVersion": processobj["cwlVersion"] }, fn=fileuri)) _convert_stdstreams_to_files(workflowobj) if preprocess_only: return document_loader, avsc_names, processobj, metadata, uri schema.validate_doc(avsc_names, processobj, document_loader, strict) if metadata.get("cwlVersion") != update.LATEST: processobj = cast( CommentedMap, cmap( update.update(processobj, document_loader, fileuri, enable_dev, metadata))) if jobobj: metadata[u"cwl:defaults"] = jobobj return document_loader, avsc_names, processobj, metadata, uri
def resolve_and_validate_document(loadingContext, workflowobj, uri, preprocess_only=False, # type: bool skip_schemas=None, # type: bool ): # type: (...) -> Tuple[LoadingContext, Text] """Validate a CWL document.""" loadingContext = loadingContext.copy() if not isinstance(workflowobj, MutableMapping): raise ValueError("workflowjobj must be a dict, got '{}': {}".format( type(workflowobj), workflowobj)) jobobj = None if "cwl:tool" in workflowobj: jobobj, _ = loadingContext.loader.resolve_all(workflowobj, uri, checklinks=loadingContext.do_validate) uri = urllib.parse.urljoin(uri, workflowobj["https://w3id.org/cwl/cwl#tool"]) del cast(dict, jobobj)["https://w3id.org/cwl/cwl#tool"] workflowobj = fetch_document(uri, loadingContext)[1] fileuri = urllib.parse.urldefrag(uri)[0] cwlVersion = workflowobj.get("cwlVersion") if not cwlVersion: fileobj = fetch_document(fileuri, loadingContext)[1] cwlVersion = fileobj.get("cwlVersion") if not cwlVersion: raise ValidationException( "No cwlVersion found. " "Use the following syntax in your CWL document to declare " "the version: cwlVersion: <version>.\n" "Note: if this is a CWL draft-2 (pre v1.0) document then it " "will need to be upgraded first.") if not isinstance(cwlVersion, string_types): with SourceLine(workflowobj, "cwlVersion", ValidationException): raise ValidationException("'cwlVersion' must be a string, " "got {}".format( type(cwlVersion))) # strip out version cwlVersion = re.sub( r"^(?:cwl:|https://w3id.org/cwl/cwl#)", "", cwlVersion) if cwlVersion not in list(ALLUPDATES): # print out all the Supported Versions of cwlVersion versions = [] for version in list(ALLUPDATES): if "dev" in version: version += " (with --enable-dev flag only)" versions.append(version) versions.sort() raise ValidationException( "The CWL reference runner no longer supports pre CWL v1.0 " "documents. Supported versions are: " "\n{}".format("\n".join(versions))) if isinstance(jobobj, CommentedMap) and "http://commonwl.org/cwltool#overrides" in jobobj: loadingContext.overrides_list.extend(resolve_overrides(jobobj, uri, uri)) del jobobj["http://commonwl.org/cwltool#overrides"] if isinstance(jobobj, CommentedMap) and "https://w3id.org/cwl/cwl#requirements" in jobobj: if cwlVersion not in ("v1.1.0-dev1",): raise ValidationException( "`cwl:requirements` in the input object is not part of CWL " "v1.0. You can adjust to use `cwltool:overrides` instead; or you " "can set the cwlVersion to v1.1.0-dev1 or greater and re-run with " "--enable-dev.") loadingContext.overrides_list.append({"overrideTarget": uri, "requirements": jobobj["https://w3id.org/cwl/cwl#requirements"]}) del jobobj["https://w3id.org/cwl/cwl#requirements"] (sch_document_loader, avsc_names) = \ process.get_schema(cwlVersion)[:2] if isinstance(avsc_names, Exception): raise avsc_names processobj = None # type: Union[CommentedMap, CommentedSeq, Text, None] document_loader = Loader(sch_document_loader.ctx, schemagraph=sch_document_loader.graph, idx=loadingContext.loader.idx, cache=sch_document_loader.cache, fetcher_constructor=loadingContext.fetcher_constructor, skip_schemas=skip_schemas) if cwlVersion == "v1.0": _add_blank_ids(workflowobj) workflowobj["id"] = fileuri processobj, metadata = document_loader.resolve_all( workflowobj, fileuri, checklinks=loadingContext.do_validate) if not isinstance(processobj, (CommentedMap, CommentedSeq)): raise ValidationException("Workflow must be a CommentedMap or CommentedSeq.") if not isinstance(metadata, CommentedMap): raise ValidationException("metadata must be a CommentedMap, was %s" % type(metadata)) _convert_stdstreams_to_files(workflowobj) if preprocess_only: return loadingContext, uri if loadingContext.do_validate: schema.validate_doc(avsc_names, processobj, document_loader, loadingContext.strict) if loadingContext.do_update: processobj = cast(CommentedMap, cmap(update.update( processobj, document_loader, fileuri, loadingContext.enable_dev, metadata))) if isinstance(processobj, MutableMapping): document_loader.idx[processobj["id"]] = processobj elif isinstance(processobj, MutableSequence): document_loader.idx[metadata["id"]] = metadata for po in processobj: document_loader.idx[po["id"]] = po if jobobj is not None: loadingContext.jobdefaults = jobobj loadingContext.loader = document_loader loadingContext.avsc_names = avsc_names loadingContext.metadata = metadata return loadingContext, uri
def validate_document(document_loader, # type: Loader workflowobj, # type: CommentedMap uri, # type: Text enable_dev=False, # type: bool strict=True, # type: bool preprocess_only=False, # type: bool fetcher_constructor=None # type: Callable[[Dict[unicode, unicode], requests.sessions.Session], Fetcher] ): # type: (...) -> Tuple[Loader, Names, Union[Dict[Text, Any], List[Dict[Text, Any]]], Dict[Text, Any], Text] """Validate a CWL document.""" if isinstance(workflowobj, list): workflowobj = { "$graph": workflowobj } if not isinstance(workflowobj, dict): raise ValueError("workflowjobj must be a dict") jobobj = None if "cwl:tool" in workflowobj: jobobj, _ = document_loader.resolve_all(workflowobj, uri) uri = urlparse.urljoin(uri, workflowobj["https://w3id.org/cwl/cwl#tool"]) del cast(dict, jobobj)["https://w3id.org/cwl/cwl#tool"] workflowobj = fetch_document(uri, fetcher_constructor=fetcher_constructor)[1] fileuri = urlparse.urldefrag(uri)[0] if "cwlVersion" in workflowobj: if not isinstance(workflowobj["cwlVersion"], (str, Text)): raise Exception("'cwlVersion' must be a string, got %s" % type(workflowobj["cwlVersion"])) workflowobj["cwlVersion"] = re.sub( r"^(?:cwl:|https://w3id.org/cwl/cwl#)", "", workflowobj["cwlVersion"]) else: _logger.warn("No cwlVersion found, treating this file as draft-2.") workflowobj["cwlVersion"] = "draft-2" if workflowobj["cwlVersion"] == "draft-2": workflowobj = cast(CommentedMap, cmap(update._draft2toDraft3dev1( workflowobj, document_loader, uri, update_steps=False))) if "@graph" in workflowobj: workflowobj["$graph"] = workflowobj["@graph"] del workflowobj["@graph"] (sch_document_loader, avsc_names) = \ process.get_schema(workflowobj["cwlVersion"])[:2] if isinstance(avsc_names, Exception): raise avsc_names processobj = None # type: Union[CommentedMap, CommentedSeq, unicode] document_loader = Loader(sch_document_loader.ctx, schemagraph=sch_document_loader.graph, idx=document_loader.idx, cache=sch_document_loader.cache, fetcher_constructor=fetcher_constructor) workflowobj["id"] = fileuri processobj, metadata = document_loader.resolve_all(workflowobj, fileuri) if not isinstance(processobj, (CommentedMap, CommentedSeq)): raise ValidationException("Workflow must be a dict or list.") if not metadata: if not isinstance(processobj, dict): raise ValidationException("Draft-2 workflows must be a dict.") metadata = cast(CommentedMap, cmap({"$namespaces": processobj.get("$namespaces", {}), "$schemas": processobj.get("$schemas", []), "cwlVersion": processobj["cwlVersion"]}, fn=fileuri)) _convert_stdstreams_to_files(workflowobj) if preprocess_only: return document_loader, avsc_names, processobj, metadata, uri schema.validate_doc(avsc_names, processobj, document_loader, strict) if metadata.get("cwlVersion") != update.LATEST: processobj = cast(CommentedMap, cmap(update.update( processobj, document_loader, fileuri, enable_dev, metadata))) if jobobj: metadata[u"cwl:defaults"] = jobobj return document_loader, avsc_names, processobj, metadata, uri
def validate_document(document_loader, workflowobj, uri, enable_dev=False, strict=True, preprocess_only=False): # type: (Loader, Dict[Text, Any], Text, bool, bool, bool) -> Tuple[Loader, Names, Union[Dict[Text, Any], List[Dict[Text, Any]]], Dict[Text, Any], Text] """Validate a CWL document.""" jobobj = None if "cwl:tool" in workflowobj: jobobj, _ = document_loader.resolve_all(workflowobj, uri) uri = urlparse.urljoin(uri, workflowobj["https://w3id.org/cwl/cwl#tool"]) del cast(dict, jobobj)["https://w3id.org/cwl/cwl#tool"] workflowobj = fetch_document(uri)[1] if isinstance(workflowobj, list): workflowobj = {"$graph": workflowobj} fileuri = urlparse.urldefrag(uri)[0] if "cwlVersion" in workflowobj: workflowobj["cwlVersion"] = re.sub( r"^(?:cwl:|https://w3id.org/cwl/cwl#)", "", workflowobj["cwlVersion"]) else: _logger.warn("No cwlVersion found, treating this file as draft-2.") workflowobj["cwlVersion"] = "draft-2" if workflowobj["cwlVersion"] == "draft-2": workflowobj = update._draft2toDraft3dev1(workflowobj, document_loader, uri, update_steps=False) if "@graph" in workflowobj: workflowobj["$graph"] = workflowobj["@graph"] del workflowobj["@graph"] (document_loader, avsc_names) = \ process.get_schema(workflowobj["cwlVersion"])[:2] if isinstance(avsc_names, Exception): raise avsc_names workflowobj["id"] = fileuri processobj, metadata = document_loader.resolve_all(workflowobj, fileuri) if not isinstance(processobj, (dict, list)): raise ValidationException("Workflow must be a dict or list.") if not metadata: if not isinstance(processobj, dict): raise ValidationException("Draft-2 workflows must be a dict.") metadata = { "$namespaces": processobj.get("$namespaces", {}), "$schemas": processobj.get("$schemas", []), "cwlVersion": processobj["cwlVersion"] } _convert_stdstreams_to_files(workflowobj) if preprocess_only: return document_loader, avsc_names, processobj, metadata, uri schema.validate_doc(avsc_names, processobj, document_loader, strict) if metadata.get("cwlVersion") != update.LATEST: processobj = update.update(processobj, document_loader, fileuri, enable_dev, metadata) if jobobj: metadata[u"cwl:defaults"] = jobobj return document_loader, avsc_names, processobj, metadata, uri
def validate_document(document_loader, workflowobj, uri, enable_dev=False, strict=True, preprocess_only=False): # type: (Loader, Dict[unicode, Any], unicode, bool, bool, bool) -> Tuple[Loader, Names, Any, Dict[str, str], unicode] """Validate a CWL document.""" jobobj = None if "cwl:tool" in workflowobj: jobobj = workflowobj uri = urlparse.urljoin(uri, jobobj["cwl:tool"]) del jobobj["cwl:tool"] workflowobj = fetch_document(uri)[1] if isinstance(workflowobj, list): workflowobj = { "$graph": workflowobj } fileuri = urlparse.urldefrag(uri)[0] if "cwlVersion" in workflowobj: workflowobj["cwlVersion"] = re.sub( r"^(?:cwl:|https://w3id.org/cwl/cwl#)", "", workflowobj["cwlVersion"]) else: _logger.warn("No cwlVersion found, treating this file as draft-2.") workflowobj["cwlVersion"] = "draft-2" if workflowobj["cwlVersion"] == "draft-2": workflowobj = update._draft2toDraft3dev1( workflowobj, document_loader, uri, update_steps=False) if "@graph" in workflowobj: workflowobj["$graph"] = workflowobj["@graph"] del workflowobj["@graph"] (document_loader, avsc_names) = \ process.get_schema(workflowobj["cwlVersion"])[:2] if isinstance(avsc_names, Exception): raise avsc_names workflowobj["id"] = fileuri processobj, metadata = document_loader.resolve_all(workflowobj, fileuri) if not metadata: metadata = {"$namespaces": processobj.get("$namespaces", {}), "$schemas": processobj.get("$schemas", []), "cwlVersion": processobj["cwlVersion"]} if preprocess_only: return document_loader, avsc_names, processobj, metadata, uri document_loader.validate_links(processobj) schema.validate_doc(avsc_names, processobj, document_loader, strict) if metadata.get("cwlVersion") != update.LATEST: processobj = update.update( processobj, document_loader, fileuri, enable_dev, metadata) if jobobj: metadata["cwl:defaults"] = jobobj return document_loader, avsc_names, processobj, metadata, uri
def resolve_and_validate_document( loadingContext, # type: LoadingContext workflowobj, # type: Union[CommentedMap, CommentedSeq] uri, # type: str preprocess_only=False, # type: bool skip_schemas=None, # type: Optional[bool] ): # type: (...) -> Tuple[LoadingContext, str] """Validate a CWL document.""" if not loadingContext.loader: raise ValueError("loadingContext must have a loader.") else: loader = loadingContext.loader loadingContext = loadingContext.copy() if not isinstance(workflowobj, MutableMapping): raise ValueError("workflowjobj must be a dict, got '{}': {}".format( type(workflowobj), workflowobj)) jobobj = None if "cwl:tool" in workflowobj: jobobj, _ = loader.resolve_all(workflowobj, uri) uri = urllib.parse.urljoin( uri, workflowobj["https://w3id.org/cwl/cwl#tool"]) del cast(Dict[str, Any], jobobj)["https://w3id.org/cwl/cwl#tool"] workflowobj = fetch_document(uri, loadingContext)[1] fileuri = urllib.parse.urldefrag(uri)[0] cwlVersion = loadingContext.metadata.get("cwlVersion") if not cwlVersion: cwlVersion = workflowobj.get("cwlVersion") if not cwlVersion and fileuri != uri: # The tool we're loading is a fragment of a bigger file. Get # the document root element and look for cwlVersion there. metadata = fetch_document(fileuri, loadingContext)[1] # type: Dict[str, Any] cwlVersion = metadata.get("cwlVersion") if not cwlVersion: raise ValidationException( "No cwlVersion found. " "Use the following syntax in your CWL document to declare " "the version: cwlVersion: <version>.\n" "Note: if this is a CWL draft-2 (pre v1.0) document then it " "will need to be upgraded first.") if not isinstance(cwlVersion, str): with SourceLine(workflowobj, "cwlVersion", ValidationException): raise ValidationException("'cwlVersion' must be a string, " "got {}".format(type(cwlVersion))) # strip out version cwlVersion = re.sub(r"^(?:cwl:|https://w3id.org/cwl/cwl#)", "", cwlVersion) if cwlVersion not in list(ALLUPDATES): # print out all the Supported Versions of cwlVersion versions = [] for version in list(ALLUPDATES): if "dev" in version: version += " (with --enable-dev flag only)" versions.append(version) versions.sort() raise ValidationException( "The CWL reference runner no longer supports pre CWL v1.0 " "documents. Supported versions are: " "\n{}".format("\n".join(versions))) if (isinstance(jobobj, CommentedMap) and "http://commonwl.org/cwltool#overrides" in jobobj): loadingContext.overrides_list.extend( resolve_overrides(jobobj, uri, uri)) del jobobj["http://commonwl.org/cwltool#overrides"] if (isinstance(jobobj, CommentedMap) and "https://w3id.org/cwl/cwl#requirements" in jobobj): if cwlVersion not in ("v1.1.0-dev1", "v1.1"): raise ValidationException( "`cwl:requirements` in the input object is not part of CWL " "v1.0. You can adjust to use `cwltool:overrides` instead; or you " "can set the cwlVersion to v1.1 or greater.") loadingContext.overrides_list.append({ "overrideTarget": uri, "requirements": jobobj["https://w3id.org/cwl/cwl#requirements"], }) del jobobj["https://w3id.org/cwl/cwl#requirements"] (sch_document_loader, avsc_names) = process.get_schema(cwlVersion)[:2] if isinstance(avsc_names, Exception): raise avsc_names processobj = None # type: Union[CommentedMap, CommentedSeq, str, None] document_loader = Loader( sch_document_loader.ctx, schemagraph=sch_document_loader.graph, idx=loader.idx, cache=sch_document_loader.cache, fetcher_constructor=loadingContext.fetcher_constructor, skip_schemas=skip_schemas, doc_cache=loadingContext.doc_cache, ) if cwlVersion == "v1.0": _add_blank_ids(workflowobj) processobj, metadata = document_loader.resolve_all(workflowobj, fileuri) if loadingContext.metadata: metadata = loadingContext.metadata if not isinstance(processobj, (CommentedMap, CommentedSeq)): raise ValidationException( "Workflow must be a CommentedMap or CommentedSeq.") if not isinstance(metadata, CommentedMap): raise ValidationException("metadata must be a CommentedMap, was %s" % type(metadata)) if isinstance(processobj, CommentedMap): uri = processobj["id"] _convert_stdstreams_to_files(workflowobj) if preprocess_only: return loadingContext, uri if loadingContext.do_validate: schema.validate_doc(avsc_names, processobj, document_loader, loadingContext.strict) # None means default behavior (do update) if loadingContext.do_update in (True, None): if "cwlVersion" not in metadata: metadata["cwlVersion"] = cwlVersion processobj = update.update(processobj, document_loader, fileuri, loadingContext.enable_dev, metadata) document_loader.idx[processobj["id"]] = processobj def update_index(pr: CommentedMap) -> None: if "id" in pr: document_loader.idx[pr["id"]] = pr visit_class(processobj, ("CommandLineTool", "Workflow", "ExpressionTool"), update_index) if isinstance(jobobj, CommentedMap): loadingContext.jobdefaults = jobobj loadingContext.loader = document_loader loadingContext.avsc_names = avsc_names loadingContext.metadata = metadata return loadingContext, uri
def validate_document(document_loader, workflowobj, uri, enable_dev=False, strict=True, preprocess_only=False): # type: (Loader, Dict[Text, Any], Text, bool, bool, bool) -> Tuple[Loader, Names, Union[Dict[Text, Any], List[Dict[Text, Any]]], Dict[Text, Any], Text] """Validate a CWL document.""" jobobj = None if "cwl:tool" in workflowobj: jobobj = workflowobj uri = urlparse.urljoin(uri, jobobj["cwl:tool"]) del jobobj["cwl:tool"] workflowobj = fetch_document(uri)[1] if isinstance(workflowobj, list): workflowobj = { "$graph": workflowobj } fileuri = urlparse.urldefrag(uri)[0] if "cwlVersion" in workflowobj: workflowobj["cwlVersion"] = re.sub( r"^(?:cwl:|https://w3id.org/cwl/cwl#)", "", workflowobj["cwlVersion"]) else: _logger.warn("No cwlVersion found, treating this file as draft-2.") workflowobj["cwlVersion"] = "draft-2" if workflowobj["cwlVersion"] == "draft-2": workflowobj = update._draft2toDraft3dev1( workflowobj, document_loader, uri, update_steps=False) if "@graph" in workflowobj: workflowobj["$graph"] = workflowobj["@graph"] del workflowobj["@graph"] (document_loader, avsc_names) = \ process.get_schema(workflowobj["cwlVersion"])[:2] if isinstance(avsc_names, Exception): raise avsc_names workflowobj["id"] = fileuri processobj, metadata = document_loader.resolve_all(workflowobj, fileuri) if not isinstance(processobj, (dict, list)): raise ValidationException("Workflow must be a dict or list.") if not metadata: if not isinstance(processobj, dict): raise ValidationException("Draft-2 workflows must be a dict.") metadata = {"$namespaces": processobj.get("$namespaces", {}), "$schemas": processobj.get("$schemas", []), "cwlVersion": processobj["cwlVersion"]} _convert_stdstreams_to_files(workflowobj) if preprocess_only: return document_loader, avsc_names, processobj, metadata, uri schema.validate_doc(avsc_names, processobj, document_loader, strict) if metadata.get("cwlVersion") != update.LATEST: processobj = update.update( processobj, document_loader, fileuri, enable_dev, metadata) if jobobj: metadata[u"cwl:defaults"] = jobobj return document_loader, avsc_names, processobj, metadata, uri
def validate_document( document_loader, # type: Loader workflowobj, # type: CommentedMap uri, # type: Text overrides, # type: List[Dict] metadata, # type: Dict[Text, Any] enable_dev=False, # type: bool strict=True, # type: bool preprocess_only=False, # type: bool fetcher_constructor=None, # type: FetcherConstructorType skip_schemas=None, # type: bool do_validate=True # type: bool ): # type: (...) -> Tuple[Loader, schema.Names, Union[Dict[Text, Any], List[Dict[Text, Any]]], Dict[Text, Any], Text] """Validate a CWL document.""" if isinstance(workflowobj, MutableSequence): workflowobj = cmap({"$graph": workflowobj}, fn=uri) if not isinstance(workflowobj, MutableMapping): raise ValueError("workflowjobj must be a dict, got '{}': {}".format( type(workflowobj), workflowobj)) jobobj = None if "cwl:tool" in workflowobj: job_loader = default_loader(fetcher_constructor) # type: ignore jobobj, _ = job_loader.resolve_all(workflowobj, uri, checklinks=do_validate) uri = urllib.parse.urljoin( uri, workflowobj["https://w3id.org/cwl/cwl#tool"]) del cast(dict, jobobj)["https://w3id.org/cwl/cwl#tool"] if isinstance(jobobj, CommentedMap ) and "http://commonwl.org/cwltool#overrides" in jobobj: overrides.extend(resolve_overrides(jobobj, uri, uri)) del jobobj["http://commonwl.org/cwltool#overrides"] workflowobj = fetch_document( uri, fetcher_constructor=fetcher_constructor)[1] fileuri = urllib.parse.urldefrag(uri)[0] if "cwlVersion" not in workflowobj: if 'cwlVersion' in metadata: workflowobj['cwlVersion'] = metadata['cwlVersion'] else: raise ValidationException( "No cwlVersion found. " "Use the following syntax in your CWL document to declare " "the version: cwlVersion: <version>.\n" "Note: if this is a CWL draft-2 (pre v1.0) document then it " "will need to be upgraded first.") if not isinstance(workflowobj["cwlVersion"], string_types): with SourceLine(workflowobj, "cwlVersion", ValidationException): raise ValidationException("'cwlVersion' must be a string, " "got {}".format( type(workflowobj["cwlVersion"]))) # strip out version workflowobj["cwlVersion"] = re.sub(r"^(?:cwl:|https://w3id.org/cwl/cwl#)", "", workflowobj["cwlVersion"]) if workflowobj["cwlVersion"] not in list(ALLUPDATES): # print out all the Supported Versions of cwlVersion versions = [] for version in list(ALLUPDATES): if "dev" in version: version += " (with --enable-dev flag only)" versions.append(version) versions.sort() raise ValidationException( "The CWL reference runner no longer supports pre CWL v1.0 " "documents. Supported versions are: " "\n{}".format("\n".join(versions))) (sch_document_loader, avsc_names) = \ process.get_schema(workflowobj["cwlVersion"])[:2] if isinstance(avsc_names, Exception): raise avsc_names processobj = None # type: Union[CommentedMap, CommentedSeq, Text, None] document_loader = Loader(sch_document_loader.ctx, schemagraph=sch_document_loader.graph, idx=document_loader.idx, cache=sch_document_loader.cache, fetcher_constructor=fetcher_constructor, skip_schemas=skip_schemas) _add_blank_ids(workflowobj) workflowobj["id"] = fileuri processobj, new_metadata = document_loader.resolve_all( workflowobj, fileuri, checklinks=do_validate) if not isinstance(processobj, (CommentedMap, CommentedSeq)): raise ValidationException("Workflow must be a dict or list.") if not new_metadata and isinstance(processobj, CommentedMap): new_metadata = cast( CommentedMap, cmap( { "$namespaces": processobj.get("$namespaces", {}), "$schemas": processobj.get("$schemas", []), "cwlVersion": processobj["cwlVersion"] }, fn=fileuri)) _convert_stdstreams_to_files(workflowobj) if preprocess_only: return document_loader, avsc_names, processobj, new_metadata, uri if do_validate: schema.validate_doc(avsc_names, processobj, document_loader, strict) if new_metadata.get("cwlVersion") != update.LATEST: processobj = cast( CommentedMap, cmap( update.update(processobj, document_loader, fileuri, enable_dev, new_metadata))) if jobobj is not None: new_metadata[u"cwl:defaults"] = jobobj if overrides: new_metadata[u"cwltool:overrides"] = overrides return document_loader, avsc_names, processobj, new_metadata, uri
def validate_document(document_loader, # type: Loader workflowobj, # type: CommentedMap uri, # type: Text overrides, # type: List[Dict] metadata, # type: Dict[Text, Any] enable_dev=False, # type: bool strict=True, # type: bool preprocess_only=False, # type: bool fetcher_constructor=None, # type: FetcherConstructorType skip_schemas=None, # type: bool do_validate=True # type: bool ): # type: (...) -> Tuple[Loader, schema.Names, Union[Dict[Text, Any], List[Dict[Text, Any]]], Dict[Text, Any], Text] """Validate a CWL document.""" if isinstance(workflowobj, MutableSequence): workflowobj = cmap({ "$graph": workflowobj }, fn=uri) if not isinstance(workflowobj, MutableMapping): raise ValueError("workflowjobj must be a dict, got '{}': {}".format( type(workflowobj), workflowobj)) jobobj = None if "cwl:tool" in workflowobj: job_loader = default_loader(fetcher_constructor) # type: ignore jobobj, _ = job_loader.resolve_all(workflowobj, uri, checklinks=do_validate) uri = urllib.parse.urljoin(uri, workflowobj["https://w3id.org/cwl/cwl#tool"]) del cast(dict, jobobj)["https://w3id.org/cwl/cwl#tool"] if isinstance(jobobj, CommentedMap) and "http://commonwl.org/cwltool#overrides" in jobobj: overrides.extend(resolve_overrides(jobobj, uri, uri)) del jobobj["http://commonwl.org/cwltool#overrides"] workflowobj = fetch_document(uri, fetcher_constructor=fetcher_constructor)[1] fileuri = urllib.parse.urldefrag(uri)[0] if "cwlVersion" not in workflowobj: if 'cwlVersion' in metadata: workflowobj['cwlVersion'] = metadata['cwlVersion'] else: raise ValidationException( "No cwlVersion found. " "Use the following syntax in your CWL document to declare " "the version: cwlVersion: <version>.\n" "Note: if this is a CWL draft-2 (pre v1.0) document then it " "will need to be upgraded first.") if not isinstance(workflowobj["cwlVersion"], string_types): with SourceLine(workflowobj, "cwlVersion", ValidationException): raise ValidationException("'cwlVersion' must be a string, " "got {}".format( type(workflowobj["cwlVersion"]))) # strip out version workflowobj["cwlVersion"] = re.sub( r"^(?:cwl:|https://w3id.org/cwl/cwl#)", "", workflowobj["cwlVersion"]) if workflowobj["cwlVersion"] not in list(ALLUPDATES): # print out all the Supported Versions of cwlVersion versions = [] for version in list(ALLUPDATES): if "dev" in version: version += " (with --enable-dev flag only)" versions.append(version) versions.sort() raise ValidationException( "The CWL reference runner no longer supports pre CWL v1.0 " "documents. Supported versions are: " "\n{}".format("\n".join(versions))) (sch_document_loader, avsc_names) = \ process.get_schema(workflowobj["cwlVersion"])[:2] if isinstance(avsc_names, Exception): raise avsc_names processobj = None # type: Union[CommentedMap, CommentedSeq, Text, None] document_loader = Loader(sch_document_loader.ctx, schemagraph=sch_document_loader.graph, idx=document_loader.idx, cache=sch_document_loader.cache, fetcher_constructor=fetcher_constructor, skip_schemas=skip_schemas) _add_blank_ids(workflowobj) workflowobj["id"] = fileuri processobj, new_metadata = document_loader.resolve_all( workflowobj, fileuri, checklinks=do_validate) if not isinstance(processobj, (CommentedMap, CommentedSeq)): raise ValidationException("Workflow must be a dict or list.") if not new_metadata and isinstance(processobj, CommentedMap): new_metadata = cast(CommentedMap, cmap( {"$namespaces": processobj.get("$namespaces", {}), "$schemas": processobj.get("$schemas", []), "cwlVersion": processobj["cwlVersion"]}, fn=fileuri)) _convert_stdstreams_to_files(workflowobj) if preprocess_only: return document_loader, avsc_names, processobj, new_metadata, uri if do_validate: schema.validate_doc(avsc_names, processobj, document_loader, strict) if new_metadata.get("cwlVersion") != update.LATEST: processobj = cast(CommentedMap, cmap(update.update( processobj, document_loader, fileuri, enable_dev, new_metadata))) if jobobj is not None: new_metadata[u"cwl:defaults"] = jobobj if overrides: new_metadata[u"cwltool:overrides"] = overrides return document_loader, avsc_names, processobj, new_metadata, uri