コード例 #1
0
ファイル: load_tool.py プロジェクト: NBISweden/cwltool
def validate_document(document_loader, workflowobj, uri,
                      enable_dev=False, strict=True, preprocess_only=False):
    # type: (Loader, Dict[unicode, Any], unicode, bool, bool, bool) -> Tuple[Loader, Names, Any, Dict[str, str], unicode]
    """Validate a CWL document."""
    jobobj = None
    if "cwl:tool" in workflowobj:
        jobobj = workflowobj
        uri = urlparse.urljoin(uri, jobobj["cwl:tool"])
        del jobobj["cwl:tool"]
        workflowobj = fetch_document(uri)[1]

    if isinstance(workflowobj, list):
        workflowobj = {
            "$graph": workflowobj
        }

    fileuri = urlparse.urldefrag(uri)[0]

    if "cwlVersion" in workflowobj:
        workflowobj["cwlVersion"] = re.sub(
            r"^(?:cwl:|https://w3id.org/cwl/cwl#)", "",
            workflowobj["cwlVersion"])
    else:
        workflowobj["cwlVersion"] = "draft-2"

    if workflowobj["cwlVersion"] == "draft-2":
        workflowobj = update._draft2toDraft3dev1(
            workflowobj, document_loader, uri, update_steps=False)
        if "@graph" in workflowobj:
            workflowobj["$graph"] = workflowobj["@graph"]
            del workflowobj["@graph"]

    (document_loader, avsc_names) = \
            process.get_schema(workflowobj["cwlVersion"])[:2]

    if isinstance(avsc_names, Exception):
        raise avsc_names

    workflowobj["id"] = fileuri
    processobj, metadata = document_loader.resolve_all(workflowobj, fileuri)

    if preprocess_only:
        return document_loader, avsc_names, processobj, metadata, uri

    document_loader.validate_links(processobj)
    schema.validate_doc(avsc_names, processobj, document_loader, strict)

    if not metadata:
        metadata = {"$namespaces": processobj.get("$namespaces", {}),
                    "$schemas": processobj.get("$schemas", []),
                    "cwlVersion": processobj["cwlVersion"]}

    if metadata.get("cwlVersion") != update.LATEST:
        processobj = update.update(
            processobj, document_loader, fileuri, enable_dev, metadata)

    if jobobj:
        metadata["cwl:defaults"] = jobobj

    return document_loader, avsc_names, processobj, metadata, uri
コード例 #2
0
def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("swagger")
    parser.add_argument("annotations")
    parser.add_argument("url")

    parser.add_argument("--dockstore-fixup",
                        action="store_true",
                        default=False)
    parser.add_argument("--print-rdf", action="store_true", default=False)
    parser.add_argument("--serve", action="store_true", default=False)
    parser.add_argument("--fuseki-path", type=str, default=".")

    args = parser.parse_args()

    with open(args.annotations) as f2:
        annotations = yaml.load(f2)

    with open(args.swagger) as f:
        sld = swg2salad.swg2salad(yaml.load(f), annotations)

    sld["$base"] = "http://ga4gh.org/schemas/tool-registry-schemas"
    sld["name"] = "file://" + os.path.realpath(args.swagger)

    document_loader, avsc_names, schema_metadata, metaschema_loader = load_schema(
        sld)

    txt = document_loader.fetch_text(
        urlparse.urljoin("file://" + os.getcwd() + "/", args.url))
    r = yaml.load(txt)

    if args.dockstore_fixup:
        dockstore_fixup(r)

    validate_doc(avsc_names, r, document_loader, True)

    sys.stderr.write("API returned valid response\n")

    toolreg = Namespace("http://ga4gh.org/schemas/tool-registry-schemas#")
    td = Namespace(
        "http://ga4gh.org/schemas/tool-registry-schemas#ToolDescriptor/")

    if args.print_rdf or args.serve:
        g = jsonld_context.makerdf(args.url, r, document_loader.ctx)
        for s, _, o in g.triples((None, td["type"], Literal("CWL"))):
            for _, _, d in g.triples((s, toolreg["descriptor"], None)):
                expand_cwl(d, unicode(s), g)

    if args.print_rdf:
        print(g.serialize(format="turtle"))

    if args.serve:
        t = tempfile.NamedTemporaryFile(suffix=".ttl")
        g.serialize(t, format="turtle")
        t.flush()
        subprocess.check_call(
            ["./fuseki-server", "--file=" + t.name, "/tools"],
            cwd=args.fuseki_path)
コード例 #3
0
def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("swagger")
    parser.add_argument("annotations")
    parser.add_argument("url")

    parser.add_argument("--print-rdf", action="store_true", default=False)
    parser.add_argument("--serve", action="store_true", default=False)
    parser.add_argument("--fuseki-path", type=str, default=".")

    args = parser.parse_args()
    warnings.simplefilter('ignore', yaml.error.UnsafeLoaderWarning)

    with open(args.annotations) as f2:
        annotations = yaml.load(f2)

    with open(args.swagger) as f:
        sld = swg2salad.swg2salad(yaml.load(f), annotations)

    sld["$base"] = "http://ga4gh.org/schemas/tool-registry-schemas"
    sld["name"] = "file://" + os.path.realpath(args.swagger)

    document_loader, avsc_names, schema_metadata, metaschema_loader = load_schema(
        cmap(sld))

    txt = document_loader.fetch_text(
        urlparse.urljoin(
            "file://" +
            os.getcwd() +
            "/",
            args.url))
    r = yaml.load(txt)

    validate_doc(avsc_names, r, document_loader, True)

    sys.stderr.write("API returned valid response\n")

    toolreg = Namespace("http://ga4gh.org/schemas/tool-registry-schemas#")
    td = Namespace(
        "http://ga4gh.org/schemas/tool-registry-schemas#ToolDescriptor/")

    if args.print_rdf or args.serve:
        g = jsonld_context.makerdf(args.url, r, document_loader.ctx)
        for s, _, o in g.triples((None, td["type"], Literal("CWL"))):
            for _, _, d in g.triples((s, toolreg["descriptor"], None)):
                expand_cwl(d, unicode(s), g)

    if args.print_rdf:
        print(g.serialize(format="turtle"))

    if args.serve:
        t = tempfile.NamedTemporaryFile(suffix=".ttl")
        g.serialize(t, format="turtle")
        t.flush()
        subprocess.check_call(
            ["./fuseki-server", "--file=" + t.name, "/tools"], cwd=args.fuseki_path)
コード例 #4
0
ファイル: schema_salad.py プロジェクト: leipzig/xd-cwl-utils
    def _schema_salad_validate(self, schema_path, document_path):
        '''
        Adapted from schema_salad main().
        :param schema_path:
        :param document_path:
        :return:
        '''

        strict_foreign_properties = False
        strict = True
        metaschema_names, metaschema_doc, metaschema_loader = get_metaschema()
        schema_uri = str(schema_path)
        if not (urlparse(schema_uri)[0]
                and urlparse(schema_uri)[0] in ['http', 'https', 'file']):
            schema_uri = file_uri(schema_uri)
        schema_raw_doc = metaschema_loader.fetch(schema_uri)

        schema_doc, schema_metadata = metaschema_loader.resolve_all(
            schema_raw_doc, schema_uri)

        # Validate schema against metaschema
        validate_doc(metaschema_names, schema_doc, metaschema_loader, True)

        # Get the json-ld context and RDFS representation from the schema
        metactx = collect_namespaces(schema_metadata)
        if "$base" in schema_metadata:
            metactx["@base"] = schema_metadata["$base"]

        (schema_ctx, rdfs) = salad_to_jsonld_context(schema_doc, metactx)

        # Create the loader that will be used to load the target document.
        document_loader = Loader(schema_ctx, skip_schemas=False)

        # Make the Avro validation that will be used to validate the target
        # document

        avsc_obj = make_avro(schema_doc, document_loader)

        avsc_names = make_avro_schema_from_avro(avsc_obj)

        # Load target document and resolve refs
        uri = str(document_path)
        document, doc_metadata = document_loader.resolve_ref(
            uri,
            strict_foreign_properties=strict_foreign_properties,
            checklinks=False
        )  # This is what's getting us around file link checking.

        validate_doc(avsc_names,
                     document,
                     document_loader,
                     strict=strict,
                     strict_foreign_properties=strict_foreign_properties)

        return
コード例 #5
0
def resolve_and_validate_document(
        loadingContext,
        workflowobj,
        uri,
        preprocess_only=False,  # type: bool
        skip_schemas=None,  # type: bool
):
    # type: (...) -> Tuple[LoadingContext, Text]
    """Validate a CWL document."""

    loadingContext = loadingContext.copy()

    if not isinstance(workflowobj, MutableMapping):
        raise ValueError("workflowjobj must be a dict, got '{}': {}".format(
            type(workflowobj), workflowobj))

    jobobj = None
    if "cwl:tool" in workflowobj:
        jobobj, _ = loadingContext.loader.resolve_all(
            workflowobj, uri, checklinks=loadingContext.do_validate)
        uri = urllib.parse.urljoin(
            uri, workflowobj["https://w3id.org/cwl/cwl#tool"])
        del cast(dict, jobobj)["https://w3id.org/cwl/cwl#tool"]

        workflowobj = fetch_document(uri, loadingContext)[1]

    fileuri = urllib.parse.urldefrag(uri)[0]

    cwlVersion = workflowobj.get("cwlVersion")
    if not cwlVersion:
        fileobj = fetch_document(fileuri, loadingContext)[1]
        cwlVersion = fileobj.get("cwlVersion")
        if not cwlVersion:
            raise ValidationException(
                "No cwlVersion found. "
                "Use the following syntax in your CWL document to declare "
                "the version: cwlVersion: <version>.\n"
                "Note: if this is a CWL draft-2 (pre v1.0) document then it "
                "will need to be upgraded first.")

    if not isinstance(cwlVersion, string_types):
        with SourceLine(workflowobj, "cwlVersion", ValidationException):
            raise ValidationException("'cwlVersion' must be a string, "
                                      "got {}".format(type(cwlVersion)))
    # strip out version
    cwlVersion = re.sub(r"^(?:cwl:|https://w3id.org/cwl/cwl#)", "", cwlVersion)
    if cwlVersion not in list(ALLUPDATES):
        # print out all the Supported Versions of cwlVersion
        versions = []
        for version in list(ALLUPDATES):
            if "dev" in version:
                version += " (with --enable-dev flag only)"
            versions.append(version)
        versions.sort()
        raise ValidationException(
            "The CWL reference runner no longer supports pre CWL v1.0 "
            "documents. Supported versions are: "
            "\n{}".format("\n".join(versions)))

    if isinstance(jobobj, CommentedMap
                  ) and "http://commonwl.org/cwltool#overrides" in jobobj:
        loadingContext.overrides_list.extend(
            resolve_overrides(jobobj, uri, uri))
        del jobobj["http://commonwl.org/cwltool#overrides"]

    if isinstance(jobobj, CommentedMap
                  ) and "https://w3id.org/cwl/cwl#requirements" in jobobj:
        if cwlVersion not in ("v1.1.0-dev1", ):
            raise ValidationException(
                "`cwl:requirements` in the input object is not part of CWL "
                "v1.0. You can adjust to use `cwltool:overrides` instead; or you "
                "can set the cwlVersion to v1.1.0-dev1 or greater and re-run with "
                "--enable-dev.")
        loadingContext.overrides_list.append({
            "overrideTarget":
            uri,
            "requirements":
            jobobj["https://w3id.org/cwl/cwl#requirements"]
        })
        del jobobj["https://w3id.org/cwl/cwl#requirements"]

    (sch_document_loader, avsc_names) = \
        process.get_schema(cwlVersion)[:2]

    if isinstance(avsc_names, Exception):
        raise avsc_names

    processobj = None  # type: Union[CommentedMap, CommentedSeq, Text, None]
    document_loader = Loader(
        sch_document_loader.ctx,
        schemagraph=sch_document_loader.graph,
        idx=loadingContext.loader.idx,
        cache=sch_document_loader.cache,
        fetcher_constructor=loadingContext.fetcher_constructor,
        skip_schemas=skip_schemas)

    if cwlVersion == "v1.0":
        _add_blank_ids(workflowobj)

    workflowobj["id"] = fileuri
    processobj, metadata = document_loader.resolve_all(
        workflowobj, fileuri, checklinks=loadingContext.do_validate)
    if not isinstance(processobj, (CommentedMap, CommentedSeq)):
        raise ValidationException(
            "Workflow must be a CommentedMap or CommentedSeq.")
    if not isinstance(metadata, CommentedMap):
        raise ValidationException("metadata must be a CommentedMap, was %s" %
                                  type(metadata))

    _convert_stdstreams_to_files(workflowobj)

    if preprocess_only:
        return loadingContext, uri

    if loadingContext.do_validate:
        schema.validate_doc(avsc_names, processobj, document_loader,
                            loadingContext.strict)

    if loadingContext.do_update:
        processobj = cast(
            CommentedMap,
            cmap(
                update.update(processobj, document_loader, fileuri,
                              loadingContext.enable_dev, metadata)))
        if isinstance(processobj, MutableMapping):
            document_loader.idx[processobj["id"]] = processobj
        elif isinstance(processobj, MutableSequence):
            document_loader.idx[metadata["id"]] = metadata
            for po in processobj:
                document_loader.idx[po["id"]] = po

    if jobobj is not None:
        loadingContext.jobdefaults = jobobj

    loadingContext.loader = document_loader
    loadingContext.avsc_names = avsc_names
    loadingContext.metadata = metadata

    return loadingContext, uri
コード例 #6
0
def validate_document(
        document_loader,  # type: Loader
        workflowobj,  # type: CommentedMap
        uri,  # type: Text
        enable_dev=False,  # type: bool
        strict=True,  # type: bool
        preprocess_only=False,  # type: bool
        fetcher_constructor=None,  # type: FetcherConstructorType
        skip_schemas=None,  # type: bool
        overrides=None,  # type: List[Dict]
        metadata=None,  # type: Optional[Dict]
):
    # type: (...) -> Tuple[Loader, Names, Union[Dict[Text, Any], List[Dict[Text, Any]]], Dict[Text, Any], Text]
    """Validate a CWL document."""

    if isinstance(workflowobj, list):
        workflowobj = cmap({"$graph": workflowobj}, fn=uri)

    if not isinstance(workflowobj, dict):
        raise ValueError("workflowjobj must be a dict, got '%s': %s" %
                         (type(workflowobj), workflowobj))

    jobobj = None
    if "cwl:tool" in workflowobj:
        job_loader = default_loader(fetcher_constructor)  # type: ignore
        jobobj, _ = job_loader.resolve_all(workflowobj, uri)
        uri = urllib.parse.urljoin(
            uri, workflowobj["https://w3id.org/cwl/cwl#tool"])
        del cast(dict, jobobj)["https://w3id.org/cwl/cwl#tool"]

        if "http://commonwl.org/cwltool#overrides" in jobobj:
            overrides.extend(resolve_overrides(jobobj, uri, uri))
            del jobobj["http://commonwl.org/cwltool#overrides"]

        workflowobj = fetch_document(
            uri, fetcher_constructor=fetcher_constructor)[1]

    fileuri = urllib.parse.urldefrag(uri)[0]
    if "cwlVersion" not in workflowobj:
        if metadata and 'cwlVersion' in metadata:
            workflowobj['cwlVersion'] = metadata['cwlVersion']
        else:
            raise ValidationException(
                "No cwlVersion found."
                "Use the following syntax in your CWL document to declare "
                "the version: cwlVersion: <version>")

    if not isinstance(workflowobj["cwlVersion"], (str, Text)):
        raise Exception("'cwlVersion' must be a string, got %s" %
                        type(workflowobj["cwlVersion"]))
    # strip out version
    workflowobj["cwlVersion"] = re.sub(r"^(?:cwl:|https://w3id.org/cwl/cwl#)",
                                       "", workflowobj["cwlVersion"])
    if workflowobj["cwlVersion"] not in list(ALLUPDATES):
        # print out all the Supported Versions of cwlVersion
        versions = list(ALLUPDATES)  # ALLUPDATES is a dict
        versions.sort()
        raise ValidationException(
            "'cwlVersion' not valid. Supported CWL versions are: \n{}".format(
                "\n".join(versions)))

    if workflowobj["cwlVersion"] == "draft-2":
        workflowobj = cast(
            CommentedMap,
            cmap(
                update._draft2toDraft3dev1(workflowobj,
                                           document_loader,
                                           uri,
                                           update_steps=False)))
        if "@graph" in workflowobj:
            workflowobj["$graph"] = workflowobj["@graph"]
            del workflowobj["@graph"]

    (sch_document_loader, avsc_names) = \
        process.get_schema(workflowobj["cwlVersion"])[:2]

    if isinstance(avsc_names, Exception):
        raise avsc_names

    processobj = None  # type: Union[CommentedMap, CommentedSeq, Text]
    document_loader = Loader(sch_document_loader.ctx,
                             schemagraph=sch_document_loader.graph,
                             idx=document_loader.idx,
                             cache=sch_document_loader.cache,
                             fetcher_constructor=fetcher_constructor,
                             skip_schemas=skip_schemas)

    _add_blank_ids(workflowobj)

    workflowobj["id"] = fileuri
    processobj, new_metadata = document_loader.resolve_all(
        workflowobj, fileuri)
    if not isinstance(processobj, (CommentedMap, CommentedSeq)):
        raise ValidationException("Workflow must be a dict or list.")

    if not new_metadata:
        if not isinstance(processobj, dict):
            raise ValidationException("Draft-2 workflows must be a dict.")
        new_metadata = cast(
            CommentedMap,
            cmap(
                {
                    "$namespaces": processobj.get("$namespaces", {}),
                    "$schemas": processobj.get("$schemas", []),
                    "cwlVersion": processobj["cwlVersion"]
                },
                fn=fileuri))

    _convert_stdstreams_to_files(workflowobj)

    if preprocess_only:
        return document_loader, avsc_names, processobj, new_metadata, uri

    schema.validate_doc(avsc_names, processobj, document_loader, strict)

    if new_metadata.get("cwlVersion") != update.LATEST:
        processobj = cast(
            CommentedMap,
            cmap(
                update.update(processobj, document_loader, fileuri, enable_dev,
                              new_metadata)))

    if jobobj:
        new_metadata[u"cwl:defaults"] = jobobj

    if overrides:
        new_metadata[u"cwltool:overrides"] = overrides

    return document_loader, avsc_names, processobj, new_metadata, uri
コード例 #7
0
ファイル: load_tool.py プロジェクト: bmeg/cwltool
def validate_document(document_loader,  # type: Loader
                      workflowobj,  # type: CommentedMap
                      uri,  # type: Text
                      enable_dev=False,  # type: bool
                      strict=True,  # type: bool
                      preprocess_only=False,  # type: bool
                      fetcher_constructor=None,  # type: FetcherConstructorType
                      skip_schemas=None,  # type: bool
                      overrides=None,  # type: List[Dict]
                      metadata=None,  # type: Optional[Dict]
                      ):
    # type: (...) -> Tuple[Loader, Names, Union[Dict[Text, Any], List[Dict[Text, Any]]], Dict[Text, Any], Text]
    """Validate a CWL document."""

    if isinstance(workflowobj, list):
        workflowobj = cmap({
            "$graph": workflowobj
        }, fn=uri)

    if not isinstance(workflowobj, dict):
        raise ValueError("workflowjobj must be a dict, got '%s': %s" % (type(workflowobj), workflowobj))

    jobobj = None
    if "cwl:tool" in workflowobj:
        job_loader = default_loader(fetcher_constructor)  # type: ignore
        jobobj, _ = job_loader.resolve_all(workflowobj, uri)
        uri = urllib.parse.urljoin(uri, workflowobj["https://w3id.org/cwl/cwl#tool"])
        del cast(dict, jobobj)["https://w3id.org/cwl/cwl#tool"]

        if "http://commonwl.org/cwltool#overrides" in jobobj:
            overrides.extend(resolve_overrides(jobobj, uri, uri))
            del jobobj["http://commonwl.org/cwltool#overrides"]

        workflowobj = fetch_document(uri, fetcher_constructor=fetcher_constructor)[1]

    fileuri = urllib.parse.urldefrag(uri)[0]
    if "cwlVersion" not in workflowobj:
        if metadata and 'cwlVersion' in metadata:
            workflowobj['cwlVersion'] = metadata['cwlVersion']
        else:
            raise ValidationException("No cwlVersion found."
                "Use the following syntax in your CWL document to declare "
                "the version: cwlVersion: <version>")

    if not isinstance(workflowobj["cwlVersion"], (str, Text)):
        raise Exception("'cwlVersion' must be a string, got %s" % type(workflowobj["cwlVersion"]))
    # strip out version
    workflowobj["cwlVersion"] = re.sub(
        r"^(?:cwl:|https://w3id.org/cwl/cwl#)", "",
        workflowobj["cwlVersion"])
    if workflowobj["cwlVersion"] not in list(ALLUPDATES):
        # print out all the Supported Versions of cwlVersion
        versions = list(ALLUPDATES)  # ALLUPDATES is a dict
        versions.sort()
        raise ValidationException("'cwlVersion' not valid. Supported CWL versions are: \n{}".format("\n".join(versions)))

    if workflowobj["cwlVersion"] == "draft-2":
        workflowobj = cast(CommentedMap, cmap(update._draft2toDraft3dev1(
            workflowobj, document_loader, uri, update_steps=False)))
        if "@graph" in workflowobj:
            workflowobj["$graph"] = workflowobj["@graph"]
            del workflowobj["@graph"]

    (sch_document_loader, avsc_names) = \
        process.get_schema(workflowobj["cwlVersion"])[:2]

    if isinstance(avsc_names, Exception):
        raise avsc_names

    processobj = None  # type: Union[CommentedMap, CommentedSeq, Text]
    document_loader = Loader(sch_document_loader.ctx, schemagraph=sch_document_loader.graph,
                             idx=document_loader.idx, cache=sch_document_loader.cache,
                             fetcher_constructor=fetcher_constructor, skip_schemas=skip_schemas)

    _add_blank_ids(workflowobj)

    workflowobj["id"] = fileuri
    processobj, new_metadata = document_loader.resolve_all(workflowobj, fileuri)
    if not isinstance(processobj, (CommentedMap, CommentedSeq)):
        raise ValidationException("Workflow must be a dict or list.")

    if not new_metadata:
        if not isinstance(processobj, dict):
            raise ValidationException("Draft-2 workflows must be a dict.")
        new_metadata = cast(CommentedMap, cmap(
            {"$namespaces": processobj.get("$namespaces", {}),
             "$schemas": processobj.get("$schemas", []),
             "cwlVersion": processobj["cwlVersion"]}, fn=fileuri))

    _convert_stdstreams_to_files(workflowobj)

    if preprocess_only:
        return document_loader, avsc_names, processobj, new_metadata, uri

    schema.validate_doc(avsc_names, processobj, document_loader, strict)

    if new_metadata.get("cwlVersion") != update.LATEST:
        processobj = cast(CommentedMap, cmap(update.update(
            processobj, document_loader, fileuri, enable_dev, new_metadata)))

    if jobobj:
        new_metadata[u"cwl:defaults"] = jobobj

    if overrides:
        new_metadata[u"cwltool:overrides"] = overrides

    return document_loader, avsc_names, processobj, new_metadata, uri
コード例 #8
0
ファイル: load_tool.py プロジェクト: BioQueue/cwltool
def validate_document(
        document_loader,  # type: Loader
        workflowobj,  # type: CommentedMap
        uri,  # type: Text
        enable_dev=False,  # type: bool
        strict=True,  # type: bool
        preprocess_only=False,  # type: bool
        fetcher_constructor=None,  # type: FetcherConstructorType
        skip_schemas=None,  # type: bool
        overrides=None,  # type: List[Dict]
        metadata=None,  # type: Optional[Dict]
        do_validate=True):
    # type: (...) -> Tuple[Loader, schema.Names, Union[Dict[Text, Any], List[Dict[Text, Any]]], Dict[Text, Any], Text]
    """Validate a CWL document."""

    if isinstance(workflowobj, list):
        workflowobj = cmap({"$graph": workflowobj}, fn=uri)

    if not isinstance(workflowobj, dict):
        raise ValueError("workflowjobj must be a dict, got '{}': {}".format(
            type(workflowobj), workflowobj))

    jobobj = None
    if "cwl:tool" in workflowobj:
        job_loader = default_loader(fetcher_constructor)  # type: ignore
        jobobj, _ = job_loader.resolve_all(workflowobj,
                                           uri,
                                           checklinks=do_validate)
        uri = urllib.parse.urljoin(
            uri, workflowobj["https://w3id.org/cwl/cwl#tool"])
        del cast(dict, jobobj)["https://w3id.org/cwl/cwl#tool"]

        if "http://commonwl.org/cwltool#overrides" in jobobj:
            overrides.extend(resolve_overrides(jobobj, uri, uri))
            del jobobj["http://commonwl.org/cwltool#overrides"]

        workflowobj = fetch_document(
            uri, fetcher_constructor=fetcher_constructor)[1]

    def var_spool_cwl_detector(
            obj,  # type: Union[Mapping, Iterable, Text]
            item=None,  # type: Optional[Any]
            obj_key=None,  # type: Optional[Any]
    ):  # type: (...)->None
        """ Detects any textual reference to /var/spool/cwl. """
        if isinstance(obj, string_types):
            if "var/spool/cwl" in obj:
                message = SourceLine(
                    item=item,
                    key=obj_key,
                    raise_type=Text,
                    include_traceback=_logger.isEnabledFor(
                        logging.DEBUG)).makeError(
                            "Non-portable reference to /var/spool/cwl found: "
                            "'{}'.\n Replace with /var/spool/cwl/ with "
                            "$(runtime.outdir).".format(obj))
                if not strict:
                    _logger.warning(message)
                else:
                    raise ValidationException(message)
            else:
                return
        elif isinstance(obj, Mapping):
            for key, value in iteritems(obj):
                var_spool_cwl_detector(value, obj, key)
        elif isinstance(obj, Iterable):
            for element in obj:
                var_spool_cwl_detector(element, obj, None)

    var_spool_cwl_detector(workflowobj)

    fileuri = urllib.parse.urldefrag(uri)[0]
    if "cwlVersion" not in workflowobj:
        if metadata and 'cwlVersion' in metadata:
            workflowobj['cwlVersion'] = metadata['cwlVersion']
        else:
            raise ValidationException(
                "No cwlVersion found. "
                "Use the following syntax in your CWL document to declare "
                "the version: cwlVersion: <version>.\n"
                "Note: if this is a CWL draft-2 (pre v1.0) document then it "
                "will need to be upgraded first.")

    if not isinstance(workflowobj["cwlVersion"], (str, Text)):
        with SourceLine(workflowobj, "cwlVersion", ValidationException):
            raise ValidationException("'cwlVersion' must be a string, "
                                      "got {}".format(
                                          type(workflowobj["cwlVersion"])))
    # strip out version
    workflowobj["cwlVersion"] = re.sub(r"^(?:cwl:|https://w3id.org/cwl/cwl#)",
                                       "", workflowobj["cwlVersion"])
    if workflowobj["cwlVersion"] not in list(ALLUPDATES):
        # print out all the Supported Versions of cwlVersion
        versions = []
        for version in list(ALLUPDATES):
            if "dev" in version:
                version += " (with --enable-dev flag only)"
            versions.append(version)
        versions.sort()
        raise ValidationException(
            "The CWL reference runner no longer supports pre CWL v1.0 "
            "documents. Supported versions are: "
            "\n{}".format("\n".join(versions)))

    (sch_document_loader, avsc_names) = \
        process.get_schema(workflowobj["cwlVersion"])[:2]

    if isinstance(avsc_names, Exception):
        raise avsc_names

    processobj = None  # type: Union[CommentedMap, CommentedSeq, Text]
    document_loader = Loader(sch_document_loader.ctx,
                             schemagraph=sch_document_loader.graph,
                             idx=document_loader.idx,
                             cache=sch_document_loader.cache,
                             fetcher_constructor=fetcher_constructor,
                             skip_schemas=skip_schemas)

    _add_blank_ids(workflowobj)

    workflowobj["id"] = fileuri
    processobj, new_metadata = document_loader.resolve_all(
        workflowobj, fileuri, checklinks=do_validate)
    if not isinstance(processobj, (CommentedMap, CommentedSeq)):
        raise ValidationException("Workflow must be a dict or list.")

    if not new_metadata:
        new_metadata = cast(
            CommentedMap,
            cmap(
                {
                    "$namespaces": processobj.get("$namespaces", {}),
                    "$schemas": processobj.get("$schemas", []),
                    "cwlVersion": processobj["cwlVersion"]
                },
                fn=fileuri))

    _convert_stdstreams_to_files(workflowobj)

    if preprocess_only:
        return document_loader, avsc_names, processobj, new_metadata, uri

    if do_validate:
        schema.validate_doc(avsc_names, processobj, document_loader, strict)

    if new_metadata.get("cwlVersion") != update.LATEST:
        processobj = cast(
            CommentedMap,
            cmap(
                update.update(processobj, document_loader, fileuri, enable_dev,
                              new_metadata)))

    if jobobj:
        new_metadata[u"cwl:defaults"] = jobobj

    if overrides:
        new_metadata[u"cwltool:overrides"] = overrides

    return document_loader, avsc_names, processobj, new_metadata, uri
コード例 #9
0
def validate_document(
    document_loader,  # type: Loader
    workflowobj,  # type: CommentedMap
    uri,  # type: Text
    enable_dev=False,  # type: bool
    strict=True,  # type: bool
    preprocess_only=False,  # type: bool
    fetcher_constructor=None
    # type: Callable[[Dict[unicode, unicode], requests.sessions.Session], Fetcher]
):
    # type: (...) -> Tuple[Loader, Names, Union[Dict[Text, Any], List[Dict[Text, Any]]], Dict[Text, Any], Text]
    """Validate a CWL document."""

    if isinstance(workflowobj, list):
        workflowobj = {"$graph": workflowobj}

    if not isinstance(workflowobj, dict):
        raise ValueError("workflowjobj must be a dict, got '%s': %s" %
                         (type(workflowobj), workflowobj))

    jobobj = None
    if "cwl:tool" in workflowobj:
        jobobj, _ = document_loader.resolve_all(workflowobj, uri)
        uri = urllib.parse.urljoin(
            uri, workflowobj["https://w3id.org/cwl/cwl#tool"])
        del cast(dict, jobobj)["https://w3id.org/cwl/cwl#tool"]
        workflowobj = fetch_document(
            uri, fetcher_constructor=fetcher_constructor)[1]

    fileuri = urllib.parse.urldefrag(uri)[0]

    if "cwlVersion" in workflowobj:
        if not isinstance(workflowobj["cwlVersion"], (str, Text)):
            raise Exception("'cwlVersion' must be a string, got %s" %
                            type(workflowobj["cwlVersion"]))
        workflowobj["cwlVersion"] = re.sub(
            r"^(?:cwl:|https://w3id.org/cwl/cwl#)", "",
            workflowobj["cwlVersion"])
    else:
        _logger.warn("No cwlVersion found, treating this file as draft-2.")
        workflowobj["cwlVersion"] = "draft-2"

    if workflowobj["cwlVersion"] == "draft-2":
        workflowobj = cast(
            CommentedMap,
            cmap(
                update._draft2toDraft3dev1(workflowobj,
                                           document_loader,
                                           uri,
                                           update_steps=False)))
        if "@graph" in workflowobj:
            workflowobj["$graph"] = workflowobj["@graph"]
            del workflowobj["@graph"]

    (sch_document_loader, avsc_names) = \
        process.get_schema(workflowobj["cwlVersion"])[:2]

    if isinstance(avsc_names, Exception):
        raise avsc_names

    processobj = None  # type: Union[CommentedMap, CommentedSeq, unicode]
    document_loader = Loader(sch_document_loader.ctx,
                             schemagraph=sch_document_loader.graph,
                             idx=document_loader.idx,
                             cache=sch_document_loader.cache,
                             fetcher_constructor=fetcher_constructor)

    _add_blank_ids(workflowobj)

    workflowobj["id"] = fileuri
    processobj, metadata = document_loader.resolve_all(workflowobj, fileuri)
    if not isinstance(processobj, (CommentedMap, CommentedSeq)):
        raise ValidationException("Workflow must be a dict or list.")

    if not metadata:
        if not isinstance(processobj, dict):
            raise ValidationException("Draft-2 workflows must be a dict.")
        metadata = cast(
            CommentedMap,
            cmap(
                {
                    "$namespaces": processobj.get("$namespaces", {}),
                    "$schemas": processobj.get("$schemas", []),
                    "cwlVersion": processobj["cwlVersion"]
                },
                fn=fileuri))

    _convert_stdstreams_to_files(workflowobj)

    if preprocess_only:
        return document_loader, avsc_names, processobj, metadata, uri

    schema.validate_doc(avsc_names, processobj, document_loader, strict)

    if metadata.get("cwlVersion") != update.LATEST:
        processobj = cast(
            CommentedMap,
            cmap(
                update.update(processobj, document_loader, fileuri, enable_dev,
                              metadata)))

    if jobobj:
        metadata[u"cwl:defaults"] = jobobj

    return document_loader, avsc_names, processobj, metadata, uri
コード例 #10
0
ファイル: load_tool.py プロジェクト: hmenager/cwltool
def resolve_and_validate_document(loadingContext,
                      workflowobj,
                      uri,
                      preprocess_only=False,     # type: bool
                      skip_schemas=None,         # type: bool
                     ):
    # type: (...) -> Tuple[LoadingContext, Text]
    """Validate a CWL document."""

    loadingContext = loadingContext.copy()

    if not isinstance(workflowobj, MutableMapping):
        raise ValueError("workflowjobj must be a dict, got '{}': {}".format(
            type(workflowobj), workflowobj))

    jobobj = None
    if "cwl:tool" in workflowobj:
        jobobj, _ = loadingContext.loader.resolve_all(workflowobj, uri, checklinks=loadingContext.do_validate)
        uri = urllib.parse.urljoin(uri, workflowobj["https://w3id.org/cwl/cwl#tool"])
        del cast(dict, jobobj)["https://w3id.org/cwl/cwl#tool"]

        workflowobj = fetch_document(uri, loadingContext)[1]

    fileuri = urllib.parse.urldefrag(uri)[0]

    cwlVersion = workflowobj.get("cwlVersion")
    if not cwlVersion:
        fileobj = fetch_document(fileuri, loadingContext)[1]
        cwlVersion = fileobj.get("cwlVersion")
        if not cwlVersion:
            raise ValidationException(
                "No cwlVersion found. "
                "Use the following syntax in your CWL document to declare "
                "the version: cwlVersion: <version>.\n"
                "Note: if this is a CWL draft-2 (pre v1.0) document then it "
                "will need to be upgraded first.")

    if not isinstance(cwlVersion, string_types):
        with SourceLine(workflowobj, "cwlVersion", ValidationException):
            raise ValidationException("'cwlVersion' must be a string, "
                                      "got {}".format(
                                          type(cwlVersion)))
    # strip out version
    cwlVersion = re.sub(
        r"^(?:cwl:|https://w3id.org/cwl/cwl#)", "",
        cwlVersion)
    if cwlVersion not in list(ALLUPDATES):
        # print out all the Supported Versions of cwlVersion
        versions = []
        for version in list(ALLUPDATES):
            if "dev" in version:
                version += " (with --enable-dev flag only)"
            versions.append(version)
        versions.sort()
        raise ValidationException(
            "The CWL reference runner no longer supports pre CWL v1.0 "
            "documents. Supported versions are: "
            "\n{}".format("\n".join(versions)))

    if isinstance(jobobj, CommentedMap) and "http://commonwl.org/cwltool#overrides" in jobobj:
        loadingContext.overrides_list.extend(resolve_overrides(jobobj, uri, uri))
        del jobobj["http://commonwl.org/cwltool#overrides"]

    if isinstance(jobobj, CommentedMap) and "https://w3id.org/cwl/cwl#requirements" in jobobj:
        if cwlVersion not in ("v1.1.0-dev1",):
            raise ValidationException(
                    "`cwl:requirements` in the input object is not part of CWL "
                    "v1.0. You can adjust to use `cwltool:overrides` instead; or you "
                    "can set the cwlVersion to v1.1.0-dev1 or greater and re-run with "
                    "--enable-dev.")
        loadingContext.overrides_list.append({"overrideTarget": uri,
                                              "requirements": jobobj["https://w3id.org/cwl/cwl#requirements"]})
        del jobobj["https://w3id.org/cwl/cwl#requirements"]

    (sch_document_loader, avsc_names) = \
        process.get_schema(cwlVersion)[:2]

    if isinstance(avsc_names, Exception):
        raise avsc_names

    processobj = None  # type: Union[CommentedMap, CommentedSeq, Text, None]
    document_loader = Loader(sch_document_loader.ctx,
                             schemagraph=sch_document_loader.graph,
                             idx=loadingContext.loader.idx,
                             cache=sch_document_loader.cache,
                             fetcher_constructor=loadingContext.fetcher_constructor,
                             skip_schemas=skip_schemas)

    if cwlVersion == "v1.0":
        _add_blank_ids(workflowobj)

    workflowobj["id"] = fileuri
    processobj, metadata = document_loader.resolve_all(
        workflowobj, fileuri, checklinks=loadingContext.do_validate)
    if not isinstance(processobj, (CommentedMap, CommentedSeq)):
        raise ValidationException("Workflow must be a CommentedMap or CommentedSeq.")
    if not isinstance(metadata, CommentedMap):
        raise ValidationException("metadata must be a CommentedMap, was %s" % type(metadata))

    _convert_stdstreams_to_files(workflowobj)

    if preprocess_only:
        return loadingContext, uri

    if loadingContext.do_validate:
        schema.validate_doc(avsc_names, processobj, document_loader, loadingContext.strict)

    if loadingContext.do_update:
        processobj = cast(CommentedMap, cmap(update.update(
            processobj, document_loader, fileuri, loadingContext.enable_dev, metadata)))
        if isinstance(processobj, MutableMapping):
            document_loader.idx[processobj["id"]] = processobj
        elif isinstance(processobj, MutableSequence):
            document_loader.idx[metadata["id"]] = metadata
            for po in processobj:
                document_loader.idx[po["id"]] = po

    if jobobj is not None:
        loadingContext.jobdefaults = jobobj

    loadingContext.loader = document_loader
    loadingContext.avsc_names = avsc_names
    loadingContext.metadata = metadata

    return loadingContext, uri
コード例 #11
0
def validate_document(document_loader,   # type: Loader
                      workflowobj,       # type: CommentedMap
                      uri,               # type: Text
                      enable_dev=False,  # type: bool
                      strict=True,       # type: bool
                      preprocess_only=False,    # type: bool
                      fetcher_constructor=None  # type: Callable[[Dict[unicode, unicode], requests.sessions.Session], Fetcher]
                      ):
    # type: (...) -> Tuple[Loader, Names, Union[Dict[Text, Any], List[Dict[Text, Any]]], Dict[Text, Any], Text]
    """Validate a CWL document."""

    if isinstance(workflowobj, list):
        workflowobj = {
            "$graph": workflowobj
        }

    if not isinstance(workflowobj, dict):
        raise ValueError("workflowjobj must be a dict")

    jobobj = None
    if "cwl:tool" in workflowobj:
        jobobj, _ = document_loader.resolve_all(workflowobj, uri)
        uri = urlparse.urljoin(uri, workflowobj["https://w3id.org/cwl/cwl#tool"])
        del cast(dict, jobobj)["https://w3id.org/cwl/cwl#tool"]
        workflowobj = fetch_document(uri, fetcher_constructor=fetcher_constructor)[1]

    fileuri = urlparse.urldefrag(uri)[0]

    if "cwlVersion" in workflowobj:
        if not isinstance(workflowobj["cwlVersion"], (str, Text)):
            raise Exception("'cwlVersion' must be a string, got %s" % type(workflowobj["cwlVersion"]))
        workflowobj["cwlVersion"] = re.sub(
            r"^(?:cwl:|https://w3id.org/cwl/cwl#)", "",
            workflowobj["cwlVersion"])
    else:
        _logger.warn("No cwlVersion found, treating this file as draft-2.")
        workflowobj["cwlVersion"] = "draft-2"

    if workflowobj["cwlVersion"] == "draft-2":
        workflowobj = cast(CommentedMap, cmap(update._draft2toDraft3dev1(
            workflowobj, document_loader, uri, update_steps=False)))
        if "@graph" in workflowobj:
            workflowobj["$graph"] = workflowobj["@graph"]
            del workflowobj["@graph"]

    (sch_document_loader, avsc_names) = \
        process.get_schema(workflowobj["cwlVersion"])[:2]

    if isinstance(avsc_names, Exception):
        raise avsc_names

    processobj = None  # type: Union[CommentedMap, CommentedSeq, unicode]
    document_loader = Loader(sch_document_loader.ctx, schemagraph=sch_document_loader.graph,
                  idx=document_loader.idx, cache=sch_document_loader.cache,
                             fetcher_constructor=fetcher_constructor)

    workflowobj["id"] = fileuri
    processobj, metadata = document_loader.resolve_all(workflowobj, fileuri)
    if not isinstance(processobj, (CommentedMap, CommentedSeq)):
        raise ValidationException("Workflow must be a dict or list.")

    if not metadata:
        if not isinstance(processobj, dict):
            raise ValidationException("Draft-2 workflows must be a dict.")
        metadata = cast(CommentedMap, cmap({"$namespaces": processobj.get("$namespaces", {}),
                         "$schemas": processobj.get("$schemas", []),
                         "cwlVersion": processobj["cwlVersion"]},
                        fn=fileuri))

    _convert_stdstreams_to_files(workflowobj)

    if preprocess_only:
        return document_loader, avsc_names, processobj, metadata, uri

    schema.validate_doc(avsc_names, processobj, document_loader, strict)

    if metadata.get("cwlVersion") != update.LATEST:
        processobj = cast(CommentedMap, cmap(update.update(
            processobj, document_loader, fileuri, enable_dev, metadata)))

    if jobobj:
        metadata[u"cwl:defaults"] = jobobj

    return document_loader, avsc_names, processobj, metadata, uri
コード例 #12
0
ファイル: load_tool.py プロジェクト: wangdi2014/cwltool
def validate_document(document_loader,
                      workflowobj,
                      uri,
                      enable_dev=False,
                      strict=True,
                      preprocess_only=False):
    # type: (Loader, Dict[Text, Any], Text, bool, bool, bool) -> Tuple[Loader, Names, Union[Dict[Text, Any], List[Dict[Text, Any]]], Dict[Text, Any], Text]
    """Validate a CWL document."""
    jobobj = None
    if "cwl:tool" in workflowobj:
        jobobj, _ = document_loader.resolve_all(workflowobj, uri)
        uri = urlparse.urljoin(uri,
                               workflowobj["https://w3id.org/cwl/cwl#tool"])
        del cast(dict, jobobj)["https://w3id.org/cwl/cwl#tool"]
        workflowobj = fetch_document(uri)[1]

    if isinstance(workflowobj, list):
        workflowobj = {"$graph": workflowobj}

    fileuri = urlparse.urldefrag(uri)[0]

    if "cwlVersion" in workflowobj:
        workflowobj["cwlVersion"] = re.sub(
            r"^(?:cwl:|https://w3id.org/cwl/cwl#)", "",
            workflowobj["cwlVersion"])
    else:
        _logger.warn("No cwlVersion found, treating this file as draft-2.")
        workflowobj["cwlVersion"] = "draft-2"

    if workflowobj["cwlVersion"] == "draft-2":
        workflowobj = update._draft2toDraft3dev1(workflowobj,
                                                 document_loader,
                                                 uri,
                                                 update_steps=False)
        if "@graph" in workflowobj:
            workflowobj["$graph"] = workflowobj["@graph"]
            del workflowobj["@graph"]

    (document_loader, avsc_names) = \
        process.get_schema(workflowobj["cwlVersion"])[:2]

    if isinstance(avsc_names, Exception):
        raise avsc_names

    workflowobj["id"] = fileuri
    processobj, metadata = document_loader.resolve_all(workflowobj, fileuri)
    if not isinstance(processobj, (dict, list)):
        raise ValidationException("Workflow must be a dict or list.")

    if not metadata:
        if not isinstance(processobj, dict):
            raise ValidationException("Draft-2 workflows must be a dict.")
        metadata = {
            "$namespaces": processobj.get("$namespaces", {}),
            "$schemas": processobj.get("$schemas", []),
            "cwlVersion": processobj["cwlVersion"]
        }

    _convert_stdstreams_to_files(workflowobj)

    if preprocess_only:
        return document_loader, avsc_names, processobj, metadata, uri

    schema.validate_doc(avsc_names, processobj, document_loader, strict)

    if metadata.get("cwlVersion") != update.LATEST:
        processobj = update.update(processobj, document_loader, fileuri,
                                   enable_dev, metadata)

    if jobobj:
        metadata[u"cwl:defaults"] = jobobj

    return document_loader, avsc_names, processobj, metadata, uri
コード例 #13
0
def validate_document(document_loader, workflowobj, uri,
                      enable_dev=False, strict=True, preprocess_only=False):
    # type: (Loader, Dict[unicode, Any], unicode, bool, bool, bool) -> Tuple[Loader, Names, Any, Dict[str, str], unicode]
    """Validate a CWL document."""
    jobobj = None
    if "cwl:tool" in workflowobj:
        jobobj = workflowobj
        uri = urlparse.urljoin(uri, jobobj["cwl:tool"])
        del jobobj["cwl:tool"]
        workflowobj = fetch_document(uri)[1]

    if isinstance(workflowobj, list):
        workflowobj = {
            "$graph": workflowobj
        }

    fileuri = urlparse.urldefrag(uri)[0]

    if "cwlVersion" in workflowobj:
        workflowobj["cwlVersion"] = re.sub(
            r"^(?:cwl:|https://w3id.org/cwl/cwl#)", "",
            workflowobj["cwlVersion"])
    else:
        _logger.warn("No cwlVersion found, treating this file as draft-2.")
        workflowobj["cwlVersion"] = "draft-2"

    if workflowobj["cwlVersion"] == "draft-2":
        workflowobj = update._draft2toDraft3dev1(
            workflowobj, document_loader, uri, update_steps=False)
        if "@graph" in workflowobj:
            workflowobj["$graph"] = workflowobj["@graph"]
            del workflowobj["@graph"]

    (document_loader, avsc_names) = \
            process.get_schema(workflowobj["cwlVersion"])[:2]

    if isinstance(avsc_names, Exception):
        raise avsc_names

    workflowobj["id"] = fileuri
    processobj, metadata = document_loader.resolve_all(workflowobj, fileuri)

    if not metadata:
        metadata = {"$namespaces": processobj.get("$namespaces", {}),
                   "$schemas": processobj.get("$schemas", []),
                   "cwlVersion": processobj["cwlVersion"]}

    if preprocess_only:
        return document_loader, avsc_names, processobj, metadata, uri

    document_loader.validate_links(processobj)
    schema.validate_doc(avsc_names, processobj, document_loader, strict)

    if metadata.get("cwlVersion") != update.LATEST:
        processobj = update.update(
            processobj, document_loader, fileuri, enable_dev, metadata)

    if jobobj:
        metadata["cwl:defaults"] = jobobj

    return document_loader, avsc_names, processobj, metadata, uri
コード例 #14
0
def resolve_and_validate_document(
        loadingContext,  # type: LoadingContext
        workflowobj,  # type: Union[CommentedMap, CommentedSeq]
        uri,  # type: str
        preprocess_only=False,  # type: bool
        skip_schemas=None,  # type: Optional[bool]
):
    # type: (...) -> Tuple[LoadingContext, str]
    """Validate a CWL document."""
    if not loadingContext.loader:
        raise ValueError("loadingContext must have a loader.")
    else:
        loader = loadingContext.loader
    loadingContext = loadingContext.copy()

    if not isinstance(workflowobj, MutableMapping):
        raise ValueError("workflowjobj must be a dict, got '{}': {}".format(
            type(workflowobj), workflowobj))

    jobobj = None
    if "cwl:tool" in workflowobj:
        jobobj, _ = loader.resolve_all(workflowobj, uri)
        uri = urllib.parse.urljoin(
            uri, workflowobj["https://w3id.org/cwl/cwl#tool"])
        del cast(Dict[str, Any], jobobj)["https://w3id.org/cwl/cwl#tool"]

        workflowobj = fetch_document(uri, loadingContext)[1]

    fileuri = urllib.parse.urldefrag(uri)[0]

    cwlVersion = loadingContext.metadata.get("cwlVersion")
    if not cwlVersion:
        cwlVersion = workflowobj.get("cwlVersion")
    if not cwlVersion and fileuri != uri:
        # The tool we're loading is a fragment of a bigger file.  Get
        # the document root element and look for cwlVersion there.
        metadata = fetch_document(fileuri,
                                  loadingContext)[1]  # type: Dict[str, Any]
        cwlVersion = metadata.get("cwlVersion")
    if not cwlVersion:
        raise ValidationException(
            "No cwlVersion found. "
            "Use the following syntax in your CWL document to declare "
            "the version: cwlVersion: <version>.\n"
            "Note: if this is a CWL draft-2 (pre v1.0) document then it "
            "will need to be upgraded first.")

    if not isinstance(cwlVersion, str):
        with SourceLine(workflowobj, "cwlVersion", ValidationException):
            raise ValidationException("'cwlVersion' must be a string, "
                                      "got {}".format(type(cwlVersion)))
    # strip out version
    cwlVersion = re.sub(r"^(?:cwl:|https://w3id.org/cwl/cwl#)", "", cwlVersion)
    if cwlVersion not in list(ALLUPDATES):
        # print out all the Supported Versions of cwlVersion
        versions = []
        for version in list(ALLUPDATES):
            if "dev" in version:
                version += " (with --enable-dev flag only)"
            versions.append(version)
        versions.sort()
        raise ValidationException(
            "The CWL reference runner no longer supports pre CWL v1.0 "
            "documents. Supported versions are: "
            "\n{}".format("\n".join(versions)))

    if (isinstance(jobobj, CommentedMap)
            and "http://commonwl.org/cwltool#overrides" in jobobj):
        loadingContext.overrides_list.extend(
            resolve_overrides(jobobj, uri, uri))
        del jobobj["http://commonwl.org/cwltool#overrides"]

    if (isinstance(jobobj, CommentedMap)
            and "https://w3id.org/cwl/cwl#requirements" in jobobj):
        if cwlVersion not in ("v1.1.0-dev1", "v1.1"):
            raise ValidationException(
                "`cwl:requirements` in the input object is not part of CWL "
                "v1.0. You can adjust to use `cwltool:overrides` instead; or you "
                "can set the cwlVersion to v1.1 or greater.")
        loadingContext.overrides_list.append({
            "overrideTarget":
            uri,
            "requirements":
            jobobj["https://w3id.org/cwl/cwl#requirements"],
        })
        del jobobj["https://w3id.org/cwl/cwl#requirements"]

    (sch_document_loader, avsc_names) = process.get_schema(cwlVersion)[:2]

    if isinstance(avsc_names, Exception):
        raise avsc_names

    processobj = None  # type: Union[CommentedMap, CommentedSeq, str, None]
    document_loader = Loader(
        sch_document_loader.ctx,
        schemagraph=sch_document_loader.graph,
        idx=loader.idx,
        cache=sch_document_loader.cache,
        fetcher_constructor=loadingContext.fetcher_constructor,
        skip_schemas=skip_schemas,
        doc_cache=loadingContext.doc_cache,
    )

    if cwlVersion == "v1.0":
        _add_blank_ids(workflowobj)

    processobj, metadata = document_loader.resolve_all(workflowobj, fileuri)
    if loadingContext.metadata:
        metadata = loadingContext.metadata
    if not isinstance(processobj, (CommentedMap, CommentedSeq)):
        raise ValidationException(
            "Workflow must be a CommentedMap or CommentedSeq.")
    if not isinstance(metadata, CommentedMap):
        raise ValidationException("metadata must be a CommentedMap, was %s" %
                                  type(metadata))

    if isinstance(processobj, CommentedMap):
        uri = processobj["id"]

    _convert_stdstreams_to_files(workflowobj)

    if preprocess_only:
        return loadingContext, uri

    if loadingContext.do_validate:
        schema.validate_doc(avsc_names, processobj, document_loader,
                            loadingContext.strict)

    # None means default behavior (do update)
    if loadingContext.do_update in (True, None):
        if "cwlVersion" not in metadata:
            metadata["cwlVersion"] = cwlVersion
        processobj = update.update(processobj, document_loader, fileuri,
                                   loadingContext.enable_dev, metadata)
        document_loader.idx[processobj["id"]] = processobj

        def update_index(pr: CommentedMap) -> None:
            if "id" in pr:
                document_loader.idx[pr["id"]] = pr

        visit_class(processobj,
                    ("CommandLineTool", "Workflow", "ExpressionTool"),
                    update_index)

    if isinstance(jobobj, CommentedMap):
        loadingContext.jobdefaults = jobobj

    loadingContext.loader = document_loader
    loadingContext.avsc_names = avsc_names
    loadingContext.metadata = metadata

    return loadingContext, uri
コード例 #15
0
ファイル: load_tool.py プロジェクト: dleehr/cwltool
def validate_document(document_loader, workflowobj, uri,
                      enable_dev=False, strict=True, preprocess_only=False):
    # type: (Loader, Dict[Text, Any], Text, bool, bool, bool) -> Tuple[Loader, Names, Union[Dict[Text, Any], List[Dict[Text, Any]]], Dict[Text, Any], Text]
    """Validate a CWL document."""
    jobobj = None
    if "cwl:tool" in workflowobj:
        jobobj = workflowobj
        uri = urlparse.urljoin(uri, jobobj["cwl:tool"])
        del jobobj["cwl:tool"]
        workflowobj = fetch_document(uri)[1]

    if isinstance(workflowobj, list):
        workflowobj = {
            "$graph": workflowobj
        }

    fileuri = urlparse.urldefrag(uri)[0]

    if "cwlVersion" in workflowobj:
        workflowobj["cwlVersion"] = re.sub(
            r"^(?:cwl:|https://w3id.org/cwl/cwl#)", "",
            workflowobj["cwlVersion"])
    else:
        _logger.warn("No cwlVersion found, treating this file as draft-2.")
        workflowobj["cwlVersion"] = "draft-2"

    if workflowobj["cwlVersion"] == "draft-2":
        workflowobj = update._draft2toDraft3dev1(
            workflowobj, document_loader, uri, update_steps=False)
        if "@graph" in workflowobj:
            workflowobj["$graph"] = workflowobj["@graph"]
            del workflowobj["@graph"]

    (document_loader, avsc_names) = \
        process.get_schema(workflowobj["cwlVersion"])[:2]

    if isinstance(avsc_names, Exception):
        raise avsc_names

    workflowobj["id"] = fileuri
    processobj, metadata = document_loader.resolve_all(workflowobj, fileuri)
    if not isinstance(processobj, (dict, list)):
        raise ValidationException("Workflow must be a dict or list.")

    if not metadata:
        if not isinstance(processobj, dict):
            raise ValidationException("Draft-2 workflows must be a dict.")
        metadata = {"$namespaces": processobj.get("$namespaces", {}),
                   "$schemas": processobj.get("$schemas", []),
                   "cwlVersion": processobj["cwlVersion"]}

    _convert_stdstreams_to_files(workflowobj)

    if preprocess_only:
        return document_loader, avsc_names, processobj, metadata, uri

    schema.validate_doc(avsc_names, processobj, document_loader, strict)

    if metadata.get("cwlVersion") != update.LATEST:
        processobj = update.update(
            processobj, document_loader, fileuri, enable_dev, metadata)

    if jobobj:
        metadata[u"cwl:defaults"] = jobobj

    return document_loader, avsc_names, processobj, metadata, uri
コード例 #16
0
def validate_document(
        document_loader,  # type: Loader
        workflowobj,  # type: CommentedMap
        uri,  # type: Text
        overrides,  # type: List[Dict]
        metadata,  # type: Dict[Text, Any]
        enable_dev=False,  # type: bool
        strict=True,  # type: bool
        preprocess_only=False,  # type: bool
        fetcher_constructor=None,  # type: FetcherConstructorType
        skip_schemas=None,  # type: bool
        do_validate=True  # type: bool
):
    # type: (...) -> Tuple[Loader, schema.Names, Union[Dict[Text, Any], List[Dict[Text, Any]]], Dict[Text, Any], Text]
    """Validate a CWL document."""

    if isinstance(workflowobj, MutableSequence):
        workflowobj = cmap({"$graph": workflowobj}, fn=uri)

    if not isinstance(workflowobj, MutableMapping):
        raise ValueError("workflowjobj must be a dict, got '{}': {}".format(
            type(workflowobj), workflowobj))

    jobobj = None
    if "cwl:tool" in workflowobj:
        job_loader = default_loader(fetcher_constructor)  # type: ignore
        jobobj, _ = job_loader.resolve_all(workflowobj,
                                           uri,
                                           checklinks=do_validate)
        uri = urllib.parse.urljoin(
            uri, workflowobj["https://w3id.org/cwl/cwl#tool"])
        del cast(dict, jobobj)["https://w3id.org/cwl/cwl#tool"]

        if isinstance(jobobj, CommentedMap
                      ) and "http://commonwl.org/cwltool#overrides" in jobobj:
            overrides.extend(resolve_overrides(jobobj, uri, uri))
            del jobobj["http://commonwl.org/cwltool#overrides"]

        workflowobj = fetch_document(
            uri, fetcher_constructor=fetcher_constructor)[1]

    fileuri = urllib.parse.urldefrag(uri)[0]
    if "cwlVersion" not in workflowobj:
        if 'cwlVersion' in metadata:
            workflowobj['cwlVersion'] = metadata['cwlVersion']
        else:
            raise ValidationException(
                "No cwlVersion found. "
                "Use the following syntax in your CWL document to declare "
                "the version: cwlVersion: <version>.\n"
                "Note: if this is a CWL draft-2 (pre v1.0) document then it "
                "will need to be upgraded first.")

    if not isinstance(workflowobj["cwlVersion"], string_types):
        with SourceLine(workflowobj, "cwlVersion", ValidationException):
            raise ValidationException("'cwlVersion' must be a string, "
                                      "got {}".format(
                                          type(workflowobj["cwlVersion"])))
    # strip out version
    workflowobj["cwlVersion"] = re.sub(r"^(?:cwl:|https://w3id.org/cwl/cwl#)",
                                       "", workflowobj["cwlVersion"])
    if workflowobj["cwlVersion"] not in list(ALLUPDATES):
        # print out all the Supported Versions of cwlVersion
        versions = []
        for version in list(ALLUPDATES):
            if "dev" in version:
                version += " (with --enable-dev flag only)"
            versions.append(version)
        versions.sort()
        raise ValidationException(
            "The CWL reference runner no longer supports pre CWL v1.0 "
            "documents. Supported versions are: "
            "\n{}".format("\n".join(versions)))

    (sch_document_loader, avsc_names) = \
        process.get_schema(workflowobj["cwlVersion"])[:2]

    if isinstance(avsc_names, Exception):
        raise avsc_names

    processobj = None  # type: Union[CommentedMap, CommentedSeq, Text, None]
    document_loader = Loader(sch_document_loader.ctx,
                             schemagraph=sch_document_loader.graph,
                             idx=document_loader.idx,
                             cache=sch_document_loader.cache,
                             fetcher_constructor=fetcher_constructor,
                             skip_schemas=skip_schemas)

    _add_blank_ids(workflowobj)

    workflowobj["id"] = fileuri
    processobj, new_metadata = document_loader.resolve_all(
        workflowobj, fileuri, checklinks=do_validate)
    if not isinstance(processobj, (CommentedMap, CommentedSeq)):
        raise ValidationException("Workflow must be a dict or list.")

    if not new_metadata and isinstance(processobj, CommentedMap):
        new_metadata = cast(
            CommentedMap,
            cmap(
                {
                    "$namespaces": processobj.get("$namespaces", {}),
                    "$schemas": processobj.get("$schemas", []),
                    "cwlVersion": processobj["cwlVersion"]
                },
                fn=fileuri))

    _convert_stdstreams_to_files(workflowobj)

    if preprocess_only:
        return document_loader, avsc_names, processobj, new_metadata, uri

    if do_validate:
        schema.validate_doc(avsc_names, processobj, document_loader, strict)

    if new_metadata.get("cwlVersion") != update.LATEST:
        processobj = cast(
            CommentedMap,
            cmap(
                update.update(processobj, document_loader, fileuri, enable_dev,
                              new_metadata)))

    if jobobj is not None:
        new_metadata[u"cwl:defaults"] = jobobj

    if overrides:
        new_metadata[u"cwltool:overrides"] = overrides

    return document_loader, avsc_names, processobj, new_metadata, uri
コード例 #17
0
ファイル: load_tool.py プロジェクト: denis-yuen/cwltool
def validate_document(document_loader,           # type: Loader
                      workflowobj,               # type: CommentedMap
                      uri,                       # type: Text
                      overrides,                 # type: List[Dict]
                      metadata,                  # type: Dict[Text, Any]
                      enable_dev=False,          # type: bool
                      strict=True,               # type: bool
                      preprocess_only=False,     # type: bool
                      fetcher_constructor=None,  # type: FetcherConstructorType
                      skip_schemas=None,         # type: bool
                      do_validate=True           # type: bool
                     ):
    # type: (...) -> Tuple[Loader, schema.Names, Union[Dict[Text, Any], List[Dict[Text, Any]]], Dict[Text, Any], Text]
    """Validate a CWL document."""

    if isinstance(workflowobj, MutableSequence):
        workflowobj = cmap({
            "$graph": workflowobj
        }, fn=uri)

    if not isinstance(workflowobj, MutableMapping):
        raise ValueError("workflowjobj must be a dict, got '{}': {}".format(
            type(workflowobj), workflowobj))

    jobobj = None
    if "cwl:tool" in workflowobj:
        job_loader = default_loader(fetcher_constructor)  # type: ignore
        jobobj, _ = job_loader.resolve_all(workflowobj, uri, checklinks=do_validate)
        uri = urllib.parse.urljoin(uri, workflowobj["https://w3id.org/cwl/cwl#tool"])
        del cast(dict, jobobj)["https://w3id.org/cwl/cwl#tool"]

        if isinstance(jobobj, CommentedMap) and "http://commonwl.org/cwltool#overrides" in jobobj:
            overrides.extend(resolve_overrides(jobobj, uri, uri))
            del jobobj["http://commonwl.org/cwltool#overrides"]

        workflowobj = fetch_document(uri, fetcher_constructor=fetcher_constructor)[1]

    fileuri = urllib.parse.urldefrag(uri)[0]
    if "cwlVersion" not in workflowobj:
        if 'cwlVersion' in metadata:
            workflowobj['cwlVersion'] = metadata['cwlVersion']
        else:
            raise ValidationException(
                "No cwlVersion found. "
                "Use the following syntax in your CWL document to declare "
                "the version: cwlVersion: <version>.\n"
                "Note: if this is a CWL draft-2 (pre v1.0) document then it "
                "will need to be upgraded first.")

    if not isinstance(workflowobj["cwlVersion"], string_types):
        with SourceLine(workflowobj, "cwlVersion", ValidationException):
            raise ValidationException("'cwlVersion' must be a string, "
                                      "got {}".format(
                                          type(workflowobj["cwlVersion"])))
    # strip out version
    workflowobj["cwlVersion"] = re.sub(
        r"^(?:cwl:|https://w3id.org/cwl/cwl#)", "",
        workflowobj["cwlVersion"])
    if workflowobj["cwlVersion"] not in list(ALLUPDATES):
        # print out all the Supported Versions of cwlVersion
        versions = []
        for version in list(ALLUPDATES):
            if "dev" in version:
                version += " (with --enable-dev flag only)"
            versions.append(version)
        versions.sort()
        raise ValidationException(
            "The CWL reference runner no longer supports pre CWL v1.0 "
            "documents. Supported versions are: "
            "\n{}".format("\n".join(versions)))

    (sch_document_loader, avsc_names) = \
        process.get_schema(workflowobj["cwlVersion"])[:2]

    if isinstance(avsc_names, Exception):
        raise avsc_names

    processobj = None  # type: Union[CommentedMap, CommentedSeq, Text, None]
    document_loader = Loader(sch_document_loader.ctx, schemagraph=sch_document_loader.graph,
                             idx=document_loader.idx, cache=sch_document_loader.cache,
                             fetcher_constructor=fetcher_constructor, skip_schemas=skip_schemas)

    _add_blank_ids(workflowobj)

    workflowobj["id"] = fileuri
    processobj, new_metadata = document_loader.resolve_all(
        workflowobj, fileuri, checklinks=do_validate)
    if not isinstance(processobj, (CommentedMap, CommentedSeq)):
        raise ValidationException("Workflow must be a dict or list.")

    if not new_metadata and isinstance(processobj, CommentedMap):
        new_metadata = cast(CommentedMap, cmap(
            {"$namespaces": processobj.get("$namespaces", {}),
             "$schemas": processobj.get("$schemas", []),
             "cwlVersion": processobj["cwlVersion"]}, fn=fileuri))

    _convert_stdstreams_to_files(workflowobj)

    if preprocess_only:
        return document_loader, avsc_names, processobj, new_metadata, uri

    if do_validate:
        schema.validate_doc(avsc_names, processobj, document_loader, strict)

    if new_metadata.get("cwlVersion") != update.LATEST:
        processobj = cast(CommentedMap, cmap(update.update(
            processobj, document_loader, fileuri, enable_dev, new_metadata)))

    if jobobj is not None:
        new_metadata[u"cwl:defaults"] = jobobj

    if overrides:
        new_metadata[u"cwltool:overrides"] = overrides

    return document_loader, avsc_names, processobj, new_metadata, uri