def validate_links(self, document):
        if isinstance(document, list):
            iterator = enumerate(document)
        elif isinstance(document, dict):
            for d in self.url_fields:
                if d in document:
                    if isinstance(document[d], basestring):
                        if document[d] not in self.idx:
                            raise validate.ValidationException("Invalid link `%s` in field `%s`" % (document[d], d))
                    elif isinstance(document[d], list):
                        for i in document[d]:
                            if isinstance(i, basestring) and i not in self.idx:
                                raise validate.ValidationException("Invalid link `%s` in field `%s`" % (i, d))
            iterator = document.iteritems()
        else:
            return

        try:
            for key, val in iterator:
                self.validate_links(val)
        except validate.ValidationException as v:
            if isinstance(key, basestring):
                raise validate.ValidationException("At field `%s`\n%s" % (key, validate.indent(str(v))))
            else:
                raise validate.ValidationException("At position %s\n%s" % (key, validate.indent(str(v))))

        return
예제 #2
0
    def validate_links(self, document):
        if isinstance(document, list):
            iterator = enumerate(document)
        elif isinstance(document, dict):
            for d in self.url_fields:
                if d in document:
                    if isinstance(document[d], basestring):
                        if document[d] not in self.idx:
                            raise validate.ValidationException("Invalid link `%s` in field `%s`" % (document[d], d))
                    elif isinstance(document[d], list):
                        for i in document[d]:
                            if isinstance(i, basestring) and i not in self.idx:
                                raise validate.ValidationException("Invalid link `%s` in field `%s`" % (i, d))
            iterator = document.iteritems()
        else:
            return

        try:
            for key, val in iterator:
                self.validate_links(val)
        except validate.ValidationException as v:
            if isinstance(key, basestring):
                raise validate.ValidationException("At field `%s`\n%s" % (key, validate.indent(str(v))))
            else:
                raise validate.ValidationException("At position %s\n%s" % (key, validate.indent(str(v))))

        return
예제 #3
0
    def validate_links(self, document):
        docid = self.getid(document)
        if docid is None:
            docid = ""

        errors = []
        if isinstance(document, list):
            iterator = enumerate(document)
        elif isinstance(document, dict):
            try:
                for d in self.url_fields:
                    if d not in self.identity_links and d in document:
                        self.validate_link(d, document[d])
            except validate.ValidationException as v:
                errors.append(v)
            iterator = document.iteritems()
        else:
            return

        for key, val in iterator:
            try:
                self.validate_links(val)
            except validate.ValidationException as v:
                if key not in self.nolinkcheck:
                    docid = self.getid(val)
                    if docid:
                        errors.append(
                            validate.ValidationException(
                                "While checking object `%s`\n%s" % (docid, validate.indent(str(v)))
                            )
                        )
                    else:
                        if isinstance(key, basestring):
                            errors.append(
                                validate.ValidationException(
                                    "While checking field `%s`\n%s" % (key, validate.indent(str(v)))
                                )
                            )
                        else:
                            errors.append(
                                validate.ValidationException(
                                    "While checking position %s\n%s" % (key, validate.indent(str(v)))
                                )
                            )

        if errors:
            if len(errors) > 1:
                raise validate.ValidationException("\n".join([str(e) for e in errors]))
            else:
                raise errors[0]
        return
    def validate_links(self, document):
        docid = self.getid(document)
        if docid is None:
            docid = ""

        errors = []
        if isinstance(document, list):
            iterator = enumerate(document)
        elif isinstance(document, dict):
            try:
                for d in self.url_fields:
                    if d not in self.identity_links and d in document:
                        self.validate_link(d, document[d])
            except validate.ValidationException as v:
                errors.append(v)
            iterator = document.iteritems()
        else:
            return

        for key, val in iterator:
            try:
                self.validate_links(val)
            except validate.ValidationException as v:
                if key not in self.nolinkcheck:
                    docid = self.getid(val)
                    if docid:
                        errors.append(
                            validate.ValidationException(
                                "While checking object `%s`\n%s" %
                                (docid, validate.indent(str(v)))))
                    else:
                        if isinstance(key, basestring):
                            errors.append(
                                validate.ValidationException(
                                    "While checking field `%s`\n%s" %
                                    (key, validate.indent(str(v)))))
                        else:
                            errors.append(
                                validate.ValidationException(
                                    "While checking position %s\n%s" %
                                    (key, validate.indent(str(v)))))

        if errors:
            if len(errors) > 1:
                raise validate.ValidationException("\n".join(
                    [str(e) for e in errors]))
            else:
                raise errors[0]
        return
def validate_doc(schema_names, validate_doc, loader, strict):
    has_root = False
    for r in schema_names.names.values():
        if r.get_prop("documentRoot"):
            has_root = True
            break

    if not has_root:
        raise validate.ValidationException(
            "No document roots defined in the schema")

    if isinstance(validate_doc, list):
        pass
    elif isinstance(validate_doc, dict):
        validate_doc = [validate_doc]
    else:
        raise validate.ValidationException("Document must be dict or list")

    anyerrors = []
    for pos, item in enumerate(validate_doc):
        errors = []
        success = False
        for r in schema_names.names.values():
            if r.get_prop("documentRoot"):
                try:
                    validate.validate_ex(
                        r,
                        item,
                        loader.identifiers,
                        strict,
                        foreign_properties=loader.foreign_properties)
                    success = True
                    break
                except validate.ValidationException as e:
                    errors.append("Could not validate as `%s` because\n%s" %
                                  (r.get_prop("name"),
                                   validate.indent(str(e), nolead=False)))
        if not success:
            objerr = "Validation error at position %i" % pos
            for ident in loader.identifiers:
                if ident in item:
                    objerr = "Validation error in object %s" % (item[ident])
                    break
            anyerrors.append("%s\n%s" %
                             (objerr, validate.indent("\n".join(errors))))
    if anyerrors:
        raise validate.ValidationException("\n".join(anyerrors))
    def resolve_all(self, document, base_url):
        if isinstance(document, list):
            iterator = enumerate(document)
        elif isinstance(document, dict):
            inc = 'include' in document
            if 'id' in document or 'import' in document or 'include' in document:
                document = self.resolve_ref(document, base_url)
            if inc:
                return document

            for d in self.url_fields:
                if d in document:
                    if isinstance(document[d], basestring):
                        document[d] = expand_url(document[d], base_url)
                    elif isinstance(document[d], list):
                        document[d] = [
                            expand_url(url, base_url) if isinstance(
                                url, basestring) else url
                            for url in document[d]
                        ]
            iterator = document.iteritems()
        else:
            return document

        for key, val in iterator:
            try:
                document[key] = self.resolve_all(val, base_url)
            except validate.ValidationException as v:
                if isinstance(key, basestring):
                    raise validate.ValidationException(
                        "Validation error in field %s:\n%s" %
                        (key, validate.indent(str(v))))
                else:
                    raise validate.ValidationException(
                        "Validation error in position %i:\n%s" %
                        (key, validate.indent(str(v))))

        return document
 def validate_requirements(self, tool, field):
     for r in tool.get(field, []):
         try:
             if self.names.get_name(r["class"], "") is None:
                 raise validate.ValidationException("Unknown requirement %s" % (r["class"]))
             validate.validate_ex(self.names.get_name(r["class"], ""), r)
             if "requirements" in r:
                 self.validate_requirements(r, "requirements")
             if "hints" in r:
                 self.validate_requirements(r, "hints")
         except validate.ValidationException as v:
             err = "While validating %s %s\n%s" % (field, r["class"], validate.indent(str(v)))
             if field == "hints":
                 _logger.warn(err)
             else:
                 raise validate.ValidationException(err)
예제 #8
0
    def __init__(self, toolpath_object, docpath):
        self.impl = toolpath_object["impl"]
        try:
            self.embedded_tool = makeTool(
                from_url(os.path.join(docpath, self.impl)), docpath)
        except validate.ValidationException as v:
            raise WorkflowException(
                "Tool definition %s failed validation:\n%s" %
                (os.path.join(docpath, self.impl), validate.indent(str(v))))

        if "id" in toolpath_object:
            self.id = toolpath_object["id"]
        else:
            self.id = "#step_" + str(random.randint(1, 1000000000))

        for i in toolpath_object["inputs"]:
            d = i["def"][len(self.impl):]
            toolid = i.get("id", self.id + "." + idk(d))
            found = False
            for a in self.embedded_tool.tool["inputs"]:
                if a["id"] == d:
                    i.update(a)
                    found = True
            if not found:
                raise WorkflowException(
                    "Did not find input '%s' in external process" % (i["def"]))

            i["id"] = toolid

        for i in toolpath_object["outputs"]:
            d = i["def"][len(self.impl):]
            toolid = i["id"]
            found = False
            for a in self.embedded_tool.tool["outputs"]:
                if a["id"] == d:
                    i.update(a)
                    found = True
            if not found:
                raise WorkflowException(
                    "Did not find output '%s' in external process" %
                    (i["def"]))

            i["id"] = toolid

        super(External, self).__init__(toolpath_object, "Process", docpath)
예제 #9
0
def validate_doc(schema_names, validate_doc, loader, strict):
    has_root = False
    for r in schema_names.names.values():
        if r.get_prop("documentRoot"):
            has_root = True
            break

    if not has_root:
        raise validate.ValidationException("No document roots defined in the schema")

    if isinstance(validate_doc, list):
        pass
    elif isinstance(validate_doc, dict):
        validate_doc = [validate_doc]
    else:
        raise validate.ValidationException("Document must be dict or list")

    anyerrors = []
    for pos, item in enumerate(validate_doc):
        errors = []
        success = False
        for r in schema_names.names.values():
            if r.get_prop("documentRoot"):
                try:
                    validate.validate_ex(r, item, loader.identifiers, strict, foreign_properties=loader.foreign_properties)
                    success = True
                    break
                except validate.ValidationException as e:
                    errors.append("Could not validate as `%s` because\n%s" % (r.get_prop("name"), validate.indent(str(e), nolead=False)))
        if not success:
            objerr = "Validation error at position %i" % pos
            for ident in loader.identifiers:
                if ident in item:
                    objerr = "Validation error in object %s" % (item[ident])
                    break
            anyerrors.append("%s\n%s" % (objerr, validate.indent("\n".join(errors))))
    if anyerrors:
        raise validate.ValidationException("\n".join(anyerrors))
    def resolve_all(self, document, base_url, file_base=None):
        loader = self
        metadata = {}
        if file_base is None:
            file_base = base_url

        if isinstance(document, dict):
            # Handle $import and $include
            if ('$import' in document or '$include' in document):
                return self.resolve_ref(document, file_base)
        elif isinstance(document, list):
            pass
        else:
            return document, metadata

        newctx = None
        if isinstance(document, dict):
            # Handle $base, $profile, $namespaces, $schemas and $graph
            if "$base" in document:
                base_url = document["$base"]

            if "$profile" in document:
                if not newctx:
                    newctx = SubLoader(self)
                prof = self.fetch(document["$profile"])
                newctx.add_namespaces(document.get("$namespaces", {}),
                                      document["$profile"])
                newctx.add_schemas(document.get("$schemas", []),
                                   document["$profile"])

            if "$namespaces" in document:
                if not newctx:
                    newctx = SubLoader(self)
                newctx.add_namespaces(document["$namespaces"])

            if "$schemas" in document:
                if not newctx:
                    newctx = SubLoader(self)
                newctx.add_schemas(document["$schemas"], file_base)

            if newctx:
                loader = newctx

            if "$graph" in document:
                metadata = {k: v for k, v in document.items() if k != "$graph"}
                document = document["$graph"]
                metadata, _ = loader.resolve_all(metadata, base_url, file_base)

        if isinstance(document, dict):
            for identifer in loader.identity_links:
                if identifer in document:
                    if isinstance(document[identifer], basestring):
                        document[identifer] = loader.expand_url(
                            document[identifer], base_url, scoped=True)
                        if document[identifer] not in loader.idx or isinstance(
                                loader.idx[document[identifer]], basestring):
                            loader.idx[document[identifer]] = document
                        base_url = document[identifer]
                    elif isinstance(document[identifer], list):
                        for n, v in enumerate(document[identifer]):
                            document[identifer][n] = loader.expand_url(
                                document[identifer][n], base_url, scoped=True)
                            if document[identifer][n] not in loader.idx:
                                loader.idx[document[identifer]
                                           [n]] = document[identifer][n]

            for d in document:
                d2 = loader.expand_url(d, "", scoped=False, vocab_term=True)
                if d != d2:
                    document[d2] = document[d]
                    del document[d]

            for d in loader.url_fields:
                if d in document:
                    if isinstance(document[d], basestring):
                        document[d] = loader.expand_url(
                            document[d],
                            base_url,
                            scoped=False,
                            vocab_term=(d in loader.vocab_fields))
                    elif isinstance(document[d], list):
                        document[d] = [
                            loader.expand_url(
                                url,
                                base_url,
                                scoped=False,
                                vocab_term=(d in loader.vocab_fields))
                            if isinstance(url, basestring) else url
                            for url in document[d]
                        ]

            try:
                for key, val in document.items():
                    document[key], _ = loader.resolve_all(
                        val, base_url, file_base)
            except validate.ValidationException as v:
                _logger.debug("loader is %s", id(loader))
                raise validate.ValidationException(
                    "(%s) (%s) Validation error in field %s:\n%s" %
                    (id(loader), file_base, key, validate.indent(str(v))))

        elif isinstance(document, list):
            i = 0
            try:
                while i < len(document):
                    val = document[i]
                    if isinstance(val, dict) and "$import" in val:
                        l, _ = loader.resolve_ref(val, file_base)
                        if isinstance(l, list):
                            del document[i]
                            for item in aslist(l):
                                document.insert(i, item)
                                i += 1
                        else:
                            document[i] = l
                            i += 1
                    else:
                        document[i], _ = loader.resolve_all(
                            val, base_url, file_base)
                        i += 1
            except validate.ValidationException as v:
                raise validate.ValidationException(
                    "(%s) (%s) Validation error in position %i:\n%s" %
                    (id(loader), file_base, i, validate.indent(str(v))))

            for identifer in loader.identity_links:
                if identifer in metadata:
                    if isinstance(metadata[identifer], basestring):
                        metadata[identifer] = loader.expand_url(
                            metadata[identifer], base_url, scoped=True)
                        loader.idx[metadata[identifer]] = document

        return document, metadata
예제 #11
0
    def resolve_all(self, document, base_url):
        loader = self
        metadata = {}

        if isinstance(document, dict):
            inc = '@include' in document
            if  '@import' in document or '@include' in document:
                document, _ = self.resolve_ref(document, base_url)
            else:
                for identifer in self.identity_links:
                    if identifer in document:
                        if isinstance(document[identifer], basestring):
                            document[identifer] = self.expand_url(document[identifer], base_url, scoped=True)
                            if document[identifer] not in self.idx or isinstance(self.idx[document[identifer]], basestring):
                                self.idx[document[identifer]] = document
                            base_url = document[identifer]
                        elif isinstance(document[identifer], list):
                            for n, v in enumerate(document[identifer]):
                                document[identifer][n] = self.expand_url(document[identifer][n], base_url, scoped=True)
                                if document[identifer][n] not in self.idx:
                                    self.idx[document[identifer][n]] = document[identifer][n]
            if inc:
                return document, {}

            if isinstance(document, dict) and "@context" in document:
                loader = Loader(self.ctx)
                loader.idx = self.idx
                loader.add_context(document["@context"])
                if "@base" in loader.ctx:
                    base_url = loader.ctx["@base"]

            if "@graph" in document:
                metadata = {k: v for k,v in document.items() if k != "@graph"}
                document = document["@graph"]
                metadata, _ = self.resolve_all(metadata, base_url)

        elif isinstance(document, list):
            pass
        else:
            return document, metadata

        try:
            if isinstance(document, dict):
                for d in document:
                    d2 = self.expand_url(d, "", scoped=False, vocab_term=True)
                    if d != d2:
                        document[d2] = document[d]
                        del document[d]

                for d in loader.url_fields:
                    if d in document:
                        if isinstance(document[d], basestring):
                            document[d] = loader.expand_url(document[d], base_url, scoped=False, vocab_term=(d in loader.vocab_fields))
                        elif isinstance(document[d], list):
                            document[d] = [loader.expand_url(url, base_url, scoped=False, vocab_term=(d in loader.vocab_fields)) if isinstance(url, basestring) else url for url in document[d] ]

                for key, val in document.items():
                    document[key], _ = loader.resolve_all(val, base_url)

            elif isinstance(document, list):
                i = 0
                while i < len(document):
                    val = document[i]
                    if isinstance(val, dict) and "@import" in val and val.get("inline"):
                        l, _ = loader.resolve_all(val, base_url)
                        del document[i]
                        for item in aslist(l):
                            document.insert(i, item)
                            i += 1
                    else:
                        document[i], _ = loader.resolve_all(val, base_url)
                        i += 1

        except validate.ValidationException as v:
            if isinstance(key, basestring):
                raise validate.ValidationException("Validation error in field %s:\n%s" % (key, validate.indent(str(v))))
            else:
                raise validate.ValidationException("Validation error in position %i:\n%s" % (key, validate.indent(str(v))))

        return document, metadata
    def resolve_all(self, document, base_url):
        if isinstance(document, list):
            iterator = enumerate(document)
        elif isinstance(document, dict):
            inc = 'include' in document
            if 'id' in document or 'import' in document or 'include' in document:
                document = self.resolve_ref(document, base_url)
            if inc:
                return document

            for d in self.url_fields:
                if d in document:
                    if isinstance(document[d], basestring):
                        document[d] = expand_url(document[d], base_url)
                    elif isinstance(document[d], list):
                        document[d] = [expand_url(url, base_url) if isinstance(url, basestring) else url for url in document[d] ]
            iterator = document.iteritems()
        else:
            return document

        for key, val in iterator:
            try:
                document[key] = self.resolve_all(val, base_url)
            except validate.ValidationException as v:
                if isinstance(key, basestring):
                    raise validate.ValidationException("Validation error in field %s:\n%s" % (key, validate.indent(str(v))))
                else:
                    raise validate.ValidationException("Validation error in position %i:\n%s" % (key, validate.indent(str(v))))

        return document
    def resolve_all(self, document, base_url, file_base=None):
        loader = self
        metadata = {}
        if file_base is None:
            file_base = base_url

        if isinstance(document, dict):
            # Handle $import and $include
            if ('$import' in document or '$include' in document):
                return self.resolve_ref(document, file_base)
        elif isinstance(document, list):
            pass
        else:
            return document, metadata

        newctx = None
        if isinstance(document, dict):
            # Handle $base, $profile, $namespaces, $schemas and $graph
            if "$base" in document:
                base_url = document["$base"]

            if "$profile" in document:
                if not newctx:
                    newctx = SubLoader(self)
                prof = self.fetch(document["$profile"])
                newctx.add_namespaces(document.get("$namespaces", {}), document["$profile"])
                newctx.add_schemas(document.get("$schemas", []), document["$profile"])

            if "$namespaces" in document:
                if not newctx:
                    newctx = SubLoader(self)
                newctx.add_namespaces(document["$namespaces"])

            if "$schemas" in document:
                if not newctx:
                    newctx = SubLoader(self)
                newctx.add_schemas(document["$schemas"], file_base)

            if newctx:
                loader = newctx

            if "$graph" in document:
                metadata = {k: v for k,v in document.items() if k != "$graph"}
                document = document["$graph"]
                metadata, _ = loader.resolve_all(metadata, base_url, file_base)

        if isinstance(document, dict):
            for identifer in loader.identity_links:
                if identifer in document:
                    if isinstance(document[identifer], basestring):
                        document[identifer] = loader.expand_url(document[identifer], base_url, scoped=True)
                        if document[identifer] not in loader.idx or isinstance(loader.idx[document[identifer]], basestring):
                            loader.idx[document[identifer]] = document
                        base_url = document[identifer]
                    elif isinstance(document[identifer], list):
                        for n, v in enumerate(document[identifer]):
                            document[identifer][n] = loader.expand_url(document[identifer][n], base_url, scoped=True)
                            if document[identifer][n] not in loader.idx:
                                loader.idx[document[identifer][n]] = document[identifer][n]

            for d in document:
                d2 = loader.expand_url(d, "", scoped=False, vocab_term=True)
                if d != d2:
                    document[d2] = document[d]
                    del document[d]

            for d in loader.url_fields:
                if d in document:
                    if isinstance(document[d], basestring):
                        document[d] = loader.expand_url(document[d], base_url, scoped=False, vocab_term=(d in loader.vocab_fields))
                    elif isinstance(document[d], list):
                        document[d] = [loader.expand_url(url, base_url, scoped=False, vocab_term=(d in loader.vocab_fields)) if isinstance(url, basestring) else url for url in document[d] ]

            try:
                for key, val in document.items():
                    document[key], _ = loader.resolve_all(val, base_url, file_base)
            except validate.ValidationException as v:
                _logger.debug("loader is %s", id(loader))
                raise validate.ValidationException("(%s) (%s) Validation error in field %s:\n%s" % (id(loader), file_base, key, validate.indent(str(v))))

        elif isinstance(document, list):
            i = 0
            try:
                while i < len(document):
                    val = document[i]
                    if isinstance(val, dict) and "$import" in val:
                        l, _ = loader.resolve_ref(val, file_base)
                        if isinstance(l, list):
                            del document[i]
                            for item in aslist(l):
                                document.insert(i, item)
                                i += 1
                        else:
                            document[i] = l
                            i += 1
                    else:
                        document[i], _ = loader.resolve_all(val, base_url, file_base)
                        i += 1
            except validate.ValidationException as v:
                raise validate.ValidationException("(%s) (%s) Validation error in position %i:\n%s" % (id(loader), file_base, i, validate.indent(str(v))))

        return document, metadata
    def __init__(self, toolpath_object, docpath):
        self.impl = toolpath_object["impl"]
        try:
            self.embedded_tool = makeTool(from_url(os.path.join(docpath, self.impl)), docpath)
        except validate.ValidationException as v:
            raise WorkflowException("Tool definition %s failed validation:\n%s" % (os.path.join(docpath, self.impl), validate.indent(str(v))))

        if "id" in toolpath_object:
            self.id = toolpath_object["id"]
        else:
            self.id = "#step_" + str(random.randint(1, 1000000000))

        for i in toolpath_object["inputs"]:
            d = i["def"][len(self.impl):]
            toolid = i.get("id", self.id + "." + idk(d))
            found = False
            for a in self.embedded_tool.tool["inputs"]:
                if a["id"] == d:
                    i.update(a)
                    found = True
            if not found:
                raise WorkflowException("Did not find input '%s' in external process" % (i["def"]))

            i["id"] = toolid

        for i in toolpath_object["outputs"]:
            d = i["def"][len(self.impl):]
            toolid = i["id"]
            found = False
            for a in self.embedded_tool.tool["outputs"]:
                if a["id"] == d:
                    i.update(a)
                    found = True
            if not found:
                raise WorkflowException("Did not find output '%s' in external process" % (i["def"]))

            i["id"] = toolid

        super(External, self).__init__(toolpath_object, "External", docpath)