Exemplo n.º 1
0
    def __init__(self, toc, j, renderlist, redirects):
        self.typedoc = StringIO.StringIO()
        self.toc = toc
        self.subs = {}
        self.docParent = {}
        self.docAfter = {}
        self.rendered = set()
        self.redirects = redirects
        self.title = None

        for t in j:
            if "extends" in t:
                for e in aslist(t["extends"]):
                    add_dictlist(self.subs, e, t["name"])
                    #if "docParent" not in t and "docAfter" not in t:
                    #    add_dictlist(self.docParent, e, t["name"])

            if t.get("docParent"):
                add_dictlist(self.docParent, t["docParent"], t["name"])

            if t.get("docChild"):
                for c in aslist(t["docChild"]):
                    add_dictlist(self.docParent, t["name"], c)

            if t.get("docAfter"):
                add_dictlist(self.docAfter, t["docAfter"], t["name"])

        _, _, metaschema_loader = schema.get_metaschema()
        alltypes = schema.extend_and_specialize(j, metaschema_loader)

        self.typemap = {}
        self.uses = {}
        self.record_refs = {}
        for t in alltypes:
            self.typemap[t["name"]] = t
            try:
                if t["type"] == "record":
                    self.record_refs[t["name"]] = []
                    for f in t.get("fields", []):
                        p = has_types(f)
                        for tp in p:
                            if tp not in self.uses:
                                self.uses[tp] = []
                            if (t["name"], f["name"]) not in self.uses[tp]:
                                _, frg1 = urlparse.urldefrag(t["name"])
                                _, frg2 = urlparse.urldefrag(f["name"])
                                self.uses[tp].append((frg1, frg2))
                            if tp not in basicTypes and tp not in self.record_refs[t["name"]]:
                                    self.record_refs[t["name"]].append(tp)
            except KeyError as e:
                _logger.error("Did not find 'type' in %s", t)
                raise

        for f in alltypes:
            if (f["name"] in renderlist or
                ((not renderlist) and
                 ("extends" not in f) and
                 ("docParent" not in f) and
                 ("docAfter" not in f))):
                self.render_type(f, 1)
    def __init__(self, toc, j, renderlist, redirects):
        self.typedoc = StringIO.StringIO()
        self.toc = toc
        self.subs = {}
        self.docParent = {}
        self.docAfter = {}
        self.rendered = set()
        self.redirects = redirects
        self.title = None

        for t in j:
            if "extends" in t:
                for e in aslist(t["extends"]):
                    add_dictlist(self.subs, e, t["name"])
                    #if "docParent" not in t and "docAfter" not in t:
                    #    add_dictlist(self.docParent, e, t["name"])

            if t.get("docParent"):
                add_dictlist(self.docParent, t["docParent"], t["name"])

            if t.get("docChild"):
                for c in aslist(t["docChild"]):
                    add_dictlist(self.docParent, t["name"], c)

            if t.get("docAfter"):
                add_dictlist(self.docAfter, t["docAfter"], t["name"])

        _, _, metaschema_loader = schema.get_metaschema()
        alltypes = schema.extend_and_specialize(j, metaschema_loader)

        self.typemap = {}
        self.uses = {}
        self.record_refs = {}
        for t in alltypes:
            self.typemap[t["name"]] = t
            try:
                if t["type"] == "record":
                    self.record_refs[t["name"]] = []
                    for f in t.get("fields", []):
                        p = has_types(f)
                        for tp in p:
                            if tp not in self.uses:
                                self.uses[tp] = []
                            if (t["name"], f["name"]) not in self.uses[tp]:
                                _, frg1 = urlparse.urldefrag(t["name"])
                                _, frg2 = urlparse.urldefrag(f["name"])
                                self.uses[tp].append((frg1, frg2))
                            if tp not in basicTypes and tp not in self.record_refs[
                                    t["name"]]:
                                self.record_refs[t["name"]].append(tp)
            except KeyError as e:
                _logger.error("Did not find 'type' in %s", t)
                raise

        for f in alltypes:
            if (f["name"] in renderlist
                    or ((not renderlist) and ("extends" not in f) and
                        ("docParent" not in f) and ("docAfter" not in f))):
                self.render_type(f, 1)
Exemplo n.º 3
0
    def __init__(self, toc, j):
        self.typedoc = StringIO.StringIO()
        self.toc = toc
        self.subs = {}
        self.docParent = {}
        self.docAfter = {}
        for t in j:
            if "extends" in t:
                for e in aslist(t["extends"]):
                    add_dictlist(self.subs, e, t["name"])
                    if "docParent" not in t and "docAfter" not in t:
                        add_dictlist(self.docParent, e, t["name"])

            if t.get("docParent"):
                add_dictlist(self.docParent, t["docParent"], t["name"])

            if t.get("docAfter"):
                add_dictlist(self.docAfter, t["docAfter"], t["name"])

        _, _, metaschema_loader = schema.get_metaschema()
        alltypes = schema.extend_and_specialize(j, metaschema_loader)

        self.typemap = {}
        self.uses = {}
        for t in alltypes:
            self.typemap[t["name"]] = t
            if t["type"] == "https://w3id.org/cwl/salad#record":
                for f in t["fields"]:
                    p = has_types(f)
                    for tp in p:
                        if tp not in self.uses:
                            self.uses[tp] = []
                        if (t["name"], f["name"]) not in self.uses[tp]:
                            _, frg1 = urlparse.urldefrag(t["name"])
                            _, frg2 = urlparse.urldefrag(f["name"])
                            self.uses[tp].append((frg1, frg2))

        for f in alltypes:
            if ("extends" not in f) and ("docParent" not in f) and ("docAfter" not in f):
                self.render_type(f, 1)
    parser.add_argument('--brand')
    parser.add_argument('--brandlink')
    parser.add_argument('--primtype', default="#PrimitiveType")

    args = parser.parse_args()

    s = []
    a = args.schema
    with open(a) as f:
        if a.endswith("md"):
            s.append({
                "name": os.path.splitext(os.path.basename(a))[0],
                "type": "documentation",
                "doc": f.read().decode("utf-8")
            })
        else:
            uri = "file://" + os.path.abspath(a)
            _, _, metaschema_loader = schema.get_metaschema()
            j, schema_metadata = metaschema_loader.resolve_ref(uri, "")
            if isinstance(j, list):
                s.extend(j)
            else:
                s.append(j)

    primitiveType = args.primtype

    redirect = {r.split("=")[0]: r.split("=")[1]
                for r in args.redirect} if args.redirect else {}
    renderlist = args.only if args.only else []
    avrold_doc(s, sys.stdout, renderlist, redirect, args.brand, args.brandlink)
Exemplo n.º 5
0
    parser.add_argument('--only', action='append')
    parser.add_argument('--redirect', action='append')
    parser.add_argument('--brand')
    parser.add_argument('--brandlink')
    parser.add_argument('--primtype', default="#PrimitiveType")

    args = parser.parse_args()

    s = []
    a = args.schema
    with open(a) as f:
        if a.endswith("md"):
            s.append({"name": os.path.splitext(os.path.basename(a))[0],
                  "type": "documentation",
                  "doc": f.read().decode("utf-8")
              })
        else:
            uri = "file://" + os.path.abspath(a)
            _, _, metaschema_loader = schema.get_metaschema()
            j, schema_metadata = metaschema_loader.resolve_ref(uri, "")
            if isinstance(j, list):
                s.extend(j)
            else:
                s.append(j)

    primitiveType = args.primtype

    redirect = {r.split("=")[0]:r.split("=")[1] for r in args.redirect} if args.redirect else {}
    renderlist = args.only if args.only else []
    avrold_doc(s, sys.stdout, renderlist, redirect, args.brand, args.brandlink)
Exemplo n.º 6
0
def main(args=None):
    if args is None:
        args = sys.argv[1:]

    parser = argparse.ArgumentParser()
    parser.add_argument("--rdf-serializer",
                        help="Output RDF serialization format used by --print-rdf (one of turtle (default), n3, nt, xml)",
                        default="turtle")

    exgroup = parser.add_mutually_exclusive_group()
    exgroup.add_argument("--print-jsonld-context", action="store_true", help="Print JSON-LD context for schema")
    exgroup.add_argument("--print-doc", action="store_true", help="Print HTML documentation from schema")
    exgroup.add_argument("--print-rdfs", action="store_true", help="Print RDF schema")
    exgroup.add_argument("--print-avro", action="store_true", help="Print Avro schema")

    exgroup.add_argument("--print-rdf", action="store_true", help="Print corresponding RDF graph for document")
    exgroup.add_argument("--print-pre", action="store_true", help="Print document after preprocessing")
    exgroup.add_argument("--print-index", action="store_true", help="Print node index")
    exgroup.add_argument("--print-metadata", action="store_true", help="Print document metadata")
    exgroup.add_argument("--version", action="store_true", help="Print version")

    parser.add_argument("--strict", action="store_true", help="Strict validation (error on unrecognized fields)")

    exgroup = parser.add_mutually_exclusive_group()
    exgroup.add_argument("--verbose", action="store_true", help="Default logging")
    exgroup.add_argument("--quiet", action="store_true", help="Only print warnings and errors.")
    exgroup.add_argument("--debug", action="store_true", help="Print even more logging")

    parser.add_argument("schema", type=str)
    parser.add_argument("document", type=str, nargs="?", default=None)

    args = parser.parse_args(args)

    if args.quiet:
        _logger.setLevel(logging.WARN)
    if args.debug:
        _logger.setLevel(logging.DEBUG)

    pkg = pkg_resources.require("schema_salad")
    if pkg:
        if args.version:
            print "%s %s" % (sys.argv[0], pkg[0].version)
            return 0
        else:
            _logger.info("%s %s", sys.argv[0], pkg[0].version)

    # Get the metaschema to validate the schema
    metaschema_names, metaschema_doc, metaschema_loader = schema.get_metaschema()

    # Load schema document and resolve refs

    schema_uri = args.schema
    if not urlparse.urlparse(schema_uri)[0]:
        schema_uri = "file://" + os.path.abspath(schema_uri)
    schema_raw_doc = metaschema_loader.fetch(schema_uri)
    schema_doc, schema_metadata = metaschema_loader.resolve_all(schema_raw_doc, schema_uri)

    # Optionally print the schema after ref resolution
    if not args.document and args.print_pre:
        print json.dumps(schema_doc, indent=4)
        return 0

    if not args.document and args.print_index:
        print json.dumps(metaschema_loader.idx.keys(), indent=4)
        return 0

    # Validate links in the schema document
    try:
        metaschema_loader.validate_links(schema_doc)
    except (validate.ValidationException) as e:
        _logger.error("Schema `%s` failed link checking:\n%s", args.schema, e, exc_info=(e if args.debug else False))
        _logger.debug("Index is %s", metaschema_loader.idx.keys())
        _logger.debug("Vocabulary is %s", metaschema_loader.vocab.keys())
        return 1

    # Validate the schema document against the metaschema
    try:
        schema.validate_doc(metaschema_names, schema_doc, metaschema_loader, args.strict)
    except validate.ValidationException as e:
        _logger.error("While validating schema `%s`:\n%s" % (args.schema, str(e)))
        return 1

    # Get the json-ld context and RDFS representation from the schema
    (schema_ctx, rdfs) = jsonld_context.salad_to_jsonld_context(schema_doc, schema_raw_doc["@context"])

    # Create the loader that will be used to load the target document.
    document_loader = Loader(schema_ctx)

    # Make the Avro validation that will be used to validate the target document
    (avsc_names, avsc_obj) = schema.make_avro_schema(schema_doc, document_loader)

    if isinstance(avsc_names, Exception):
        _logger.error("Schema `%s` error:\n%s", args.schema, avsc_names, exc_info=(avsc_names if args.debug else False))
        if args.print_avro:
            print json.dumps(avsc_obj, indent=4)
        return 1

    # Optionally print Avro-compatible schema from schema
    if args.print_avro:
        print json.dumps(avsc_obj, indent=4)
        return 0

    # Optionally print the json-ld context from the schema
    if args.print_jsonld_context:
        j = {"@context": schema_ctx}
        print json.dumps(j, indent=4, sort_keys=True)
        return 0

    # Optionally print the RDFS graph from the schema
    if args.print_rdfs:
        print(rdfs.serialize(format=args.rdf_serializer))
        return 0

    # Optionally create documentation page from the schema
    if args.print_doc:
        makedoc.avrold_doc(schema_doc, sys.stdout)
        return 0

    if args.print_metadata and not args.document:
        print json.dumps(schema_metadata, indent=4)
        return 0

    # If no document specified, all done.
    if not args.document:
        print "Schema `%s` is valid" % args.schema
        return 0

    # Load target document and resolve refs
    try:
        uri = args.document
        if not urlparse.urlparse(uri)[0]:
            doc = "file://" + os.path.abspath(uri)
        document, doc_metadata = document_loader.resolve_ref(uri)
    except (validate.ValidationException, RuntimeError) as e:
        _logger.error("Document `%s` failed validation:\n%s", args.document, e, exc_info=(e if args.debug else False))
        return 1

    # Optionally print the document after ref resolution
    if args.print_pre:
        print json.dumps(document, indent=4)
        return 0

    if args.print_index:
        print json.dumps(document_loader.idx.keys(), indent=4)
        return 0

    # Validate links in the target document
    try:
        document_loader.validate_links(document)
    except (validate.ValidationException) as e:
        _logger.error("Document `%s` failed link checking:\n%s", args.document, e, exc_info=(e if args.debug else False))
        _logger.debug("Index is %s", json.dumps(document_loader.idx.keys(), indent=4))
        return 1

    # Validate the schema document against the metaschema
    try:
        schema.validate_doc(avsc_names, document, document_loader, args.strict)
    except validate.ValidationException as e:
        _logger.error("While validating document `%s`:\n%s" % (args.document, str(e)))
        return 1

    # Optionally convert the document to RDF
    if args.print_rdf:
        printrdf(args.document, document, schema_ctx, args.rdf_serializer)
        return 0

    if args.print_metadata:
        print json.dumps(doc_metadata, indent=4)
        return 0

    print "Document `%s` is valid" % args.document

    return 0
def main(args=None):
    if args is None:
        args = sys.argv[1:]

    parser = argparse.ArgumentParser()
    parser.add_argument("--rdf-serializer",
                        help="Output RDF serialization format used by --print-rdf (one of turtle (default), n3, nt, xml)",
                        default="turtle")

    exgroup = parser.add_mutually_exclusive_group()
    exgroup.add_argument("--print-jsonld-context", action="store_true", help="Print JSON-LD context for schema")
    exgroup.add_argument("--print-doc", action="store_true", help="Print HTML documentation from schema")
    exgroup.add_argument("--print-rdfs", action="store_true", help="Print RDF schema")
    exgroup.add_argument("--print-avro", action="store_true", help="Print Avro schema")

    exgroup.add_argument("--print-rdf", action="store_true", help="Print corresponding RDF graph for document")
    exgroup.add_argument("--print-pre", action="store_true", help="Print document after preprocessing")
    exgroup.add_argument("--print-index", action="store_true", help="Print node index")
    exgroup.add_argument("--print-metadata", action="store_true", help="Print document metadata")
    exgroup.add_argument("--version", action="store_true", help="Print version")

    exgroup = parser.add_mutually_exclusive_group()
    exgroup.add_argument("--strict", action="store_true", help="Strict validation (unrecognized or out of place fields are error)",
                         default=True, dest="strict")
    exgroup.add_argument("--non-strict", action="store_false", help="Lenient validation (ignore unrecognized fields)",
                         default=True, dest="strict")

    exgroup = parser.add_mutually_exclusive_group()
    exgroup.add_argument("--verbose", action="store_true", help="Default logging")
    exgroup.add_argument("--quiet", action="store_true", help="Only print warnings and errors.")
    exgroup.add_argument("--debug", action="store_true", help="Print even more logging")

    parser.add_argument("schema", type=str)
    parser.add_argument("document", type=str, nargs="?", default=None)

    args = parser.parse_args(args)

    if args.quiet:
        _logger.setLevel(logging.WARN)
    if args.debug:
        _logger.setLevel(logging.DEBUG)

    pkg = pkg_resources.require("schema_salad")
    if pkg:
        if args.version:
            print "%s %s" % (sys.argv[0], pkg[0].version)
            return 0
        else:
            _logger.info("%s %s", sys.argv[0], pkg[0].version)

    # Get the metaschema to validate the schema
    metaschema_names, metaschema_doc, metaschema_loader = schema.get_metaschema()

    # Load schema document and resolve refs

    schema_uri = args.schema
    if not urlparse.urlparse(schema_uri)[0]:
        schema_uri = "file://" + os.path.abspath(schema_uri)
    schema_raw_doc = metaschema_loader.fetch(schema_uri)
    schema_doc, schema_metadata = metaschema_loader.resolve_all(schema_raw_doc, schema_uri)

    # Optionally print the schema after ref resolution
    if not args.document and args.print_pre:
        print json.dumps(schema_doc, indent=4)
        return 0

    if not args.document and args.print_index:
        print json.dumps(metaschema_loader.idx.keys(), indent=4)
        return 0

    # Validate links in the schema document
    try:
        metaschema_loader.validate_links(schema_doc)
    except (validate.ValidationException) as e:
        _logger.error("Schema `%s` failed link checking:\n%s", args.schema, e, exc_info=(e if args.debug else False))
        _logger.debug("Index is %s", metaschema_loader.idx.keys())
        _logger.debug("Vocabulary is %s", metaschema_loader.vocab.keys())
        return 1

    # Validate the schema document against the metaschema
    try:
        schema.validate_doc(metaschema_names, schema_doc, metaschema_loader, args.strict)
    except validate.ValidationException as e:
        _logger.error("While validating schema `%s`:\n%s" % (args.schema, str(e)))
        return 1

    # Get the json-ld context and RDFS representation from the schema
    metactx = {}
    if isinstance(schema_raw_doc, dict):
        metactx = schema_raw_doc.get("$namespaces", {})
        if "$base" in schema_raw_doc:
            metactx["@base"] = schema_raw_doc["$base"]
    (schema_ctx, rdfs) = jsonld_context.salad_to_jsonld_context(schema_doc, metactx)

    # Create the loader that will be used to load the target document.
    document_loader = Loader(schema_ctx)

    # Make the Avro validation that will be used to validate the target document
    (avsc_names, avsc_obj) = schema.make_avro_schema(schema_doc, document_loader)

    if isinstance(avsc_names, Exception):
        _logger.error("Schema `%s` error:\n%s", args.schema, avsc_names, exc_info=(avsc_names if args.debug else False))
        if args.print_avro:
            print json.dumps(avsc_obj, indent=4)
        return 1

    # Optionally print Avro-compatible schema from schema
    if args.print_avro:
        print json.dumps(avsc_obj, indent=4)
        return 0

    # Optionally print the json-ld context from the schema
    if args.print_jsonld_context:
        j = {"@context": schema_ctx}
        print json.dumps(j, indent=4, sort_keys=True)
        return 0

    # Optionally print the RDFS graph from the schema
    if args.print_rdfs:
        print(rdfs.serialize(format=args.rdf_serializer))
        return 0

    # Optionally create documentation page from the schema
    if args.print_doc:
        makedoc.avrold_doc(schema_doc, sys.stdout)
        return 0

    if args.print_metadata and not args.document:
        print json.dumps(schema_metadata, indent=4)
        return 0

    # If no document specified, all done.
    if not args.document:
        print "Schema `%s` is valid" % args.schema
        return 0

    # Load target document and resolve refs
    try:
        uri = args.document
        if not urlparse.urlparse(uri)[0]:
            doc = "file://" + os.path.abspath(uri)
        document, doc_metadata = document_loader.resolve_ref(uri)
    except (validate.ValidationException, RuntimeError) as e:
        _logger.error("Document `%s` failed validation:\n%s", args.document, e, exc_info=(e if args.debug else False))
        return 1

    # Optionally print the document after ref resolution
    if args.print_pre:
        print json.dumps(document, indent=4)
        return 0

    if args.print_index:
        print json.dumps(document_loader.idx.keys(), indent=4)
        return 0

    # Validate links in the target document
    try:
        document_loader.validate_links(document)
    except (validate.ValidationException) as e:
        _logger.error("Document `%s` failed link checking:\n%s", args.document, e, exc_info=(e if args.debug else False))
        _logger.debug("Index is %s", json.dumps(document_loader.idx.keys(), indent=4))
        return 1

    # Validate the schema document against the metaschema
    try:
        schema.validate_doc(avsc_names, document, document_loader, args.strict)
    except validate.ValidationException as e:
        _logger.error("While validating document `%s`:\n%s" % (args.document, str(e)))
        return 1

    # Optionally convert the document to RDF
    if args.print_rdf:
        printrdf(args.document, document, schema_ctx, args.rdf_serializer)
        return 0

    if args.print_metadata:
        print json.dumps(doc_metadata, indent=4)
        return 0

    print "Document `%s` is valid" % args.document

    return 0