def __init__(self, toc, j, renderlist, redirects): self.typedoc = StringIO.StringIO() self.toc = toc self.subs = {} self.docParent = {} self.docAfter = {} self.rendered = set() self.redirects = redirects self.title = None for t in j: if "extends" in t: for e in aslist(t["extends"]): add_dictlist(self.subs, e, t["name"]) #if "docParent" not in t and "docAfter" not in t: # add_dictlist(self.docParent, e, t["name"]) if t.get("docParent"): add_dictlist(self.docParent, t["docParent"], t["name"]) if t.get("docChild"): for c in aslist(t["docChild"]): add_dictlist(self.docParent, t["name"], c) if t.get("docAfter"): add_dictlist(self.docAfter, t["docAfter"], t["name"]) _, _, metaschema_loader = schema.get_metaschema() alltypes = schema.extend_and_specialize(j, metaschema_loader) self.typemap = {} self.uses = {} self.record_refs = {} for t in alltypes: self.typemap[t["name"]] = t try: if t["type"] == "record": self.record_refs[t["name"]] = [] for f in t.get("fields", []): p = has_types(f) for tp in p: if tp not in self.uses: self.uses[tp] = [] if (t["name"], f["name"]) not in self.uses[tp]: _, frg1 = urlparse.urldefrag(t["name"]) _, frg2 = urlparse.urldefrag(f["name"]) self.uses[tp].append((frg1, frg2)) if tp not in basicTypes and tp not in self.record_refs[t["name"]]: self.record_refs[t["name"]].append(tp) except KeyError as e: _logger.error("Did not find 'type' in %s", t) raise for f in alltypes: if (f["name"] in renderlist or ((not renderlist) and ("extends" not in f) and ("docParent" not in f) and ("docAfter" not in f))): self.render_type(f, 1)
def __init__(self, toc, j, renderlist, redirects): self.typedoc = StringIO.StringIO() self.toc = toc self.subs = {} self.docParent = {} self.docAfter = {} self.rendered = set() self.redirects = redirects self.title = None for t in j: if "extends" in t: for e in aslist(t["extends"]): add_dictlist(self.subs, e, t["name"]) #if "docParent" not in t and "docAfter" not in t: # add_dictlist(self.docParent, e, t["name"]) if t.get("docParent"): add_dictlist(self.docParent, t["docParent"], t["name"]) if t.get("docChild"): for c in aslist(t["docChild"]): add_dictlist(self.docParent, t["name"], c) if t.get("docAfter"): add_dictlist(self.docAfter, t["docAfter"], t["name"]) _, _, metaschema_loader = schema.get_metaschema() alltypes = schema.extend_and_specialize(j, metaschema_loader) self.typemap = {} self.uses = {} self.record_refs = {} for t in alltypes: self.typemap[t["name"]] = t try: if t["type"] == "record": self.record_refs[t["name"]] = [] for f in t.get("fields", []): p = has_types(f) for tp in p: if tp not in self.uses: self.uses[tp] = [] if (t["name"], f["name"]) not in self.uses[tp]: _, frg1 = urlparse.urldefrag(t["name"]) _, frg2 = urlparse.urldefrag(f["name"]) self.uses[tp].append((frg1, frg2)) if tp not in basicTypes and tp not in self.record_refs[ t["name"]]: self.record_refs[t["name"]].append(tp) except KeyError as e: _logger.error("Did not find 'type' in %s", t) raise for f in alltypes: if (f["name"] in renderlist or ((not renderlist) and ("extends" not in f) and ("docParent" not in f) and ("docAfter" not in f))): self.render_type(f, 1)
def __init__(self, toc, j): self.typedoc = StringIO.StringIO() self.toc = toc self.subs = {} self.docParent = {} self.docAfter = {} for t in j: if "extends" in t: for e in aslist(t["extends"]): add_dictlist(self.subs, e, t["name"]) if "docParent" not in t and "docAfter" not in t: add_dictlist(self.docParent, e, t["name"]) if t.get("docParent"): add_dictlist(self.docParent, t["docParent"], t["name"]) if t.get("docAfter"): add_dictlist(self.docAfter, t["docAfter"], t["name"]) _, _, metaschema_loader = schema.get_metaschema() alltypes = schema.extend_and_specialize(j, metaschema_loader) self.typemap = {} self.uses = {} for t in alltypes: self.typemap[t["name"]] = t if t["type"] == "https://w3id.org/cwl/salad#record": for f in t["fields"]: p = has_types(f) for tp in p: if tp not in self.uses: self.uses[tp] = [] if (t["name"], f["name"]) not in self.uses[tp]: _, frg1 = urlparse.urldefrag(t["name"]) _, frg2 = urlparse.urldefrag(f["name"]) self.uses[tp].append((frg1, frg2)) for f in alltypes: if ("extends" not in f) and ("docParent" not in f) and ("docAfter" not in f): self.render_type(f, 1)
parser.add_argument('--brand') parser.add_argument('--brandlink') parser.add_argument('--primtype', default="#PrimitiveType") args = parser.parse_args() s = [] a = args.schema with open(a) as f: if a.endswith("md"): s.append({ "name": os.path.splitext(os.path.basename(a))[0], "type": "documentation", "doc": f.read().decode("utf-8") }) else: uri = "file://" + os.path.abspath(a) _, _, metaschema_loader = schema.get_metaschema() j, schema_metadata = metaschema_loader.resolve_ref(uri, "") if isinstance(j, list): s.extend(j) else: s.append(j) primitiveType = args.primtype redirect = {r.split("=")[0]: r.split("=")[1] for r in args.redirect} if args.redirect else {} renderlist = args.only if args.only else [] avrold_doc(s, sys.stdout, renderlist, redirect, args.brand, args.brandlink)
parser.add_argument('--only', action='append') parser.add_argument('--redirect', action='append') parser.add_argument('--brand') parser.add_argument('--brandlink') parser.add_argument('--primtype', default="#PrimitiveType") args = parser.parse_args() s = [] a = args.schema with open(a) as f: if a.endswith("md"): s.append({"name": os.path.splitext(os.path.basename(a))[0], "type": "documentation", "doc": f.read().decode("utf-8") }) else: uri = "file://" + os.path.abspath(a) _, _, metaschema_loader = schema.get_metaschema() j, schema_metadata = metaschema_loader.resolve_ref(uri, "") if isinstance(j, list): s.extend(j) else: s.append(j) primitiveType = args.primtype redirect = {r.split("=")[0]:r.split("=")[1] for r in args.redirect} if args.redirect else {} renderlist = args.only if args.only else [] avrold_doc(s, sys.stdout, renderlist, redirect, args.brand, args.brandlink)
def main(args=None): if args is None: args = sys.argv[1:] parser = argparse.ArgumentParser() parser.add_argument("--rdf-serializer", help="Output RDF serialization format used by --print-rdf (one of turtle (default), n3, nt, xml)", default="turtle") exgroup = parser.add_mutually_exclusive_group() exgroup.add_argument("--print-jsonld-context", action="store_true", help="Print JSON-LD context for schema") exgroup.add_argument("--print-doc", action="store_true", help="Print HTML documentation from schema") exgroup.add_argument("--print-rdfs", action="store_true", help="Print RDF schema") exgroup.add_argument("--print-avro", action="store_true", help="Print Avro schema") exgroup.add_argument("--print-rdf", action="store_true", help="Print corresponding RDF graph for document") exgroup.add_argument("--print-pre", action="store_true", help="Print document after preprocessing") exgroup.add_argument("--print-index", action="store_true", help="Print node index") exgroup.add_argument("--print-metadata", action="store_true", help="Print document metadata") exgroup.add_argument("--version", action="store_true", help="Print version") parser.add_argument("--strict", action="store_true", help="Strict validation (error on unrecognized fields)") exgroup = parser.add_mutually_exclusive_group() exgroup.add_argument("--verbose", action="store_true", help="Default logging") exgroup.add_argument("--quiet", action="store_true", help="Only print warnings and errors.") exgroup.add_argument("--debug", action="store_true", help="Print even more logging") parser.add_argument("schema", type=str) parser.add_argument("document", type=str, nargs="?", default=None) args = parser.parse_args(args) if args.quiet: _logger.setLevel(logging.WARN) if args.debug: _logger.setLevel(logging.DEBUG) pkg = pkg_resources.require("schema_salad") if pkg: if args.version: print "%s %s" % (sys.argv[0], pkg[0].version) return 0 else: _logger.info("%s %s", sys.argv[0], pkg[0].version) # Get the metaschema to validate the schema metaschema_names, metaschema_doc, metaschema_loader = schema.get_metaschema() # Load schema document and resolve refs schema_uri = args.schema if not urlparse.urlparse(schema_uri)[0]: schema_uri = "file://" + os.path.abspath(schema_uri) schema_raw_doc = metaschema_loader.fetch(schema_uri) schema_doc, schema_metadata = metaschema_loader.resolve_all(schema_raw_doc, schema_uri) # Optionally print the schema after ref resolution if not args.document and args.print_pre: print json.dumps(schema_doc, indent=4) return 0 if not args.document and args.print_index: print json.dumps(metaschema_loader.idx.keys(), indent=4) return 0 # Validate links in the schema document try: metaschema_loader.validate_links(schema_doc) except (validate.ValidationException) as e: _logger.error("Schema `%s` failed link checking:\n%s", args.schema, e, exc_info=(e if args.debug else False)) _logger.debug("Index is %s", metaschema_loader.idx.keys()) _logger.debug("Vocabulary is %s", metaschema_loader.vocab.keys()) return 1 # Validate the schema document against the metaschema try: schema.validate_doc(metaschema_names, schema_doc, metaschema_loader, args.strict) except validate.ValidationException as e: _logger.error("While validating schema `%s`:\n%s" % (args.schema, str(e))) return 1 # Get the json-ld context and RDFS representation from the schema (schema_ctx, rdfs) = jsonld_context.salad_to_jsonld_context(schema_doc, schema_raw_doc["@context"]) # Create the loader that will be used to load the target document. document_loader = Loader(schema_ctx) # Make the Avro validation that will be used to validate the target document (avsc_names, avsc_obj) = schema.make_avro_schema(schema_doc, document_loader) if isinstance(avsc_names, Exception): _logger.error("Schema `%s` error:\n%s", args.schema, avsc_names, exc_info=(avsc_names if args.debug else False)) if args.print_avro: print json.dumps(avsc_obj, indent=4) return 1 # Optionally print Avro-compatible schema from schema if args.print_avro: print json.dumps(avsc_obj, indent=4) return 0 # Optionally print the json-ld context from the schema if args.print_jsonld_context: j = {"@context": schema_ctx} print json.dumps(j, indent=4, sort_keys=True) return 0 # Optionally print the RDFS graph from the schema if args.print_rdfs: print(rdfs.serialize(format=args.rdf_serializer)) return 0 # Optionally create documentation page from the schema if args.print_doc: makedoc.avrold_doc(schema_doc, sys.stdout) return 0 if args.print_metadata and not args.document: print json.dumps(schema_metadata, indent=4) return 0 # If no document specified, all done. if not args.document: print "Schema `%s` is valid" % args.schema return 0 # Load target document and resolve refs try: uri = args.document if not urlparse.urlparse(uri)[0]: doc = "file://" + os.path.abspath(uri) document, doc_metadata = document_loader.resolve_ref(uri) except (validate.ValidationException, RuntimeError) as e: _logger.error("Document `%s` failed validation:\n%s", args.document, e, exc_info=(e if args.debug else False)) return 1 # Optionally print the document after ref resolution if args.print_pre: print json.dumps(document, indent=4) return 0 if args.print_index: print json.dumps(document_loader.idx.keys(), indent=4) return 0 # Validate links in the target document try: document_loader.validate_links(document) except (validate.ValidationException) as e: _logger.error("Document `%s` failed link checking:\n%s", args.document, e, exc_info=(e if args.debug else False)) _logger.debug("Index is %s", json.dumps(document_loader.idx.keys(), indent=4)) return 1 # Validate the schema document against the metaschema try: schema.validate_doc(avsc_names, document, document_loader, args.strict) except validate.ValidationException as e: _logger.error("While validating document `%s`:\n%s" % (args.document, str(e))) return 1 # Optionally convert the document to RDF if args.print_rdf: printrdf(args.document, document, schema_ctx, args.rdf_serializer) return 0 if args.print_metadata: print json.dumps(doc_metadata, indent=4) return 0 print "Document `%s` is valid" % args.document return 0
def main(args=None): if args is None: args = sys.argv[1:] parser = argparse.ArgumentParser() parser.add_argument("--rdf-serializer", help="Output RDF serialization format used by --print-rdf (one of turtle (default), n3, nt, xml)", default="turtle") exgroup = parser.add_mutually_exclusive_group() exgroup.add_argument("--print-jsonld-context", action="store_true", help="Print JSON-LD context for schema") exgroup.add_argument("--print-doc", action="store_true", help="Print HTML documentation from schema") exgroup.add_argument("--print-rdfs", action="store_true", help="Print RDF schema") exgroup.add_argument("--print-avro", action="store_true", help="Print Avro schema") exgroup.add_argument("--print-rdf", action="store_true", help="Print corresponding RDF graph for document") exgroup.add_argument("--print-pre", action="store_true", help="Print document after preprocessing") exgroup.add_argument("--print-index", action="store_true", help="Print node index") exgroup.add_argument("--print-metadata", action="store_true", help="Print document metadata") exgroup.add_argument("--version", action="store_true", help="Print version") exgroup = parser.add_mutually_exclusive_group() exgroup.add_argument("--strict", action="store_true", help="Strict validation (unrecognized or out of place fields are error)", default=True, dest="strict") exgroup.add_argument("--non-strict", action="store_false", help="Lenient validation (ignore unrecognized fields)", default=True, dest="strict") exgroup = parser.add_mutually_exclusive_group() exgroup.add_argument("--verbose", action="store_true", help="Default logging") exgroup.add_argument("--quiet", action="store_true", help="Only print warnings and errors.") exgroup.add_argument("--debug", action="store_true", help="Print even more logging") parser.add_argument("schema", type=str) parser.add_argument("document", type=str, nargs="?", default=None) args = parser.parse_args(args) if args.quiet: _logger.setLevel(logging.WARN) if args.debug: _logger.setLevel(logging.DEBUG) pkg = pkg_resources.require("schema_salad") if pkg: if args.version: print "%s %s" % (sys.argv[0], pkg[0].version) return 0 else: _logger.info("%s %s", sys.argv[0], pkg[0].version) # Get the metaschema to validate the schema metaschema_names, metaschema_doc, metaschema_loader = schema.get_metaschema() # Load schema document and resolve refs schema_uri = args.schema if not urlparse.urlparse(schema_uri)[0]: schema_uri = "file://" + os.path.abspath(schema_uri) schema_raw_doc = metaschema_loader.fetch(schema_uri) schema_doc, schema_metadata = metaschema_loader.resolve_all(schema_raw_doc, schema_uri) # Optionally print the schema after ref resolution if not args.document and args.print_pre: print json.dumps(schema_doc, indent=4) return 0 if not args.document and args.print_index: print json.dumps(metaschema_loader.idx.keys(), indent=4) return 0 # Validate links in the schema document try: metaschema_loader.validate_links(schema_doc) except (validate.ValidationException) as e: _logger.error("Schema `%s` failed link checking:\n%s", args.schema, e, exc_info=(e if args.debug else False)) _logger.debug("Index is %s", metaschema_loader.idx.keys()) _logger.debug("Vocabulary is %s", metaschema_loader.vocab.keys()) return 1 # Validate the schema document against the metaschema try: schema.validate_doc(metaschema_names, schema_doc, metaschema_loader, args.strict) except validate.ValidationException as e: _logger.error("While validating schema `%s`:\n%s" % (args.schema, str(e))) return 1 # Get the json-ld context and RDFS representation from the schema metactx = {} if isinstance(schema_raw_doc, dict): metactx = schema_raw_doc.get("$namespaces", {}) if "$base" in schema_raw_doc: metactx["@base"] = schema_raw_doc["$base"] (schema_ctx, rdfs) = jsonld_context.salad_to_jsonld_context(schema_doc, metactx) # Create the loader that will be used to load the target document. document_loader = Loader(schema_ctx) # Make the Avro validation that will be used to validate the target document (avsc_names, avsc_obj) = schema.make_avro_schema(schema_doc, document_loader) if isinstance(avsc_names, Exception): _logger.error("Schema `%s` error:\n%s", args.schema, avsc_names, exc_info=(avsc_names if args.debug else False)) if args.print_avro: print json.dumps(avsc_obj, indent=4) return 1 # Optionally print Avro-compatible schema from schema if args.print_avro: print json.dumps(avsc_obj, indent=4) return 0 # Optionally print the json-ld context from the schema if args.print_jsonld_context: j = {"@context": schema_ctx} print json.dumps(j, indent=4, sort_keys=True) return 0 # Optionally print the RDFS graph from the schema if args.print_rdfs: print(rdfs.serialize(format=args.rdf_serializer)) return 0 # Optionally create documentation page from the schema if args.print_doc: makedoc.avrold_doc(schema_doc, sys.stdout) return 0 if args.print_metadata and not args.document: print json.dumps(schema_metadata, indent=4) return 0 # If no document specified, all done. if not args.document: print "Schema `%s` is valid" % args.schema return 0 # Load target document and resolve refs try: uri = args.document if not urlparse.urlparse(uri)[0]: doc = "file://" + os.path.abspath(uri) document, doc_metadata = document_loader.resolve_ref(uri) except (validate.ValidationException, RuntimeError) as e: _logger.error("Document `%s` failed validation:\n%s", args.document, e, exc_info=(e if args.debug else False)) return 1 # Optionally print the document after ref resolution if args.print_pre: print json.dumps(document, indent=4) return 0 if args.print_index: print json.dumps(document_loader.idx.keys(), indent=4) return 0 # Validate links in the target document try: document_loader.validate_links(document) except (validate.ValidationException) as e: _logger.error("Document `%s` failed link checking:\n%s", args.document, e, exc_info=(e if args.debug else False)) _logger.debug("Index is %s", json.dumps(document_loader.idx.keys(), indent=4)) return 1 # Validate the schema document against the metaschema try: schema.validate_doc(avsc_names, document, document_loader, args.strict) except validate.ValidationException as e: _logger.error("While validating document `%s`:\n%s" % (args.document, str(e))) return 1 # Optionally convert the document to RDF if args.print_rdf: printrdf(args.document, document, schema_ctx, args.rdf_serializer) return 0 if args.print_metadata: print json.dumps(doc_metadata, indent=4) return 0 print "Document `%s` is valid" % args.document return 0