def __init__(self, toc, j, renderlist, redirects):
self.typedoc = StringIO.StringIO()
self.toc = toc
self.subs = {}
self.docParent = {}
self.docAfter = {}
self.rendered = set()
self.redirects = redirects
self.title = None
for t in j:
if "extends" in t:
for e in aslist(t["extends"]):
add_dictlist(self.subs, e, t["name"])
#if "docParent" not in t and "docAfter" not in t:
# add_dictlist(self.docParent, e, t["name"])
if t.get("docParent"):
add_dictlist(self.docParent, t["docParent"], t["name"])
if t.get("docChild"):
for c in aslist(t["docChild"]):
add_dictlist(self.docParent, t["name"], c)
if t.get("docAfter"):
add_dictlist(self.docAfter, t["docAfter"], t["name"])
_, _, metaschema_loader = schema.get_metaschema()
alltypes = schema.extend_and_specialize(j, metaschema_loader)
self.typemap = {}
self.uses = {}
self.record_refs = {}
for t in alltypes:
self.typemap[t["name"]] = t
try:
if t["type"] == "record":
self.record_refs[t["name"]] = []
for f in t.get("fields", []):
p = has_types(f)
for tp in p:
if tp not in self.uses:
self.uses[tp] = []
if (t["name"], f["name"]) not in self.uses[tp]:
_, frg1 = urlparse.urldefrag(t["name"])
_, frg2 = urlparse.urldefrag(f["name"])
self.uses[tp].append((frg1, frg2))
if tp not in basicTypes and tp not in self.record_refs[t["name"]]:
self.record_refs[t["name"]].append(tp)
except KeyError as e:
_logger.error("Did not find 'type' in %s", t)
raise
for f in alltypes:
if (f["name"] in renderlist or
((not renderlist) and
("extends" not in f) and
("docParent" not in f) and
("docAfter" not in f))):
self.render_type(f, 1)
def __init__(self, toc, j, renderlist, redirects):
self.typedoc = StringIO.StringIO()
self.toc = toc
self.subs = {}
self.docParent = {}
self.docAfter = {}
self.rendered = set()
self.redirects = redirects
self.title = None
for t in j:
if "extends" in t:
for e in aslist(t["extends"]):
add_dictlist(self.subs, e, t["name"])
#if "docParent" not in t and "docAfter" not in t:
# add_dictlist(self.docParent, e, t["name"])
if t.get("docParent"):
add_dictlist(self.docParent, t["docParent"], t["name"])
if t.get("docChild"):
for c in aslist(t["docChild"]):
add_dictlist(self.docParent, t["name"], c)
if t.get("docAfter"):
add_dictlist(self.docAfter, t["docAfter"], t["name"])
_, _, metaschema_loader = schema.get_metaschema()
alltypes = schema.extend_and_specialize(j, metaschema_loader)
self.typemap = {}
self.uses = {}
self.record_refs = {}
for t in alltypes:
self.typemap[t["name"]] = t
try:
if t["type"] == "record":
self.record_refs[t["name"]] = []
for f in t.get("fields", []):
p = has_types(f)
for tp in p:
if tp not in self.uses:
self.uses[tp] = []
if (t["name"], f["name"]) not in self.uses[tp]:
_, frg1 = urlparse.urldefrag(t["name"])
_, frg2 = urlparse.urldefrag(f["name"])
self.uses[tp].append((frg1, frg2))
if tp not in basicTypes and tp not in self.record_refs[
t["name"]]:
self.record_refs[t["name"]].append(tp)
except KeyError as e:
_logger.error("Did not find 'type' in %s", t)
raise
for f in alltypes:
if (f["name"] in renderlist
or ((not renderlist) and ("extends" not in f) and
("docParent" not in f) and ("docAfter" not in f))):
self.render_type(f, 1)
def __init__(self, toc, j):
self.typedoc = StringIO.StringIO()
self.toc = toc
self.subs = {}
self.docParent = {}
self.docAfter = {}
for t in j:
if "extends" in t:
for e in aslist(t["extends"]):
add_dictlist(self.subs, e, t["name"])
if "docParent" not in t and "docAfter" not in t:
add_dictlist(self.docParent, e, t["name"])
if t.get("docParent"):
add_dictlist(self.docParent, t["docParent"], t["name"])
if t.get("docAfter"):
add_dictlist(self.docAfter, t["docAfter"], t["name"])
_, _, metaschema_loader = schema.get_metaschema()
alltypes = schema.extend_and_specialize(j, metaschema_loader)
self.typemap = {}
self.uses = {}
for t in alltypes:
self.typemap[t["name"]] = t
if t["type"] == "https://w3id.org/cwl/salad#record":
for f in t["fields"]:
p = has_types(f)
for tp in p:
if tp not in self.uses:
self.uses[tp] = []
if (t["name"], f["name"]) not in self.uses[tp]:
_, frg1 = urlparse.urldefrag(t["name"])
_, frg2 = urlparse.urldefrag(f["name"])
self.uses[tp].append((frg1, frg2))
for f in alltypes:
if ("extends" not in f) and ("docParent" not in f) and ("docAfter" not in f):
self.render_type(f, 1)
parser.add_argument('--brand')
parser.add_argument('--brandlink')
parser.add_argument('--primtype', default="#PrimitiveType")
args = parser.parse_args()
s = []
a = args.schema
with open(a) as f:
if a.endswith("md"):
s.append({
"name": os.path.splitext(os.path.basename(a))[0],
"type": "documentation",
"doc": f.read().decode("utf-8")
})
else:
uri = "file://" + os.path.abspath(a)
_, _, metaschema_loader = schema.get_metaschema()
j, schema_metadata = metaschema_loader.resolve_ref(uri, "")
if isinstance(j, list):
s.extend(j)
else:
s.append(j)
primitiveType = args.primtype
redirect = {r.split("=")[0]: r.split("=")[1]
for r in args.redirect} if args.redirect else {}
renderlist = args.only if args.only else []
avrold_doc(s, sys.stdout, renderlist, redirect, args.brand, args.brandlink)
parser.add_argument('--only', action='append')
parser.add_argument('--redirect', action='append')
parser.add_argument('--brand')
parser.add_argument('--brandlink')
parser.add_argument('--primtype', default="#PrimitiveType")
args = parser.parse_args()
s = []
a = args.schema
with open(a) as f:
if a.endswith("md"):
s.append({"name": os.path.splitext(os.path.basename(a))[0],
"type": "documentation",
"doc": f.read().decode("utf-8")
})
else:
uri = "file://" + os.path.abspath(a)
_, _, metaschema_loader = schema.get_metaschema()
j, schema_metadata = metaschema_loader.resolve_ref(uri, "")
if isinstance(j, list):
s.extend(j)
else:
s.append(j)
primitiveType = args.primtype
redirect = {r.split("=")[0]:r.split("=")[1] for r in args.redirect} if args.redirect else {}
renderlist = args.only if args.only else []
avrold_doc(s, sys.stdout, renderlist, redirect, args.brand, args.brandlink)
def main(args=None):
if args is None:
args = sys.argv[1:]
parser = argparse.ArgumentParser()
parser.add_argument("--rdf-serializer",
help="Output RDF serialization format used by --print-rdf (one of turtle (default), n3, nt, xml)",
default="turtle")
exgroup = parser.add_mutually_exclusive_group()
exgroup.add_argument("--print-jsonld-context", action="store_true", help="Print JSON-LD context for schema")
exgroup.add_argument("--print-doc", action="store_true", help="Print HTML documentation from schema")
exgroup.add_argument("--print-rdfs", action="store_true", help="Print RDF schema")
exgroup.add_argument("--print-avro", action="store_true", help="Print Avro schema")
exgroup.add_argument("--print-rdf", action="store_true", help="Print corresponding RDF graph for document")
exgroup.add_argument("--print-pre", action="store_true", help="Print document after preprocessing")
exgroup.add_argument("--print-index", action="store_true", help="Print node index")
exgroup.add_argument("--print-metadata", action="store_true", help="Print document metadata")
exgroup.add_argument("--version", action="store_true", help="Print version")
parser.add_argument("--strict", action="store_true", help="Strict validation (error on unrecognized fields)")
exgroup = parser.add_mutually_exclusive_group()
exgroup.add_argument("--verbose", action="store_true", help="Default logging")
exgroup.add_argument("--quiet", action="store_true", help="Only print warnings and errors.")
exgroup.add_argument("--debug", action="store_true", help="Print even more logging")
parser.add_argument("schema", type=str)
parser.add_argument("document", type=str, nargs="?", default=None)
args = parser.parse_args(args)
if args.quiet:
_logger.setLevel(logging.WARN)
if args.debug:
_logger.setLevel(logging.DEBUG)
pkg = pkg_resources.require("schema_salad")
if pkg:
if args.version:
print "%s %s" % (sys.argv[0], pkg[0].version)
return 0
else:
_logger.info("%s %s", sys.argv[0], pkg[0].version)
# Get the metaschema to validate the schema
metaschema_names, metaschema_doc, metaschema_loader = schema.get_metaschema()
# Load schema document and resolve refs
schema_uri = args.schema
if not urlparse.urlparse(schema_uri)[0]:
schema_uri = "file://" + os.path.abspath(schema_uri)
schema_raw_doc = metaschema_loader.fetch(schema_uri)
schema_doc, schema_metadata = metaschema_loader.resolve_all(schema_raw_doc, schema_uri)
# Optionally print the schema after ref resolution
if not args.document and args.print_pre:
print json.dumps(schema_doc, indent=4)
return 0
if not args.document and args.print_index:
print json.dumps(metaschema_loader.idx.keys(), indent=4)
return 0
# Validate links in the schema document
try:
metaschema_loader.validate_links(schema_doc)
except (validate.ValidationException) as e:
_logger.error("Schema `%s` failed link checking:\n%s", args.schema, e, exc_info=(e if args.debug else False))
_logger.debug("Index is %s", metaschema_loader.idx.keys())
_logger.debug("Vocabulary is %s", metaschema_loader.vocab.keys())
return 1
# Validate the schema document against the metaschema
try:
schema.validate_doc(metaschema_names, schema_doc, metaschema_loader, args.strict)
except validate.ValidationException as e:
_logger.error("While validating schema `%s`:\n%s" % (args.schema, str(e)))
return 1
# Get the json-ld context and RDFS representation from the schema
(schema_ctx, rdfs) = jsonld_context.salad_to_jsonld_context(schema_doc, schema_raw_doc["@context"])
# Create the loader that will be used to load the target document.
document_loader = Loader(schema_ctx)
# Make the Avro validation that will be used to validate the target document
(avsc_names, avsc_obj) = schema.make_avro_schema(schema_doc, document_loader)
if isinstance(avsc_names, Exception):
_logger.error("Schema `%s` error:\n%s", args.schema, avsc_names, exc_info=(avsc_names if args.debug else False))
if args.print_avro:
print json.dumps(avsc_obj, indent=4)
return 1
# Optionally print Avro-compatible schema from schema
if args.print_avro:
print json.dumps(avsc_obj, indent=4)
return 0
# Optionally print the json-ld context from the schema
if args.print_jsonld_context:
j = {"@context": schema_ctx}
print json.dumps(j, indent=4, sort_keys=True)
return 0
# Optionally print the RDFS graph from the schema
if args.print_rdfs:
print(rdfs.serialize(format=args.rdf_serializer))
return 0
# Optionally create documentation page from the schema
if args.print_doc:
makedoc.avrold_doc(schema_doc, sys.stdout)
return 0
if args.print_metadata and not args.document:
print json.dumps(schema_metadata, indent=4)
return 0
# If no document specified, all done.
if not args.document:
print "Schema `%s` is valid" % args.schema
return 0
# Load target document and resolve refs
try:
uri = args.document
if not urlparse.urlparse(uri)[0]:
doc = "file://" + os.path.abspath(uri)
document, doc_metadata = document_loader.resolve_ref(uri)
except (validate.ValidationException, RuntimeError) as e:
_logger.error("Document `%s` failed validation:\n%s", args.document, e, exc_info=(e if args.debug else False))
return 1
# Optionally print the document after ref resolution
if args.print_pre:
print json.dumps(document, indent=4)
return 0
if args.print_index:
print json.dumps(document_loader.idx.keys(), indent=4)
return 0
# Validate links in the target document
try:
document_loader.validate_links(document)
except (validate.ValidationException) as e:
_logger.error("Document `%s` failed link checking:\n%s", args.document, e, exc_info=(e if args.debug else False))
_logger.debug("Index is %s", json.dumps(document_loader.idx.keys(), indent=4))
return 1
# Validate the schema document against the metaschema
try:
schema.validate_doc(avsc_names, document, document_loader, args.strict)
except validate.ValidationException as e:
_logger.error("While validating document `%s`:\n%s" % (args.document, str(e)))
return 1
# Optionally convert the document to RDF
if args.print_rdf:
printrdf(args.document, document, schema_ctx, args.rdf_serializer)
return 0
if args.print_metadata:
print json.dumps(doc_metadata, indent=4)
return 0
print "Document `%s` is valid" % args.document
return 0
def main(args=None):
if args is None:
args = sys.argv[1:]
parser = argparse.ArgumentParser()
parser.add_argument("--rdf-serializer",
help="Output RDF serialization format used by --print-rdf (one of turtle (default), n3, nt, xml)",
default="turtle")
exgroup = parser.add_mutually_exclusive_group()
exgroup.add_argument("--print-jsonld-context", action="store_true", help="Print JSON-LD context for schema")
exgroup.add_argument("--print-doc", action="store_true", help="Print HTML documentation from schema")
exgroup.add_argument("--print-rdfs", action="store_true", help="Print RDF schema")
exgroup.add_argument("--print-avro", action="store_true", help="Print Avro schema")
exgroup.add_argument("--print-rdf", action="store_true", help="Print corresponding RDF graph for document")
exgroup.add_argument("--print-pre", action="store_true", help="Print document after preprocessing")
exgroup.add_argument("--print-index", action="store_true", help="Print node index")
exgroup.add_argument("--print-metadata", action="store_true", help="Print document metadata")
exgroup.add_argument("--version", action="store_true", help="Print version")
exgroup = parser.add_mutually_exclusive_group()
exgroup.add_argument("--strict", action="store_true", help="Strict validation (unrecognized or out of place fields are error)",
default=True, dest="strict")
exgroup.add_argument("--non-strict", action="store_false", help="Lenient validation (ignore unrecognized fields)",
default=True, dest="strict")
exgroup = parser.add_mutually_exclusive_group()
exgroup.add_argument("--verbose", action="store_true", help="Default logging")
exgroup.add_argument("--quiet", action="store_true", help="Only print warnings and errors.")
exgroup.add_argument("--debug", action="store_true", help="Print even more logging")
parser.add_argument("schema", type=str)
parser.add_argument("document", type=str, nargs="?", default=None)
args = parser.parse_args(args)
if args.quiet:
_logger.setLevel(logging.WARN)
if args.debug:
_logger.setLevel(logging.DEBUG)
pkg = pkg_resources.require("schema_salad")
if pkg:
if args.version:
print "%s %s" % (sys.argv[0], pkg[0].version)
return 0
else:
_logger.info("%s %s", sys.argv[0], pkg[0].version)
# Get the metaschema to validate the schema
metaschema_names, metaschema_doc, metaschema_loader = schema.get_metaschema()
# Load schema document and resolve refs
schema_uri = args.schema
if not urlparse.urlparse(schema_uri)[0]:
schema_uri = "file://" + os.path.abspath(schema_uri)
schema_raw_doc = metaschema_loader.fetch(schema_uri)
schema_doc, schema_metadata = metaschema_loader.resolve_all(schema_raw_doc, schema_uri)
# Optionally print the schema after ref resolution
if not args.document and args.print_pre:
print json.dumps(schema_doc, indent=4)
return 0
if not args.document and args.print_index:
print json.dumps(metaschema_loader.idx.keys(), indent=4)
return 0
# Validate links in the schema document
try:
metaschema_loader.validate_links(schema_doc)
except (validate.ValidationException) as e:
_logger.error("Schema `%s` failed link checking:\n%s", args.schema, e, exc_info=(e if args.debug else False))
_logger.debug("Index is %s", metaschema_loader.idx.keys())
_logger.debug("Vocabulary is %s", metaschema_loader.vocab.keys())
return 1
# Validate the schema document against the metaschema
try:
schema.validate_doc(metaschema_names, schema_doc, metaschema_loader, args.strict)
except validate.ValidationException as e:
_logger.error("While validating schema `%s`:\n%s" % (args.schema, str(e)))
return 1
# Get the json-ld context and RDFS representation from the schema
metactx = {}
if isinstance(schema_raw_doc, dict):
metactx = schema_raw_doc.get("$namespaces", {})
if "$base" in schema_raw_doc:
metactx["@base"] = schema_raw_doc["$base"]
(schema_ctx, rdfs) = jsonld_context.salad_to_jsonld_context(schema_doc, metactx)
# Create the loader that will be used to load the target document.
document_loader = Loader(schema_ctx)
# Make the Avro validation that will be used to validate the target document
(avsc_names, avsc_obj) = schema.make_avro_schema(schema_doc, document_loader)
if isinstance(avsc_names, Exception):
_logger.error("Schema `%s` error:\n%s", args.schema, avsc_names, exc_info=(avsc_names if args.debug else False))
if args.print_avro:
print json.dumps(avsc_obj, indent=4)
return 1
# Optionally print Avro-compatible schema from schema
if args.print_avro:
print json.dumps(avsc_obj, indent=4)
return 0
# Optionally print the json-ld context from the schema
if args.print_jsonld_context:
j = {"@context": schema_ctx}
print json.dumps(j, indent=4, sort_keys=True)
return 0
# Optionally print the RDFS graph from the schema
if args.print_rdfs:
print(rdfs.serialize(format=args.rdf_serializer))
return 0
# Optionally create documentation page from the schema
if args.print_doc:
makedoc.avrold_doc(schema_doc, sys.stdout)
return 0
if args.print_metadata and not args.document:
print json.dumps(schema_metadata, indent=4)
return 0
# If no document specified, all done.
if not args.document:
print "Schema `%s` is valid" % args.schema
return 0
# Load target document and resolve refs
try:
uri = args.document
if not urlparse.urlparse(uri)[0]:
doc = "file://" + os.path.abspath(uri)
document, doc_metadata = document_loader.resolve_ref(uri)
except (validate.ValidationException, RuntimeError) as e:
_logger.error("Document `%s` failed validation:\n%s", args.document, e, exc_info=(e if args.debug else False))
return 1
# Optionally print the document after ref resolution
if args.print_pre:
print json.dumps(document, indent=4)
return 0
if args.print_index:
print json.dumps(document_loader.idx.keys(), indent=4)
return 0
# Validate links in the target document
try:
document_loader.validate_links(document)
except (validate.ValidationException) as e:
_logger.error("Document `%s` failed link checking:\n%s", args.document, e, exc_info=(e if args.debug else False))
_logger.debug("Index is %s", json.dumps(document_loader.idx.keys(), indent=4))
return 1
# Validate the schema document against the metaschema
try:
schema.validate_doc(avsc_names, document, document_loader, args.strict)
except validate.ValidationException as e:
_logger.error("While validating document `%s`:\n%s" % (args.document, str(e)))
return 1
# Optionally convert the document to RDF
if args.print_rdf:
printrdf(args.document, document, schema_ctx, args.rdf_serializer)
return 0
if args.print_metadata:
print json.dumps(doc_metadata, indent=4)
return 0
print "Document `%s` is valid" % args.document
return 0