def print_inheritance(doc, stream): # type: (List[Dict[Text, Any]], IO) -> None stream.write("digraph {\n") for d in doc: if d["type"] == "record": label = shortname(d["name"]) if len(d.get("fields", [])) > 0: label += "\\n* %s\\l" % ("\\l* ".join(shortname(f["name"]) for f in d.get("fields", []))) stream.write("\"%s\" [shape=%s label=\"%s\"];\n" % (shortname(d["name"]), "ellipse" if d.get("abstract") else "box", label)) if "extends" in d: for e in aslist(d["extends"]): stream.write("\"%s\" -> \"%s\";\n" % (shortname(e), shortname(d["name"]))) stream.write("}\n")
def print_inheritance(doc, stream): # type: (List[Dict[Text, Any]], IO) -> None stream.write("digraph {\n") for d in doc: if d["type"] == "record": label = shortname(d["name"]) if len(d.get("fields", [])) > 0: label += "\\n* %s\\l" % ("\\l* ".join( shortname(f["name"]) for f in d.get("fields", []))) stream.write("\"%s\" [shape=%s label=\"%s\"];\n" % (shortname( d["name"]), "ellipse" if d.get("abstract") else "box", label)) if "extends" in d: for e in aslist(d["extends"]): stream.write("\"%s\" -> \"%s\";\n" % (shortname(e), shortname(d["name"]))) stream.write("}\n")
def print_inheritance(doc: List[Dict[str, Any]], stream: IO[Any]) -> None: """Write a Grapviz inheritance graph for the supplied document.""" stream.write("digraph {\n") for entry in doc: if entry["type"] == "record": label = name = shortname(entry["name"]) fields = entry.get("fields", []) if fields: label += "\\n* {}\\l".format("\\l* ".join( shortname(field["name"]) for field in fields)) shape = "ellipse" if entry.get("abstract") else "box" stream.write(f'"{name}" [shape={shape} label="{label}"];\n') if "extends" in entry: for target in aslist(entry["extends"]): stream.write(f'"{shortname(target)}" -> "{name}";\n') stream.write("}\n")
def print_inheritance(doc, stream): # type: (List[Dict[Text, Any]], IO[Any]) -> None """Write a Grapviz inheritance graph for the supplied document.""" stream.write("digraph {\n") for entry in doc: if entry["type"] == "record": label = name = shortname(entry["name"]) fields = entry.get("fields", []) if fields: label += "\\n* %s\\l" % ("\\l* ".join( shortname(field["name"]) for field in fields)) shape = "ellipse" if entry.get("abstract") else "box" stream.write("\"%s\" [shape=%s label=\"%s\"];\n" % (name, shape, label)) if "extends" in entry: for target in aslist(entry["extends"]): stream.write("\"%s\" -> \"%s\";\n" % (shortname(target), name)) stream.write("}\n")
def extend_and_specialize( items: List[Dict[str, Any]], loader: Loader ) -> List[Dict[str, Any]]: """ Apply 'extend' and 'specialize' to fully materialize derived record types. """ items2 = deepcopy_strip(items) types = {i["name"]: i for i in items2} # type: Dict[str, Any] results = [] for stype in items2: if "extends" in stype: specs = {} # type: Dict[str, str] if "specialize" in stype: for spec in aslist(stype["specialize"]): specs[spec["specializeFrom"]] = spec["specializeTo"] exfields = [] # type: List[str] exsym = [] # type: List[str] for ex in aslist(stype["extends"]): if ex not in types: raise ValidationException( "Extends {} in {} refers to invalid base type.".format( stype["extends"], stype["name"] ) ) basetype = copy.copy(types[ex]) if stype["type"] == "record": if specs: basetype["fields"] = replace_type( basetype.get("fields", []), specs, loader, set() ) for field in basetype.get("fields", []): if "inherited_from" not in field: field["inherited_from"] = ex exfields.extend(basetype.get("fields", [])) elif stype["type"] == "enum": exsym.extend(basetype.get("symbols", [])) if stype["type"] == "record": stype = copy.copy(stype) exfields.extend(stype.get("fields", [])) stype["fields"] = exfields fieldnames = set() # type: Set[str] for field in stype["fields"]: if field["name"] in fieldnames: raise ValidationException( "Field name {} appears twice in {}".format( field["name"], stype["name"] ) ) else: fieldnames.add(field["name"]) elif stype["type"] == "enum": stype = copy.copy(stype) exsym.extend(stype.get("symbols", [])) stype["symbol"] = exsym types[stype["name"]] = stype results.append(stype) ex_types = {} for result in results: ex_types[result["name"]] = result extended_by = {} # type: Dict[str, str] for result in results: if "extends" in result: for ex in aslist(result["extends"]): if ex_types[ex].get("abstract"): add_dictlist(extended_by, ex, ex_types[result["name"]]) add_dictlist(extended_by, avro_name(ex), ex_types[ex]) for result in results: if result.get("abstract") and result["name"] not in extended_by: raise ValidationException( "{} is abstract but missing a concrete subtype".format(result["name"]) ) for result in results: if "fields" in result: result["fields"] = replace_type( result["fields"], extended_by, loader, set() ) return results
def extend_and_specialize(items, loader): # type: (List[Dict[Text, Any]], Loader) -> List[Dict[Text, Any]] """Apply 'extend' and 'specialize' to fully materialize derived record types.""" items = deepcopy_strip(items) types = {t["name"]: t for t in items} # type: Dict[Text, Any] n = [] for t in items: if "extends" in t: spec = {} # type: Dict[Text, Text] if "specialize" in t: for sp in aslist(t["specialize"]): spec[sp["specializeFrom"]] = sp["specializeTo"] exfields = [] # type: List[Text] exsym = [] # type: List[Text] for ex in aslist(t["extends"]): if ex not in types: raise Exception( "Extends %s in %s refers to invalid base type" % (t["extends"], t["name"])) basetype = copy.copy(types[ex]) if t["type"] == "record": if len(spec) > 0: basetype["fields"] = replace_type( basetype.get("fields", []), spec, loader, set()) for f in basetype.get("fields", []): if "inherited_from" not in f: f["inherited_from"] = ex exfields.extend(basetype.get("fields", [])) elif t["type"] == "enum": exsym.extend(basetype.get("symbols", [])) if t["type"] == "record": t = copy.copy(t) exfields.extend(t.get("fields", [])) t["fields"] = exfields fieldnames = set() # type: Set[Text] for field in t["fields"]: if field["name"] in fieldnames: raise validate.ValidationException( "Field name %s appears twice in %s" % (field["name"], t["name"])) else: fieldnames.add(field["name"]) elif t["type"] == "enum": t = copy.copy(t) exsym.extend(t.get("symbols", [])) t["symbol"] = exsym types[t["name"]] = t n.append(t) ex_types = {} for t in n: ex_types[t["name"]] = t extended_by = {} # type: Dict[Text, Text] for t in n: if "extends" in t: for ex in aslist(t["extends"]): if ex_types[ex].get("abstract"): add_dictlist(extended_by, ex, ex_types[t["name"]]) add_dictlist(extended_by, avro_name(ex), ex_types[ex]) for t in n: if t.get("abstract") and t["name"] not in extended_by: raise validate.ValidationException( "%s is abstract but missing a concrete subtype" % t["name"]) for t in n: if "fields" in t: t["fields"] = replace_type(t["fields"], extended_by, loader, set()) return n
def extend_and_specialize(items: List[Dict[str, Any]], loader: Loader) -> List[Dict[str, Any]]: """ Apply 'extend' and 'specialize' to fully materialize derived record types. """ items2 = deepcopy_strip(items) types = {i["name"]: i for i in items2} # type: Dict[str, Any] results = [] for stype in items2: if "extends" in stype: specs = {} # type: Dict[str, str] if "specialize" in stype: for spec in aslist(stype["specialize"]): specs[spec["specializeFrom"]] = spec["specializeTo"] exfields = [] # type: List[Any] exsym = [] # type: List[str] for ex in aslist(stype["extends"]): if ex not in types: raise ValidationException( "Extends {} in {} refers to invalid base type.".format( stype["extends"], stype["name"])) basetype = copy.copy(types[ex]) if stype["type"] == "record": if specs: basetype["fields"] = replace_type( basetype.get("fields", []), specs, loader, set()) for field in basetype.get("fields", []): if "inherited_from" not in field: field["inherited_from"] = ex exfields.extend(basetype.get("fields", [])) elif stype["type"] == "enum": exsym.extend(basetype.get("symbols", [])) if stype["type"] == "record": stype = copy.copy(stype) combined_fields = [] fields = stype.get("fields", []) # We use short names here so that if a type inherits a field # (e.g. Child#id) from a parent (Parent#id) we avoid adding # the same field twice (previously we had just # ``exfields.extends(stype.fields)``). sns_fields = { shortname(field["name"]): field for field in fields } sns_exfields = { shortname(exfield["name"]): exfield for exfield in exfields } # N.B.: This could be simpler. We could have a single loop # to create the list of fields. The reason for this more # convoluted solution is to make sure we keep the order # of ``exfields`` first, and then the type fields. That's # because we have unit tests that rely on the order that # fields are written. Codegen output changes as well. # We are relying on the insertion order preserving # property of python dicts (i.e. relyig on Py3.5+). # First pass adding the exfields. for sn_exfield, exfield in sns_exfields.items(): field = sns_fields.get(sn_exfield, None) if field is None: field = exfield else: # make sure field name has not been used yet if not is_subtype(exfield["type"], field["type"]): raise SchemaParseException( f"Field name {field['name']} already in use with " "incompatible type. " f"{field['type']} vs {exfield['type']}.") combined_fields.append(field) # Second pass, now add the ones that are specific to the subtype. for field in sns_fields.values(): if field not in combined_fields: combined_fields.append(field) stype["fields"] = combined_fields fieldnames = set() # type: Set[str] for field in stype["fields"]: if field["name"] in fieldnames: raise ValidationException( "Field name {} appears twice in {}".format( field["name"], stype["name"])) else: fieldnames.add(field["name"]) elif stype["type"] == "enum": stype = copy.copy(stype) exsym.extend(stype.get("symbols", [])) stype["symbols"] = exsym types[stype["name"]] = stype results.append(stype) ex_types = {} for result in results: ex_types[result["name"]] = result extended_by = {} # type: Dict[str, str] for result in results: if "extends" in result: for ex in aslist(result["extends"]): if ex_types[ex].get("abstract"): add_dictlist(extended_by, ex, ex_types[result["name"]]) add_dictlist(extended_by, validate.avro_type_name(ex), ex_types[ex]) for result in results: if result.get("abstract") and result["name"] not in extended_by: raise ValidationException( "{} is abstract but missing a concrete subtype".format( result["name"])) for result in results: if "fields" in result: result["fields"] = replace_type(result["fields"], extended_by, loader, set()) return results
def process_type(t, # type: Dict[str, Any] g, # type: Graph context, # type: ContextType defaultBase, # type: str namespaces, # type: Dict[str, rdflib.namespace.Namespace] defaultPrefix # type: str ): # type: (...) -> None if t["type"] not in ("record", "enum"): return if "name" in t: recordname = t["name"] _logger.debug("Processing %s %s\n", t.get("type"), t) classnode = URIRef(recordname) g.add((classnode, RDF.type, RDFS.Class)) split = urllib.parse.urlsplit(recordname) predicate = recordname if t.get("inVocab", True): if split.scheme: (ns, ln) = rdflib.namespace.split_uri(six.text_type(recordname)) predicate = recordname recordname = ln else: predicate = "%s:%s" % (defaultPrefix, recordname) if context.get(recordname, predicate) != predicate: raise Exception("Predicate collision on '%s', '%s' != '%s'" % ( recordname, context[recordname], predicate)) if not recordname: raise Exception() _logger.debug("Adding to context '%s' %s (%s)", recordname, predicate, type(predicate)) context[recordname] = predicate if t["type"] == "record": for i in t.get("fields", []): fieldname = i["name"] _logger.debug("Processing field %s", i) v = pred(t, i, fieldname, context, defaultPrefix, namespaces) # type: Union[Dict[Any, Any], Text, None] if isinstance(v, six.string_types): v = v if v[0] != "@" else None elif v is not None: v = v["_@id"] if v.get("_@id", "@")[0] != "@" else None if bool(v): (ns, ln) = rdflib.namespace.split_uri(six.text_type(v)) if ns[0:-1] in namespaces: propnode = namespaces[ns[0:-1]][ln] else: propnode = URIRef(v) g.add((propnode, RDF.type, RDF.Property)) g.add((propnode, RDFS.domain, classnode)) # TODO generate range from datatype. if isinstance(i["type"], dict): process_type(i["type"], g, context, defaultBase, namespaces, defaultPrefix) if "extends" in t: for e in aslist(t["extends"]): g.add((classnode, RDFS.subClassOf, URIRef(e))) elif t["type"] == "enum": _logger.debug("Processing enum %s", t.get("name")) for i in t["symbols"]: pred(t, None, i, context, defaultBase, namespaces)
def process_type( t, # type: Dict[str, Any] g, # type: Graph context, # type: ContextType defaultBase, # type: str namespaces, # type: Dict[str, rdflib.namespace.Namespace] defaultPrefix # type: str ): # type: (...) -> None if t["type"] not in ("record", "enum"): return if "name" in t: recordname = t["name"] _logger.debug("Processing %s %s\n", t.get("type"), t) classnode = URIRef(recordname) g.add((classnode, RDF.type, RDFS.Class)) split = urllib.parse.urlsplit(recordname) predicate = recordname if t.get("inVocab", True): if split.scheme: (ns, ln) = rdflib.namespace.split_uri(six.text_type(recordname)) predicate = recordname recordname = ln else: predicate = "%s:%s" % (defaultPrefix, recordname) if context.get(recordname, predicate) != predicate: raise Exception("Predicate collision on '%s', '%s' != '%s'" % (recordname, context[recordname], predicate)) if not recordname: raise Exception() _logger.debug("Adding to context '%s' %s (%s)", recordname, predicate, type(predicate)) context[recordname] = predicate if t["type"] == "record": for i in t.get("fields", []): fieldname = i["name"] _logger.debug("Processing field %s", i) v = pred(t, i, fieldname, context, defaultPrefix, namespaces) # type: Union[Dict[Any, Any], Text, None] if isinstance(v, six.string_types): v = v if v[0] != "@" else None elif v is not None: v = v["_@id"] if v.get("_@id", "@")[0] != "@" else None if bool(v): (ns, ln) = rdflib.namespace.split_uri(six.text_type(v)) if ns[0:-1] in namespaces: propnode = namespaces[ns[0:-1]][ln] else: propnode = URIRef(v) g.add((propnode, RDF.type, RDF.Property)) g.add((propnode, RDFS.domain, classnode)) # TODO generate range from datatype. if isinstance(i["type"], dict): process_type(i["type"], g, context, defaultBase, namespaces, defaultPrefix) if "extends" in t: for e in aslist(t["extends"]): g.add((classnode, RDFS.subClassOf, URIRef(e))) elif t["type"] == "enum": _logger.debug("Processing enum %s", t.get("name")) for i in t["symbols"]: pred(t, None, i, context, defaultBase, namespaces)
def extend_and_specialize(items, loader): # type: (List[Dict[Text, Any]], Loader) -> List[Dict[Text, Any]] """Apply 'extend' and 'specialize' to fully materialize derived record types.""" items = deepcopy_strip(items) types = {t["name"]: t for t in items} # type: Dict[Text, Any] n = [] for t in items: if "extends" in t: spec = {} # type: Dict[Text, Text] if "specialize" in t: for sp in aslist(t["specialize"]): spec[sp["specializeFrom"]] = sp["specializeTo"] exfields = [] # type: List[Text] exsym = [] # type: List[Text] for ex in aslist(t["extends"]): if ex not in types: raise Exception("Extends %s in %s refers to invalid base type" % ( t["extends"], t["name"])) basetype = copy.copy(types[ex]) if t["type"] == "record": if len(spec) > 0: basetype["fields"] = replace_type( basetype.get("fields", []), spec, loader, set()) for f in basetype.get("fields", []): if "inherited_from" not in f: f["inherited_from"] = ex exfields.extend(basetype.get("fields", [])) elif t["type"] == "enum": exsym.extend(basetype.get("symbols", [])) if t["type"] == "record": t = copy.copy(t) exfields.extend(t.get("fields", [])) t["fields"] = exfields fieldnames = set() # type: Set[Text] for field in t["fields"]: if field["name"] in fieldnames: raise validate.ValidationException( "Field name %s appears twice in %s" % (field["name"], t["name"])) else: fieldnames.add(field["name"]) elif t["type"] == "enum": t = copy.copy(t) exsym.extend(t.get("symbols", [])) t["symbol"] = exsym types[t["name"]] = t n.append(t) ex_types = {} for t in n: ex_types[t["name"]] = t extended_by = {} # type: Dict[Text, Text] for t in n: if "extends" in t: for ex in aslist(t["extends"]): if ex_types[ex].get("abstract"): add_dictlist(extended_by, ex, ex_types[t["name"]]) add_dictlist(extended_by, avro_name(ex), ex_types[ex]) for t in n: if t.get("abstract") and t["name"] not in extended_by: raise validate.ValidationException( "%s is abstract but missing a concrete subtype" % t["name"]) for t in n: if "fields" in t: t["fields"] = replace_type(t["fields"], extended_by, loader, set()) return n