def print_inheritance(doc, stream):
# type: (List[Dict[Text, Any]], IO) -> None
stream.write("digraph {\n")
for d in doc:
if d["type"] == "record":
label = shortname(d["name"])
if len(d.get("fields", [])) > 0:
label += "\\n* %s\\l" % ("\\l* ".join(shortname(f["name"]) for f in d.get("fields", [])))
stream.write("\"%s\" [shape=%s label=\"%s\"];\n" % (shortname(d["name"]), "ellipse" if d.get("abstract") else "box", label))
if "extends" in d:
for e in aslist(d["extends"]):
stream.write("\"%s\" -> \"%s\";\n" % (shortname(e), shortname(d["name"])))
stream.write("}\n")
def print_inheritance(doc, stream):
# type: (List[Dict[Text, Any]], IO) -> None
stream.write("digraph {\n")
for d in doc:
if d["type"] == "record":
label = shortname(d["name"])
if len(d.get("fields", [])) > 0:
label += "\\n* %s\\l" % ("\\l* ".join(
shortname(f["name"]) for f in d.get("fields", [])))
stream.write("\"%s\" [shape=%s label=\"%s\"];\n" % (shortname(
d["name"]), "ellipse" if d.get("abstract") else "box", label))
if "extends" in d:
for e in aslist(d["extends"]):
stream.write("\"%s\" -> \"%s\";\n" %
(shortname(e), shortname(d["name"])))
stream.write("}\n")
def print_inheritance(doc: List[Dict[str, Any]], stream: IO[Any]) -> None:
"""Write a Grapviz inheritance graph for the supplied document."""
stream.write("digraph {\n")
for entry in doc:
if entry["type"] == "record":
label = name = shortname(entry["name"])
fields = entry.get("fields", [])
if fields:
label += "\\n* {}\\l".format("\\l* ".join(
shortname(field["name"]) for field in fields))
shape = "ellipse" if entry.get("abstract") else "box"
stream.write(f'"{name}" [shape={shape} label="{label}"];\n')
if "extends" in entry:
for target in aslist(entry["extends"]):
stream.write(f'"{shortname(target)}" -> "{name}";\n')
stream.write("}\n")
def print_inheritance(doc, stream):
# type: (List[Dict[Text, Any]], IO[Any]) -> None
"""Write a Grapviz inheritance graph for the supplied document."""
stream.write("digraph {\n")
for entry in doc:
if entry["type"] == "record":
label = name = shortname(entry["name"])
fields = entry.get("fields", [])
if fields:
label += "\\n* %s\\l" % ("\\l* ".join(
shortname(field["name"]) for field in fields))
shape = "ellipse" if entry.get("abstract") else "box"
stream.write("\"%s\" [shape=%s label=\"%s\"];\n" %
(name, shape, label))
if "extends" in entry:
for target in aslist(entry["extends"]):
stream.write("\"%s\" -> \"%s\";\n" %
(shortname(target), name))
stream.write("}\n")
def extend_and_specialize(
items: List[Dict[str, Any]], loader: Loader
) -> List[Dict[str, Any]]:
"""
Apply 'extend' and 'specialize' to fully materialize derived record types.
"""
items2 = deepcopy_strip(items)
types = {i["name"]: i for i in items2} # type: Dict[str, Any]
results = []
for stype in items2:
if "extends" in stype:
specs = {} # type: Dict[str, str]
if "specialize" in stype:
for spec in aslist(stype["specialize"]):
specs[spec["specializeFrom"]] = spec["specializeTo"]
exfields = [] # type: List[str]
exsym = [] # type: List[str]
for ex in aslist(stype["extends"]):
if ex not in types:
raise ValidationException(
"Extends {} in {} refers to invalid base type.".format(
stype["extends"], stype["name"]
)
)
basetype = copy.copy(types[ex])
if stype["type"] == "record":
if specs:
basetype["fields"] = replace_type(
basetype.get("fields", []), specs, loader, set()
)
for field in basetype.get("fields", []):
if "inherited_from" not in field:
field["inherited_from"] = ex
exfields.extend(basetype.get("fields", []))
elif stype["type"] == "enum":
exsym.extend(basetype.get("symbols", []))
if stype["type"] == "record":
stype = copy.copy(stype)
exfields.extend(stype.get("fields", []))
stype["fields"] = exfields
fieldnames = set() # type: Set[str]
for field in stype["fields"]:
if field["name"] in fieldnames:
raise ValidationException(
"Field name {} appears twice in {}".format(
field["name"], stype["name"]
)
)
else:
fieldnames.add(field["name"])
elif stype["type"] == "enum":
stype = copy.copy(stype)
exsym.extend(stype.get("symbols", []))
stype["symbol"] = exsym
types[stype["name"]] = stype
results.append(stype)
ex_types = {}
for result in results:
ex_types[result["name"]] = result
extended_by = {} # type: Dict[str, str]
for result in results:
if "extends" in result:
for ex in aslist(result["extends"]):
if ex_types[ex].get("abstract"):
add_dictlist(extended_by, ex, ex_types[result["name"]])
add_dictlist(extended_by, avro_name(ex), ex_types[ex])
for result in results:
if result.get("abstract") and result["name"] not in extended_by:
raise ValidationException(
"{} is abstract but missing a concrete subtype".format(result["name"])
)
for result in results:
if "fields" in result:
result["fields"] = replace_type(
result["fields"], extended_by, loader, set()
)
return results
def extend_and_specialize(items, loader):
# type: (List[Dict[Text, Any]], Loader) -> List[Dict[Text, Any]]
"""Apply 'extend' and 'specialize' to fully materialize derived record
types."""
items = deepcopy_strip(items)
types = {t["name"]: t for t in items} # type: Dict[Text, Any]
n = []
for t in items:
if "extends" in t:
spec = {} # type: Dict[Text, Text]
if "specialize" in t:
for sp in aslist(t["specialize"]):
spec[sp["specializeFrom"]] = sp["specializeTo"]
exfields = [] # type: List[Text]
exsym = [] # type: List[Text]
for ex in aslist(t["extends"]):
if ex not in types:
raise Exception(
"Extends %s in %s refers to invalid base type" %
(t["extends"], t["name"]))
basetype = copy.copy(types[ex])
if t["type"] == "record":
if len(spec) > 0:
basetype["fields"] = replace_type(
basetype.get("fields", []), spec, loader, set())
for f in basetype.get("fields", []):
if "inherited_from" not in f:
f["inherited_from"] = ex
exfields.extend(basetype.get("fields", []))
elif t["type"] == "enum":
exsym.extend(basetype.get("symbols", []))
if t["type"] == "record":
t = copy.copy(t)
exfields.extend(t.get("fields", []))
t["fields"] = exfields
fieldnames = set() # type: Set[Text]
for field in t["fields"]:
if field["name"] in fieldnames:
raise validate.ValidationException(
"Field name %s appears twice in %s" %
(field["name"], t["name"]))
else:
fieldnames.add(field["name"])
elif t["type"] == "enum":
t = copy.copy(t)
exsym.extend(t.get("symbols", []))
t["symbol"] = exsym
types[t["name"]] = t
n.append(t)
ex_types = {}
for t in n:
ex_types[t["name"]] = t
extended_by = {} # type: Dict[Text, Text]
for t in n:
if "extends" in t:
for ex in aslist(t["extends"]):
if ex_types[ex].get("abstract"):
add_dictlist(extended_by, ex, ex_types[t["name"]])
add_dictlist(extended_by, avro_name(ex), ex_types[ex])
for t in n:
if t.get("abstract") and t["name"] not in extended_by:
raise validate.ValidationException(
"%s is abstract but missing a concrete subtype" % t["name"])
for t in n:
if "fields" in t:
t["fields"] = replace_type(t["fields"], extended_by, loader, set())
return n
def extend_and_specialize(items: List[Dict[str, Any]],
loader: Loader) -> List[Dict[str, Any]]:
"""
Apply 'extend' and 'specialize' to fully materialize derived record types.
"""
items2 = deepcopy_strip(items)
types = {i["name"]: i for i in items2} # type: Dict[str, Any]
results = []
for stype in items2:
if "extends" in stype:
specs = {} # type: Dict[str, str]
if "specialize" in stype:
for spec in aslist(stype["specialize"]):
specs[spec["specializeFrom"]] = spec["specializeTo"]
exfields = [] # type: List[Any]
exsym = [] # type: List[str]
for ex in aslist(stype["extends"]):
if ex not in types:
raise ValidationException(
"Extends {} in {} refers to invalid base type.".format(
stype["extends"], stype["name"]))
basetype = copy.copy(types[ex])
if stype["type"] == "record":
if specs:
basetype["fields"] = replace_type(
basetype.get("fields", []), specs, loader, set())
for field in basetype.get("fields", []):
if "inherited_from" not in field:
field["inherited_from"] = ex
exfields.extend(basetype.get("fields", []))
elif stype["type"] == "enum":
exsym.extend(basetype.get("symbols", []))
if stype["type"] == "record":
stype = copy.copy(stype)
combined_fields = []
fields = stype.get("fields", [])
# We use short names here so that if a type inherits a field
# (e.g. Child#id) from a parent (Parent#id) we avoid adding
# the same field twice (previously we had just
# ``exfields.extends(stype.fields)``).
sns_fields = {
shortname(field["name"]): field
for field in fields
}
sns_exfields = {
shortname(exfield["name"]): exfield
for exfield in exfields
}
# N.B.: This could be simpler. We could have a single loop
# to create the list of fields. The reason for this more
# convoluted solution is to make sure we keep the order
# of ``exfields`` first, and then the type fields. That's
# because we have unit tests that rely on the order that
# fields are written. Codegen output changes as well.
# We are relying on the insertion order preserving
# property of python dicts (i.e. relyig on Py3.5+).
# First pass adding the exfields.
for sn_exfield, exfield in sns_exfields.items():
field = sns_fields.get(sn_exfield, None)
if field is None:
field = exfield
else:
# make sure field name has not been used yet
if not is_subtype(exfield["type"], field["type"]):
raise SchemaParseException(
f"Field name {field['name']} already in use with "
"incompatible type. "
f"{field['type']} vs {exfield['type']}.")
combined_fields.append(field)
# Second pass, now add the ones that are specific to the subtype.
for field in sns_fields.values():
if field not in combined_fields:
combined_fields.append(field)
stype["fields"] = combined_fields
fieldnames = set() # type: Set[str]
for field in stype["fields"]:
if field["name"] in fieldnames:
raise ValidationException(
"Field name {} appears twice in {}".format(
field["name"], stype["name"]))
else:
fieldnames.add(field["name"])
elif stype["type"] == "enum":
stype = copy.copy(stype)
exsym.extend(stype.get("symbols", []))
stype["symbols"] = exsym
types[stype["name"]] = stype
results.append(stype)
ex_types = {}
for result in results:
ex_types[result["name"]] = result
extended_by = {} # type: Dict[str, str]
for result in results:
if "extends" in result:
for ex in aslist(result["extends"]):
if ex_types[ex].get("abstract"):
add_dictlist(extended_by, ex, ex_types[result["name"]])
add_dictlist(extended_by, validate.avro_type_name(ex),
ex_types[ex])
for result in results:
if result.get("abstract") and result["name"] not in extended_by:
raise ValidationException(
"{} is abstract but missing a concrete subtype".format(
result["name"]))
for result in results:
if "fields" in result:
result["fields"] = replace_type(result["fields"], extended_by,
loader, set())
return results
def process_type(t, # type: Dict[str, Any]
g, # type: Graph
context, # type: ContextType
defaultBase, # type: str
namespaces, # type: Dict[str, rdflib.namespace.Namespace]
defaultPrefix # type: str
):
# type: (...) -> None
if t["type"] not in ("record", "enum"):
return
if "name" in t:
recordname = t["name"]
_logger.debug("Processing %s %s\n", t.get("type"), t)
classnode = URIRef(recordname)
g.add((classnode, RDF.type, RDFS.Class))
split = urllib.parse.urlsplit(recordname)
predicate = recordname
if t.get("inVocab", True):
if split.scheme:
(ns, ln) = rdflib.namespace.split_uri(six.text_type(recordname))
predicate = recordname
recordname = ln
else:
predicate = "%s:%s" % (defaultPrefix, recordname)
if context.get(recordname, predicate) != predicate:
raise Exception("Predicate collision on '%s', '%s' != '%s'" % (
recordname, context[recordname], predicate))
if not recordname:
raise Exception()
_logger.debug("Adding to context '%s' %s (%s)",
recordname, predicate, type(predicate))
context[recordname] = predicate
if t["type"] == "record":
for i in t.get("fields", []):
fieldname = i["name"]
_logger.debug("Processing field %s", i)
v = pred(t, i, fieldname, context, defaultPrefix,
namespaces) # type: Union[Dict[Any, Any], Text, None]
if isinstance(v, six.string_types):
v = v if v[0] != "@" else None
elif v is not None:
v = v["_@id"] if v.get("_@id", "@")[0] != "@" else None
if bool(v):
(ns, ln) = rdflib.namespace.split_uri(six.text_type(v))
if ns[0:-1] in namespaces:
propnode = namespaces[ns[0:-1]][ln]
else:
propnode = URIRef(v)
g.add((propnode, RDF.type, RDF.Property))
g.add((propnode, RDFS.domain, classnode))
# TODO generate range from datatype.
if isinstance(i["type"], dict):
process_type(i["type"], g, context, defaultBase,
namespaces, defaultPrefix)
if "extends" in t:
for e in aslist(t["extends"]):
g.add((classnode, RDFS.subClassOf, URIRef(e)))
elif t["type"] == "enum":
_logger.debug("Processing enum %s", t.get("name"))
for i in t["symbols"]:
pred(t, None, i, context, defaultBase, namespaces)
def process_type(
t, # type: Dict[str, Any]
g, # type: Graph
context, # type: ContextType
defaultBase, # type: str
namespaces, # type: Dict[str, rdflib.namespace.Namespace]
defaultPrefix # type: str
):
# type: (...) -> None
if t["type"] not in ("record", "enum"):
return
if "name" in t:
recordname = t["name"]
_logger.debug("Processing %s %s\n", t.get("type"), t)
classnode = URIRef(recordname)
g.add((classnode, RDF.type, RDFS.Class))
split = urllib.parse.urlsplit(recordname)
predicate = recordname
if t.get("inVocab", True):
if split.scheme:
(ns,
ln) = rdflib.namespace.split_uri(six.text_type(recordname))
predicate = recordname
recordname = ln
else:
predicate = "%s:%s" % (defaultPrefix, recordname)
if context.get(recordname, predicate) != predicate:
raise Exception("Predicate collision on '%s', '%s' != '%s'" %
(recordname, context[recordname], predicate))
if not recordname:
raise Exception()
_logger.debug("Adding to context '%s' %s (%s)", recordname, predicate,
type(predicate))
context[recordname] = predicate
if t["type"] == "record":
for i in t.get("fields", []):
fieldname = i["name"]
_logger.debug("Processing field %s", i)
v = pred(t, i, fieldname, context, defaultPrefix,
namespaces) # type: Union[Dict[Any, Any], Text, None]
if isinstance(v, six.string_types):
v = v if v[0] != "@" else None
elif v is not None:
v = v["_@id"] if v.get("_@id", "@")[0] != "@" else None
if bool(v):
(ns, ln) = rdflib.namespace.split_uri(six.text_type(v))
if ns[0:-1] in namespaces:
propnode = namespaces[ns[0:-1]][ln]
else:
propnode = URIRef(v)
g.add((propnode, RDF.type, RDF.Property))
g.add((propnode, RDFS.domain, classnode))
# TODO generate range from datatype.
if isinstance(i["type"], dict):
process_type(i["type"], g, context, defaultBase, namespaces,
defaultPrefix)
if "extends" in t:
for e in aslist(t["extends"]):
g.add((classnode, RDFS.subClassOf, URIRef(e)))
elif t["type"] == "enum":
_logger.debug("Processing enum %s", t.get("name"))
for i in t["symbols"]:
pred(t, None, i, context, defaultBase, namespaces)
def extend_and_specialize(items, loader):
# type: (List[Dict[Text, Any]], Loader) -> List[Dict[Text, Any]]
"""Apply 'extend' and 'specialize' to fully materialize derived record
types."""
items = deepcopy_strip(items)
types = {t["name"]: t for t in items} # type: Dict[Text, Any]
n = []
for t in items:
if "extends" in t:
spec = {} # type: Dict[Text, Text]
if "specialize" in t:
for sp in aslist(t["specialize"]):
spec[sp["specializeFrom"]] = sp["specializeTo"]
exfields = [] # type: List[Text]
exsym = [] # type: List[Text]
for ex in aslist(t["extends"]):
if ex not in types:
raise Exception("Extends %s in %s refers to invalid base type" % (
t["extends"], t["name"]))
basetype = copy.copy(types[ex])
if t["type"] == "record":
if len(spec) > 0:
basetype["fields"] = replace_type(
basetype.get("fields", []), spec, loader, set())
for f in basetype.get("fields", []):
if "inherited_from" not in f:
f["inherited_from"] = ex
exfields.extend(basetype.get("fields", []))
elif t["type"] == "enum":
exsym.extend(basetype.get("symbols", []))
if t["type"] == "record":
t = copy.copy(t)
exfields.extend(t.get("fields", []))
t["fields"] = exfields
fieldnames = set() # type: Set[Text]
for field in t["fields"]:
if field["name"] in fieldnames:
raise validate.ValidationException(
"Field name %s appears twice in %s" % (field["name"], t["name"]))
else:
fieldnames.add(field["name"])
elif t["type"] == "enum":
t = copy.copy(t)
exsym.extend(t.get("symbols", []))
t["symbol"] = exsym
types[t["name"]] = t
n.append(t)
ex_types = {}
for t in n:
ex_types[t["name"]] = t
extended_by = {} # type: Dict[Text, Text]
for t in n:
if "extends" in t:
for ex in aslist(t["extends"]):
if ex_types[ex].get("abstract"):
add_dictlist(extended_by, ex, ex_types[t["name"]])
add_dictlist(extended_by, avro_name(ex), ex_types[ex])
for t in n:
if t.get("abstract") and t["name"] not in extended_by:
raise validate.ValidationException(
"%s is abstract but missing a concrete subtype" % t["name"])
for t in n:
if "fields" in t:
t["fields"] = replace_type(t["fields"], extended_by, loader, set())
return n
