def extend_avro(items):
types = {t["name"]: t for t in items}
n = []
for t in items:
if "extends" in t:
r = copy.deepcopy(types[t["extends"]])
r["name"] = t["name"]
if "specialize" in t:
r["fields"] = replace_type(r["fields"], t["specialize"])
for f in r["fields"]:
if "inherited_from" not in f:
f["inherited_from"] = t["extends"]
r["fields"].extend(t.get("fields", []))
for y in [x for x in r["fields"] if x["name"] == "class"]:
y["type"] = {"type": "enum",
"symbols": [r["name"]],
"name": r["name"]+"_class",
}
y["doc"] = "Must be `%s` to indicate this is a %s object." % (r["name"], r["name"])
r["extends"] = t["extends"]
r["abstract"] = t.get("abstract", False)
r["doc"] = t.get("doc", "")
types[t["name"]] = r
t = r
n.append(t)
ex_types = {t["name"]: t for t in n}
extended_by = {}
for t in n:
if "extends" in t and ex_types[t["extends"]].get("abstract"):
add_dictlist(extended_by, t["extends"], ex_types[t["name"]])
for t in n:
if "fields" in t:
t["fields"] = replace_type(t["fields"], extended_by)
n = replace_type(n, ex_types)
return n
def extend_and_specialize(items, loader):
"""Apply 'extend' and 'specialize' to fully materialize derived record
types."""
types = {t["name"]: t for t in items}
n = []
for t in items:
t = copy.deepcopy(t)
if "extends" in t:
if "specialize" in t:
spec = {
sp["specializeFrom"]: sp["specializeTo"]
for sp in aslist(t["specialize"])
}
else:
spec = {}
exfields = []
exsym = []
for ex in aslist(t["extends"]):
if ex not in types:
raise Exception(
"Extends %s in %s refers to invalid base type" %
(t["extends"], t["name"]))
basetype = copy.deepcopy(types[ex])
if t["type"] == "record":
if spec:
basetype["fields"] = replace_type(
basetype.get("fields", []), spec, loader, set())
for f in basetype.get("fields", []):
if "inherited_from" not in f:
f["inherited_from"] = ex
exfields.extend(basetype.get("fields", []))
elif t["type"] == "enum":
exsym.extend(basetype.get("symbols", []))
if t["type"] == "record":
exfields.extend(t.get("fields", []))
t["fields"] = exfields
fieldnames = set()
for field in t["fields"]:
if field["name"] in fieldnames:
raise validate.ValidationException(
"Field name %s appears twice in %s" %
(field["name"], t["name"]))
else:
fieldnames.add(field["name"])
for y in [x for x in t["fields"] if x["name"] == "class"]:
y["type"] = {
"type": "enum",
"symbols": [r["name"]],
"name": r["name"] + "_class",
}
y["doc"] = "Must be `%s` to indicate this is a %s object." % (
r["name"], r["name"])
elif t["type"] == "enum":
exsym.extend(t.get("symbols", []))
t["symbol"] = exsym
types[t["name"]] = t
n.append(t)
ex_types = {t["name"]: t for t in n}
extended_by = {}
for t in n:
if "extends" in t:
for ex in aslist(t["extends"]):
if ex_types[ex].get("abstract"):
add_dictlist(extended_by, ex, ex_types[t["name"]])
add_dictlist(extended_by, avro_name(ex), ex_types[ex])
for t in n:
if "fields" in t:
t["fields"] = replace_type(t["fields"], extended_by, loader, set())
return n
def extend_and_specialize(items, loader):
"""Apply 'extend' and 'specialize' to fully materialize derived record
types."""
types = {t["name"]: t for t in items}
n = []
for t in items:
t = copy.deepcopy(t)
if "extends" in t:
if "specialize" in t:
spec = {sp["specializeFrom"]: sp["specializeTo"] for sp in aslist(t["specialize"])}
else:
spec = {}
exfields = []
exsym = []
for ex in aslist(t["extends"]):
if ex not in types:
raise Exception("Extends %s in %s refers to invalid base type" % (t["extends"], t["name"]))
basetype = copy.deepcopy(types[ex])
if t["type"] == "record":
if spec:
basetype["fields"] = replace_type(basetype.get("fields", []), spec, loader, set())
for f in basetype.get("fields", []):
if "inherited_from" not in f:
f["inherited_from"] = ex
exfields.extend(basetype.get("fields", []))
elif t["type"] == "enum":
exsym.extend(basetype.get("symbols", []))
if t["type"] == "record":
exfields.extend(t.get("fields", []))
t["fields"] = exfields
fieldnames = set()
for field in t["fields"]:
if field["name"] in fieldnames:
raise validate.ValidationException("Field name %s appears twice in %s" % (field["name"], t["name"]))
else:
fieldnames.add(field["name"])
for y in [x for x in t["fields"] if x["name"] == "class"]:
y["type"] = {"type": "enum",
"symbols": [r["name"]],
"name": r["name"]+"_class",
}
y["doc"] = "Must be `%s` to indicate this is a %s object." % (r["name"], r["name"])
elif t["type"] == "enum":
exsym.extend(t.get("symbols", []))
t["symbol"] = exsym
types[t["name"]] = t
n.append(t)
ex_types = {t["name"]: t for t in n}
extended_by = {}
for t in n:
if "extends" in t:
for ex in aslist(t["extends"]):
if ex_types[ex].get("abstract"):
add_dictlist(extended_by, ex, ex_types[t["name"]])
add_dictlist(extended_by, avro_name(ex), ex_types[ex])
for t in n:
if "fields" in t:
t["fields"] = replace_type(t["fields"], extended_by, loader, set())
return n
