def extend_and_specialize( items: List[Dict[str, Any]], loader: Loader ) -> List[Dict[str, Any]]: """ Apply 'extend' and 'specialize' to fully materialize derived record types. """ items2 = deepcopy_strip(items) types = {i["name"]: i for i in items2} # type: Dict[str, Any] results = [] for stype in items2: if "extends" in stype: specs = {} # type: Dict[str, str] if "specialize" in stype: for spec in aslist(stype["specialize"]): specs[spec["specializeFrom"]] = spec["specializeTo"] exfields = [] # type: List[str] exsym = [] # type: List[str] for ex in aslist(stype["extends"]): if ex not in types: raise ValidationException( "Extends {} in {} refers to invalid base type.".format( stype["extends"], stype["name"] ) ) basetype = copy.copy(types[ex]) if stype["type"] == "record": if specs: basetype["fields"] = replace_type( basetype.get("fields", []), specs, loader, set() ) for field in basetype.get("fields", []): if "inherited_from" not in field: field["inherited_from"] = ex exfields.extend(basetype.get("fields", [])) elif stype["type"] == "enum": exsym.extend(basetype.get("symbols", [])) if stype["type"] == "record": stype = copy.copy(stype) exfields.extend(stype.get("fields", [])) stype["fields"] = exfields fieldnames = set() # type: Set[str] for field in stype["fields"]: if field["name"] in fieldnames: raise ValidationException( "Field name {} appears twice in {}".format( field["name"], stype["name"] ) ) else: fieldnames.add(field["name"]) elif stype["type"] == "enum": stype = copy.copy(stype) exsym.extend(stype.get("symbols", [])) stype["symbol"] = exsym types[stype["name"]] = stype results.append(stype) ex_types = {} for result in results: ex_types[result["name"]] = result extended_by = {} # type: Dict[str, str] for result in results: if "extends" in result: for ex in aslist(result["extends"]): if ex_types[ex].get("abstract"): add_dictlist(extended_by, ex, ex_types[result["name"]]) add_dictlist(extended_by, avro_name(ex), ex_types[ex]) for result in results: if result.get("abstract") and result["name"] not in extended_by: raise ValidationException( "{} is abstract but missing a concrete subtype".format(result["name"]) ) for result in results: if "fields" in result: result["fields"] = replace_type( result["fields"], extended_by, loader, set() ) return results
def extend_and_specialize(items: List[Dict[str, Any]], loader: Loader) -> List[Dict[str, Any]]: """ Apply 'extend' and 'specialize' to fully materialize derived record types. """ items2 = deepcopy_strip(items) types = {i["name"]: i for i in items2} # type: Dict[str, Any] results = [] for stype in items2: if "extends" in stype: specs = {} # type: Dict[str, str] if "specialize" in stype: for spec in aslist(stype["specialize"]): specs[spec["specializeFrom"]] = spec["specializeTo"] exfields = [] # type: List[Any] exsym = [] # type: List[str] for ex in aslist(stype["extends"]): if ex not in types: raise ValidationException( "Extends {} in {} refers to invalid base type.".format( stype["extends"], stype["name"])) basetype = copy.copy(types[ex]) if stype["type"] == "record": if specs: basetype["fields"] = replace_type( basetype.get("fields", []), specs, loader, set()) for field in basetype.get("fields", []): if "inherited_from" not in field: field["inherited_from"] = ex exfields.extend(basetype.get("fields", [])) elif stype["type"] == "enum": exsym.extend(basetype.get("symbols", [])) if stype["type"] == "record": stype = copy.copy(stype) combined_fields = [] fields = stype.get("fields", []) # We use short names here so that if a type inherits a field # (e.g. Child#id) from a parent (Parent#id) we avoid adding # the same field twice (previously we had just # ``exfields.extends(stype.fields)``). sns_fields = { shortname(field["name"]): field for field in fields } sns_exfields = { shortname(exfield["name"]): exfield for exfield in exfields } # N.B.: This could be simpler. We could have a single loop # to create the list of fields. The reason for this more # convoluted solution is to make sure we keep the order # of ``exfields`` first, and then the type fields. That's # because we have unit tests that rely on the order that # fields are written. Codegen output changes as well. # We are relying on the insertion order preserving # property of python dicts (i.e. relyig on Py3.5+). # First pass adding the exfields. for sn_exfield, exfield in sns_exfields.items(): field = sns_fields.get(sn_exfield, None) if field is None: field = exfield else: # make sure field name has not been used yet if not is_subtype(exfield["type"], field["type"]): raise SchemaParseException( f"Field name {field['name']} already in use with " "incompatible type. " f"{field['type']} vs {exfield['type']}.") combined_fields.append(field) # Second pass, now add the ones that are specific to the subtype. for field in sns_fields.values(): if field not in combined_fields: combined_fields.append(field) stype["fields"] = combined_fields fieldnames = set() # type: Set[str] for field in stype["fields"]: if field["name"] in fieldnames: raise ValidationException( "Field name {} appears twice in {}".format( field["name"], stype["name"])) else: fieldnames.add(field["name"]) elif stype["type"] == "enum": stype = copy.copy(stype) exsym.extend(stype.get("symbols", [])) stype["symbols"] = exsym types[stype["name"]] = stype results.append(stype) ex_types = {} for result in results: ex_types[result["name"]] = result extended_by = {} # type: Dict[str, str] for result in results: if "extends" in result: for ex in aslist(result["extends"]): if ex_types[ex].get("abstract"): add_dictlist(extended_by, ex, ex_types[result["name"]]) add_dictlist(extended_by, validate.avro_type_name(ex), ex_types[ex]) for result in results: if result.get("abstract") and result["name"] not in extended_by: raise ValidationException( "{} is abstract but missing a concrete subtype".format( result["name"])) for result in results: if "fields" in result: result["fields"] = replace_type(result["fields"], extended_by, loader, set()) return results
def extend_and_specialize(items, loader): # type: (List[Dict[Text, Any]], Loader) -> List[Dict[Text, Any]] """Apply 'extend' and 'specialize' to fully materialize derived record types.""" items = deepcopy_strip(items) types = {t["name"]: t for t in items} # type: Dict[Text, Any] n = [] for t in items: if "extends" in t: spec = {} # type: Dict[Text, Text] if "specialize" in t: for sp in aslist(t["specialize"]): spec[sp["specializeFrom"]] = sp["specializeTo"] exfields = [] # type: List[Text] exsym = [] # type: List[Text] for ex in aslist(t["extends"]): if ex not in types: raise Exception( "Extends %s in %s refers to invalid base type" % (t["extends"], t["name"])) basetype = copy.copy(types[ex]) if t["type"] == "record": if len(spec) > 0: basetype["fields"] = replace_type( basetype.get("fields", []), spec, loader, set()) for f in basetype.get("fields", []): if "inherited_from" not in f: f["inherited_from"] = ex exfields.extend(basetype.get("fields", [])) elif t["type"] == "enum": exsym.extend(basetype.get("symbols", [])) if t["type"] == "record": t = copy.copy(t) exfields.extend(t.get("fields", [])) t["fields"] = exfields fieldnames = set() # type: Set[Text] for field in t["fields"]: if field["name"] in fieldnames: raise validate.ValidationException( "Field name %s appears twice in %s" % (field["name"], t["name"])) else: fieldnames.add(field["name"]) elif t["type"] == "enum": t = copy.copy(t) exsym.extend(t.get("symbols", [])) t["symbol"] = exsym types[t["name"]] = t n.append(t) ex_types = {} for t in n: ex_types[t["name"]] = t extended_by = {} # type: Dict[Text, Text] for t in n: if "extends" in t: for ex in aslist(t["extends"]): if ex_types[ex].get("abstract"): add_dictlist(extended_by, ex, ex_types[t["name"]]) add_dictlist(extended_by, avro_name(ex), ex_types[ex]) for t in n: if t.get("abstract") and t["name"] not in extended_by: raise validate.ValidationException( "%s is abstract but missing a concrete subtype" % t["name"]) for t in n: if "fields" in t: t["fields"] = replace_type(t["fields"], extended_by, loader, set()) return n
def extend_and_specialize(items, loader): # type: (List[Dict[Text, Any]], Loader) -> List[Dict[Text, Any]] """Apply 'extend' and 'specialize' to fully materialize derived record types.""" items = deepcopy_strip(items) types = {t["name"]: t for t in items} # type: Dict[Text, Any] n = [] for t in items: if "extends" in t: spec = {} # type: Dict[Text, Text] if "specialize" in t: for sp in aslist(t["specialize"]): spec[sp["specializeFrom"]] = sp["specializeTo"] exfields = [] # type: List[Text] exsym = [] # type: List[Text] for ex in aslist(t["extends"]): if ex not in types: raise Exception("Extends %s in %s refers to invalid base type" % ( t["extends"], t["name"])) basetype = copy.copy(types[ex]) if t["type"] == "record": if len(spec) > 0: basetype["fields"] = replace_type( basetype.get("fields", []), spec, loader, set()) for f in basetype.get("fields", []): if "inherited_from" not in f: f["inherited_from"] = ex exfields.extend(basetype.get("fields", [])) elif t["type"] == "enum": exsym.extend(basetype.get("symbols", [])) if t["type"] == "record": t = copy.copy(t) exfields.extend(t.get("fields", [])) t["fields"] = exfields fieldnames = set() # type: Set[Text] for field in t["fields"]: if field["name"] in fieldnames: raise validate.ValidationException( "Field name %s appears twice in %s" % (field["name"], t["name"])) else: fieldnames.add(field["name"]) elif t["type"] == "enum": t = copy.copy(t) exsym.extend(t.get("symbols", [])) t["symbol"] = exsym types[t["name"]] = t n.append(t) ex_types = {} for t in n: ex_types[t["name"]] = t extended_by = {} # type: Dict[Text, Text] for t in n: if "extends" in t: for ex in aslist(t["extends"]): if ex_types[ex].get("abstract"): add_dictlist(extended_by, ex, ex_types[t["name"]]) add_dictlist(extended_by, avro_name(ex), ex_types[ex]) for t in n: if t.get("abstract") and t["name"] not in extended_by: raise validate.ValidationException( "%s is abstract but missing a concrete subtype" % t["name"]) for t in n: if "fields" in t: t["fields"] = replace_type(t["fields"], extended_by, loader, set()) return n