def test_serde_to_avro(pytestconfig, json_filename):
# In this test, we want to read in from JSON -> MCE object.
# Next we serialize from MCE to Avro and then deserialize back to MCE.
# Finally, we want to compare the two MCE objects.
json_path = pytestconfig.rootpath / json_filename
mces = list(iterate_mce_file(str(json_path)))
# Serialize to Avro.
parsed_schema = fastavro.parse_schema(json.loads(SCHEMA_JSON_STR))
fo = io.BytesIO()
out_records = [mce.to_obj(tuples=True) for mce in mces]
fastavro.writer(fo, parsed_schema, out_records)
# Deserialized from Avro.
fo.seek(0)
in_records = list(fastavro.reader(fo))
in_mces = [
MetadataChangeEventClass.from_obj(record, tuples=True)
for record in in_records
]
# Check diff
assert len(mces) == len(in_mces)
for i in range(len(mces)):
assert str(mces[i]) == str(in_mces[i])
def assert_entity_mce_aspect(entity_urn: str, aspect: Any, aspect_type: Type,
file: str) -> int:
test_output = load_json_file(file)
entity_type = Urn.create_from_string(entity_urn).get_type()
assert isinstance(test_output, list)
# mce urns
mces: List[MetadataChangeEventClass] = [
MetadataChangeEventClass.from_obj(x) for x in test_output
if _get_filter(mce=True, entity_type=entity_type)(x)
and _get_element(x, _get_mce_urn_path_spec(entity_type)) == entity_urn
]
matches = 0
for mce in mces:
for a in mce.proposedSnapshot.aspects:
if isinstance(a, aspect_type):
assert a == aspect
matches = matches + 1
return matches
def read_mces(path: os.PathLike) -> List[MetadataChangeEventClass]:
with open(path) as f:
objs = json.load(f)
mces = [MetadataChangeEventClass.from_obj(obj) for obj in objs]
return mces