def test_simple_schema_promotion(self):
field_alias_reader = parse(
json.dumps({
"name":
"foo",
"type":
"record",
"fields": [{
"type": "int",
"name": "bar",
"aliases": ["f1"]
}],
}))
record_alias_reader = parse(
json.dumps({
"name": "other",
"type": "record",
"fields": [{
"type": "int",
"name": "f1"
}],
"aliases": ["foo"],
}))
writer = parse(
json.dumps({
"name":
"foo",
"type":
"record",
"fields": [
{
"type": "int",
"name": "f1"
},
{
"type": "string",
"name": "f2",
},
],
}))
# alias testing
res = ReaderWriterCompatibilityChecker().get_compatibility(
field_alias_reader, writer)
self.assertIs(res.compatibility, SchemaCompatibilityType.compatible,
res.locations)
res = ReaderWriterCompatibilityChecker().get_compatibility(
record_alias_reader, writer)
self.assertIs(res.compatibility, SchemaCompatibilityType.compatible,
res.locations)
def test_schema_compatibility_fixed_size_mismatch(self):
incompatible_fixed_pairs = [
(FIXED_4_BYTES, FIXED_8_BYTES, "expected: 8, found: 4", "/size"),
(FIXED_8_BYTES, FIXED_4_BYTES, "expected: 4, found: 8", "/size"),
(
A_DINT_B_DFIXED_8_BYTES_RECORD1,
A_DINT_B_DFIXED_4_BYTES_RECORD1,
"expected: 4, found: 8",
"/fields/1/type/size",
),
(
A_DINT_B_DFIXED_4_BYTES_RECORD1,
A_DINT_B_DFIXED_8_BYTES_RECORD1,
"expected: 8, found: 4",
"/fields/1/type/size",
),
]
for (reader, writer, message, location) in incompatible_fixed_pairs:
result = ReaderWriterCompatibilityChecker().get_compatibility(
reader, writer)
self.assertIs(result.compatibility,
SchemaCompatibilityType.incompatible)
self.assertIn(
location,
result.locations,
f"expected {location}, found {result}",
)
self.assertIn(
message,
result.messages,
f"expected {location}, found {result}",
)
def test_schema_compatibility_missing_union_branch(self):
incompatible_pairs = [
(INT_UNION_SCHEMA, INT_STRING_UNION_SCHEMA, {"reader union lacking writer type: STRING"}, {"/1"}),
(STRING_UNION_SCHEMA, INT_STRING_UNION_SCHEMA, {"reader union lacking writer type: INT"}, {"/0"}),
(INT_UNION_SCHEMA, UNION_INT_RECORD1, {"reader union lacking writer type: RECORD"}, {"/1"}),
(INT_UNION_SCHEMA, UNION_INT_RECORD2, {"reader union lacking writer type: RECORD"}, {"/1"}),
(UNION_INT_RECORD1, UNION_INT_RECORD2, {"reader union lacking writer type: RECORD"}, {"/1"}),
(INT_UNION_SCHEMA, UNION_INT_ENUM1_AB, {"reader union lacking writer type: ENUM"}, {"/1"}),
(INT_UNION_SCHEMA, UNION_INT_FIXED_4_BYTES, {"reader union lacking writer type: FIXED"}, {"/1"}),
(INT_UNION_SCHEMA, UNION_INT_BOOLEAN, {"reader union lacking writer type: BOOLEAN"}, {"/1"}),
(INT_UNION_SCHEMA, LONG_UNION_SCHEMA, {"reader union lacking writer type: LONG"}, {"/0"}),
(INT_UNION_SCHEMA, FLOAT_UNION_SCHEMA, {"reader union lacking writer type: FLOAT"}, {"/0"}),
(INT_UNION_SCHEMA, DOUBLE_UNION_SCHEMA, {"reader union lacking writer type: DOUBLE"}, {"/0"}),
(INT_UNION_SCHEMA, BYTES_UNION_SCHEMA, {"reader union lacking writer type: BYTES"}, {"/0"}),
(INT_UNION_SCHEMA, UNION_INT_ARRAY_INT, {"reader union lacking writer type: ARRAY"}, {"/1"}),
(INT_UNION_SCHEMA, UNION_INT_MAP_INT, {"reader union lacking writer type: MAP"}, {"/1"}),
(INT_UNION_SCHEMA, UNION_INT_NULL, {"reader union lacking writer type: NULL"}, {"/1"}),
(
INT_UNION_SCHEMA, INT_LONG_FLOAT_DOUBLE_UNION_SCHEMA, {
"reader union lacking writer type: LONG", "reader union lacking writer type: FLOAT",
"reader union lacking writer type: DOUBLE"
}, {"/1", "/2", "/3"}
),
(
A_DINT_B_DINT_UNION_RECORD1, A_DINT_B_DINT_STRING_UNION_RECORD1, {"reader union lacking writer type: STRING"},
{"/fields/1/type/1"}
),
]
for (reader, writer, message, location) in incompatible_pairs:
result = ReaderWriterCompatibilityChecker().get_compatibility(reader, writer)
self.assertIs(result.compatibility, SchemaCompatibilityType.incompatible)
self.assertEqual(result.messages, message)
self.assertEqual(result.locations, location)
def test_schema_compatibility_reader_field_missing_default_value(self):
incompatible_pairs = [
(A_INT_RECORD1, EMPTY_RECORD1, "a", "/fields/0"),
(A_INT_B_DINT_RECORD1, EMPTY_RECORD1, "a", "/fields/0"),
]
for (reader, writer, message, location) in incompatible_pairs:
result = ReaderWriterCompatibilityChecker().get_compatibility(reader, writer)
self.assertIs(result.compatibility, SchemaCompatibilityType.incompatible)
self.assertEqual(len(result.messages), 1)
self.assertEqual(len(result.locations), 1)
self.assertEqual(message, "".join(result.messages))
self.assertEqual(location, "".join(result.locations))
def test_schema_compatibility_name_mismatch(self):
incompatible_pairs = [
(ENUM1_AB_SCHEMA, ENUM2_AB_SCHEMA, "expected: Enum2", "/name"),
(EMPTY_RECORD2, EMPTY_RECORD1, "expected: Record1", "/name"),
(FIXED_4_BYTES, FIXED_4_ANOTHER_NAME, "expected: AnotherName", "/name"),
(A_DINT_B_DENUM_1_RECORD1, A_DINT_B_DENUM_2_RECORD1, "expected: Enum2", "/fields/1/type/name")]
for (reader, writer, message, location) in incompatible_pairs:
result = ReaderWriterCompatibilityChecker().get_compatibility(reader, writer)
self.assertIs(result.compatibility, SchemaCompatibilityType.incompatible)
self.assertIn(message, result.messages)
self.assertIn(location, result.locations)
def test_schema_compatibility_missing_enum_symbols(self):
incompatible_pairs = [
# str(set) representation
(ENUM1_AB_SCHEMA, ENUM1_ABC_SCHEMA, "{'C'}", "/symbols"),
(ENUM1_BC_SCHEMA, ENUM1_ABC_SCHEMA, "{'A'}", "/symbols"),
(RECORD1_WITH_ENUM_AB, RECORD1_WITH_ENUM_ABC, "{'C'}", "/fields/0/type/symbols"),
]
for (reader, writer, message, location) in incompatible_pairs:
result = ReaderWriterCompatibilityChecker().get_compatibility(reader, writer)
self.assertIs(result.compatibility, SchemaCompatibilityType.incompatible)
self.assertIn(message, result.messages)
self.assertIn(location, result.locations)
def test_schema_compatibility_reader_field_missing_default_value():
incompatible_pairs = [
(A_INT_RECORD1, EMPTY_RECORD1, "a", "/fields/0"),
(A_INT_B_DINT_RECORD1, EMPTY_RECORD1, "a", "/fields/0"),
]
for (reader, writer, message, location) in incompatible_pairs:
result = ReaderWriterCompatibilityChecker().get_compatibility(
reader, writer)
assert result.compatibility is SchemaCompatibilityType.incompatible
assert len(result.messages) == 1 and len(result.locations) == 1
assert message == "".join(result.messages)
assert location == "".join(result.locations)
def test_schema_compatibility_fixed_size_mismatch():
incompatible_fixed_pairs = [
(FIXED_4_BYTES, FIXED_8_BYTES, "expected: 8, found: 4", "/size"),
(FIXED_8_BYTES, FIXED_4_BYTES, "expected: 4, found: 8", "/size"),
(A_DINT_B_DFIXED_8_BYTES_RECORD1, A_DINT_B_DFIXED_4_BYTES_RECORD1,
"expected: 4, found: 8", "/fields/1/type/size"),
(A_DINT_B_DFIXED_4_BYTES_RECORD1, A_DINT_B_DFIXED_8_BYTES_RECORD1,
"expected: 8, found: 4", "/fields/1/type/size"),
]
for (reader, writer, message, location) in incompatible_fixed_pairs:
result = ReaderWriterCompatibilityChecker().get_compatibility(
reader, writer)
assert result.compatibility is SchemaCompatibilityType.incompatible
assert location in result.locations, "expected {}, found {}".format(
location, result)
assert message in result.messages, "expected {}, found {}".format(
location, result)
def test_schema_compatibility_type_mismatch(self):
incompatible_pairs = [
(NULL_SCHEMA, INT_SCHEMA, "reader type: NULL not compatible with writer type: INT", "/"),
(NULL_SCHEMA, LONG_SCHEMA, "reader type: NULL not compatible with writer type: LONG", "/"),
(BOOLEAN_SCHEMA, INT_SCHEMA, "reader type: BOOLEAN not compatible with writer type: INT", "/"),
(INT_SCHEMA, NULL_SCHEMA, "reader type: INT not compatible with writer type: NULL", "/"),
(INT_SCHEMA, BOOLEAN_SCHEMA, "reader type: INT not compatible with writer type: BOOLEAN", "/"),
(INT_SCHEMA, LONG_SCHEMA, "reader type: INT not compatible with writer type: LONG", "/"),
(INT_SCHEMA, FLOAT_SCHEMA, "reader type: INT not compatible with writer type: FLOAT", "/"),
(INT_SCHEMA, DOUBLE_SCHEMA, "reader type: INT not compatible with writer type: DOUBLE", "/"),
(LONG_SCHEMA, FLOAT_SCHEMA, "reader type: LONG not compatible with writer type: FLOAT", "/"),
(LONG_SCHEMA, DOUBLE_SCHEMA, "reader type: LONG not compatible with writer type: DOUBLE", "/"),
(FLOAT_SCHEMA, DOUBLE_SCHEMA, "reader type: FLOAT not compatible with writer type: DOUBLE", "/"),
(DOUBLE_SCHEMA, STRING_SCHEMA, "reader type: DOUBLE not compatible with writer type: STRING", "/"),
(FIXED_4_BYTES, STRING_SCHEMA, "reader type: FIXED not compatible with writer type: STRING", "/"),
(STRING_SCHEMA, BOOLEAN_SCHEMA, "reader type: STRING not compatible with writer type: BOOLEAN", "/"),
(STRING_SCHEMA, INT_SCHEMA, "reader type: STRING not compatible with writer type: INT", "/"),
(BYTES_SCHEMA, NULL_SCHEMA, "reader type: BYTES not compatible with writer type: NULL", "/"),
(BYTES_SCHEMA, INT_SCHEMA, "reader type: BYTES not compatible with writer type: INT", "/"),
(A_INT_RECORD1, INT_SCHEMA, "reader type: RECORD not compatible with writer type: INT", "/"),
(INT_ARRAY_SCHEMA, LONG_ARRAY_SCHEMA, "reader type: INT not compatible with writer type: LONG", "/items"),
(INT_MAP_SCHEMA, INT_ARRAY_SCHEMA, "reader type: MAP not compatible with writer type: ARRAY", "/"),
(INT_ARRAY_SCHEMA, INT_MAP_SCHEMA, "reader type: ARRAY not compatible with writer type: MAP", "/"),
(INT_MAP_SCHEMA, LONG_MAP_SCHEMA, "reader type: INT not compatible with writer type: LONG", "/values"),
(INT_SCHEMA, ENUM2_AB_SCHEMA, "reader type: INT not compatible with writer type: ENUM", "/"),
(ENUM2_AB_SCHEMA, INT_SCHEMA, "reader type: ENUM not compatible with writer type: INT", "/"),
(
FLOAT_SCHEMA, INT_LONG_FLOAT_DOUBLE_UNION_SCHEMA, "reader type: FLOAT not compatible with writer type: DOUBLE",
"/"
),
(LONG_SCHEMA, INT_FLOAT_UNION_SCHEMA, "reader type: LONG not compatible with writer type: FLOAT", "/"),
(INT_SCHEMA, INT_FLOAT_UNION_SCHEMA, "reader type: INT not compatible with writer type: FLOAT", "/"),
# (INT_LIST_RECORD, LONG_LIST_RECORD, "reader type: INT not compatible with writer type: LONG", "/fields/0/type"),
(NULL_SCHEMA, INT_SCHEMA, "reader type: NULL not compatible with writer type: INT", "/"),
]
for (reader, writer, message, location) in incompatible_pairs:
result = ReaderWriterCompatibilityChecker().get_compatibility(reader, writer)
self.assertIs(result.compatibility, SchemaCompatibilityType.incompatible)
self.assertIn(message, result.messages)
self.assertIn(location, result.locations)
def are_compatible(self, reader: Schema, writer: Schema) -> bool:
return ReaderWriterCompatibilityChecker().get_compatibility(
reader, writer).compatibility is SchemaCompatibilityType.compatible
def test_schema_compatibility_type_mismatch(self):
incompatible_pairs = [
(
NULL_SCHEMA,
INT_SCHEMA,
"reader type: null not compatible with writer type: int",
"/",
),
(
NULL_SCHEMA,
LONG_SCHEMA,
"reader type: null not compatible with writer type: long",
"/",
),
(
BOOLEAN_SCHEMA,
INT_SCHEMA,
"reader type: boolean not compatible with writer type: int",
"/",
),
(
INT_SCHEMA,
NULL_SCHEMA,
"reader type: int not compatible with writer type: null",
"/",
),
(
INT_SCHEMA,
BOOLEAN_SCHEMA,
"reader type: int not compatible with writer type: boolean",
"/",
),
(
INT_SCHEMA,
LONG_SCHEMA,
"reader type: int not compatible with writer type: long",
"/",
),
(
INT_SCHEMA,
FLOAT_SCHEMA,
"reader type: int not compatible with writer type: float",
"/",
),
(
INT_SCHEMA,
DOUBLE_SCHEMA,
"reader type: int not compatible with writer type: double",
"/",
),
(
LONG_SCHEMA,
FLOAT_SCHEMA,
"reader type: long not compatible with writer type: float",
"/",
),
(
LONG_SCHEMA,
DOUBLE_SCHEMA,
"reader type: long not compatible with writer type: double",
"/",
),
(
FLOAT_SCHEMA,
DOUBLE_SCHEMA,
"reader type: float not compatible with writer type: double",
"/",
),
(
DOUBLE_SCHEMA,
STRING_SCHEMA,
"reader type: double not compatible with writer type: string",
"/",
),
(
FIXED_4_BYTES,
STRING_SCHEMA,
"reader type: fixed not compatible with writer type: string",
"/",
),
(
STRING_SCHEMA,
BOOLEAN_SCHEMA,
"reader type: string not compatible with writer type: boolean",
"/",
),
(
STRING_SCHEMA,
INT_SCHEMA,
"reader type: string not compatible with writer type: int",
"/",
),
(
BYTES_SCHEMA,
NULL_SCHEMA,
"reader type: bytes not compatible with writer type: null",
"/",
),
(
BYTES_SCHEMA,
INT_SCHEMA,
"reader type: bytes not compatible with writer type: int",
"/",
),
(
A_INT_RECORD1,
INT_SCHEMA,
"reader type: record not compatible with writer type: int",
"/",
),
(
INT_ARRAY_SCHEMA,
LONG_ARRAY_SCHEMA,
"reader type: int not compatible with writer type: long",
"/items",
),
(
INT_MAP_SCHEMA,
INT_ARRAY_SCHEMA,
"reader type: map not compatible with writer type: array",
"/",
),
(
INT_ARRAY_SCHEMA,
INT_MAP_SCHEMA,
"reader type: array not compatible with writer type: map",
"/",
),
(
INT_MAP_SCHEMA,
LONG_MAP_SCHEMA,
"reader type: int not compatible with writer type: long",
"/values",
),
(
INT_SCHEMA,
ENUM2_AB_SCHEMA,
"reader type: int not compatible with writer type: enum",
"/",
),
(
ENUM2_AB_SCHEMA,
INT_SCHEMA,
"reader type: enum not compatible with writer type: int",
"/",
),
(
FLOAT_SCHEMA,
INT_LONG_FLOAT_DOUBLE_UNION_SCHEMA,
"reader type: float not compatible with writer type: double",
"/",
),
(
LONG_SCHEMA,
INT_FLOAT_UNION_SCHEMA,
"reader type: long not compatible with writer type: float",
"/",
),
(
INT_SCHEMA,
INT_FLOAT_UNION_SCHEMA,
"reader type: int not compatible with writer type: float",
"/",
),
# (INT_LIST_RECORD, LONG_LIST_RECORD, "reader type: int not compatible with writer type: long", "/fields/0/type"),
(
NULL_SCHEMA,
INT_SCHEMA,
"reader type: null not compatible with writer type: int",
"/",
),
]
for (reader, writer, message, location) in incompatible_pairs:
result = ReaderWriterCompatibilityChecker().get_compatibility(
reader, writer)
self.assertIs(result.compatibility,
SchemaCompatibilityType.incompatible)
self.assertIn(message, result.messages)
self.assertIn(location, result.locations)