def parse_avro(schema_str: str): # pylint: disable=inconsistent-return-statements try: ts = TypedSchema(parse_avro_schema_definition(schema_str), SchemaType.AVRO, schema_str) return ts except SchemaParseException as e: raise InvalidSchema from e
def test_union_to_simple_comparison(field): writer = { "type": "record", "name": "name", "namespace": "namespace", "fields": [field] } reader = { "type": "record", "name": "name", "namespace": "namespace", "fields": [{ "type": ["null", field["type"]], "name": "fn", }] } reader = parse_avro_schema_definition(json.dumps(reader)) writer = parse_avro_schema_definition(json.dumps(writer)) assert are_compatible(reader, writer)
def test_schema_compatibility(): # testValidateSchemaPairMissingField writer = parse_avro_schema_definition( json.dumps({ "type": "record", "name": "Record", "fields": [{ "name": "oldField1", "type": "int" }, { "name": "oldField2", "type": "string" }] })) reader = parse_avro_schema_definition( json.dumps({ "type": "record", "name": "Record", "fields": [{ "name": "oldField1", "type": "int" }] })) assert are_compatible(reader, writer) # testValidateSchemaPairMissingSecondField reader = parse_avro_schema_definition( json.dumps({ "type": "record", "name": "Record", "fields": [{ "name": "oldField2", "type": "string" }] })) assert are_compatible(reader, writer) # testValidateSchemaPairAllFields reader = parse_avro_schema_definition( json.dumps({ "type": "record", "name": "Record", "fields": [{ "name": "oldField1", "type": "int" }, { "name": "oldField2", "type": "string" }] })) assert are_compatible(reader, writer) # testValidateSchemaNewFieldWithDefault reader = parse_avro_schema_definition( json.dumps({ "type": "record", "name": "Record", "fields": [{ "name": "oldField1", "type": "int" }, { "name": "newField2", "type": "int", "default": 42 }] })) assert are_compatible(reader, writer) # testValidateSchemaNewField reader = parse_avro_schema_definition( json.dumps({ "type": "record", "name": "Record", "fields": [{ "name": "oldField1", "type": "int" }, { "name": "newField2", "type": "int" }] })) assert not are_compatible(reader, writer) # testValidateArrayWriterSchema writer = parse_avro_schema_definition( json.dumps({ "type": "array", "items": { "type": "string" } })) reader = parse_avro_schema_definition( json.dumps({ "type": "array", "items": { "type": "string" } })) assert are_compatible(reader, writer) reader = parse_avro_schema_definition( json.dumps({ "type": "map", "values": { "type": "string" } })) assert not are_compatible(reader, writer) # testValidatePrimitiveWriterSchema writer = parse_avro_schema_definition(json.dumps({"type": "string"})) reader = parse_avro_schema_definition(json.dumps({"type": "string"})) assert are_compatible(reader, writer) reader = parse_avro_schema_definition(json.dumps({"type": "int"})) assert not are_compatible(reader, writer) # testUnionReaderWriterSubsetIncompatibility # cannot have a union as a top level data type, so im cheating a bit here writer = parse_avro_schema_definition( json.dumps({ "name": "Record", "type": "record", "fields": [{ "name": "f1", "type": ["int", "string", "long"] }] })) reader = parse_avro_schema_definition( json.dumps({ "name": "Record", "type": "record", "fields": [{ "name": "f1", "type": ["int", "string"] }] })) reader = reader.fields[0].type writer = writer.fields[0].type assert isinstance(reader, UnionSchema) assert isinstance(writer, UnionSchema) assert not are_compatible(reader, writer) # testReaderWriterCompatibility compatible_reader_writer_test_cases = [ (BOOLEAN_SCHEMA, BOOLEAN_SCHEMA), (INT_SCHEMA, INT_SCHEMA), (LONG_SCHEMA, INT_SCHEMA), (LONG_SCHEMA, LONG_SCHEMA), (FLOAT_SCHEMA, INT_SCHEMA), (FLOAT_SCHEMA, LONG_SCHEMA), (DOUBLE_SCHEMA, LONG_SCHEMA), (DOUBLE_SCHEMA, INT_SCHEMA), (DOUBLE_SCHEMA, FLOAT_SCHEMA), (STRING_SCHEMA, STRING_SCHEMA), (BYTES_SCHEMA, BYTES_SCHEMA), (STRING_SCHEMA, BYTES_SCHEMA), (BYTES_SCHEMA, STRING_SCHEMA), (INT_ARRAY_SCHEMA, INT_ARRAY_SCHEMA), (LONG_ARRAY_SCHEMA, INT_ARRAY_SCHEMA), (INT_MAP_SCHEMA, INT_MAP_SCHEMA), (LONG_MAP_SCHEMA, INT_MAP_SCHEMA), (ENUM1_AB_SCHEMA, ENUM1_AB_SCHEMA), (ENUM1_ABC_SCHEMA, ENUM1_AB_SCHEMA), # Union related pairs (EMPTY_UNION_SCHEMA, EMPTY_UNION_SCHEMA), (FLOAT_UNION_SCHEMA, EMPTY_UNION_SCHEMA), (FLOAT_UNION_SCHEMA, INT_UNION_SCHEMA), (FLOAT_UNION_SCHEMA, LONG_UNION_SCHEMA), (FLOAT_UNION_SCHEMA, INT_LONG_UNION_SCHEMA), (INT_UNION_SCHEMA, INT_UNION_SCHEMA), (INT_STRING_UNION_SCHEMA, STRING_INT_UNION_SCHEMA), (INT_UNION_SCHEMA, EMPTY_UNION_SCHEMA), (LONG_UNION_SCHEMA, EMPTY_UNION_SCHEMA), (LONG_UNION_SCHEMA, INT_UNION_SCHEMA), (FLOAT_UNION_SCHEMA, INT_UNION_SCHEMA), (DOUBLE_UNION_SCHEMA, INT_UNION_SCHEMA), (FLOAT_UNION_SCHEMA, LONG_UNION_SCHEMA), (DOUBLE_UNION_SCHEMA, LONG_UNION_SCHEMA), (FLOAT_UNION_SCHEMA, EMPTY_UNION_SCHEMA), (DOUBLE_UNION_SCHEMA, FLOAT_UNION_SCHEMA), (STRING_UNION_SCHEMA, EMPTY_UNION_SCHEMA), (STRING_UNION_SCHEMA, BYTES_UNION_SCHEMA), (BYTES_UNION_SCHEMA, EMPTY_UNION_SCHEMA), (BYTES_UNION_SCHEMA, STRING_UNION_SCHEMA), (DOUBLE_UNION_SCHEMA, INT_FLOAT_UNION_SCHEMA), # Readers capable of reading all branches of a union are compatible (FLOAT_SCHEMA, INT_FLOAT_UNION_SCHEMA), (LONG_SCHEMA, INT_LONG_UNION_SCHEMA), (DOUBLE_SCHEMA, INT_FLOAT_UNION_SCHEMA), (DOUBLE_SCHEMA, INT_LONG_FLOAT_DOUBLE_UNION_SCHEMA), # Special case of singleton unions: (FLOAT_SCHEMA, FLOAT_UNION_SCHEMA), (INT_UNION_SCHEMA, INT_SCHEMA), (INT_SCHEMA, INT_UNION_SCHEMA), # Fixed types (FIXED_4_BYTES, FIXED_4_BYTES), # Tests involving records: (EMPTY_RECORD1, EMPTY_RECORD1), (EMPTY_RECORD1, A_INT_RECORD1), (A_INT_RECORD1, A_INT_RECORD1), (A_DINT_RECORD1, A_INT_RECORD1), (A_DINT_RECORD1, A_DINT_RECORD1), (A_INT_RECORD1, A_DINT_RECORD1), (A_LONG_RECORD1, A_INT_RECORD1), (A_INT_RECORD1, A_INT_B_INT_RECORD1), (A_DINT_RECORD1, A_INT_B_INT_RECORD1), (A_INT_B_DINT_RECORD1, A_INT_RECORD1), (A_DINT_B_DINT_RECORD1, EMPTY_RECORD1), (A_DINT_B_DINT_RECORD1, A_INT_RECORD1), (A_INT_B_INT_RECORD1, A_DINT_B_DINT_RECORD1), ( parse_avro_schema_definition(json.dumps({"type": "null"})), parse_avro_schema_definition(json.dumps({"type": "null"})), ), (INT_LIST_RECORD, INT_LIST_RECORD), (LONG_LIST_RECORD, LONG_LIST_RECORD), (LONG_LIST_RECORD, INT_LIST_RECORD), (NULL_SCHEMA, NULL_SCHEMA), (ENUM_AB_ENUM_DEFAULT_A_RECORD, ENUM_ABC_ENUM_DEFAULT_A_RECORD), (ENUM_AB_FIELD_DEFAULT_A_ENUM_DEFAULT_B_RECORD, ENUM_ABC_FIELD_DEFAULT_B_ENUM_DEFAULT_A_RECORD), (NS_RECORD1, NS_RECORD2), ] for (reader, writer) in compatible_reader_writer_test_cases: assert are_compatible(reader, writer)
def test_simple_schema_promotion(): reader = parse_avro_schema_definition( json.dumps({ "name": "foo", "type": "record", "fields": [{ "type": "int", "name": "f1" }] })) field_alias_reader = parse_avro_schema_definition( json.dumps({ "name": "foo", "type": "record", "fields": [{ "type": "int", "name": "bar", "aliases": ["f1"] }] })) record_alias_reader = parse_avro_schema_definition( json.dumps({ "name": "other", "type": "record", "fields": [{ "type": "int", "name": "f1" }], "aliases": ["foo"] })) writer = parse_avro_schema_definition( json.dumps({ "name": "foo", "type": "record", "fields": [{ "type": "int", "name": "f1" }, { "type": "string", "name": "f2", }] })) # alias testing res = ReaderWriterCompatibilityChecker().get_compatibility( field_alias_reader, writer) assert res.compatibility is SchemaCompatibilityType.compatible, res.locations res = ReaderWriterCompatibilityChecker().get_compatibility( record_alias_reader, writer) assert res.compatibility is SchemaCompatibilityType.compatible, res.locations res = ReaderWriterCompatibilityChecker().get_compatibility(reader, writer) assert res == SchemaCompatibilityResult.compatible(), res res = ReaderWriterCompatibilityChecker().get_compatibility(writer, reader) assert res != SchemaCompatibilityResult.compatible(), res writer = parse_avro_schema_definition( json.dumps({ "type": "record", "name": "CA", "namespace": "ns1", "fields": [{ "type": "string", "name": "provider" }, { "type": ["null", "string"], "name": "name", "default": None }, { "type": ["null", "string"], "name": "phone", "default": None }, { "type": ["null", "string"], "name": "email", "default": None }, { "type": ["null", "string"], "name": "reference", "default": None }, { "type": ["null", "double"], "name": "price", "default": None }] })) reader = parse_avro_schema_definition( json.dumps({ "type": "record", "name": "CA", "namespace": "ns1", "fields": [{ "type": "string", "name": "provider" }, { "type": ["null", "string"], "name": "name", "default": None }, { "type": ["null", "string"], "name": "phone", "default": None }, { "type": ["null", "string"], "name": "email", "default": None }, { "type": ["null", "string"], "name": "reference", "default": None }, { "type": ["null", "double"], "name": "price", "default": None }, { "type": ["null", "string"], "name": "status_date", "default": None }] })) res = ReaderWriterCompatibilityChecker().get_compatibility(writer=writer, reader=reader) assert res == SchemaCompatibilityResult.compatible(), res
""" These are duplicates of other test_schema.py tests, but do not make use of the registry client fixture and are here for debugging and speed, and as an initial sanity check """ from avro.schema import ArraySchema, Field, MapSchema, RecordSchema, Schema, UnionSchema from karapace.avro_compatibility import (parse_avro_schema_definition, ReaderWriterCompatibilityChecker, SchemaCompatibilityResult, SchemaCompatibilityType) import json import pytest # Schemas defined in AvroCompatibilityTest.java. Used here to ensure compatibility with the schema-registry schema1 = parse_avro_schema_definition( '{"type":"record","name":"myrecord","fields":[{"type":"string","name":"f1"}]}' ) schema2 = parse_avro_schema_definition( '{"type":"record","name":"myrecord","fields":[{"type":"string","name":"f1"},{"type":"string",' '"name":"f2","default":"foo"}]}') schema3 = parse_avro_schema_definition( '{"type":"record","name":"myrecord","fields":[{"type":"string","name":"f1"},{"type":"string","name":"f2"}]}' ) schema4 = parse_avro_schema_definition( '{"type":"record","name":"myrecord","fields":[{"type":"string","name":"f1_new","aliases":["f1"]}]}' ) schema6 = parse_avro_schema_definition( '{"type":"record","name":"myrecord","fields":[{"type":["null","string"],"name":"f1","doc":"doc of f1"}]}' ) schema7 = parse_avro_schema_definition( '{"type":"record","name":"myrecord","fields":[{"type":["null","string","int"],"name":"f1","doc":"doc of f1"}]}'