class TestMessageSerializer(unittest.TestCase):
def setUp(self):
# need to set up the serializer
self.client = MockSchemaRegistryClient()
self.ms = MessageSerializer(self.client)
def assertMessageIsSame(self, message, expected, schema_id, schema):
self.assertTrue(message)
self.assertTrue(len(message) > 5)
magic, sid = struct.unpack('>bI', message[0:5])
self.assertEqual(magic, 0)
self.assertEqual(sid, schema_id)
decoded_msg, decoded_schema = self.ms.decode_message(message)
self.assertTrue(decoded_msg)
self.assertEqual(decoded_msg, expected)
self.assertEqual(decoded_schema, schema)
def test_encode_with_schema_id(self):
adv = avro.loads(data_gen.ADVANCED_SCHEMA)
basic = avro.loads(data_gen.BASIC_SCHEMA)
subject = 'test'
schema_id = self.client.register(subject, basic)
records = data_gen.BASIC_ITEMS
for record in records:
message = self.ms.encode_record_with_schema_id(schema_id, record)
self.assertMessageIsSame(message, record, schema_id, basic)
subject = 'test_adv'
adv_schema_id = self.client.register(subject, adv)
self.assertNotEqual(adv_schema_id, schema_id)
records = data_gen.ADVANCED_ITEMS
for record in records:
message = self.ms.encode_record_with_schema_id(
adv_schema_id, record)
self.assertMessageIsSame(message, record, adv_schema_id, adv)
def test_encode_record_with_schema(self):
topic = 'test'
basic = avro.loads(data_gen.BASIC_SCHEMA)
subject = 'test-value'
schema_id = self.client.register(subject, basic)
records = data_gen.BASIC_ITEMS
for record in records:
message = self.ms.encode_record_with_schema(topic, basic, record)
self.assertMessageIsSame(message, record, schema_id, basic)
def test_decode_none(self):
""""null/None messages should decode to None"""
self.assertIsNone(self.ms.decode_message(None))
def hash_func(self):
return hash(str(self))
class TestMessageSerializer(unittest.TestCase):
def setUp(self):
# need to set up the serializer
self.client = MockSchemaRegistryClient()
self.ms = MessageSerializer(self.client)
def assertMessageIsSame(self, message, expected, schema_id):
self.assertTrue(message)
self.assertTrue(len(message) > 5)
magic, sid = struct.unpack('>bI', message[0:5])
self.assertEqual(magic, 0)
self.assertEqual(sid, schema_id)
decoded = self.ms.decode_message(message)
self.assertTrue(decoded)
self.assertEqual(decoded, expected)
def test_encode_with_schema_id(self):
adv = avro.loads(data_gen.ADVANCED_SCHEMA)
basic = avro.loads(data_gen.BASIC_SCHEMA)
subject = 'test'
schema_id = self.client.register(subject, basic)
records = data_gen.BASIC_ITEMS
for record in records:
message = self.ms.encode_record_with_schema_id(schema_id, record)
self.assertMessageIsSame(message, record, schema_id)
subject = 'test_adv'
adv_schema_id = self.client.register(subject, adv)
self.assertNotEqual(adv_schema_id, schema_id)
records = data_gen.ADVANCED_ITEMS
for record in records:
message = self.ms.encode_record_with_schema_id(adv_schema_id, record)
self.assertMessageIsSame(message, record, adv_schema_id)
def test_encode_record_with_schema(self):
topic = 'test'
basic = avro.loads(data_gen.BASIC_SCHEMA)
subject = 'test-value'
schema_id = self.client.register(subject, basic)
records = data_gen.BASIC_ITEMS
for record in records:
message = self.ms.encode_record_with_schema(topic, basic, record)
self.assertMessageIsSame(message, record, schema_id)
def test_decode_none(self):
""""null/None messages should decode to None"""
self.assertIsNone(self.ms.decode_message(None))
def hash_func(self):
return hash(str(self))
class TestMessageSerializer(unittest.TestCase):
def setUp(self):
# need to set up the serializer
# Make RecordSchema and PrimitiveSchema hashable
schema.RecordSchema.__hash__ = self.hash_func
schema.PrimitiveSchema.__hash__ = self.hash_func
self.client = MockSchemaRegistryClient()
self.ms = MessageSerializer(self.client)
def assertMessageIsSame(self, message, expected, schema_id):
self.assertTrue(message)
self.assertTrue(len(message) > 5)
magic, sid = struct.unpack('>bI', message[0:5])
self.assertEqual(magic, 0)
self.assertEqual(sid, schema_id)
decoded = self.ms.decode_message(message)
self.assertTrue(decoded)
self.assertEqual(decoded, expected)
def test_encode_with_schema_id(self):
adv = util.parse_schema_from_string(data_gen.ADVANCED_SCHEMA)
basic = util.parse_schema_from_string(data_gen.BASIC_SCHEMA)
subject = 'test'
schema_id = self.client.register(subject, basic)
records = data_gen.BASIC_ITEMS
for record in records:
message = self.ms.encode_record_with_schema_id(schema_id, record)
self.assertMessageIsSame(message, record, schema_id)
subject = 'test_adv'
adv_schema_id = self.client.register(subject, adv)
self.assertNotEqual(adv_schema_id, schema_id)
records = data_gen.ADVANCED_ITEMS
for record in records:
message = self.ms.encode_record_with_schema_id(
adv_schema_id, record)
self.assertMessageIsSame(message, record, adv_schema_id)
def test_encode_record_with_schema(self):
topic = 'test'
basic = util.parse_schema_from_string(data_gen.BASIC_SCHEMA)
subject = 'test-value'
schema_id = self.client.register(subject, basic)
records = data_gen.BASIC_ITEMS
for record in records:
message = self.ms.encode_record_with_schema(topic, basic, record)
self.assertMessageIsSame(message, record, schema_id)
def hash_func(self):
return hash(str(self))
class _AvroIORegistry:
def __init__(self, schema_registry_url):
"""Private implementation class for Avro IO using the registry"""
log.info(
f"Using registry with schema_url/id {schema_registry_url}/{config.SCHEMA_ID}"
)
try:
self.client = CachedSchemaRegistryClient(url=schema_registry_url)
self.schema = self.client.get_by_id(config.SCHEMA_ID)
self.serializer = MessageSerializer(self.client)
except:
raise ValueError("Client id or schema id not found")
def decode(self, bytes):
return self.serializer.decode_message(bytes)
def encode(self, record):
return self.serializer.encode_record_with_schema_id(
config.SCHEMA_ID, record)
class TestMessageSerializer(unittest.TestCase):
def setUp(self):
# need to set up the serializer
self.client = MockSchemaRegistryClient()
self.ms = MessageSerializer(self.client)
def assertMessageIsSame(self, message, expected, schema_id):
self.assertTrue(message)
self.assertTrue(len(message) > 5)
magic, sid = struct.unpack('>bI', message[0:5])
self.assertEqual(magic, 0)
self.assertEqual(sid, schema_id)
decoded = self.ms.decode_message(message)
self.assertTrue(decoded)
self.assertEqual(decoded, expected)
def test_encode_with_schema_id(self):
adv = avro.loads(data_gen.ADVANCED_SCHEMA)
basic = avro.loads(data_gen.BASIC_SCHEMA)
subject = 'test'
schema_id = self.client.register(subject, basic)
records = data_gen.BASIC_ITEMS
for record in records:
message = self.ms.encode_record_with_schema_id(schema_id, record)
self.assertMessageIsSame(message, record, schema_id)
subject = 'test_adv'
adv_schema_id = self.client.register(subject, adv)
self.assertNotEqual(adv_schema_id, schema_id)
records = data_gen.ADVANCED_ITEMS
for record in records:
message = self.ms.encode_record_with_schema_id(
adv_schema_id, record)
self.assertMessageIsSame(message, record, adv_schema_id)
def test_encode_record_with_schema(self):
topic = 'test'
basic = avro.loads(data_gen.BASIC_SCHEMA)
subject = 'test-value'
schema_id = self.client.register(subject, basic)
records = data_gen.BASIC_ITEMS
for record in records:
message = self.ms.encode_record_with_schema(topic, basic, record)
self.assertMessageIsSame(message, record, schema_id)
def test_decode_none(self):
""""null/None messages should decode to None"""
self.assertIsNone(self.ms.decode_message(None))
def test_decode_with_schema(self):
topic = 'test_specific'
schema_v1 = avro.loads(
data_gen.load_schema_file('evolution_schema_v1.avsc'))
schema_v2 = avro.loads(
data_gen.load_schema_file('evolution_schema_v2.avsc'))
dsv1 = SpecificRecordMessageDeserializer(self.client,
value_schema=schema_v1)
dsv2 = SpecificRecordMessageDeserializer(self.client,
value_schema=schema_v2)
record_v1 = {"name": "suzyq", "age": 27}
record_v2 = dict(record_v1)
record_v2['gender'] = 'NONE'
encoded_v1 = self.ms.encode_record_with_schema(topic, schema_v1,
record_v1)
decoded_v1_v1 = dsv1.decode_message(encoded_v1, is_key=False)
self.assertDictEqual(record_v1, decoded_v1_v1)
decoded_v1_v2 = dsv2.decode_message(encoded_v1, is_key=False)
self.assertDictEqual(record_v2, decoded_v1_v2)
encoded_v2 = self.ms.encode_record_with_schema(topic, schema_v2,
record_v2)
decoded_v2_v2 = dsv2.decode_message(encoded_v2, is_key=False)
self.assertDictEqual(record_v2, decoded_v2_v2)
decoded_v2_v1 = dsv1.decode_message(encoded_v2, is_key=False)
self.assertDictEqual(record_v1, decoded_v2_v1)
def hash_func(self):
return hash(str(self))
class TestMessageSerializer(unittest.TestCase):
def setUp(self):
# need to set up the serializer
self.client = MockSchemaRegistryClient()
self.ms = MessageSerializer(self.client)
def assertMessageIsSame(self, message, expected, schema_id):
self.assertTrue(message)
self.assertTrue(len(message) > 5)
magic, sid = struct.unpack('>bI', message[0:5])
self.assertEqual(magic, 0)
self.assertEqual(sid, schema_id)
decoded = self.ms.decode_message(message)
self.assertTrue(decoded)
self.assertEqual(decoded, expected)
def test_encode_with_schema_id(self):
adv = avro.loads(data_gen.ADVANCED_SCHEMA)
basic = avro.loads(data_gen.BASIC_SCHEMA)
subject = 'test'
schema_id = self.client.register(subject, basic)
records = data_gen.BASIC_ITEMS
for record in records:
message = self.ms.encode_record_with_schema_id(schema_id, record)
self.assertMessageIsSame(message, record, schema_id)
subject = 'test_adv'
adv_schema_id = self.client.register(subject, adv)
self.assertNotEqual(adv_schema_id, schema_id)
records = data_gen.ADVANCED_ITEMS
for record in records:
message = self.ms.encode_record_with_schema_id(
adv_schema_id, record)
self.assertMessageIsSame(message, record, adv_schema_id)
def test_encode_record_with_schema(self):
topic = 'test'
basic = avro.loads(data_gen.BASIC_SCHEMA)
subject = 'test-value'
schema_id = self.client.register(subject, basic)
records = data_gen.BASIC_ITEMS
for record in records:
message = self.ms.encode_record_with_schema(topic, basic, record)
self.assertMessageIsSame(message, record, schema_id)
@skipIf(version_info < (3, ),
'unittest.mock.patch not available in Python 2')
def test_encode_record_with_schema_sets_writers_cache_once(self):
topic = 'test'
basic = avro.loads(data_gen.BASIC_SCHEMA)
subject = 'test-value'
self.client.register(subject, basic)
records = data_gen.BASIC_ITEMS
with patch.object(self.ms, "_get_encoder_func") as encoder_func_mock:
for record in records:
self.ms.encode_record_with_schema(topic, basic, record)
encoder_func_mock.assert_called_once_with(basic)
def test_decode_none(self):
"""null/None messages should decode to None"""
self.assertIsNone(self.ms.decode_message(None))
def hash_func(self):
return hash(str(self))
subject = schema.fullname # == "my.test.value"
# io.confluent.kafka.serializers.subject.TopicRecordNameStrategy:
# The subject name is <topic>-<type>, where <topic> is the Kafka topic name, and <type> is the fully-qualified
# name of the Avro record type of the message. This setting also allows any number of event types in the same topic,
# and further constrains the compatibility check to the current topic only.
# subject = topic + '-' + schema.fullname # == "avro-python-producer-topic-my.test.value"
# get registered schema id from the schema_registry
schema_id = schema_registry.register(subject, schema)
for i in range(5):
key = "key-" + str(i)
value = "value-" + str(i)
record_value = avro_serde.encode_record_with_schema_id(
schema_id=schema_id,
record={
"name": value,
"type": "avro"
},
is_key=False,
)
producer.produce(topic, key=key.encode('utf-8'), value=record_value)
print("Produced:", key, record_value)
producer.flush()
print("End: avro-python-producer")
class TestMessageSerializer(unittest.TestCase):
def setUp(self):
# need to set up the serializer
self.client = MockSchemaRegistryClient()
self.ms = MessageSerializer(self.client)
def assertMessageIsSame(self, message, expected, schema_id):
self.assertTrue(message)
self.assertTrue(len(message) > 5)
magic, sid = struct.unpack('>bI', message[0:5])
self.assertEqual(magic, 0)
self.assertEqual(sid, schema_id)
decoded = self.ms.decode_message(message)
self.assertTrue(decoded)
self.assertEqual(decoded, expected)
def test_encode_with_schema_id(self):
adv = avro.loads(data_gen.ADVANCED_SCHEMA)
basic = avro.loads(data_gen.BASIC_SCHEMA)
subject = 'test'
schema_id = self.client.register(subject, basic)
records = data_gen.BASIC_ITEMS
for record in records:
message = self.ms.encode_record_with_schema_id(schema_id, record)
self.assertMessageIsSame(message, record, schema_id)
subject = 'test_adv'
adv_schema_id = self.client.register(subject, adv)
self.assertNotEqual(adv_schema_id, schema_id)
records = data_gen.ADVANCED_ITEMS
for record in records:
message = self.ms.encode_record_with_schema_id(adv_schema_id, record)
self.assertMessageIsSame(message, record, adv_schema_id)
def test_encode_record_with_schema(self):
topic = 'test'
basic = avro.loads(data_gen.BASIC_SCHEMA)
subject = 'test-value'
schema_id = self.client.register(subject, basic)
records = data_gen.BASIC_ITEMS
for record in records:
message = self.ms.encode_record_with_schema(topic, basic, record)
self.assertMessageIsSame(message, record, schema_id)
def test_decode_none(self):
""""null/None messages should decode to None"""
self.assertIsNone(self.ms.decode_message(None))
def test__get_subject_for_key_with_topic_name_strategy(self):
basic = avro.loads(data_gen.BASIC_SCHEMA)
topic = "topic"
self.ms.registry_client.key_subject_name_strategy_func = topic_name_strategy # noqa
subject = self.ms._get_subject(topic=topic, schema=basic, is_key=True)
expected = "topic-key"
self.assertEqual(expected, subject)
def test__get_subject_for_key_with_record_name_strategy(self):
basic = avro.loads(data_gen.BASIC_SCHEMA)
topic = "topic"
self.ms.registry_client.key_subject_name_strategy_func = record_name_strategy # noqa
subject = self.ms._get_subject(topic=topic, schema=basic, is_key=True)
expected = "python.test.basic.basic"
self.assertEqual(expected, subject)
def test__get_subject_for_key_with_topic_record_name_strategy(self):
basic = avro.loads(data_gen.BASIC_SCHEMA)
topic = "topic"
self.ms.registry_client.key_subject_name_strategy_func = topic_record_name_strategy # noqa
subject = self.ms._get_subject(topic=topic, schema=basic, is_key=True)
expected = "topic-python.test.basic.basic"
self.assertEqual(expected, subject)
def test__get_subject_for_value_with_topic_name_strategy(self):
basic = avro.loads(data_gen.BASIC_SCHEMA)
topic = "topic"
self.ms.registry_client.value_subject_name_strategy_func = topic_name_strategy # noqa
subject = self.ms._get_subject(topic=topic, schema=basic, is_key=False)
expected = "topic-value"
self.assertEqual(expected, subject)
def test__get_subject_for_value_with_record_name_strategy(self):
basic = avro.loads(data_gen.BASIC_SCHEMA)
topic = "topic"
self.ms.registry_client.value_subject_name_strategy_func = record_name_strategy # noqa
subject = self.ms._get_subject(topic=topic, schema=basic, is_key=False)
expected = "python.test.basic.basic"
self.assertEqual(expected, subject)
def test__get_subject_for_value_with_topic_record_name_strategy(self):
basic = avro.loads(data_gen.BASIC_SCHEMA)
topic = "topic"
self.ms.registry_client.value_subject_name_strategy_func = topic_record_name_strategy # noqa
subject = self.ms._get_subject(topic=topic, schema=basic, is_key=False)
expected = "topic-python.test.basic.basic"
self.assertEqual(expected, subject)
def hash_func(self):
return hash(str(self))
