def __init__(self, *args, **kwargs):
super(MockServer, self).__init__(*args, **kwargs)
self.counts = {}
self.registry = MockSchemaRegistryClient()
self.schema_cache = {}
self.all_routes = {
'GET': [(r"/schemas/ids/(\d+)", 'get_schema_by_id'),
(r"/subjects/(\w+)/versions/latest", 'get_latest')],
'POST': [(r"/subjects/(\w+)/versions", 'register'),
(r"/subjects/(\w+)", 'get_version')]
}
class TestMessageSerializer(unittest.TestCase):
def setUp(self):
# need to set up the serializer
self.client = MockSchemaRegistryClient()
self.ms = MessageSerializer(self.client)
def assertMessageIsSame(self, message, expected, schema_id, schema):
self.assertTrue(message)
self.assertTrue(len(message) > 5)
magic, sid = struct.unpack('>bI', message[0:5])
self.assertEqual(magic, 0)
self.assertEqual(sid, schema_id)
decoded_msg, decoded_schema = self.ms.decode_message(message)
self.assertTrue(decoded_msg)
self.assertEqual(decoded_msg, expected)
self.assertEqual(decoded_schema, schema)
def test_encode_with_schema_id(self):
adv = avro.loads(data_gen.ADVANCED_SCHEMA)
basic = avro.loads(data_gen.BASIC_SCHEMA)
subject = 'test'
schema_id = self.client.register(subject, basic)
records = data_gen.BASIC_ITEMS
for record in records:
message = self.ms.encode_record_with_schema_id(schema_id, record)
self.assertMessageIsSame(message, record, schema_id, basic)
subject = 'test_adv'
adv_schema_id = self.client.register(subject, adv)
self.assertNotEqual(adv_schema_id, schema_id)
records = data_gen.ADVANCED_ITEMS
for record in records:
message = self.ms.encode_record_with_schema_id(
adv_schema_id, record)
self.assertMessageIsSame(message, record, adv_schema_id, adv)
def test_encode_record_with_schema(self):
topic = 'test'
basic = avro.loads(data_gen.BASIC_SCHEMA)
subject = 'test-value'
schema_id = self.client.register(subject, basic)
records = data_gen.BASIC_ITEMS
for record in records:
message = self.ms.encode_record_with_schema(topic, basic, record)
self.assertMessageIsSame(message, record, schema_id, basic)
def test_decode_none(self):
""""null/None messages should decode to None"""
self.assertIsNone(self.ms.decode_message(None))
def hash_func(self):
return hash(str(self))
class TestMessageSerializer(unittest.TestCase):
def setUp(self):
# need to set up the serializer
self.client = MockSchemaRegistryClient()
self.ms = MessageSerializer(self.client)
def assertMessageIsSame(self, message, expected, schema_id):
self.assertTrue(message)
self.assertTrue(len(message) > 5)
magic, sid = struct.unpack('>bI', message[0:5])
self.assertEqual(magic, 0)
self.assertEqual(sid, schema_id)
decoded = self.ms.decode_message(message)
self.assertTrue(decoded)
self.assertEqual(decoded, expected)
def test_encode_with_schema_id(self):
adv = avro.loads(data_gen.ADVANCED_SCHEMA)
basic = avro.loads(data_gen.BASIC_SCHEMA)
subject = 'test'
schema_id = self.client.register(subject, basic)
records = data_gen.BASIC_ITEMS
for record in records:
message = self.ms.encode_record_with_schema_id(schema_id, record)
self.assertMessageIsSame(message, record, schema_id)
subject = 'test_adv'
adv_schema_id = self.client.register(subject, adv)
self.assertNotEqual(adv_schema_id, schema_id)
records = data_gen.ADVANCED_ITEMS
for record in records:
message = self.ms.encode_record_with_schema_id(adv_schema_id, record)
self.assertMessageIsSame(message, record, adv_schema_id)
def test_encode_record_with_schema(self):
topic = 'test'
basic = avro.loads(data_gen.BASIC_SCHEMA)
subject = 'test-value'
schema_id = self.client.register(subject, basic)
records = data_gen.BASIC_ITEMS
for record in records:
message = self.ms.encode_record_with_schema(topic, basic, record)
self.assertMessageIsSame(message, record, schema_id)
def test_decode_none(self):
""""null/None messages should decode to None"""
self.assertIsNone(self.ms.decode_message(None))
def hash_func(self):
return hash(str(self))
def test_produce_with_empty_key_value_with_schema(self):
value_schema = avro.load(os.path.join(avsc_dir, "basic_schema.avsc"))
schema_registry = MockSchemaRegistryClient()
producer = AvroProducer({},
schema_registry=schema_registry,
default_value_schema=value_schema)
producer.produce(topic='test', value={'name': 'abc'}, key='')
def test_produce_with_custom_registry(self):
schema_registry = MockSchemaRegistryClient()
value_schema = avro.load(os.path.join(avsc_dir, "basic_schema.avsc"))
producer = AvroProducer({}, schema_registry=schema_registry)
producer.produce(topic='test',
value={"name": 'abc"'},
value_schema=value_schema,
key='mykey')
def test_produce_with_empty_key_no_schema(self):
value_schema = avro.load(os.path.join(avsc_dir,
"primitive_float.avsc"))
schema_registry = MockSchemaRegistryClient()
producer = AvroProducer({},
schema_registry=schema_registry,
default_value_schema=value_schema)
with self.assertRaises(KeySerializerError):
producer.produce(topic='test', value=0.0, key='')
def test_explicit_topic_subject_name_strategy(self):
schema_registry = MockSchemaRegistryClient()
producer = AvroProducer(
config={},
schema_registry=schema_registry,
key_subject_name_strategy=strategies.topic_name_strategy,
value_subject_name_strategy=strategies.topic_name_strategy)
serializer = producer._serializer
assert serializer.key_subject_name_strategy is strategies.topic_name_strategy
assert serializer.value_subject_name_strategy is strategies.topic_name_strategy
def test_differing_key_and_value_subject_name_strategies(self):
schema_registry = MockSchemaRegistryClient()
producer = AvroProducer(
config={},
schema_registry=schema_registry,
key_subject_name_strategy=strategies.record_name_strategy,
value_subject_name_strategy=strategies.topic_record_name_strategy)
serializer = producer._serializer
assert serializer.key_subject_name_strategy is strategies.record_name_strategy
assert serializer.value_subject_name_strategy is strategies.topic_record_name_strategy
def test_produce_with_empty_key_value_with_schema(self):
key_schema = avro.load(os.path.join(avsc_dir, "primitive_string.avsc"))
value_schema = avro.load(os.path.join(avsc_dir,
"primitive_float.avsc"))
schema_registry = MockSchemaRegistryClient()
producer = AvroProducer({},
schema_registry=schema_registry,
default_key_schema=key_schema,
default_value_schema=value_schema)
producer.produce(topic='test', value=0.0, key='')
class TestMessageSerializer(unittest.TestCase):
def setUp(self):
# need to set up the serializer
self.client = MockSchemaRegistryClient()
self.ms = MessageSerializer(self.client)
def assertMessageIsSame(self, message, expected, schema_id):
self.assertTrue(message)
self.assertTrue(len(message) > 5)
magic, sid = struct.unpack('>bI', message[0:5])
self.assertEqual(magic, 0)
self.assertEqual(sid, schema_id)
decoded = self.ms.decode_message(message)
self.assertTrue(decoded)
self.assertEqual(decoded, expected)
def test_encode_with_schema_id(self):
adv = avro.loads(data_gen.ADVANCED_SCHEMA)
basic = avro.loads(data_gen.BASIC_SCHEMA)
subject = 'test'
schema_id = self.client.register(subject, basic)
records = data_gen.BASIC_ITEMS
for record in records:
message = self.ms.encode_record_with_schema_id(schema_id, record)
self.assertMessageIsSame(message, record, schema_id)
subject = 'test_adv'
adv_schema_id = self.client.register(subject, adv)
self.assertNotEqual(adv_schema_id, schema_id)
records = data_gen.ADVANCED_ITEMS
for record in records:
message = self.ms.encode_record_with_schema_id(
adv_schema_id, record)
self.assertMessageIsSame(message, record, adv_schema_id)
def test_encode_record_with_schema(self):
topic = 'test'
basic = avro.loads(data_gen.BASIC_SCHEMA)
subject = 'test-value'
schema_id = self.client.register(subject, basic)
records = data_gen.BASIC_ITEMS
for record in records:
message = self.ms.encode_record_with_schema(topic, basic, record)
self.assertMessageIsSame(message, record, schema_id)
def test_decode_none(self):
""""null/None messages should decode to None"""
self.assertIsNone(self.ms.decode_message(None))
def test_decode_with_schema(self):
topic = 'test_specific'
schema_v1 = avro.loads(
data_gen.load_schema_file('evolution_schema_v1.avsc'))
schema_v2 = avro.loads(
data_gen.load_schema_file('evolution_schema_v2.avsc'))
dsv1 = SpecificRecordMessageDeserializer(self.client,
value_schema=schema_v1)
dsv2 = SpecificRecordMessageDeserializer(self.client,
value_schema=schema_v2)
record_v1 = {"name": "suzyq", "age": 27}
record_v2 = dict(record_v1)
record_v2['gender'] = 'NONE'
encoded_v1 = self.ms.encode_record_with_schema(topic, schema_v1,
record_v1)
decoded_v1_v1 = dsv1.decode_message(encoded_v1, is_key=False)
self.assertDictEqual(record_v1, decoded_v1_v1)
decoded_v1_v2 = dsv2.decode_message(encoded_v1, is_key=False)
self.assertDictEqual(record_v2, decoded_v1_v2)
encoded_v2 = self.ms.encode_record_with_schema(topic, schema_v2,
record_v2)
decoded_v2_v2 = dsv2.decode_message(encoded_v2, is_key=False)
self.assertDictEqual(record_v2, decoded_v2_v2)
decoded_v2_v1 = dsv1.decode_message(encoded_v2, is_key=False)
self.assertDictEqual(record_v1, decoded_v2_v1)
def hash_func(self):
return hash(str(self))
class TestMessageSerializer(unittest.TestCase):
def setUp(self):
# need to set up the serializer
self.client = MockSchemaRegistryClient()
self.ms = MessageSerializer(self.client)
def assertMessageIsSame(self, message, expected, schema_id):
self.assertTrue(message)
self.assertTrue(len(message) > 5)
magic, sid = struct.unpack('>bI', message[0:5])
self.assertEqual(magic, 0)
self.assertEqual(sid, schema_id)
decoded = self.ms.decode_message(message)
self.assertTrue(decoded)
self.assertEqual(decoded, expected)
def test_encode_with_schema_id(self):
adv = avro.loads(data_gen.ADVANCED_SCHEMA)
basic = avro.loads(data_gen.BASIC_SCHEMA)
subject = 'test'
schema_id = self.client.register(subject, basic)
records = data_gen.BASIC_ITEMS
for record in records:
message = self.ms.encode_record_with_schema_id(schema_id, record)
self.assertMessageIsSame(message, record, schema_id)
subject = 'test_adv'
adv_schema_id = self.client.register(subject, adv)
self.assertNotEqual(adv_schema_id, schema_id)
records = data_gen.ADVANCED_ITEMS
for record in records:
message = self.ms.encode_record_with_schema_id(
adv_schema_id, record)
self.assertMessageIsSame(message, record, adv_schema_id)
def test_encode_record_with_schema(self):
topic = 'test'
basic = avro.loads(data_gen.BASIC_SCHEMA)
subject = 'test-value'
schema_id = self.client.register(subject, basic)
records = data_gen.BASIC_ITEMS
for record in records:
message = self.ms.encode_record_with_schema(topic, basic, record)
self.assertMessageIsSame(message, record, schema_id)
@skipIf(version_info < (3, ),
'unittest.mock.patch not available in Python 2')
def test_encode_record_with_schema_sets_writers_cache_once(self):
topic = 'test'
basic = avro.loads(data_gen.BASIC_SCHEMA)
subject = 'test-value'
self.client.register(subject, basic)
records = data_gen.BASIC_ITEMS
with patch.object(self.ms, "_get_encoder_func") as encoder_func_mock:
for record in records:
self.ms.encode_record_with_schema(topic, basic, record)
encoder_func_mock.assert_called_once_with(basic)
def test_decode_none(self):
"""null/None messages should decode to None"""
self.assertIsNone(self.ms.decode_message(None))
def hash_func(self):
return hash(str(self))
class TestMessageSerializer(unittest.TestCase):
def setUp(self):
# need to set up the serializer
self.client = MockSchemaRegistryClient()
self.ms = MessageSerializer(self.client)
def assertMessageIsSame(self, message, expected, schema_id):
self.assertTrue(message)
self.assertTrue(len(message) > 5)
magic, sid = struct.unpack('>bI', message[0:5])
self.assertEqual(magic, 0)
self.assertEqual(sid, schema_id)
decoded = self.ms.decode_message(message)
self.assertTrue(decoded)
self.assertEqual(decoded, expected)
def test_encode_with_schema_id(self):
adv = avro.loads(data_gen.ADVANCED_SCHEMA)
basic = avro.loads(data_gen.BASIC_SCHEMA)
subject = 'test'
schema_id = self.client.register(subject, basic)
records = data_gen.BASIC_ITEMS
for record in records:
message = self.ms.encode_record_with_schema_id(schema_id, record)
self.assertMessageIsSame(message, record, schema_id)
subject = 'test_adv'
adv_schema_id = self.client.register(subject, adv)
self.assertNotEqual(adv_schema_id, schema_id)
records = data_gen.ADVANCED_ITEMS
for record in records:
message = self.ms.encode_record_with_schema_id(adv_schema_id, record)
self.assertMessageIsSame(message, record, adv_schema_id)
def test_encode_record_with_schema(self):
topic = 'test'
basic = avro.loads(data_gen.BASIC_SCHEMA)
subject = 'test-value'
schema_id = self.client.register(subject, basic)
records = data_gen.BASIC_ITEMS
for record in records:
message = self.ms.encode_record_with_schema(topic, basic, record)
self.assertMessageIsSame(message, record, schema_id)
def test_decode_none(self):
""""null/None messages should decode to None"""
self.assertIsNone(self.ms.decode_message(None))
def test__get_subject_for_key_with_topic_name_strategy(self):
basic = avro.loads(data_gen.BASIC_SCHEMA)
topic = "topic"
self.ms.registry_client.key_subject_name_strategy_func = topic_name_strategy # noqa
subject = self.ms._get_subject(topic=topic, schema=basic, is_key=True)
expected = "topic-key"
self.assertEqual(expected, subject)
def test__get_subject_for_key_with_record_name_strategy(self):
basic = avro.loads(data_gen.BASIC_SCHEMA)
topic = "topic"
self.ms.registry_client.key_subject_name_strategy_func = record_name_strategy # noqa
subject = self.ms._get_subject(topic=topic, schema=basic, is_key=True)
expected = "python.test.basic.basic"
self.assertEqual(expected, subject)
def test__get_subject_for_key_with_topic_record_name_strategy(self):
basic = avro.loads(data_gen.BASIC_SCHEMA)
topic = "topic"
self.ms.registry_client.key_subject_name_strategy_func = topic_record_name_strategy # noqa
subject = self.ms._get_subject(topic=topic, schema=basic, is_key=True)
expected = "topic-python.test.basic.basic"
self.assertEqual(expected, subject)
def test__get_subject_for_value_with_topic_name_strategy(self):
basic = avro.loads(data_gen.BASIC_SCHEMA)
topic = "topic"
self.ms.registry_client.value_subject_name_strategy_func = topic_name_strategy # noqa
subject = self.ms._get_subject(topic=topic, schema=basic, is_key=False)
expected = "topic-value"
self.assertEqual(expected, subject)
def test__get_subject_for_value_with_record_name_strategy(self):
basic = avro.loads(data_gen.BASIC_SCHEMA)
topic = "topic"
self.ms.registry_client.value_subject_name_strategy_func = record_name_strategy # noqa
subject = self.ms._get_subject(topic=topic, schema=basic, is_key=False)
expected = "python.test.basic.basic"
self.assertEqual(expected, subject)
def test__get_subject_for_value_with_topic_record_name_strategy(self):
basic = avro.loads(data_gen.BASIC_SCHEMA)
topic = "topic"
self.ms.registry_client.value_subject_name_strategy_func = topic_record_name_strategy # noqa
subject = self.ms._get_subject(topic=topic, schema=basic, is_key=False)
expected = "topic-python.test.basic.basic"
self.assertEqual(expected, subject)
def hash_func(self):
return hash(str(self))
def setUp(self):
# need to set up the serializer
self.client = MockSchemaRegistryClient()
self.ms = MessageSerializer(self.client)
class MockServer(HTTPSERVER.HTTPServer, object):
def __init__(self, *args, **kwargs):
super(MockServer, self).__init__(*args, **kwargs)
self.counts = {}
self.registry = MockSchemaRegistryClient()
self.schema_cache = {}
self.all_routes = {
'GET': [(r"/schemas/ids/(\d+)", 'get_schema_by_id'),
(r"/subjects/(\w+)/versions/latest", 'get_latest')],
'POST': [(r"/subjects/(\w+)/versions", 'register'),
(r"/subjects/(\w+)", 'get_version')]
}
def _send_response(self, resp, status, body):
resp.send_response(status)
resp.send_header("Content-Type", "application/json")
resp.end_headers()
resp.wfile.write(json.dumps(body).encode())
resp.finish()
def _create_error(self, msg, status=400, err_code=1):
return (status, {"error_code": err_code, "message": msg})
def _run_routes(self, req):
self.add_count((req.command, req.path))
routes = self.all_routes.get(req.command, [])
for r in routes:
m = re.match(r[0], req.path)
if m:
func = getattr(self, r[1])
status, body = func(req, m.groups())
return self._send_response(req, status, body)
# here means we got a bad req
status, body = self._create_error("bad path specified")
self._send_response(req, status, body)
def get_schema_by_id(self, req, groups):
schema_id = int(groups[0])
schema = self.registry.get_by_id(schema_id)
if not schema:
return self._create_error("schema not found", 404)
result = {"schema": json.dumps(schema.to_json())}
return (200, result)
def _get_identity_schema(self, avro_schema):
# normalized
schema_str = json.dumps(avro_schema.to_json())
if schema_str in self.schema_cache:
return self.schema_cache[schema_str]
self.schema_cache[schema_str] = avro_schema
return avro_schema
def _get_schema_from_body(self, req):
length = int(req.headers['content-length'])
data = req.rfile.read(length)
data = json.loads(data.decode("utf-8"))
schema = data.get("schema", None)
if not schema:
return None
try:
avro_schema = avro.loads(schema)
return self._get_identity_schema(avro_schema)
except:
return None
def register(self, req, groups):
avro_schema = self._get_schema_from_body(req)
if not avro_schema:
return self._create_error("Invalid avro schema", 422, 42201)
subject = groups[0]
schema_id = self.registry.register(subject, avro_schema)
return (200, {'id': schema_id})
def get_version(self, req, groups):
avro_schema = self._get_schema_from_body(req)
if not avro_schema:
return self._create_error("Invalid avro schema", 422, 42201)
subject = groups[0]
version = self.registry.get_version(subject, avro_schema)
if version == -1:
return self._create_error("Not found", 404)
schema_id = self.registry.get_id_for_schema(subject, avro_schema)
result = {
"schema": json.dumps(avro_schema.to_json()),
"subject": subject,
"id": schema_id,
"version": version
}
return (200, result)
def get_latest(self, req, groups):
subject = groups[0]
schema_id, avro_schema, version = self.registry.get_latest_schema(
subject)
if schema_id is None:
return self._create_error("Not found", 404)
result = {
"schema": json.dumps(avro_schema.to_json()),
"subject": subject,
"id": schema_id,
"version": version
}
return (200, result)
def add_count(self, path):
if path not in self.counts:
self.counts[path] = 0
self.counts[path] += 1
def test_produce_with_empty_value_no_schema(self):
schema_registry = MockSchemaRegistryClient()
producer = AvroProducer({}, schema_registry=schema_registry)
with self.assertRaises(ValueSerializerError):
producer.produce(topic='test', value='', key='not empty')
def setUp(self):
# need to set up the serializer
self.client = MockSchemaRegistryClient()
self.ms = MessageSerializer(self.client)
def test_produce_with_custom_registry_and_registry_url(self):
schema_registry = MockSchemaRegistryClient()
with self.assertRaises(ValueError):
AvroProducer({'schema.registry.url': 'http://127.0.0.1:9001'},
schema_registry=schema_registry)
def setUp(self):
self.client = MockSchemaRegistryClient()