def __init__(self, *args, **kwargs): super(MockServer, self).__init__(*args, **kwargs) self.counts = {} self.registry = MockSchemaRegistryClient() self.schema_cache = {} self.all_routes = { 'GET': [(r"/schemas/ids/(\d+)", 'get_schema_by_id'), (r"/subjects/(\w+)/versions/latest", 'get_latest')], 'POST': [(r"/subjects/(\w+)/versions", 'register'), (r"/subjects/(\w+)", 'get_version')] }
class TestMessageSerializer(unittest.TestCase): def setUp(self): # need to set up the serializer self.client = MockSchemaRegistryClient() self.ms = MessageSerializer(self.client) def assertMessageIsSame(self, message, expected, schema_id, schema): self.assertTrue(message) self.assertTrue(len(message) > 5) magic, sid = struct.unpack('>bI', message[0:5]) self.assertEqual(magic, 0) self.assertEqual(sid, schema_id) decoded_msg, decoded_schema = self.ms.decode_message(message) self.assertTrue(decoded_msg) self.assertEqual(decoded_msg, expected) self.assertEqual(decoded_schema, schema) def test_encode_with_schema_id(self): adv = avro.loads(data_gen.ADVANCED_SCHEMA) basic = avro.loads(data_gen.BASIC_SCHEMA) subject = 'test' schema_id = self.client.register(subject, basic) records = data_gen.BASIC_ITEMS for record in records: message = self.ms.encode_record_with_schema_id(schema_id, record) self.assertMessageIsSame(message, record, schema_id, basic) subject = 'test_adv' adv_schema_id = self.client.register(subject, adv) self.assertNotEqual(adv_schema_id, schema_id) records = data_gen.ADVANCED_ITEMS for record in records: message = self.ms.encode_record_with_schema_id( adv_schema_id, record) self.assertMessageIsSame(message, record, adv_schema_id, adv) def test_encode_record_with_schema(self): topic = 'test' basic = avro.loads(data_gen.BASIC_SCHEMA) subject = 'test-value' schema_id = self.client.register(subject, basic) records = data_gen.BASIC_ITEMS for record in records: message = self.ms.encode_record_with_schema(topic, basic, record) self.assertMessageIsSame(message, record, schema_id, basic) def test_decode_none(self): """"null/None messages should decode to None""" self.assertIsNone(self.ms.decode_message(None)) def hash_func(self): return hash(str(self))
class TestMessageSerializer(unittest.TestCase): def setUp(self): # need to set up the serializer self.client = MockSchemaRegistryClient() self.ms = MessageSerializer(self.client) def assertMessageIsSame(self, message, expected, schema_id): self.assertTrue(message) self.assertTrue(len(message) > 5) magic, sid = struct.unpack('>bI', message[0:5]) self.assertEqual(magic, 0) self.assertEqual(sid, schema_id) decoded = self.ms.decode_message(message) self.assertTrue(decoded) self.assertEqual(decoded, expected) def test_encode_with_schema_id(self): adv = avro.loads(data_gen.ADVANCED_SCHEMA) basic = avro.loads(data_gen.BASIC_SCHEMA) subject = 'test' schema_id = self.client.register(subject, basic) records = data_gen.BASIC_ITEMS for record in records: message = self.ms.encode_record_with_schema_id(schema_id, record) self.assertMessageIsSame(message, record, schema_id) subject = 'test_adv' adv_schema_id = self.client.register(subject, adv) self.assertNotEqual(adv_schema_id, schema_id) records = data_gen.ADVANCED_ITEMS for record in records: message = self.ms.encode_record_with_schema_id(adv_schema_id, record) self.assertMessageIsSame(message, record, adv_schema_id) def test_encode_record_with_schema(self): topic = 'test' basic = avro.loads(data_gen.BASIC_SCHEMA) subject = 'test-value' schema_id = self.client.register(subject, basic) records = data_gen.BASIC_ITEMS for record in records: message = self.ms.encode_record_with_schema(topic, basic, record) self.assertMessageIsSame(message, record, schema_id) def test_decode_none(self): """"null/None messages should decode to None""" self.assertIsNone(self.ms.decode_message(None)) def hash_func(self): return hash(str(self))
def test_produce_with_empty_key_value_with_schema(self): value_schema = avro.load(os.path.join(avsc_dir, "basic_schema.avsc")) schema_registry = MockSchemaRegistryClient() producer = AvroProducer({}, schema_registry=schema_registry, default_value_schema=value_schema) producer.produce(topic='test', value={'name': 'abc'}, key='')
def test_produce_with_custom_registry(self): schema_registry = MockSchemaRegistryClient() value_schema = avro.load(os.path.join(avsc_dir, "basic_schema.avsc")) producer = AvroProducer({}, schema_registry=schema_registry) producer.produce(topic='test', value={"name": 'abc"'}, value_schema=value_schema, key='mykey')
def test_produce_with_empty_key_no_schema(self): value_schema = avro.load(os.path.join(avsc_dir, "primitive_float.avsc")) schema_registry = MockSchemaRegistryClient() producer = AvroProducer({}, schema_registry=schema_registry, default_value_schema=value_schema) with self.assertRaises(KeySerializerError): producer.produce(topic='test', value=0.0, key='')
def test_explicit_topic_subject_name_strategy(self): schema_registry = MockSchemaRegistryClient() producer = AvroProducer( config={}, schema_registry=schema_registry, key_subject_name_strategy=strategies.topic_name_strategy, value_subject_name_strategy=strategies.topic_name_strategy) serializer = producer._serializer assert serializer.key_subject_name_strategy is strategies.topic_name_strategy assert serializer.value_subject_name_strategy is strategies.topic_name_strategy
def test_differing_key_and_value_subject_name_strategies(self): schema_registry = MockSchemaRegistryClient() producer = AvroProducer( config={}, schema_registry=schema_registry, key_subject_name_strategy=strategies.record_name_strategy, value_subject_name_strategy=strategies.topic_record_name_strategy) serializer = producer._serializer assert serializer.key_subject_name_strategy is strategies.record_name_strategy assert serializer.value_subject_name_strategy is strategies.topic_record_name_strategy
def test_produce_with_empty_key_value_with_schema(self): key_schema = avro.load(os.path.join(avsc_dir, "primitive_string.avsc")) value_schema = avro.load(os.path.join(avsc_dir, "primitive_float.avsc")) schema_registry = MockSchemaRegistryClient() producer = AvroProducer({}, schema_registry=schema_registry, default_key_schema=key_schema, default_value_schema=value_schema) producer.produce(topic='test', value=0.0, key='')
class TestMessageSerializer(unittest.TestCase): def setUp(self): # need to set up the serializer self.client = MockSchemaRegistryClient() self.ms = MessageSerializer(self.client) def assertMessageIsSame(self, message, expected, schema_id): self.assertTrue(message) self.assertTrue(len(message) > 5) magic, sid = struct.unpack('>bI', message[0:5]) self.assertEqual(magic, 0) self.assertEqual(sid, schema_id) decoded = self.ms.decode_message(message) self.assertTrue(decoded) self.assertEqual(decoded, expected) def test_encode_with_schema_id(self): adv = avro.loads(data_gen.ADVANCED_SCHEMA) basic = avro.loads(data_gen.BASIC_SCHEMA) subject = 'test' schema_id = self.client.register(subject, basic) records = data_gen.BASIC_ITEMS for record in records: message = self.ms.encode_record_with_schema_id(schema_id, record) self.assertMessageIsSame(message, record, schema_id) subject = 'test_adv' adv_schema_id = self.client.register(subject, adv) self.assertNotEqual(adv_schema_id, schema_id) records = data_gen.ADVANCED_ITEMS for record in records: message = self.ms.encode_record_with_schema_id( adv_schema_id, record) self.assertMessageIsSame(message, record, adv_schema_id) def test_encode_record_with_schema(self): topic = 'test' basic = avro.loads(data_gen.BASIC_SCHEMA) subject = 'test-value' schema_id = self.client.register(subject, basic) records = data_gen.BASIC_ITEMS for record in records: message = self.ms.encode_record_with_schema(topic, basic, record) self.assertMessageIsSame(message, record, schema_id) def test_decode_none(self): """"null/None messages should decode to None""" self.assertIsNone(self.ms.decode_message(None)) def test_decode_with_schema(self): topic = 'test_specific' schema_v1 = avro.loads( data_gen.load_schema_file('evolution_schema_v1.avsc')) schema_v2 = avro.loads( data_gen.load_schema_file('evolution_schema_v2.avsc')) dsv1 = SpecificRecordMessageDeserializer(self.client, value_schema=schema_v1) dsv2 = SpecificRecordMessageDeserializer(self.client, value_schema=schema_v2) record_v1 = {"name": "suzyq", "age": 27} record_v2 = dict(record_v1) record_v2['gender'] = 'NONE' encoded_v1 = self.ms.encode_record_with_schema(topic, schema_v1, record_v1) decoded_v1_v1 = dsv1.decode_message(encoded_v1, is_key=False) self.assertDictEqual(record_v1, decoded_v1_v1) decoded_v1_v2 = dsv2.decode_message(encoded_v1, is_key=False) self.assertDictEqual(record_v2, decoded_v1_v2) encoded_v2 = self.ms.encode_record_with_schema(topic, schema_v2, record_v2) decoded_v2_v2 = dsv2.decode_message(encoded_v2, is_key=False) self.assertDictEqual(record_v2, decoded_v2_v2) decoded_v2_v1 = dsv1.decode_message(encoded_v2, is_key=False) self.assertDictEqual(record_v1, decoded_v2_v1) def hash_func(self): return hash(str(self))
class TestMessageSerializer(unittest.TestCase): def setUp(self): # need to set up the serializer self.client = MockSchemaRegistryClient() self.ms = MessageSerializer(self.client) def assertMessageIsSame(self, message, expected, schema_id): self.assertTrue(message) self.assertTrue(len(message) > 5) magic, sid = struct.unpack('>bI', message[0:5]) self.assertEqual(magic, 0) self.assertEqual(sid, schema_id) decoded = self.ms.decode_message(message) self.assertTrue(decoded) self.assertEqual(decoded, expected) def test_encode_with_schema_id(self): adv = avro.loads(data_gen.ADVANCED_SCHEMA) basic = avro.loads(data_gen.BASIC_SCHEMA) subject = 'test' schema_id = self.client.register(subject, basic) records = data_gen.BASIC_ITEMS for record in records: message = self.ms.encode_record_with_schema_id(schema_id, record) self.assertMessageIsSame(message, record, schema_id) subject = 'test_adv' adv_schema_id = self.client.register(subject, adv) self.assertNotEqual(adv_schema_id, schema_id) records = data_gen.ADVANCED_ITEMS for record in records: message = self.ms.encode_record_with_schema_id( adv_schema_id, record) self.assertMessageIsSame(message, record, adv_schema_id) def test_encode_record_with_schema(self): topic = 'test' basic = avro.loads(data_gen.BASIC_SCHEMA) subject = 'test-value' schema_id = self.client.register(subject, basic) records = data_gen.BASIC_ITEMS for record in records: message = self.ms.encode_record_with_schema(topic, basic, record) self.assertMessageIsSame(message, record, schema_id) @skipIf(version_info < (3, ), 'unittest.mock.patch not available in Python 2') def test_encode_record_with_schema_sets_writers_cache_once(self): topic = 'test' basic = avro.loads(data_gen.BASIC_SCHEMA) subject = 'test-value' self.client.register(subject, basic) records = data_gen.BASIC_ITEMS with patch.object(self.ms, "_get_encoder_func") as encoder_func_mock: for record in records: self.ms.encode_record_with_schema(topic, basic, record) encoder_func_mock.assert_called_once_with(basic) def test_decode_none(self): """null/None messages should decode to None""" self.assertIsNone(self.ms.decode_message(None)) def hash_func(self): return hash(str(self))
class TestMessageSerializer(unittest.TestCase): def setUp(self): # need to set up the serializer self.client = MockSchemaRegistryClient() self.ms = MessageSerializer(self.client) def assertMessageIsSame(self, message, expected, schema_id): self.assertTrue(message) self.assertTrue(len(message) > 5) magic, sid = struct.unpack('>bI', message[0:5]) self.assertEqual(magic, 0) self.assertEqual(sid, schema_id) decoded = self.ms.decode_message(message) self.assertTrue(decoded) self.assertEqual(decoded, expected) def test_encode_with_schema_id(self): adv = avro.loads(data_gen.ADVANCED_SCHEMA) basic = avro.loads(data_gen.BASIC_SCHEMA) subject = 'test' schema_id = self.client.register(subject, basic) records = data_gen.BASIC_ITEMS for record in records: message = self.ms.encode_record_with_schema_id(schema_id, record) self.assertMessageIsSame(message, record, schema_id) subject = 'test_adv' adv_schema_id = self.client.register(subject, adv) self.assertNotEqual(adv_schema_id, schema_id) records = data_gen.ADVANCED_ITEMS for record in records: message = self.ms.encode_record_with_schema_id(adv_schema_id, record) self.assertMessageIsSame(message, record, adv_schema_id) def test_encode_record_with_schema(self): topic = 'test' basic = avro.loads(data_gen.BASIC_SCHEMA) subject = 'test-value' schema_id = self.client.register(subject, basic) records = data_gen.BASIC_ITEMS for record in records: message = self.ms.encode_record_with_schema(topic, basic, record) self.assertMessageIsSame(message, record, schema_id) def test_decode_none(self): """"null/None messages should decode to None""" self.assertIsNone(self.ms.decode_message(None)) def test__get_subject_for_key_with_topic_name_strategy(self): basic = avro.loads(data_gen.BASIC_SCHEMA) topic = "topic" self.ms.registry_client.key_subject_name_strategy_func = topic_name_strategy # noqa subject = self.ms._get_subject(topic=topic, schema=basic, is_key=True) expected = "topic-key" self.assertEqual(expected, subject) def test__get_subject_for_key_with_record_name_strategy(self): basic = avro.loads(data_gen.BASIC_SCHEMA) topic = "topic" self.ms.registry_client.key_subject_name_strategy_func = record_name_strategy # noqa subject = self.ms._get_subject(topic=topic, schema=basic, is_key=True) expected = "python.test.basic.basic" self.assertEqual(expected, subject) def test__get_subject_for_key_with_topic_record_name_strategy(self): basic = avro.loads(data_gen.BASIC_SCHEMA) topic = "topic" self.ms.registry_client.key_subject_name_strategy_func = topic_record_name_strategy # noqa subject = self.ms._get_subject(topic=topic, schema=basic, is_key=True) expected = "topic-python.test.basic.basic" self.assertEqual(expected, subject) def test__get_subject_for_value_with_topic_name_strategy(self): basic = avro.loads(data_gen.BASIC_SCHEMA) topic = "topic" self.ms.registry_client.value_subject_name_strategy_func = topic_name_strategy # noqa subject = self.ms._get_subject(topic=topic, schema=basic, is_key=False) expected = "topic-value" self.assertEqual(expected, subject) def test__get_subject_for_value_with_record_name_strategy(self): basic = avro.loads(data_gen.BASIC_SCHEMA) topic = "topic" self.ms.registry_client.value_subject_name_strategy_func = record_name_strategy # noqa subject = self.ms._get_subject(topic=topic, schema=basic, is_key=False) expected = "python.test.basic.basic" self.assertEqual(expected, subject) def test__get_subject_for_value_with_topic_record_name_strategy(self): basic = avro.loads(data_gen.BASIC_SCHEMA) topic = "topic" self.ms.registry_client.value_subject_name_strategy_func = topic_record_name_strategy # noqa subject = self.ms._get_subject(topic=topic, schema=basic, is_key=False) expected = "topic-python.test.basic.basic" self.assertEqual(expected, subject) def hash_func(self): return hash(str(self))
def setUp(self): # need to set up the serializer self.client = MockSchemaRegistryClient() self.ms = MessageSerializer(self.client)
class MockServer(HTTPSERVER.HTTPServer, object): def __init__(self, *args, **kwargs): super(MockServer, self).__init__(*args, **kwargs) self.counts = {} self.registry = MockSchemaRegistryClient() self.schema_cache = {} self.all_routes = { 'GET': [(r"/schemas/ids/(\d+)", 'get_schema_by_id'), (r"/subjects/(\w+)/versions/latest", 'get_latest')], 'POST': [(r"/subjects/(\w+)/versions", 'register'), (r"/subjects/(\w+)", 'get_version')] } def _send_response(self, resp, status, body): resp.send_response(status) resp.send_header("Content-Type", "application/json") resp.end_headers() resp.wfile.write(json.dumps(body).encode()) resp.finish() def _create_error(self, msg, status=400, err_code=1): return (status, {"error_code": err_code, "message": msg}) def _run_routes(self, req): self.add_count((req.command, req.path)) routes = self.all_routes.get(req.command, []) for r in routes: m = re.match(r[0], req.path) if m: func = getattr(self, r[1]) status, body = func(req, m.groups()) return self._send_response(req, status, body) # here means we got a bad req status, body = self._create_error("bad path specified") self._send_response(req, status, body) def get_schema_by_id(self, req, groups): schema_id = int(groups[0]) schema = self.registry.get_by_id(schema_id) if not schema: return self._create_error("schema not found", 404) result = {"schema": json.dumps(schema.to_json())} return (200, result) def _get_identity_schema(self, avro_schema): # normalized schema_str = json.dumps(avro_schema.to_json()) if schema_str in self.schema_cache: return self.schema_cache[schema_str] self.schema_cache[schema_str] = avro_schema return avro_schema def _get_schema_from_body(self, req): length = int(req.headers['content-length']) data = req.rfile.read(length) data = json.loads(data.decode("utf-8")) schema = data.get("schema", None) if not schema: return None try: avro_schema = avro.loads(schema) return self._get_identity_schema(avro_schema) except: return None def register(self, req, groups): avro_schema = self._get_schema_from_body(req) if not avro_schema: return self._create_error("Invalid avro schema", 422, 42201) subject = groups[0] schema_id = self.registry.register(subject, avro_schema) return (200, {'id': schema_id}) def get_version(self, req, groups): avro_schema = self._get_schema_from_body(req) if not avro_schema: return self._create_error("Invalid avro schema", 422, 42201) subject = groups[0] version = self.registry.get_version(subject, avro_schema) if version == -1: return self._create_error("Not found", 404) schema_id = self.registry.get_id_for_schema(subject, avro_schema) result = { "schema": json.dumps(avro_schema.to_json()), "subject": subject, "id": schema_id, "version": version } return (200, result) def get_latest(self, req, groups): subject = groups[0] schema_id, avro_schema, version = self.registry.get_latest_schema( subject) if schema_id is None: return self._create_error("Not found", 404) result = { "schema": json.dumps(avro_schema.to_json()), "subject": subject, "id": schema_id, "version": version } return (200, result) def add_count(self, path): if path not in self.counts: self.counts[path] = 0 self.counts[path] += 1
def test_produce_with_empty_value_no_schema(self): schema_registry = MockSchemaRegistryClient() producer = AvroProducer({}, schema_registry=schema_registry) with self.assertRaises(ValueSerializerError): producer.produce(topic='test', value='', key='not empty')
def setUp(self): # need to set up the serializer self.client = MockSchemaRegistryClient() self.ms = MessageSerializer(self.client)
def test_produce_with_custom_registry_and_registry_url(self): schema_registry = MockSchemaRegistryClient() with self.assertRaises(ValueError): AvroProducer({'schema.registry.url': 'http://127.0.0.1:9001'}, schema_registry=schema_registry)
def setUp(self): self.client = MockSchemaRegistryClient()