class TestMessageSerializer(unittest.TestCase): def setUp(self): # need to set up the serializer self.client = MockSchemaRegistryClient() self.ms = MessageSerializer(self.client) def assertMessageIsSame(self, message, expected, schema_id): self.assertTrue(message) self.assertTrue(len(message) > 5) magic,sid = struct.unpack('>bI',message[0:5]) self.assertEqual(magic, 0) self.assertEqual(sid, schema_id) decoded = self.ms.decode_message(message) self.assertTrue(decoded) self.assertEqual(decoded, expected) def test_encode_with_schema_id(self): adv = Util.parse_schema_from_string(data_gen.ADVANCED_SCHEMA) basic = Util.parse_schema_from_string(data_gen.BASIC_SCHEMA) subject = 'test' schema_id = self.client.register(subject, basic) records = data_gen.BASIC_ITEMS for record in records: message = self.ms.encode_record_with_schema_id(schema_id, record) self.assertMessageIsSame(message, record, schema_id) subject = 'test_adv' adv_schema_id = self.client.register(subject, adv) self.assertNotEqual(adv_schema_id, schema_id) records = data_gen.ADVANCED_ITEMS for record in records: message = self.ms.encode_record_with_schema_id(adv_schema_id, record) self.assertMessageIsSame(message, record, adv_schema_id) def test_encode_record_for_topic(self): topic = 'test' basic = Util.parse_schema_from_string(data_gen.BASIC_SCHEMA) subject = 'test-value' schema_id = self.client.register(subject, basic) records = data_gen.BASIC_ITEMS for record in records: message = self.ms.encode_record_for_topic(topic, record) self.assertMessageIsSame(message, record ,schema_id) def test_encode_record_with_schema(self): topic = 'test' basic = Util.parse_schema_from_string(data_gen.BASIC_SCHEMA) subject = 'test-value' schema_id = self.client.register(subject, basic) records = data_gen.BASIC_ITEMS for record in records: message = self.ms.encode_record_with_schema(topic, basic, record) self.assertMessageIsSame(message, record ,schema_id)
class TestMessageSerializer(unittest.TestCase): def setUp(self): # need to set up the serializer self.client = MockSchemaRegistryClient() self.ms = MessageSerializer(self.client) def assertMessageIsSame(self, message, expected, schema_id): self.assertTrue(message) self.assertTrue(len(message) > 5) magic, sid = struct.unpack('>bI', message[0:5]) self.assertEqual(magic, 0) self.assertEqual(sid, schema_id) decoded = self.ms.decode_message(message) self.assertTrue(decoded) self.assertEqual(decoded, expected) def test_encode_with_schema_id(self): adv = Util.parse_schema_from_string(data_gen.ADVANCED_SCHEMA) basic = Util.parse_schema_from_string(data_gen.BASIC_SCHEMA) subject = 'test' schema_id = self.client.register(subject, basic) records = data_gen.BASIC_ITEMS for record in records: message = self.ms.encode_record_with_schema_id(schema_id, record) self.assertMessageIsSame(message, record, schema_id) subject = 'test_adv' adv_schema_id = self.client.register(subject, adv) self.assertNotEqual(adv_schema_id, schema_id) records = data_gen.ADVANCED_ITEMS for record in records: message = self.ms.encode_record_with_schema_id( adv_schema_id, record) self.assertMessageIsSame(message, record, adv_schema_id) def test_encode_record_for_topic(self): topic = 'test' basic = Util.parse_schema_from_string(data_gen.BASIC_SCHEMA) subject = 'test-value' schema_id = self.client.register(subject, basic) records = data_gen.BASIC_ITEMS for record in records: message = self.ms.encode_record_for_topic(topic, record) self.assertMessageIsSame(message, record, schema_id) def test_encode_record_with_schema(self): topic = 'test' basic = Util.parse_schema_from_string(data_gen.BASIC_SCHEMA) subject = 'test-value' schema_id = self.client.register(subject, basic) records = data_gen.BASIC_ITEMS for record in records: message = self.ms.encode_record_with_schema(topic, basic, record) self.assertMessageIsSame(message, record, schema_id)
class MockServer(http.server.HTTPServer, object): def __init__(self, *args, **kwargs): super(MockServer, self).__init__(*args, **kwargs) self.counts = { } self.registry = MockSchemaRegistryClient() self.schema_cache = { } self.all_routes = { 'GET' : [ (r"/schemas/ids/(\d+)", 'get_schema_by_id'), (r"/subjects/(\w+)/versions/latest", 'get_latest') ], 'POST' : [ (r"/subjects/(\w+)/versions", 'register'), (r"/subjects/(\w+)", 'get_version') ] } def _send_response(self, resp, status, body): resp.send_response(status) resp.send_header("Content-Type","application/json") resp.end_headers() resp.wfile.write(json.dumps(body)) resp.finish() def _create_error(self, msg, status=400, err_code=1): return (status, { "error_code" : err_code, "message" : msg }) def _run_routes(self, req): self.add_count((req.command, req.path)) routes = self.all_routes.get(req.command, []) for r in routes: m = re.match(r[0], req.path) if m: func = getattr(self, r[1]) status,body = func(req, m.groups()) return self._send_response(req, status, body) # here means we got a bad req status,body = self._create_error("bad path specified") self._send_response(req, status, body) def get_schema_by_id(self, req, groups): schema_id = int(groups[0]) schema = self.registry.get_by_id(schema_id) if not schema: return self._create_error("schema not found", 404) result = { "schema" : json.dumps(schema.to_json()) } return (200,result) def _get_identity_schema(self, avro_schema): # normalized schema_str = json.dumps(avro_schema.to_json()) if schema_str in self.schema_cache: return self.schema_cache[schema_str] self.schema_cache[schema_str] = avro_schema return avro_schema def _get_schema_from_body(self, req): length = int(req.headers.getheader('content-length')) data = req.rfile.read(length) data = json.loads(data) schema = data.get("schema",None) if not schema: return None try: avro_schema = Util.parse_schema_from_string(schema) return self._get_identity_schema(avro_schema) except: return None def register(self, req, groups): avro_schema = self._get_schema_from_body(req) if not avro_schema: return self._create_error("Invalid avro schema", 422, 42201) subject = groups[0] schema_id = self.registry.register(subject, avro_schema) return (200, {'id' : schema_id }) def get_version(self, req, groups): avro_schema = self._get_schema_from_body(req) if not avro_schema: return self._create_error("Invalid avro schema", 422, 42201) subject = groups[0] version = self.registry.get_version(subject, avro_schema) if version == -1: return self._create_error("Not found", 404) schema_id = self.registry.get_id_for_schema(subject, avro_schema) result = { "schema" : json.dumps(avro_schema.to_json()), "subject" : subject, "id" : schema_id, "version" : version } return (200, result) def get_latest(self, req, groups): subject = groups[0] schema_id,avro_schema,version = self.registry.get_latest_schema(subject) if schema_id == None: return self._create_error("Not found", 404) result = { "schema" : json.dumps(avro_schema.to_json()), "subject" : subject, "id" : schema_id, "version" : version } return (200, result) def add_count(self, path): if path not in self.counts: self.counts[path] = 0 self.counts[path] += 1
class MockServer(http.server.HTTPServer, object): def __init__(self, *args, **kwargs): super(MockServer, self).__init__(*args, **kwargs) self.counts = {} self.registry = MockSchemaRegistryClient() self.schema_cache = {} self.all_routes = { 'GET': [(r"/schemas/ids/(\d+)", 'get_schema_by_id'), (r"/subjects/(\w+)/versions/latest", 'get_latest')], 'POST': [(r"/subjects/(\w+)/versions", 'register'), (r"/subjects/(\w+)", 'get_version')] } def _send_response(self, resp, status, body): resp.send_response(status) resp.send_header("Content-Type", "application/json") resp.end_headers() response = BytesIO() response.write(json.dumps(body).encode('utf-8')) resp.wfile.write(response.getvalue()) def _create_error(self, msg, status=400, err_code=1): return (status, {"error_code": err_code, "message": msg}) def _run_routes(self, req): self.add_count((req.command, req.path)) routes = self.all_routes.get(req.command, []) for r in routes: m = re.match(r[0], req.path) if m: func = getattr(self, r[1]) status, body = func(req, m.groups()) return self._send_response(req, status, body) # here means we got a bad req status, body = self._create_error("bad path specified") self._send_response(req, status, body) def get_schema_by_id(self, req, groups): schema_id = int(groups[0]) schema = self.registry.get_by_id(schema_id) if not schema: return self._create_error("schema not found", 404) result = {"schema": json.dumps(schema.to_json())} return (200, result) def _get_identity_schema(self, avro_schema): # normalized schema_str = json.dumps(avro_schema.to_json()) if schema_str in self.schema_cache: return self.schema_cache[schema_str] self.schema_cache[schema_str] = avro_schema return avro_schema def _get_schema_from_body(self, req): length = int(req.headers['content-length']) data = req.rfile.read(length) data = json.loads(data) schema = data.get("schema", None) if not schema: return None try: avro_schema = Util.parse_schema_from_string(schema) return self._get_identity_schema(avro_schema) except: return None def register(self, req, groups): avro_schema = self._get_schema_from_body(req) if not avro_schema: return self._create_error("Invalid avro schema", 422, 42201) subject = groups[0] schema_id = self.registry.register(subject, avro_schema) return (200, {'id': schema_id}) def get_version(self, req, groups): avro_schema = self._get_schema_from_body(req) if not avro_schema: return self._create_error("Invalid avro schema", 422, 42201) subject = groups[0] version = self.registry.get_version(subject, avro_schema) if version == -1: return self._create_error("Not found", 404) schema_id = self.registry.get_id_for_schema(subject, avro_schema) result = { "schema": json.dumps(avro_schema.to_json()), "subject": subject, "id": schema_id, "version": version } return (200, result) def get_latest(self, req, groups): subject = groups[0] schema_id, avro_schema, version = self.registry.get_latest_schema( subject) if schema_id == None: return self._create_error("Not found", 404) result = { "schema": json.dumps(avro_schema.to_json()), "subject": subject, "id": schema_id, "version": version } return (200, result) def add_count(self, path): if path not in self.counts: self.counts[path] = 0 self.counts[path] += 1