class TestMessageSerializer(unittest.TestCase):
def setUp(self):
# need to set up the serializer
self.client = MockSchemaRegistryClient()
self.ms = MessageSerializer(self.client)
def assertMessageIsSame(self, message, expected, schema_id):
self.assertTrue(message)
self.assertTrue(len(message) > 5)
magic,sid = struct.unpack('>bI',message[0:5])
self.assertEqual(magic, 0)
self.assertEqual(sid, schema_id)
decoded = self.ms.decode_message(message)
self.assertTrue(decoded)
self.assertEqual(decoded, expected)
def test_encode_with_schema_id(self):
adv = Util.parse_schema_from_string(data_gen.ADVANCED_SCHEMA)
basic = Util.parse_schema_from_string(data_gen.BASIC_SCHEMA)
subject = 'test'
schema_id = self.client.register(subject, basic)
records = data_gen.BASIC_ITEMS
for record in records:
message = self.ms.encode_record_with_schema_id(schema_id, record)
self.assertMessageIsSame(message, record, schema_id)
subject = 'test_adv'
adv_schema_id = self.client.register(subject, adv)
self.assertNotEqual(adv_schema_id, schema_id)
records = data_gen.ADVANCED_ITEMS
for record in records:
message = self.ms.encode_record_with_schema_id(adv_schema_id, record)
self.assertMessageIsSame(message, record, adv_schema_id)
def test_encode_record_for_topic(self):
topic = 'test'
basic = Util.parse_schema_from_string(data_gen.BASIC_SCHEMA)
subject = 'test-value'
schema_id = self.client.register(subject, basic)
records = data_gen.BASIC_ITEMS
for record in records:
message = self.ms.encode_record_for_topic(topic, record)
self.assertMessageIsSame(message, record ,schema_id)
def test_encode_record_with_schema(self):
topic = 'test'
basic = Util.parse_schema_from_string(data_gen.BASIC_SCHEMA)
subject = 'test-value'
schema_id = self.client.register(subject, basic)
records = data_gen.BASIC_ITEMS
for record in records:
message = self.ms.encode_record_with_schema(topic, basic, record)
self.assertMessageIsSame(message, record ,schema_id)
class TestMessageSerializer(unittest.TestCase):
def setUp(self):
# need to set up the serializer
self.client = MockSchemaRegistryClient()
self.ms = MessageSerializer(self.client)
def assertMessageIsSame(self, message, expected, schema_id):
self.assertTrue(message)
self.assertTrue(len(message) > 5)
magic, sid = struct.unpack('>bI', message[0:5])
self.assertEqual(magic, 0)
self.assertEqual(sid, schema_id)
decoded = self.ms.decode_message(message)
self.assertTrue(decoded)
self.assertEqual(decoded, expected)
def test_encode_with_schema_id(self):
adv = Util.parse_schema_from_string(data_gen.ADVANCED_SCHEMA)
basic = Util.parse_schema_from_string(data_gen.BASIC_SCHEMA)
subject = 'test'
schema_id = self.client.register(subject, basic)
records = data_gen.BASIC_ITEMS
for record in records:
message = self.ms.encode_record_with_schema_id(schema_id, record)
self.assertMessageIsSame(message, record, schema_id)
subject = 'test_adv'
adv_schema_id = self.client.register(subject, adv)
self.assertNotEqual(adv_schema_id, schema_id)
records = data_gen.ADVANCED_ITEMS
for record in records:
message = self.ms.encode_record_with_schema_id(
adv_schema_id, record)
self.assertMessageIsSame(message, record, adv_schema_id)
def test_encode_record_for_topic(self):
topic = 'test'
basic = Util.parse_schema_from_string(data_gen.BASIC_SCHEMA)
subject = 'test-value'
schema_id = self.client.register(subject, basic)
records = data_gen.BASIC_ITEMS
for record in records:
message = self.ms.encode_record_for_topic(topic, record)
self.assertMessageIsSame(message, record, schema_id)
def test_encode_record_with_schema(self):
topic = 'test'
basic = Util.parse_schema_from_string(data_gen.BASIC_SCHEMA)
subject = 'test-value'
schema_id = self.client.register(subject, basic)
records = data_gen.BASIC_ITEMS
for record in records:
message = self.ms.encode_record_with_schema(topic, basic, record)
self.assertMessageIsSame(message, record, schema_id)
def __init__(self, *args, **kwargs):
super(MockServer, self).__init__(*args, **kwargs)
self.counts = {}
self.registry = MockSchemaRegistryClient()
self.schema_cache = {}
self.all_routes = {
'GET': [(r"/schemas/ids/(\d+)", 'get_schema_by_id'),
(r"/subjects/(\w+)/versions/latest", 'get_latest')],
'POST': [(r"/subjects/(\w+)/versions", 'register'),
(r"/subjects/(\w+)", 'get_version')]
}
def __init__(self, *args, **kwargs):
super(MockServer, self).__init__(*args, **kwargs)
self.counts = { }
self.registry = MockSchemaRegistryClient()
self.schema_cache = { }
self.all_routes = {
'GET' : [
(r"/schemas/ids/(\d+)", 'get_schema_by_id'),
(r"/subjects/(\w+)/versions/latest", 'get_latest')
],
'POST' : [
(r"/subjects/(\w+)/versions", 'register'),
(r"/subjects/(\w+)", 'get_version')
]
}
class MockServer(http.server.HTTPServer, object):
def __init__(self, *args, **kwargs):
super(MockServer, self).__init__(*args, **kwargs)
self.counts = { }
self.registry = MockSchemaRegistryClient()
self.schema_cache = { }
self.all_routes = {
'GET' : [
(r"/schemas/ids/(\d+)", 'get_schema_by_id'),
(r"/subjects/(\w+)/versions/latest", 'get_latest')
],
'POST' : [
(r"/subjects/(\w+)/versions", 'register'),
(r"/subjects/(\w+)", 'get_version')
]
}
def _send_response(self, resp, status, body):
resp.send_response(status)
resp.send_header("Content-Type","application/json")
resp.end_headers()
resp.wfile.write(json.dumps(body))
resp.finish()
def _create_error(self, msg, status=400, err_code=1):
return (status, {
"error_code" : err_code,
"message" : msg
})
def _run_routes(self, req):
self.add_count((req.command, req.path))
routes = self.all_routes.get(req.command, [])
for r in routes:
m = re.match(r[0], req.path)
if m:
func = getattr(self, r[1])
status,body = func(req, m.groups())
return self._send_response(req, status, body)
# here means we got a bad req
status,body = self._create_error("bad path specified")
self._send_response(req, status, body)
def get_schema_by_id(self, req, groups):
schema_id = int(groups[0])
schema = self.registry.get_by_id(schema_id)
if not schema:
return self._create_error("schema not found", 404)
result = {
"schema" : json.dumps(schema.to_json())
}
return (200,result)
def _get_identity_schema(self, avro_schema):
# normalized
schema_str = json.dumps(avro_schema.to_json())
if schema_str in self.schema_cache:
return self.schema_cache[schema_str]
self.schema_cache[schema_str] = avro_schema
return avro_schema
def _get_schema_from_body(self, req):
length = int(req.headers.getheader('content-length'))
data = req.rfile.read(length)
data = json.loads(data)
schema = data.get("schema",None)
if not schema:
return None
try:
avro_schema = Util.parse_schema_from_string(schema)
return self._get_identity_schema(avro_schema)
except:
return None
def register(self, req, groups):
avro_schema = self._get_schema_from_body(req)
if not avro_schema:
return self._create_error("Invalid avro schema", 422, 42201)
subject = groups[0]
schema_id = self.registry.register(subject, avro_schema)
return (200, {'id' : schema_id })
def get_version(self, req, groups):
avro_schema = self._get_schema_from_body(req)
if not avro_schema:
return self._create_error("Invalid avro schema", 422, 42201)
subject = groups[0]
version = self.registry.get_version(subject, avro_schema)
if version == -1:
return self._create_error("Not found", 404)
schema_id = self.registry.get_id_for_schema(subject, avro_schema)
result = {
"schema" : json.dumps(avro_schema.to_json()),
"subject" : subject,
"id" : schema_id,
"version" : version
}
return (200, result)
def get_latest(self, req, groups):
subject = groups[0]
schema_id,avro_schema,version = self.registry.get_latest_schema(subject)
if schema_id == None:
return self._create_error("Not found", 404)
result = {
"schema" : json.dumps(avro_schema.to_json()),
"subject" : subject,
"id" : schema_id,
"version" : version
}
return (200, result)
def add_count(self, path):
if path not in self.counts:
self.counts[path] = 0
self.counts[path] += 1
class MockServer(http.server.HTTPServer, object):
def __init__(self, *args, **kwargs):
super(MockServer, self).__init__(*args, **kwargs)
self.counts = {}
self.registry = MockSchemaRegistryClient()
self.schema_cache = {}
self.all_routes = {
'GET': [(r"/schemas/ids/(\d+)", 'get_schema_by_id'),
(r"/subjects/(\w+)/versions/latest", 'get_latest')],
'POST': [(r"/subjects/(\w+)/versions", 'register'),
(r"/subjects/(\w+)", 'get_version')]
}
def _send_response(self, resp, status, body):
resp.send_response(status)
resp.send_header("Content-Type", "application/json")
resp.end_headers()
response = BytesIO()
response.write(json.dumps(body).encode('utf-8'))
resp.wfile.write(response.getvalue())
def _create_error(self, msg, status=400, err_code=1):
return (status, {"error_code": err_code, "message": msg})
def _run_routes(self, req):
self.add_count((req.command, req.path))
routes = self.all_routes.get(req.command, [])
for r in routes:
m = re.match(r[0], req.path)
if m:
func = getattr(self, r[1])
status, body = func(req, m.groups())
return self._send_response(req, status, body)
# here means we got a bad req
status, body = self._create_error("bad path specified")
self._send_response(req, status, body)
def get_schema_by_id(self, req, groups):
schema_id = int(groups[0])
schema = self.registry.get_by_id(schema_id)
if not schema:
return self._create_error("schema not found", 404)
result = {"schema": json.dumps(schema.to_json())}
return (200, result)
def _get_identity_schema(self, avro_schema):
# normalized
schema_str = json.dumps(avro_schema.to_json())
if schema_str in self.schema_cache:
return self.schema_cache[schema_str]
self.schema_cache[schema_str] = avro_schema
return avro_schema
def _get_schema_from_body(self, req):
length = int(req.headers['content-length'])
data = req.rfile.read(length)
data = json.loads(data)
schema = data.get("schema", None)
if not schema:
return None
try:
avro_schema = Util.parse_schema_from_string(schema)
return self._get_identity_schema(avro_schema)
except:
return None
def register(self, req, groups):
avro_schema = self._get_schema_from_body(req)
if not avro_schema:
return self._create_error("Invalid avro schema", 422, 42201)
subject = groups[0]
schema_id = self.registry.register(subject, avro_schema)
return (200, {'id': schema_id})
def get_version(self, req, groups):
avro_schema = self._get_schema_from_body(req)
if not avro_schema:
return self._create_error("Invalid avro schema", 422, 42201)
subject = groups[0]
version = self.registry.get_version(subject, avro_schema)
if version == -1:
return self._create_error("Not found", 404)
schema_id = self.registry.get_id_for_schema(subject, avro_schema)
result = {
"schema": json.dumps(avro_schema.to_json()),
"subject": subject,
"id": schema_id,
"version": version
}
return (200, result)
def get_latest(self, req, groups):
subject = groups[0]
schema_id, avro_schema, version = self.registry.get_latest_schema(
subject)
if schema_id == None:
return self._create_error("Not found", 404)
result = {
"schema": json.dumps(avro_schema.to_json()),
"subject": subject,
"id": schema_id,
"version": version
}
return (200, result)
def add_count(self, path):
if path not in self.counts:
self.counts[path] = 0
self.counts[path] += 1
def setUp(self):
# need to set up the serializer
self.client = MockSchemaRegistryClient()
self.ms = MessageSerializer(self.client)
def setUp(self):
# need to set up the serializer
self.client = MockSchemaRegistryClient()
self.ms = MessageSerializer(self.client)
def setUp(self):
self.client = MockSchemaRegistryClient()
