async def test_get_schema_by_id() -> None:
"""Test the RegistryApi.get_schema_by_id method."""
# Body that we expect the registry API to return given the request.
input_schema = json.dumps(
{
"type": "record",
"name": "schema1",
"namespace": "test-schemas",
"fields": [{"name": "a", "type": "int"}],
}
)
expected_body = json.dumps({"schema": input_schema}).encode("utf-8")
client = MockRegistryApi(body=expected_body)
schema = await client.get_schema_by_id(1)
# Check that the schema was parsed
assert schema["name"] == "test-schemas.schema1"
cached_schema = client.schema_cache[1]
assert "__fastavro_parsed" in cached_schema
# Check that the schem was cached
assert cached_schema["name"] == "test-schemas.schema1"
# Check the request
assert client.url == "http://registry:8081/schemas/ids/1"
assert client.method == "GET"
async def test_polyserializer_given_schema() -> None:
"""Test the PolyySerializer class, given a schema itself."""
schema = {
"type":
"record",
"name":
"schema1",
"namespace":
"test-schemas",
"fields": [
{
"name": "a",
"type": "int"
},
{
"name": "b",
"type": "string"
},
],
}
body = json.dumps({"id": 1}).encode("utf-8")
client = MockRegistryApi(body=body)
serializer = PolySerializer(registry=client)
deserializer = Deserializer(registry=client)
message = {"a": 42, "b": "hello"}
data = await serializer.serialize(message, schema=schema)
response = await deserializer.deserialize(data)
assert response["id"] == 1
assert response["message"] == message
async def test_registryapi_delete() -> None:
"""Test RegistryApi.put().
"""
client = MockRegistryApi(url="http://registry:8081")
await client.delete("/a{/b}", url_vars={"b": "hello"})
assert client.method == "DELETE"
assert client.url == "http://registry:8081/a/hello"
async def test_parse_schema(efd_client):
"""Test the EfdClient._parse_schema method."""
# Body that we expect the registry API to return given the request.
expected_body = {
"schema":
json.dumps({
"name":
"schema1",
"type":
"record",
"fields": [{
"name": "a",
"type": "int",
"description": "Description 1",
"units": "second"
}, {
"name": "b",
"type": "double",
"description": "Description 2",
"units": "meter"
}, {
"name": "c",
"type": "float",
"description": "Description 3",
"units": "gram"
}, {
"name": "d",
"type": "string",
"description": "Description 4",
"units": "torr"
}],
}),
"subject":
"schema1",
"version":
1,
"id":
2,
}
body = json.dumps(expected_body).encode("utf-8")
client = MockRegistryApi(body=body)
schema = await client.get_schema_by_subject("schema1")
result = efd_client._parse_schema("schema1", schema)
assert isinstance(result, pd.DataFrame)
for i, l in enumerate('abcd'):
assert result['name'][i] == l
for i in range(4):
assert result['description'][i] == f'Description {i+1}'
assert 'units' in result.columns
assert 'aunits' in result.columns
assert 'type' not in result.columns
for unit in result['aunits']:
assert isinstance(unit, u.UnitBase)
async def test_registryapi_put() -> None:
"""Test RegistryApi.put()."""
expected_data = {"key": "value"}
client = MockRegistryApi(
url="http://registry:8081",
body=json.dumps(expected_data).encode("utf-8"),
)
response = await client.put("/a{/b}", url_vars={"b": "hello"}, data={})
assert response == expected_data
assert client.method == "PUT"
assert client.url == "http://registry:8081/a/hello"
async def test_registryapi_get_json() -> None:
"""Test client with a regular GET call with a JSON response."""
expected_data = {"hello": "world"}
client = MockRegistryApi(
url="http://registry:8081",
body=json.dumps(expected_data).encode("utf-8"),
)
response = await client.get(
"/subjects{/subject}/versions", url_vars={"subject": "helloworld"}
)
assert response == expected_data
assert client.method == "GET"
async def test_registryapi_get_empty() -> None:
"""Test client with a regular GET call with empty response."""
client = MockRegistryApi(url="http://registry:8081")
response = await client.get(
"/subjects{/subject}/versions", url_vars={"subject": "helloworld"}
)
assert response is None
# Check URL formatting
assert client.url == "http://registry:8081/subjects/helloworld/versions"
# Check headers
assert client.headers["accept"] == make_headers()["accept"]
assert client.headers["content-length"] == "0"
assert client.method == "GET"
async def test_get_schema_by_subject() -> None:
"""Test the RegistryApi.get_schema_by_subject method."""
# Body that we expect the registry API to return given the request.
expected_body = {
"schema": json.dumps(
{
"type": "record",
"name": "schema1",
"namespace": "test-schemas",
"fields": [{"name": "a", "type": "int"}],
}
),
"subject": "schema1",
"version": 1,
"id": 2,
}
client = MockRegistryApi(body=json.dumps(expected_body).encode("utf-8"))
result = await client.get_schema_by_subject("schema1")
# Check the request
assert (
client.url == "http://registry:8081/subjects/schema1/versions/latest"
)
assert client.method == "GET"
# Check that the schema was parsed
assert result["schema"]["name"] == "test-schemas.schema1"
assert "__fastavro_parsed" in result["schema"]
# Check other content of the result
assert result["version"] == 1
assert result["id"] == 2
assert result["subject"] == "schema1"
# Check that the schema got cached
assert ("schema1", 1) in client.subject_cache
# Get that schema, purely using the cache
result2 = await client.get_schema_by_subject("schema1", version=1)
assert result == result2
async def test_serializer() -> None:
"""Test the Serializer class."""
client = MockRegistryApi(body=json.dumps({"id": 1}).encode("utf-8"))
schema1 = {
"type":
"record",
"name":
"schema1",
"namespace":
"test-schemas",
"fields": [
{
"name": "a",
"type": "int"
},
{
"name": "b",
"type": "string"
},
],
}
serializer = await Serializer.register(registry=client, schema=schema1)
assert serializer.id == 1
message = {"a": 1, "b": "helloworld"}
data = serializer(message)
assert isinstance(data, bytes)
# Check that the message can be deserialized
# First, the wire format prefix
unpacked_id, unpacked_body = unpack_wire_format_data(data)
assert unpacked_id == serializer.id
# Second, the message
unpacked_schema = client.schema_cache[unpacked_id]
message_fh = BytesIO(unpacked_body)
message_fh.seek(0)
unpacked_message = fastavro.schemaless_reader(message_fh, unpacked_schema)
assert unpacked_message == message
async def test_register_schema() -> None:
"""Test the RegistryApi.register_schema() method."""
input_schema = {
"type": "record",
"name": "schema1",
"namespace": "test-schemas",
"fields": [{"name": "a", "type": "int"}],
}
# Body that we expect the registry API to return given the request.
expected_body = json.dumps({"id": 1}).encode("utf-8")
client = MockRegistryApi(url="http://registry:8081", body=expected_body)
schema_id = await client.register_schema(input_schema)
assert schema_id == 1
# Test details of the request itself
assert client.method == "POST"
assert (
client.url
== "http://registry:8081/subjects/test-schemas.schema1/versions"
)
sent_json = json.loads(client.body)
assert "schema" in sent_json
sent_schema = json.loads(sent_json["schema"])
assert "__fastavro_parsed" not in sent_schema
assert sent_schema["name"] == "test-schemas.schema1"
# Check that the schema is in the cache and is parsed
# Value of type "Union[int, Dict[str, Any]]" is not indexable
cached_schema = client.schema_cache[1]
assert cached_schema["name"] == "test-schemas.schema1"
assert "__fastavro_parsed" in cached_schema
# Make a second call to get the schema out
new_schema_id = await client.register_schema(input_schema)
assert new_schema_id == schema_id
async def test_deserializer() -> None:
"""Test the Deserializer class."""
# First schema
schema1 = {
"type":
"record",
"name":
"schema1",
"namespace":
"test-schemas",
"fields": [
{
"name": "a",
"type": "int"
},
{
"name": "b",
"type": "string"
},
],
}
schema1_id = 1
# Second schema
schema2 = {
"type":
"record",
"name":
"schema2",
"namespace":
"test-schemas",
"fields": [
{
"name": "c",
"type": "int"
},
{
"name": "d",
"type": "string"
},
],
}
schema2_id = 2
# insert the schemas directly into the cache
client = MockRegistryApi()
client.schema_cache.insert(schema1, schema1_id)
client.schema_cache.insert(schema2, schema2_id)
# Serialize a couple messages, using the serializer. Since the schema
# is cached it won't need a mocked response body.
serializer1 = await Serializer.register(registry=client, schema=schema1)
serializer2 = await Serializer.register(registry=client, schema=schema2)
message_1 = {"a": 42, "b": "hello"}
data_1 = serializer1(message_1)
message_2 = {"c": 13, "d": "bonjour"}
data_2 = serializer2(message_2)
# Deserialization
deserializer = Deserializer(registry=client)
response_1 = await deserializer.deserialize(data_1)
assert response_1["id"] == schema1_id
assert response_1["message"] == message_1
assert "schema" not in response_1
response_2 = await deserializer.deserialize(data_2, include_schema=True)
assert response_2["id"] == schema2_id
assert response_2["message"] == message_2
assert "schema" in response_2
assert response_2["schema"]["name"] == "test-schemas.schema2"
async def test_polyserializer_given_id() -> None:
"""Test the PolySerializer class, given schema IDs."""
# First schema
schema1 = {
"type":
"record",
"name":
"schema1",
"namespace":
"test-schemas",
"fields": [
{
"name": "a",
"type": "int"
},
{
"name": "b",
"type": "string"
},
],
}
schema1_id = 1
# Second schema
schema2 = {
"type":
"record",
"name":
"schema2",
"namespace":
"test-schemas",
"fields": [
{
"name": "c",
"type": "int"
},
{
"name": "d",
"type": "string"
},
],
}
schema2_id = 2
# insert the schemas directly into the cache
client = MockRegistryApi()
client.schema_cache.insert(schema1, schema1_id)
client.schema_cache.insert(schema2, schema2_id)
serializer = PolySerializer(registry=client)
message_1 = {"a": 42, "b": "hello"}
data_1 = await serializer.serialize(message_1, schema_id=schema1_id)
message_2 = {"c": 13, "d": "bonjour"}
data_2 = await serializer.serialize(message_2, schema_id=schema2_id)
# Deserialization
deserializer = Deserializer(registry=client)
response_1 = await deserializer.deserialize(data_1)
assert response_1["id"] == schema1_id
assert response_1["message"] == message_1
response_2 = await deserializer.deserialize(data_2)
assert response_2["id"] == schema2_id
assert response_2["message"] == message_2