def test_nested_schema_with_register_codec(client):
nested_schema = schema.AvroSchema(data_gen.NESTED_SCHENA)
order_schema = schema.AvroSchema(data_gen.ORDER_SCHENA)
customer_serializer = serializer.FaustSerializer(client,
"test-nested-schema",
nested_schema)
order_serializer = serializer.FaustSerializer(client, "test-order-schema",
order_schema)
faust.serializers.codecs.register("customer_serializer",
customer_serializer)
faust.serializers.codecs.register("order_serializer", order_serializer)
class Order(faust.Record, serializer="order_serializer"):
uid: int
class Customer(faust.Record, serializer="customer_serializer"):
name: str
uid: int
order: Order
payload = data_gen.create_nested_schema()
customer = Customer(**payload)
message_encoded = customer.dumps()
assert message_encoded
assert len(message_encoded) > 5
assert isinstance(message_encoded, bytes)
message_decoded = Customer.loads(message_encoded)
assert message_decoded == customer
async def test_multi_register(client):
"""
Register two different schemas under the same subject
with backwards compatibility
"""
version_1 = schema.AvroSchema(data_gen.USER_V1)
version_2 = schema.AvroSchema(data_gen.USER_V2)
subject = "test-user-schema"
id1 = await client.register(subject, version_1)
latest_schema_1 = await client.get_schema(subject)
await client.check_version(subject, version_1)
id2 = await client.register(subject, version_2)
latest_schema_2 = await client.get_schema(subject)
await client.check_version(subject, version_2)
assert id1 != id2
assert latest_schema_1 != latest_schema_2
# ensure version is higher
assert latest_schema_1.version < latest_schema_2.version
await client.register(subject, version_1)
latest_schema_3 = await client.get_schema(subject)
assert latest_schema_2 == latest_schema_3
async def test_delete_subject(async_client, user_schema_v3):
subject = "subject-to-delete"
versions = [schema.AvroSchema(data_gen.USER_V1), schema.AvroSchema(data_gen.USER_V2)]
for version in versions:
await async_client.register(subject, version)
assert len(await async_client.delete_subject(subject)) == len(versions)
async def test_register_with_logical_types(async_client):
parsed = schema.AvroSchema(data_gen.LOGICAL_TYPES_SCHEMA)
schema_id = await async_client.register("test-logical-types-schema",
parsed)
assert schema_id > 0
assert len(async_client.id_to_schema) == 1
async def test_dupe_register(async_client):
parsed = schema.AvroSchema(data_gen.BASIC_SCHEMA)
subject = "test-basic-schema"
schema_id = await async_client.register(subject, parsed)
# Verify we had a check version call
async_client.assert_url_suffix(0, "/subjects/%s" % subject)
async_client.assert_method(0, "POST")
# Verify that we had a register call
async_client.assert_url_suffix(1, "/subjects/%s/versions" % subject)
async_client.assert_method(1, "POST")
assert len(async_client.request_calls) == 2
assert schema_id > 0
latest = await async_client.get_schema(subject)
async_client.assert_url_suffix(2, "/subjects/%s/versions/latest" % subject)
async_client.assert_method(2, "GET")
assert len(async_client.request_calls) == 3
# register again under same subject
dupe_id = await async_client.register(subject, parsed)
assert schema_id == dupe_id
# Served from cache
assert len(async_client.request_calls) == 3
dupe_latest = await async_client.get_schema(subject)
assert latest == dupe_latest
def _get_encoder_func(
self, avro_schema: typing.Union[schema.AvroSchema,
str]) -> typing.Callable:
if isinstance(avro_schema, str):
avro_schema = schema.AvroSchema(json.loads(avro_schema))
return lambda record, fp: schemaless_writer(fp, avro_schema.schema,
record) # type: ignore
def lambda_handler(event, context):
responseData = {}
print(event)
if 'SCHEMA_REEGISTRY_URI' in os.environ:
SchemaRegistryURI = os.environ['SCHEMA_REEGISTRY_URI']
print('SchemaRegistryURI is: {}'.format(SchemaRegistryURI))
else:
responseData['cause'] = "Schema registry URI not provided"
cfnresponse.send(event, context, cfnresponse.FAILED, responseData)
if 'SchemaConfig' in event['ResourceProperties']:
json_content = json.loads(event['ResourceProperties']['SchemaConfig'])
print("Loaded AVRO schema from file {}".format(json_content))
else:
responseData['status'] = "Location of Schema file not provided"
responseData['cause'] = event['ResourceProperties']
print('SchemaFile not provided')
cfnresponse.send(event, context, cfnresponse.FAILED, responseData)
if event['RequestType'] == 'Create':
if 'type' in json_content:
print("Looks like we got valide AVRO schema")
try:
client = SchemaRegistryClient(url=SchemaRegistryURI)
except Exception as e:
responseData['cause'] = "Schema registry client cannot be created {} \
".format(e.__class__.__name__)
print(responseData['cause'])
cfnresponse.send(event, context, cfnresponse.FAILED, responseData)
print('Schema registry client created')
avro_schema = schema.AvroSchema(json_content)
print('AVRO schema object created')
###Get subjects, just for debug
list_registered_subjects = requests.get(SchemaRegistryURI + "/subjects")
print("Native http rest response, list of current subjects {}, \
".format(list_registered_subjects.json()))
try:
schema_id = client.register(json_content['name'] + "-value", avro_schema, timeout=10)
responseData['cause'] = "Schema {} created successfully, schema id is {} \
".format(json_content['name'], schema_id)
print(responseData['cause'])
cfnresponse.send(event, context, cfnresponse.SUCCESS, responseData)
except Exception as e:
responseData['cause'] = "Some exception in request to schema register happened, {}\
".format(format(e.__class__.__name__))
print (responseData['cause'])
cfnresponse.send(event, context, cfnresponse.FAILED, responseData)
else:
responseData['cause'] = 'Provided file not an AVRO schema, \
there are no /"type/" field in request object'
print(responseData['cause'])
cfnresponse.send(event, context, cfnresponse.FAILED, responseData)
else:
#if event['RequestType'] == 'Delete' or event['RequestType'] == 'Update':
responseData['cause'] = 'CloudFormation Delete and Update method not implemented, \
just return cnfresponse=SUCCSESS without any job/modifications'
cfnresponse.send(event, context, cfnresponse.SUCCESS, responseData)
async def test_multi_subject_register(client):
parsed = schema.AvroSchema(data_gen.BASIC_SCHEMA)
schema_id = await client.register("test-basic-schema", parsed)
assert schema_id > 0
# register again under different subject
dupe_id = await client.register("test-basic-schema-backup", parsed)
assert schema_id == dupe_id
assert len(client.id_to_schema) == 1
def test_getters(client):
parsed = schema_loader.AvroSchema(data_gen.BASIC_SCHEMA)
subject = "subject-does-not-exist"
parsed_basic = schema_loader.AvroSchema(data_gen.BASIC_SCHEMA)
client.register("test-basic-schema", parsed_basic)
schema = client.get_by_id(1)
assert schema is not None
latest = client.get_schema(subject)
assert latest is None
# register schema
schema_id = client.register(subject, parsed)
latest = client.get_schema(subject)
fetched = client.get_by_id(schema_id)
assert fetched == parsed
def test_encode_record_with_schema(client, message_serializer):
topic = "test"
basic = schema.AvroSchema(data_gen.BASIC_SCHEMA)
subject = "test-value"
schema_id = client.register(subject, basic)
records = data_gen.BASIC_ITEMS
for record in records:
message = message_serializer.encode_record_with_schema(topic, basic, record)
assertMessageIsSame(message, record, schema_id, message_serializer)
async def test_compatibility(client, user_schema_v3):
"""
Test the compatibility of a new User Schema against the User schema version 2.
"""
subject = "test-user-schema"
version_2 = schema.AvroSchema(data_gen.USER_V2)
await client.register(subject, version_2)
compatibility = await client.test_compatibility(subject, user_schema_v3)
assert compatibility
def test_flat_schema(client):
advance_schema = schema.AvroSchema(data_gen.ADVANCED_SCHEMA)
subject = "test-advance-schema"
client.register(subject, advance_schema)
schema_version = client.get_schema(subject)
parsed_schema = schema_version.schema
parsed_schema.schema.pop("__fastavro_parsed")
assert schema_version.schema.flat_schema == parsed_schema.schema
def test_encode_with_schema_id(client, message_serializer):
basic = schema.AvroSchema(data_gen.BASIC_SCHEMA)
subject = "test-basic-schema"
schema_id = client.register(subject, basic)
records = data_gen.BASIC_ITEMS
for record in records:
message = message_serializer.encode_record_with_schema_id(schema_id, record)
assertMessageIsSame(message, record, schema_id, message_serializer)
adv = schema.AvroSchema(data_gen.ADVANCED_SCHEMA)
subject = "test-advance-schema"
adv_schema_id = client.register(subject, adv)
assert adv_schema_id != schema_id
records = data_gen.ADVANCED_ITEMS
for record in records:
message = message_serializer.encode_record_with_schema_id(adv_schema_id, record)
assertMessageIsSame(message, record, adv_schema_id, message_serializer)
def __enter__(self):
self.server = ServerThread(self.port)
self.server.start()
with open(os.path.join(os.path.dirname(__file__), 'data/test.avsc'),
'r') as json_string:
json_data = json.load(json_string)
av_schema = schema.AvroSchema(json_data)
self.schema_id = self.client.register('test', av_schema)
return self
def test_nested_schema(client):
nested_schema = schema.AvroSchema(data_gen.NESTED_SCHENA)
faust_serializer = serializer.FaustSerializer(client, "test-nested-schema", nested_schema)
record = data_gen.create_nested_schema()
message_encoded = faust_serializer._dumps(record)
assert message_encoded
assert len(message_encoded) > 5
assert isinstance(message_encoded, bytes)
message_decoded = faust_serializer._loads(message_encoded)
assert message_decoded == record
def test_pickelable(client):
unpickled_client = pickle.loads(pickle.dumps(client))
assert client == unpickled_client
# make sure that is possible to do client operations with unpickled_client
subject = "test-basic-schema"
parsed = schema.AvroSchema(data_gen.BASIC_SCHEMA)
unpickled_client.get_subjects()
schema_id = unpickled_client.register(subject, parsed)
assert schema_id > 0
assert unpickled_client.delete_subject(subject)
async def test_dupe_register(client):
parsed = schema.AvroSchema(data_gen.BASIC_SCHEMA)
subject = "test-basic-schema"
schema_id = await client.register(subject, parsed)
assert schema_id > 0
latest = await client.get_schema(subject)
# register again under same subject
dupe_id = await client.register(subject, parsed)
assert schema_id == dupe_id
dupe_latest = await client.get_schema(subject)
assert latest == dupe_latest
def user_schema_v3():
"""
The user V2 is:
{
"type": "record",
"name": "User",
"aliases": ["UserKey"],
"fields": [
{"name": "name", "type": "string"},
{"name": "favorite_number", "type": ["int", "null"], "default": 42},
{"name": "favorite_color", "type": ["string", "null"], "default": "purple"}
]
}
"""
return schema.AvroSchema(flat_schemas.get("user_schema_v3"))
async def test_getters(async_client):
subject = "test-basic-schema"
parsed_basic = schema_loader.AvroSchema(data_gen.BASIC_SCHEMA)
await async_client.register(subject, parsed_basic)
schema = await async_client.get_by_id(1)
assert schema is not None
subject = "subject-does-not-exist"
latest = await async_client.get_schema(subject)
assert latest is None
schema_id = await async_client.register(subject, parsed_basic)
latest = await async_client.get_schema(subject)
fetched = await async_client.get_by_id(schema_id)
assert fetched == parsed_basic
async def test_encode_logical_types(client, message_serializer):
logical_types_schema = schema.AvroSchema(data_gen.LOGICAL_TYPES_SCHEMA)
subject = "test-logical-types-schema"
schema_id = await client.register(subject, logical_types_schema)
record = data_gen.create_logical_item()
message = await message_serializer.encode_record_with_schema_id(
schema_id, record)
decoded = await message_serializer.decode_message(message)
decoded_datetime = decoded.get("metadata").get("timestamp")
timestamp = record.get("metadata").get("timestamp")
decoded_total = decoded.get("metadata").get("total")
total = record.get("metadata").get("total")
assert timestamp == decoded_datetime.replace(tzinfo=None)
assert total == decoded_total
from schema_registry.client import SchemaRegistryClient, schema
from schema_registry.serializers import FaustSerializer
SCHEMA_REGISTRY_URL = "http://localhost:8081"
# from faust_project.users.models import AdvanceUserModel
# Initialize Schema Registry Client
client = SchemaRegistryClient(url=SCHEMA_REGISTRY_URL)
avro_medicare_key_schema = schema.AvroSchema({
"type": "record",
"name": "providerKey",
"namespace": "org.science.medicare",
"fields": [
{
"name": "npi",
"type": "int"
}
],
"connect.name": "org.science.medicare.providerKey"
})
avro_medicare_value_schema = schema.AvroSchema(
"""
{"type":"record","name":"provider","namespace":"org.science.medicare","fields":[{"name":"npi","type":"int"},
{"name":"nppes_provider_last_org_name","type":["null","string"],"default":"null"},
{"name":"nppes_provider_first_name","type":["null","string"],"default":"null"},{"name":"nppes_provider_mi",
"type":["null","string"],"default":"null"},{"name":"nppes_credentials","type":["null","string"],
"default":"null"},{"name":"nppes_provider_gender","type":["null","string"],"default":"null"},
{"name":"nppes_entity_code","type":["null","string"],"default":"null"},{"name":"nppes_provider_street1",
"type":["null","string"],"default":"null"},{"name":"nppes_provider_street2","type":["null","string"],
async def test_context(client):
async with client as c:
parsed = schema.AvroSchema(data_gen.BASIC_SCHEMA)
schema_id = await c.register("test-basic-schema", parsed)
assert schema_id > 0
assert len(c.id_to_schema) == 1
def test_schema_from_string():
parsed = schema.AvroSchema(data_gen.BASIC_SCHEMA)
assert isinstance(parsed, schema.AvroSchema)
def country_schema():
return schema.AvroSchema(flat_schemas.get("country_schema"))
from simple_settings import settings
from example.users.models import AdvanceUserModel
# Initialize Schema Registry Client
client = SchemaRegistryClient(url=settings.SCHEMA_REGISTRY_URL)
avro_user_schema = schema.AvroSchema({
"type":
"record",
"namespace":
"com.example",
"name":
"AvroUsers",
"fields": [{
"name": "first_name",
"type": "string"
}, {
"name": "last_name",
"type": "string"
}]
})
avro_user_serializer = FaustSerializer(client, "users", avro_user_schema)
# example of how to use it with dataclasses-avroschema
avro_advance_user_serializer = FaustSerializer(client, "advance_users",
AdvanceUserModel.avro_schema())
async def add_value_schema(topic: str, s: dict) -> int:
subject = f"{topic}-value"
return await REGISTRY.register(subject,
schema.AvroSchema(s),
timeout=apply_timeout)
async def test_register(client):
parsed = schema.AvroSchema(data_gen.BASIC_SCHEMA)
schema_id = await client.register("test-basic-schema", parsed)
assert schema_id > 0
assert len(client.id_to_schema) == 1
client = SchemaRegistryClient(
url="http://" + os.environ.get("SCHEMA_REGISTRY", "localhost:8081"))
# schema that we want to use. For this example we
# are using a dict, but this schema could be located in a file called avro_user_schema.avsc
avro_ticker_schema = schema.AvroSchema({
"type":
"record",
"namespace":
"akira.data",
"name":
"AvroTicker",
"fields": [{
"name": "data",
"type": "string"
}, {
"name": "symbol",
"type": "string"
}, {
"name": "index",
"type": "string",
"doc": "should be in datetime format"
}]
})
schema_id = client.register(avro_ticker_schema.name.lower(),
avro_ticker_schema)
avro_ticker_serializer = FaustSerializer(client,
avro_ticker_schema.name.lower(),
avro_ticker_schema)
## Needs testing
client = SchemaRegistryClient(url="http://127.0.0.1:8081")
deployment_schema = {
"type":
"record",
"namespace":
"com.kubertenes",
"name":
"AvroDeployment",
"fields": [
{
"name": "image",
"type": "string"
},
{
"name": "replicas",
"type": "int"
},
{
"name": "port",
"type": "int"
},
],
}
avro_schema = schema.AvroSchema(deployment_schema)
schema_id = client.register("test-deployment", avro_schema)
def deployment_schema():
return schema.AvroSchema(flat_schemas.get("deployment_schema"))
