def test_nested_schema_with_register_codec(client): nested_schema = schema.AvroSchema(data_gen.NESTED_SCHENA) order_schema = schema.AvroSchema(data_gen.ORDER_SCHENA) customer_serializer = serializer.FaustSerializer(client, "test-nested-schema", nested_schema) order_serializer = serializer.FaustSerializer(client, "test-order-schema", order_schema) faust.serializers.codecs.register("customer_serializer", customer_serializer) faust.serializers.codecs.register("order_serializer", order_serializer) class Order(faust.Record, serializer="order_serializer"): uid: int class Customer(faust.Record, serializer="customer_serializer"): name: str uid: int order: Order payload = data_gen.create_nested_schema() customer = Customer(**payload) message_encoded = customer.dumps() assert message_encoded assert len(message_encoded) > 5 assert isinstance(message_encoded, bytes) message_decoded = Customer.loads(message_encoded) assert message_decoded == customer
async def test_multi_register(client): """ Register two different schemas under the same subject with backwards compatibility """ version_1 = schema.AvroSchema(data_gen.USER_V1) version_2 = schema.AvroSchema(data_gen.USER_V2) subject = "test-user-schema" id1 = await client.register(subject, version_1) latest_schema_1 = await client.get_schema(subject) await client.check_version(subject, version_1) id2 = await client.register(subject, version_2) latest_schema_2 = await client.get_schema(subject) await client.check_version(subject, version_2) assert id1 != id2 assert latest_schema_1 != latest_schema_2 # ensure version is higher assert latest_schema_1.version < latest_schema_2.version await client.register(subject, version_1) latest_schema_3 = await client.get_schema(subject) assert latest_schema_2 == latest_schema_3
async def test_delete_subject(async_client, user_schema_v3): subject = "subject-to-delete" versions = [schema.AvroSchema(data_gen.USER_V1), schema.AvroSchema(data_gen.USER_V2)] for version in versions: await async_client.register(subject, version) assert len(await async_client.delete_subject(subject)) == len(versions)
async def test_register_with_logical_types(async_client): parsed = schema.AvroSchema(data_gen.LOGICAL_TYPES_SCHEMA) schema_id = await async_client.register("test-logical-types-schema", parsed) assert schema_id > 0 assert len(async_client.id_to_schema) == 1
async def test_dupe_register(async_client): parsed = schema.AvroSchema(data_gen.BASIC_SCHEMA) subject = "test-basic-schema" schema_id = await async_client.register(subject, parsed) # Verify we had a check version call async_client.assert_url_suffix(0, "/subjects/%s" % subject) async_client.assert_method(0, "POST") # Verify that we had a register call async_client.assert_url_suffix(1, "/subjects/%s/versions" % subject) async_client.assert_method(1, "POST") assert len(async_client.request_calls) == 2 assert schema_id > 0 latest = await async_client.get_schema(subject) async_client.assert_url_suffix(2, "/subjects/%s/versions/latest" % subject) async_client.assert_method(2, "GET") assert len(async_client.request_calls) == 3 # register again under same subject dupe_id = await async_client.register(subject, parsed) assert schema_id == dupe_id # Served from cache assert len(async_client.request_calls) == 3 dupe_latest = await async_client.get_schema(subject) assert latest == dupe_latest
def _get_encoder_func( self, avro_schema: typing.Union[schema.AvroSchema, str]) -> typing.Callable: if isinstance(avro_schema, str): avro_schema = schema.AvroSchema(json.loads(avro_schema)) return lambda record, fp: schemaless_writer(fp, avro_schema.schema, record) # type: ignore
def lambda_handler(event, context): responseData = {} print(event) if 'SCHEMA_REEGISTRY_URI' in os.environ: SchemaRegistryURI = os.environ['SCHEMA_REEGISTRY_URI'] print('SchemaRegistryURI is: {}'.format(SchemaRegistryURI)) else: responseData['cause'] = "Schema registry URI not provided" cfnresponse.send(event, context, cfnresponse.FAILED, responseData) if 'SchemaConfig' in event['ResourceProperties']: json_content = json.loads(event['ResourceProperties']['SchemaConfig']) print("Loaded AVRO schema from file {}".format(json_content)) else: responseData['status'] = "Location of Schema file not provided" responseData['cause'] = event['ResourceProperties'] print('SchemaFile not provided') cfnresponse.send(event, context, cfnresponse.FAILED, responseData) if event['RequestType'] == 'Create': if 'type' in json_content: print("Looks like we got valide AVRO schema") try: client = SchemaRegistryClient(url=SchemaRegistryURI) except Exception as e: responseData['cause'] = "Schema registry client cannot be created {} \ ".format(e.__class__.__name__) print(responseData['cause']) cfnresponse.send(event, context, cfnresponse.FAILED, responseData) print('Schema registry client created') avro_schema = schema.AvroSchema(json_content) print('AVRO schema object created') ###Get subjects, just for debug list_registered_subjects = requests.get(SchemaRegistryURI + "/subjects") print("Native http rest response, list of current subjects {}, \ ".format(list_registered_subjects.json())) try: schema_id = client.register(json_content['name'] + "-value", avro_schema, timeout=10) responseData['cause'] = "Schema {} created successfully, schema id is {} \ ".format(json_content['name'], schema_id) print(responseData['cause']) cfnresponse.send(event, context, cfnresponse.SUCCESS, responseData) except Exception as e: responseData['cause'] = "Some exception in request to schema register happened, {}\ ".format(format(e.__class__.__name__)) print (responseData['cause']) cfnresponse.send(event, context, cfnresponse.FAILED, responseData) else: responseData['cause'] = 'Provided file not an AVRO schema, \ there are no /"type/" field in request object' print(responseData['cause']) cfnresponse.send(event, context, cfnresponse.FAILED, responseData) else: #if event['RequestType'] == 'Delete' or event['RequestType'] == 'Update': responseData['cause'] = 'CloudFormation Delete and Update method not implemented, \ just return cnfresponse=SUCCSESS without any job/modifications' cfnresponse.send(event, context, cfnresponse.SUCCESS, responseData)
async def test_multi_subject_register(client): parsed = schema.AvroSchema(data_gen.BASIC_SCHEMA) schema_id = await client.register("test-basic-schema", parsed) assert schema_id > 0 # register again under different subject dupe_id = await client.register("test-basic-schema-backup", parsed) assert schema_id == dupe_id assert len(client.id_to_schema) == 1
def test_getters(client): parsed = schema_loader.AvroSchema(data_gen.BASIC_SCHEMA) subject = "subject-does-not-exist" parsed_basic = schema_loader.AvroSchema(data_gen.BASIC_SCHEMA) client.register("test-basic-schema", parsed_basic) schema = client.get_by_id(1) assert schema is not None latest = client.get_schema(subject) assert latest is None # register schema schema_id = client.register(subject, parsed) latest = client.get_schema(subject) fetched = client.get_by_id(schema_id) assert fetched == parsed
def test_encode_record_with_schema(client, message_serializer): topic = "test" basic = schema.AvroSchema(data_gen.BASIC_SCHEMA) subject = "test-value" schema_id = client.register(subject, basic) records = data_gen.BASIC_ITEMS for record in records: message = message_serializer.encode_record_with_schema(topic, basic, record) assertMessageIsSame(message, record, schema_id, message_serializer)
async def test_compatibility(client, user_schema_v3): """ Test the compatibility of a new User Schema against the User schema version 2. """ subject = "test-user-schema" version_2 = schema.AvroSchema(data_gen.USER_V2) await client.register(subject, version_2) compatibility = await client.test_compatibility(subject, user_schema_v3) assert compatibility
def test_flat_schema(client): advance_schema = schema.AvroSchema(data_gen.ADVANCED_SCHEMA) subject = "test-advance-schema" client.register(subject, advance_schema) schema_version = client.get_schema(subject) parsed_schema = schema_version.schema parsed_schema.schema.pop("__fastavro_parsed") assert schema_version.schema.flat_schema == parsed_schema.schema
def test_encode_with_schema_id(client, message_serializer): basic = schema.AvroSchema(data_gen.BASIC_SCHEMA) subject = "test-basic-schema" schema_id = client.register(subject, basic) records = data_gen.BASIC_ITEMS for record in records: message = message_serializer.encode_record_with_schema_id(schema_id, record) assertMessageIsSame(message, record, schema_id, message_serializer) adv = schema.AvroSchema(data_gen.ADVANCED_SCHEMA) subject = "test-advance-schema" adv_schema_id = client.register(subject, adv) assert adv_schema_id != schema_id records = data_gen.ADVANCED_ITEMS for record in records: message = message_serializer.encode_record_with_schema_id(adv_schema_id, record) assertMessageIsSame(message, record, adv_schema_id, message_serializer)
def __enter__(self): self.server = ServerThread(self.port) self.server.start() with open(os.path.join(os.path.dirname(__file__), 'data/test.avsc'), 'r') as json_string: json_data = json.load(json_string) av_schema = schema.AvroSchema(json_data) self.schema_id = self.client.register('test', av_schema) return self
def test_nested_schema(client): nested_schema = schema.AvroSchema(data_gen.NESTED_SCHENA) faust_serializer = serializer.FaustSerializer(client, "test-nested-schema", nested_schema) record = data_gen.create_nested_schema() message_encoded = faust_serializer._dumps(record) assert message_encoded assert len(message_encoded) > 5 assert isinstance(message_encoded, bytes) message_decoded = faust_serializer._loads(message_encoded) assert message_decoded == record
def test_pickelable(client): unpickled_client = pickle.loads(pickle.dumps(client)) assert client == unpickled_client # make sure that is possible to do client operations with unpickled_client subject = "test-basic-schema" parsed = schema.AvroSchema(data_gen.BASIC_SCHEMA) unpickled_client.get_subjects() schema_id = unpickled_client.register(subject, parsed) assert schema_id > 0 assert unpickled_client.delete_subject(subject)
async def test_dupe_register(client): parsed = schema.AvroSchema(data_gen.BASIC_SCHEMA) subject = "test-basic-schema" schema_id = await client.register(subject, parsed) assert schema_id > 0 latest = await client.get_schema(subject) # register again under same subject dupe_id = await client.register(subject, parsed) assert schema_id == dupe_id dupe_latest = await client.get_schema(subject) assert latest == dupe_latest
def user_schema_v3(): """ The user V2 is: { "type": "record", "name": "User", "aliases": ["UserKey"], "fields": [ {"name": "name", "type": "string"}, {"name": "favorite_number", "type": ["int", "null"], "default": 42}, {"name": "favorite_color", "type": ["string", "null"], "default": "purple"} ] } """ return schema.AvroSchema(flat_schemas.get("user_schema_v3"))
async def test_getters(async_client): subject = "test-basic-schema" parsed_basic = schema_loader.AvroSchema(data_gen.BASIC_SCHEMA) await async_client.register(subject, parsed_basic) schema = await async_client.get_by_id(1) assert schema is not None subject = "subject-does-not-exist" latest = await async_client.get_schema(subject) assert latest is None schema_id = await async_client.register(subject, parsed_basic) latest = await async_client.get_schema(subject) fetched = await async_client.get_by_id(schema_id) assert fetched == parsed_basic
async def test_encode_logical_types(client, message_serializer): logical_types_schema = schema.AvroSchema(data_gen.LOGICAL_TYPES_SCHEMA) subject = "test-logical-types-schema" schema_id = await client.register(subject, logical_types_schema) record = data_gen.create_logical_item() message = await message_serializer.encode_record_with_schema_id( schema_id, record) decoded = await message_serializer.decode_message(message) decoded_datetime = decoded.get("metadata").get("timestamp") timestamp = record.get("metadata").get("timestamp") decoded_total = decoded.get("metadata").get("total") total = record.get("metadata").get("total") assert timestamp == decoded_datetime.replace(tzinfo=None) assert total == decoded_total
from schema_registry.client import SchemaRegistryClient, schema from schema_registry.serializers import FaustSerializer SCHEMA_REGISTRY_URL = "http://localhost:8081" # from faust_project.users.models import AdvanceUserModel # Initialize Schema Registry Client client = SchemaRegistryClient(url=SCHEMA_REGISTRY_URL) avro_medicare_key_schema = schema.AvroSchema({ "type": "record", "name": "providerKey", "namespace": "org.science.medicare", "fields": [ { "name": "npi", "type": "int" } ], "connect.name": "org.science.medicare.providerKey" }) avro_medicare_value_schema = schema.AvroSchema( """ {"type":"record","name":"provider","namespace":"org.science.medicare","fields":[{"name":"npi","type":"int"}, {"name":"nppes_provider_last_org_name","type":["null","string"],"default":"null"}, {"name":"nppes_provider_first_name","type":["null","string"],"default":"null"},{"name":"nppes_provider_mi", "type":["null","string"],"default":"null"},{"name":"nppes_credentials","type":["null","string"], "default":"null"},{"name":"nppes_provider_gender","type":["null","string"],"default":"null"}, {"name":"nppes_entity_code","type":["null","string"],"default":"null"},{"name":"nppes_provider_street1", "type":["null","string"],"default":"null"},{"name":"nppes_provider_street2","type":["null","string"],
async def test_context(client): async with client as c: parsed = schema.AvroSchema(data_gen.BASIC_SCHEMA) schema_id = await c.register("test-basic-schema", parsed) assert schema_id > 0 assert len(c.id_to_schema) == 1
def test_schema_from_string(): parsed = schema.AvroSchema(data_gen.BASIC_SCHEMA) assert isinstance(parsed, schema.AvroSchema)
def country_schema(): return schema.AvroSchema(flat_schemas.get("country_schema"))
from simple_settings import settings from example.users.models import AdvanceUserModel # Initialize Schema Registry Client client = SchemaRegistryClient(url=settings.SCHEMA_REGISTRY_URL) avro_user_schema = schema.AvroSchema({ "type": "record", "namespace": "com.example", "name": "AvroUsers", "fields": [{ "name": "first_name", "type": "string" }, { "name": "last_name", "type": "string" }] }) avro_user_serializer = FaustSerializer(client, "users", avro_user_schema) # example of how to use it with dataclasses-avroschema avro_advance_user_serializer = FaustSerializer(client, "advance_users", AdvanceUserModel.avro_schema())
async def add_value_schema(topic: str, s: dict) -> int: subject = f"{topic}-value" return await REGISTRY.register(subject, schema.AvroSchema(s), timeout=apply_timeout)
async def test_register(client): parsed = schema.AvroSchema(data_gen.BASIC_SCHEMA) schema_id = await client.register("test-basic-schema", parsed) assert schema_id > 0 assert len(client.id_to_schema) == 1
client = SchemaRegistryClient( url="http://" + os.environ.get("SCHEMA_REGISTRY", "localhost:8081")) # schema that we want to use. For this example we # are using a dict, but this schema could be located in a file called avro_user_schema.avsc avro_ticker_schema = schema.AvroSchema({ "type": "record", "namespace": "akira.data", "name": "AvroTicker", "fields": [{ "name": "data", "type": "string" }, { "name": "symbol", "type": "string" }, { "name": "index", "type": "string", "doc": "should be in datetime format" }] }) schema_id = client.register(avro_ticker_schema.name.lower(), avro_ticker_schema) avro_ticker_serializer = FaustSerializer(client, avro_ticker_schema.name.lower(), avro_ticker_schema)
## Needs testing client = SchemaRegistryClient(url="http://127.0.0.1:8081") deployment_schema = { "type": "record", "namespace": "com.kubertenes", "name": "AvroDeployment", "fields": [ { "name": "image", "type": "string" }, { "name": "replicas", "type": "int" }, { "name": "port", "type": "int" }, ], } avro_schema = schema.AvroSchema(deployment_schema) schema_id = client.register("test-deployment", avro_schema)
def deployment_schema(): return schema.AvroSchema(flat_schemas.get("deployment_schema"))