async def test_parse_invalid_type(self, schemaregistry_fully_qualified_namespace, schemaregistry_group, **kwargs): sr_client = self.create_client( fully_qualified_namespace=schemaregistry_fully_qualified_namespace) sr_avro_serializer = AvroSerializer(client=sr_client, group_name=schemaregistry_group, auto_register_schemas=True) schema_no_type = """{ "name": "User", "namespace":"example.avro", "fields":[{"name":"name","type":"string"}] }""" with pytest.raises(SchemaParseError): # caught avro SchemaParseError await sr_avro_serializer.serialize({"name": u"Ben"}, schema=schema_no_type) schema_wrong_type_type = """{ "name":"User", "type":1, "namespace":"example.avro", "fields":[{"name":"name","type":"string"}] }""" with pytest.raises(SchemaParseError): await sr_avro_serializer.serialize({"name": u"Ben"}, schema=schema_wrong_type_type)
async def test_basic_sr_avro_serializer_without_auto_register_schemas( self, schemaregistry_fully_qualified_namespace, schemaregistry_group, **kwargs): sr_client = self.create_client( schemaregistry_fully_qualified_namespace) sr_avro_serializer = AvroSerializer(client=sr_client, group_name=schemaregistry_group, auto_register_schemas=True) async with sr_client: schema_str = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}""" schema_str = "{\"type\": \"record\", \"name\": \"User\", \"namespace\": \"example.avro\", \"fields\": [{\"type\": \"string\", \"name\": \"name\"}, {\"type\": [\"int\", \"null\"], \"name\": \"favorite_number\"}, {\"type\": [\"string\", \"null\"], \"name\": \"favorite_color\"}]}" schema = avro.schema.parse(schema_str) dict_data = { "name": u"Ben", "favorite_number": 7, "favorite_color": u"red" } encoded_data = await sr_avro_serializer.serialize( dict_data, schema=schema_str) assert encoded_data[0:4] == b'\0\0\0\0' schema_properties = await sr_client.get_schema_properties( schemaregistry_group, schema.fullname, str(schema), "Avro") schema_id = schema_properties.id assert encoded_data[4:36] == schema_id.encode("utf-8") decoded_data = await sr_avro_serializer.deserialize(encoded_data) assert decoded_data["name"] == u"Ben" assert decoded_data["favorite_number"] == 7 assert decoded_data["favorite_color"] == u"red"
async def test_parse_fixed_types(self, schemaregistry_fully_qualified_namespace, schemaregistry_group, **kwargs): sr_client = self.create_client( fully_qualified_namespace=schemaregistry_fully_qualified_namespace) sr_avro_serializer = AvroSerializer(client=sr_client, group_name=schemaregistry_group, auto_register_schemas=True) # avro bug: should give warning from IgnoredLogicalType error since precision < 0 #fixed_type_ignore_logical_type_error = """{"type": "fixed", "size": 4, "namespace":"example.avro", "name":"User", "precision": -1}""" #await sr_avro_serializer.serialize({}, schema=fixed_type_ignore_logical_type_error) schema_no_size = """{"type": "fixed", "name":"User"}""" with pytest.raises(SchemaParseError): # caught AvroException await sr_avro_serializer.serialize({}, schema=schema_no_size) schema_no_name = """{"type": "fixed", "size": 3}""" with pytest.raises(SchemaParseError): # caught SchemaParseError await sr_avro_serializer.serialize({}, schema=schema_no_name) schema_wrong_name = """{"type": "fixed", "name": 1, "size": 3}""" with pytest.raises(SchemaParseError): # caught SchemaParseError await sr_avro_serializer.serialize({}, schema=schema_wrong_name) schema_wrong_namespace = """{"type": "fixed", "name": "User", "size": 3, "namespace": 1}""" with pytest.raises(SchemaParseError): # caught SchemaParseError await sr_avro_serializer.serialize({}, schema=schema_wrong_namespace)
async def main(): schema_registry = SchemaRegistryClient( fully_qualified_namespace=SCHEMAREGISTRY_FULLY_QUALIFIED_NAMESPACE, credential=token_credential) serializer = AvroSerializer(client=schema_registry, group_name=GROUP_NAME, auto_register_schemas=True) bytes_data_ben, bytes_data_alice = await serialize(serializer) dict_data_ben = await deserialize(serializer, bytes_data_ben) dict_data_alice = await deserialize(serializer, bytes_data_alice) await serializer.close() await token_credential.close()
async def test_serialize_primitive( self, schemaregistry_fully_qualified_namespace, schemaregistry_group, **kwargs): sr_client = self.create_client( fully_qualified_namespace=schemaregistry_fully_qualified_namespace) sr_avro_serializer = AvroSerializer(client=sr_client, group_name=schemaregistry_group, auto_register_schemas=True) null_type = """{"type": "null"}""" encoded_data = await sr_avro_serializer.serialize(None, schema=null_type) assert len(encoded_data) == 36 # assert no data encoded
async def test_parse_primitive_types( self, schemaregistry_fully_qualified_namespace, schemaregistry_group, **kwargs): sr_client = self.create_client( fully_qualified_namespace=schemaregistry_fully_qualified_namespace) sr_avro_serializer = AvroSerializer(client=sr_client, group_name=schemaregistry_group, auto_register_schemas=True) primitive_string = "string" with pytest.raises(SchemaParseError) as e: await sr_avro_serializer.serialize("hello", schema=primitive_string)
async def test_parse_error_schema_as_record( self, schemaregistry_fully_qualified_namespace, schemaregistry_group, **kwargs): sr_client = self.create_client( fully_qualified_namespace=schemaregistry_fully_qualified_namespace) sr_avro_serializer = AvroSerializer(client=sr_client, group_name=schemaregistry_group, auto_register_schemas=True) schema_error_type = """{ "name":"User", "namespace":"example.avro.error", "type":"error", "fields":[{"name":"name","type":"string"}] }""" encoded_data = await sr_avro_serializer.serialize( {"name": u"Ben"}, schema=schema_error_type) schema_id = encoded_data[4:36].decode("utf-8") registered_schema = await sr_client.get_schema(schema_id) decoded_registered_schema = json.loads(registered_schema.definition) assert decoded_registered_schema["type"] == "error"
async def test_parse_invalid_json_string( self, schemaregistry_fully_qualified_namespace, schemaregistry_group, **kwargs): sr_client = self.create_client( fully_qualified_namespace=schemaregistry_fully_qualified_namespace) sr_avro_serializer = AvroSerializer(client=sr_client, group_name=schemaregistry_group, auto_register_schemas=True) invalid_schema = { "name": "User", "type": "record", "namespace": "example.avro", "fields": [{ "name": "name", "type": "string" }] } invalid_schema_string = "{}".format(invalid_schema) with pytest.raises(SchemaParseError): # caught avro SchemaParseError await sr_avro_serializer.serialize({"name": u"Ben"}, schema=invalid_schema_string)
async def test_serialize_record(self, schemaregistry_fully_qualified_namespace, schemaregistry_group, **kwargs): sr_client = self.create_client( fully_qualified_namespace=schemaregistry_fully_qualified_namespace) sr_avro_serializer = AvroSerializer(client=sr_client, group_name=schemaregistry_group, auto_register_schemas=True) # add below to schema later if possible # {"name":"example.innerrec","type":"record","fields":[{"name":"a","type":"int"}]}, # {"name":"innerenum","type":"enum","symbols":["FOO", "BAR"]}, # {"name":"innerarray","type":"array","items":"int"}, # {"name":"innermap","type":"map","values":"int"}, # {"name":"innerfixed","type":"fixed","size":74} schema_record = """{ "name":"User", "namespace":"example.avro.populatedrecord", "type":"record", "fields":[ {"name":"name","type":"string"}, {"name":"age","type":"int"}, {"name":"married","type":"boolean"}, {"name":"height","type":"float"}, {"name":"randb","type":"bytes"} ] }""" data = { "name": u"Ben", "age": 3, "married": False, "height": 13.5, "randb": b"\u00FF" } encoded_data = await sr_avro_serializer.serialize(data, schema=schema_record) decoded_data = await sr_avro_serializer.deserialize(encoded_data) assert decoded_data == data
SCHEMAREGISTRY_FULLY_QUALIFIED_NAMESPACE = os.environ[ 'SCHEMAREGISTRY_FULLY_QUALIFIED_NAMESPACE'] GROUP_NAME = os.environ['SCHEMAREGISTRY_GROUP'] # create an EventHubConsumerClient instance eventhub_consumer = EventHubConsumerClient.from_connection_string( conn_str=EVENTHUB_CONNECTION_STR, consumer_group='$Default', eventhub_name=EVENTHUB_NAME, ) # create a AvroSerializer instance azure_credential = DefaultAzureCredential() avro_serializer = AvroSerializer(client=SchemaRegistryClient( fully_qualified_namespace=SCHEMAREGISTRY_FULLY_QUALIFIED_NAMESPACE, credential=azure_credential), group_name=GROUP_NAME, auto_register_schemas=True) async def on_event(partition_context, event): print("Received event from partition: {}.".format( partition_context.partition_id)) bytes_payload = b"".join(b for b in event.body) print('The received bytes of the EventData is {}.'.format(bytes_payload)) # Use the deserialize method to convert bytes to dict object. # The deserialize method would extract the schema id from the payload, and automatically retrieve the Avro Schema # from the Schema Registry Service. The schema would be cached locally for future usage. deserialized_data = await avro_serializer.deserialize(bytes_payload)
async def test_parse_record_fields( self, schemaregistry_fully_qualified_namespace, schemaregistry_group, **kwargs): sr_client = self.create_client( fully_qualified_namespace=schemaregistry_fully_qualified_namespace) sr_avro_serializer = AvroSerializer(client=sr_client, group_name=schemaregistry_group, auto_register_schemas=True) schema_no_fields = """{ "name":"User", "namespace":"example.avro", "type":"record" }""" with pytest.raises(SchemaParseError): await sr_avro_serializer.serialize({"name": u"Ben"}, schema=schema_no_fields) schema_wrong_type_fields = """{ "name":"User", "namespace":"example.avro", "type":"record" "fields": "hello" }""" with pytest.raises(SchemaParseError): await sr_avro_serializer.serialize({"name": u"Ben"}, schema=schema_wrong_type_fields) schema_wrong_field_item_type = """{ "name":"User", "namespace":"example.avro", "type":"record" "fields": ["hello"] }""" with pytest.raises(SchemaParseError): await sr_avro_serializer.serialize( {"name": u"Ben"}, schema=schema_wrong_field_item_type) schema_record_field_no_name = """{ "name":"User", "namespace":"example.avro", "type":"record", "fields":[{"type":"string"}] }""" with pytest.raises(SchemaParseError): await sr_avro_serializer.serialize( {"name": u"Ben"}, schema=schema_record_field_no_name) schema_record_field_wrong_type_name = """{ "name":"User", "namespace":"example.avro", "type":"record", "fields":[{"name": 1, "type":"string"}] }""" with pytest.raises(SchemaParseError): await sr_avro_serializer.serialize( {"name": u"Ben"}, schema=schema_record_field_wrong_type_name) schema_record_field_with_invalid_order = """{ "name":"User", "namespace":"example.avro.order", "type":"record", "fields":[{"name":"name","type":"string","order":"fake_order"}] }""" with pytest.raises(SchemaParseError): await sr_avro_serializer.serialize( {"name": u"Ben"}, schema=schema_record_field_with_invalid_order) schema_record_duplicate_fields = """{ "name":"User", "namespace":"example.avro", "type":"record", "fields":[{"name":"name","type":"string"}, {"name":"name","type":"string"}] }""" with pytest.raises(SchemaParseError): await sr_avro_serializer.serialize( {"name": u"Ben"}, schema=schema_record_duplicate_fields) schema_field_type_invalid = """{ "name":"User", "namespace":"example.avro", "type":"record", "fields":[{"name":"name","type":1}] }""" with pytest.raises(SchemaParseError): await sr_avro_serializer.serialize( {"name": u"Ben"}, schema=schema_field_type_invalid)
async def test_parse_record_name(self, schemaregistry_fully_qualified_namespace, schemaregistry_group, **kwargs): sr_client = self.create_client( fully_qualified_namespace=schemaregistry_fully_qualified_namespace) sr_avro_serializer = AvroSerializer(client=sr_client, group_name=schemaregistry_group, auto_register_schemas=True) schema_name_has_dot = """{ "namespace": "thrownaway", "name":"User.avro", "type":"record", "fields":[{"name":"name","type":"string"}] }""" encoded_schema = await sr_avro_serializer.serialize( {"name": u"Ben"}, schema=schema_name_has_dot) schema_id = encoded_schema[4:36].decode("utf-8") registered_schema = await sr_client.get_schema(schema_id) decoded_registered_schema = json.loads(registered_schema.definition) assert decoded_registered_schema["name"] == "User.avro" assert decoded_registered_schema["namespace"] == "thrownaway" schema_name_no_namespace = """{ "name":"User", "type":"record", "fields":[{"name":"name","type":"string"}] }""" encoded_schema = await sr_avro_serializer.serialize( {"name": u"Ben"}, schema=schema_name_no_namespace) schema_id = encoded_schema[4:36].decode("utf-8") registered_schema = await sr_client.get_schema(schema_id) decoded_registered_schema = json.loads(registered_schema.definition) assert decoded_registered_schema["name"] == "User" assert "namespace" not in decoded_registered_schema schema_invalid_fullname = """{ "name":"abc", "type":"record", "namespace":"9example.avro", "fields":[{"name":"name","type":"string"}] }""" with pytest.raises(SchemaParseError): await sr_avro_serializer.serialize({"name": u"Ben"}, schema=schema_invalid_fullname) schema_invalid_name_in_fullname = """{ "name":"1abc", "type":"record", "fields":[{"name":"name","type":"string"}] }""" with pytest.raises(SchemaParseError): await sr_avro_serializer.serialize( {"name": u"Ben"}, schema=schema_invalid_name_in_fullname) schema_invalid_name_reserved_type = """{ "name":"record", "type":"record", "fields":[{"name":"name","type":"string"}] }""" with pytest.raises(SchemaParseError): await sr_avro_serializer.serialize( {"name": u"Ben"}, schema=schema_invalid_name_reserved_type) schema_wrong_type_name = """{ "name":1, "type":"record", "namespace":"example.avro", "fields":[{"name":"name","type":"string"}] }""" with pytest.raises(SchemaParseError): await sr_avro_serializer.serialize({"name": u"Ben"}, schema=schema_wrong_type_name) schema_no_name = """{ "namespace":"example.avro", "type":"record", "fields":[{"name":"name","type":"string"}] }""" with pytest.raises(SchemaParseError): await sr_avro_serializer.serialize({"name": u"Ben"}, schema=schema_no_name)