def test_user_record(self):
"""
This schema example is from documentation http://avro.apache.org/docs/1.7.6/gettingstartedpython.html
"""
schema_dict = {
"namespace": "example.avro",
"type": "record",
"name": "User",
"fields": [
{"name": "name", "type": "string"},
{"name": "favorite_number", "type": ["int", "null"]},
{"name": "favorite_color", "type": ["string", "null"]}
]
}
avro_schema = make_avsc_object(schema_dict, avro.schema.Names())
serializer = AvroJsonSerializer(avro_schema)
deserializer = AvroJsonDeserializer(avro_schema)
alyssa = {"name": "Alyssa", "favorite_number": 256}
alyssa_full = {"name": "Alyssa", "favorite_number": 256, "favorite_color": None}
alyssa_json = """{"name":"Alyssa","favorite_number":{"int":256},"favorite_color":null}"""
self.assertEquals(serializer.to_json(alyssa), alyssa_json)
self.assertEquals(deserializer.from_json(alyssa_json), alyssa_full)
ben = {"name": "Ben", "favorite_number": 7, "favorite_color": "red"}
ben_json = """{"name":"Ben","favorite_number":{"int":7},"favorite_color":{"string":"red"}}"""
self.assertEquals(serializer.to_json(ben), ben_json)
self.assertEquals(deserializer.from_json(ben_json), ben)
lion = {"name": "Lion"}
lion_full = {"name": "Lion", "favorite_number": None, "favorite_color": None}
lion_json = """{"name":"Lion","favorite_number":null,"favorite_color":null}"""
self.assertEquals(serializer.to_json(lion), lion_json)
self.assertEquals(deserializer.from_json(lion_json), lion_full)
class AvroSerializer(EventSerializer):
def __init__(self, schema: Schema):
self._schema: Schema = schema
self._serializer: Union[DatumWriter, AvroJsonSerializer] = None
def serialize(self, event, serialization_type: SerializationType) -> bytes:
if serialization_type is SerializationType.AVRO_BINARY:
return self._serialize_avro_binary(event)
elif serialization_type is SerializationType.AVRO_JSON:
return self._serialize_avro_json(event)
else:
raise UnsupportedSerializationTypeException(
f"The serialization type {serialization_type} is not supported for Avro"
)
def _serialize_avro_binary(self, event_data: object) -> bytes:
if self._serializer is None or not isinstance(self._serializer,
DatumWriter):
self._serializer = avro.io.DatumWriter(self._schema)
bytes_writer = BytesIO()
encoder = avro.io.BinaryEncoder(bytes_writer)
self._serializer.write(event_data, encoder)
bytes_writer.flush()
return bytes_writer.getvalue()
def _serialize_avro_json(self, event_data: object) -> bytes:
if self._serializer is None or not isinstance(self._serializer,
AvroJsonSerializer):
self._serializer = AvroJsonSerializer(self._schema)
return self._serializer.to_json(event_data).encode("utf-8")
def test_user_record(self):
"""
This schema example is from documentation http://avro.apache.org/docs/1.7.6/gettingstartedpython.html
"""
schema_dict = {
"namespace":
"example.avro",
"type":
"record",
"name":
"User",
"fields": [{
"name": "name",
"type": "string"
}, {
"name": "favorite_number",
"type": ["int", "null"]
}, {
"name": "favorite_color",
"type": ["string", "null"]
}]
}
avro_schema = make_avsc_object(schema_dict, avro.schema.Names())
serializer = AvroJsonSerializer(avro_schema)
deserializer = AvroJsonDeserializer(avro_schema)
alyssa = {"name": "Alyssa", "favorite_number": 256}
alyssa_full = {
"name": "Alyssa",
"favorite_number": 256,
"favorite_color": None
}
alyssa_json = """{"name":"Alyssa","favorite_number":{"int":256},"favorite_color":null}"""
self.assertEquals(serializer.to_json(alyssa), alyssa_json)
self.assertEquals(deserializer.from_json(alyssa_json), alyssa_full)
ben = {"name": "Ben", "favorite_number": 7, "favorite_color": "red"}
ben_json = """{"name":"Ben","favorite_number":{"int":7},"favorite_color":{"string":"red"}}"""
self.assertEquals(serializer.to_json(ben), ben_json)
self.assertEquals(deserializer.from_json(ben_json), ben)
lion = {"name": "Lion"}
lion_full = {
"name": "Lion",
"favorite_number": None,
"favorite_color": None
}
lion_json = """{"name":"Lion","favorite_number":null,"favorite_color":null}"""
self.assertEquals(serializer.to_json(lion), lion_json)
self.assertEquals(deserializer.from_json(lion_json), lion_full)
def dict_to_json(data: Dict):
# to JSON
# avro_schema = schema.SchemaFromJSONData(schema_dict)
avro_schema = schema.Parse(open("rate.avsc", "rb").read())
serializer = AvroJsonSerializer(avro_schema)
json_str = serializer.to_json(data)
pretty_print(json_str)
def test_user_record(self):
"""
This schema example is from documentation http://avro.apache.org/docs/1.7.6/gettingstartedpython.html
"""
schema_dict = {
"namespace":
"example.avro",
"type":
"record",
"name":
"User",
"fields": [{
"name": "name",
"type": "string"
}, {
"name": "favorite_number",
"type": ["int", "null"]
}, {
"name": "favorite_color",
"type": ["string", "null"]
}]
}
avro_schema = avro.schema.make_avsc_object(schema_dict,
avro.schema.Names())
serializer = AvroJsonSerializer(avro_schema)
self.assertEquals(
serializer.to_json({
"name": "Alyssa",
"favorite_number": 256
}),
"""{"name":"Alyssa","favorite_number":{"int":256},"favorite_color":null}"""
)
self.assertEquals(
serializer.to_json({
"name": "Ben",
"favorite_number": 7,
"favorite_color": "red"
}),
"""{"name":"Ben","favorite_number":{"int":7},"favorite_color":{"string":"red"}}"""
)
self.assertEquals(
serializer.to_json({"name": "Lion"}),
"""{"name":"Lion","favorite_number":null,"favorite_color":null}""")
def test_fixed_ascii(self):
schema_dict = {
"namespace": "example.avro",
"type": "record",
"name": "WithFixed",
"fields": [self.FIELD_FIXED]
}
data = {"ffixed": b"fixed text here!"}
avro_schema = make_avsc_object(schema_dict, avro.schema.Names())
serializer = AvroJsonSerializer(avro_schema)
json_data = serializer.to_json(data)
self.assertEquals(json_data, """{"ffixed":"fixed text here!"}""")
def test_user_record(self):
"""
This schema example is from documentation http://avro.apache.org/docs/1.7.6/gettingstartedpython.html
"""
schema_dict = {
"namespace": "example.avro",
"type": "record",
"name": "User",
"fields": [
{"name": "name", "type": "string"},
{"name": "favorite_number", "type": ["int", "null"]},
{"name": "favorite_color", "type": ["string", "null"]}
]
}
avro_schema = avro.schema.make_avsc_object(schema_dict, avro.schema.Names())
serializer = AvroJsonSerializer(avro_schema)
self.assertEquals(serializer.to_json({"name": "Alyssa", "favorite_number": 256}),
"""{"name":"Alyssa","favorite_number":{"int":256},"favorite_color":null}""")
self.assertEquals(serializer.to_json({"name": "Ben", "favorite_number": 7, "favorite_color": "red"}),
"""{"name":"Ben","favorite_number":{"int":7},"favorite_color":{"string":"red"}}""")
self.assertEquals(serializer.to_json({"name": "Lion"}),
"""{"name":"Lion","favorite_number":null,"favorite_color":null}""")
def test_fixed_ascii(self):
schema_dict = {
"namespace": "example.avro",
"type": "record",
"name": "WithFixed",
"fields": [
self.FIELD_FIXED
]
}
data = {"ffixed": "fixed text here!"}
avro_schema = avro.schema.make_avsc_object(schema_dict, avro.schema.Names())
serializer = AvroJsonSerializer(avro_schema)
json_data = serializer.to_json(data)
self.assertEquals(json_data, """{"ffixed":"fixed text here!"}""")
def test_fixed_non_ascii(self):
schema_dict = {
"namespace": "example.avro",
"type": "record",
"name": "WithFixed",
"fields": [
self.FIELD_FIXED
]
}
data = {"ffixed": "(~^\xfbzoW\x13p\x19!4\x0b+\x00\x00"}
avro_schema = avro.schema.make_avsc_object(schema_dict, avro.schema.Names())
serializer = AvroJsonSerializer(avro_schema)
json_data = serializer.to_json(data)
self.assertEquals(json_data, """{"ffixed":"(~^\\u00fbzoW\\u0013p\\u0019!4\\u000b+\\u0000\\u0000"}""")
def test_fixed_non_ascii(self):
schema_dict = {
"namespace": "example.avro",
"type": "record",
"name": "WithFixed",
"fields": [self.FIELD_FIXED]
}
data = {"ffixed": b"(~^\xfbzoW\x13p\x19!4\x0b+\x00\x00"}
avro_schema = make_avsc_object(schema_dict, avro.schema.Names())
serializer = AvroJsonSerializer(avro_schema)
json_data = serializer.to_json(data)
self.assertEquals(
json_data,
"""{"ffixed":"(~^\\u00fbzoW\\u0013p\\u0019!4\\u000b+\\u0000\\u0000"}"""
)
def test_nested_union_records(self):
schema_dict = {
"namespace":
"nested",
"name":
"OuterType",
"type":
"record",
"fields": [{
"name":
"outer",
"type": [
"null", {
"name":
"MiddleType",
"type":
"record",
"fields": [{
"name":
"middle",
"type": [
"null", {
"name": "InnerType",
"type": "record",
"fields": [{
"name": "inner",
"type": "int"
}]
}
]
}]
}
]
}]
}
data1 = {"outer": {"middle": {"inner": 1}}}
data2 = {"outer": {"middle": None}}
avro1 = """{"outer":{"nested.MiddleType":{"middle":{"nested.InnerType":{"inner":1}}}}}"""
avro2 = """{"outer":{"nested.MiddleType":{"middle":null}}}"""
avro_schema = make_avsc_object(schema_dict, avro.schema.Names())
serializer = AvroJsonSerializer(avro_schema)
self.assertEquals(serializer.to_json(data1), avro1)
self.assertEquals(serializer.to_json(data2), avro2)
deserializer = AvroJsonDeserializer(avro_schema)
self.assertEquals(deserializer.from_json(avro1), data1)
self.assertEquals(deserializer.from_json(avro2), data2)
def test_bytes_field_ascii(self):
schema_dict = {
"namespace": "example.avro",
"type": "record",
"name": "WithFixed",
"fields": [{
"type": "bytes",
"name": "fbytes"
}]
}
data = {"fbytes": b"this is some long bytes field"}
avro_schema = make_avsc_object(schema_dict, avro.schema.Names())
serializer = AvroJsonSerializer(avro_schema)
json_data = serializer.to_json(data)
self.assertEquals(json_data,
"""{"fbytes":"this is some long bytes field"}""")
def test_bytes_field_ascii(self):
schema_dict = {
"namespace": "example.avro",
"type": "record",
"name": "WithFixed",
"fields": [
{
"type": "bytes",
"name": "fbytes"
}
]
}
data = {"fbytes": "this is some long bytes field"}
avro_schema = avro.schema.make_avsc_object(schema_dict, avro.schema.Names())
serializer = AvroJsonSerializer(avro_schema)
json_data = serializer.to_json(data)
self.assertEquals(json_data, """{"fbytes":"this is some long bytes field"}""")
def test_bytes_field_non_ascii(self):
schema_dict = {
"namespace": "example.avro",
"type": "record",
"name": "WithFixed",
"fields": [
{
"type": "bytes",
"name": "fbytes"
}
]
}
data = {"fbytes": b"(~^\xfbzoW\x13p\x19!4\x0b+\x00\x00\x0b+\x00\x00"}
avro_schema = make_avsc_object(schema_dict, avro.schema.Names())
serializer = AvroJsonSerializer(avro_schema)
avro_json = serializer.to_json(data)
self.assertEquals(avro_json, """{"fbytes":"(~^\\u00fbzoW\\u0013p\\u0019!4\\u000b+\\u0000\\u0000\\u000b+\\u0000\\u0000"}""")
json_data = AvroJsonDeserializer(avro_schema).from_json(avro_json)
self.assertEquals(json_data, data)
def test_bytes_field_non_ascii(self):
schema_dict = {
"namespace": "example.avro",
"type": "record",
"name": "WithFixed",
"fields": [{
"type": "bytes",
"name": "fbytes"
}]
}
data = {"fbytes": b"(~^\xfbzoW\x13p\x19!4\x0b+\x00\x00\x0b+\x00\x00"}
avro_schema = make_avsc_object(schema_dict, avro.schema.Names())
serializer = AvroJsonSerializer(avro_schema)
avro_json = serializer.to_json(data)
self.assertEquals(
avro_json,
"""{"fbytes":"(~^\\u00fbzoW\\u0013p\\u0019!4\\u000b+\\u0000\\u0000\\u000b+\\u0000\\u0000"}"""
)
json_data = AvroJsonDeserializer(avro_schema).from_json(avro_json)
self.assertEquals(json_data, data)
def test_nested_union_records(self):
schema_dict = {
"namespace": "nested",
"name": "OuterType",
"type": "record",
"fields": [{
"name": "outer",
"type": ["null", {
"name": "MiddleType",
"type": "record",
"fields": [{
"name": "middle",
"type": ["null", {
"name": "InnerType",
"type": "record",
"fields": [{
"name": "inner",
"type": "int"
}]
}]
}]
}]
}]
}
data1 = {"outer": {"middle": {"inner": 1}}}
data2 = {"outer": {"middle": None}}
avro1 = """{"outer":{"nested.MiddleType":{"middle":{"nested.InnerType":{"inner":1}}}}}"""
avro2 = """{"outer":{"nested.MiddleType":{"middle":null}}}"""
avro_schema = make_avsc_object(schema_dict, avro.schema.Names())
serializer = AvroJsonSerializer(avro_schema)
self.assertEquals(serializer.to_json(data1), avro1)
self.assertEquals(serializer.to_json(data2), avro2)
deserializer = AvroJsonDeserializer(avro_schema)
self.assertEquals(deserializer.from_json(avro1), data1)
self.assertEquals(deserializer.from_json(avro2), data2)
def serialize(schema, data):
avro_schema = SchemaFromJSONData(schema, avro.schema.Names())
serializer = AvroJsonSerializer(avro_schema)
return serializer.to_json(data)
def serialize_to_json(self, result_dict):
avro_schema = make_avsc_object(self.avro_schema, avro.schema.Names())
serializer = AvroJsonSerializer(avro_schema)
return serializer.to_json(result_dict)
"uint64_t",
{"name": "int8_t", "type":"fixed", "size":1},
{"name": "int16_t", "type":"fixed", "size":2},
{"name": "int32_t", "type":"fixed", "size":4},
"int64_t"]},
{"name": "tags", "type": ["null", {"type": "map", "values": "string"}]},
{"name": "metadata", "type": ["null", {"type": "map", "values": "string"}]}
]
}
# Create avro schema and serializer for schema
avro_schema = avro.schema.make_avsc_object(schema, avro.schema.Names())
serializer = AvroJsonSerializer(avro_schema)
# Create record (JSON)
record = {
"metric": 'hey',
"timestamp": 983993882838,
"value": 9,
"tags":
{
"hey": "there",
}
}
# Serialize JSON record, this will add proper union formatting (this will return a string)
record = serializer.to_json(record)
# Add proper 'value' wrapper around record (must json.loads(record) because record is a string)
message = {'value':json.loads(record)}
print "BELOW MESSAGE IS FORMATTED CORRECTLY, can be added to list and sent to databus REST"
print message
