def test_rejects_empty_avro_representation_for_writing(
self,
avro_schema_object
):
writer = AvroStringWriter(schema=avro_schema_object)
with pytest.raises(AvroTypeException):
writer.encode(message_avro_representation=None)
def test_publish_message_with_keys(self, message_with_pkeys, producer):
expected_keys_avro_json = {
"type":
"record",
"namespace":
"yelp.data_pipeline",
"name":
"primary_keys",
"doc":
"Represents primary keys present in Message payload.",
"fields": [
{
"type": "string",
"name": "field2",
"doc": "test",
"pkey": 1
},
{
"type": "int",
"name": "field1",
"doc": "test",
"pkey": 2
},
{
"type": "int",
"name": "field3",
"doc": "test",
"pkey": 3
},
]
}
expected_keys = {
"field2": message_with_pkeys.payload_data["field2"],
"field1": message_with_pkeys.payload_data["field1"],
"field3": message_with_pkeys.payload_data["field3"]
}
with capture_new_messages(message_with_pkeys.topic) as get_messages:
producer.publish(message_with_pkeys)
producer.flush()
offsets_and_messages = get_messages()
assert len(offsets_and_messages) == 1
dp_message = create_from_offset_and_message(offsets_and_messages[0])
assert dp_message.keys == expected_keys
avro_string_writer = AvroStringWriter(schema=expected_keys_avro_json)
expected_encoded_keys = avro_string_writer.encode(
message_avro_representation=expected_keys)
assert offsets_and_messages[0].message.key == expected_encoded_keys
avro_string_reader = AvroStringReader(
reader_schema=expected_keys_avro_json,
writer_schema=expected_keys_avro_json)
decoded_keys = avro_string_reader.decode(
encoded_message=offsets_and_messages[0].message.key)
assert decoded_keys == expected_keys
def test_payload_valid_for_writing_reading(self, avro_schema_object):
payload_data = generate_payload_data(avro_schema_object)
writer = AvroStringWriter(schema=avro_schema_object)
reader = AvroStringReader(
reader_schema=avro_schema_object,
writer_schema=avro_schema_object
)
encoded_payload = writer.encode(payload_data)
decoded_payload = reader.decode(encoded_payload)
assert decoded_payload == payload_data
def meta_attribute_payload(
self,
meta_attribute_avro_schema_json,
meta_attribute_payload_data
):
writer = AvroStringWriter(
schema=meta_attribute_avro_schema_json
)
return writer.encode(
message_avro_representation=meta_attribute_payload_data
)
def test_publish_message_with_keys(
self,
message_with_pkeys,
producer
):
expected_keys_avro_json = {
"type": "record",
"namespace": "yelp.data_pipeline",
"name": "primary_keys",
"doc": "Represents primary keys present in Message payload.",
"fields": [
{"type": "string", "name": "field2", "doc": "test", "pkey": 1},
{"type": "int", "name": "field1", "doc": "test", "pkey": 2},
{"type": "int", "name": "field3", "doc": "test", "pkey": 3},
]
}
expected_keys = {
"field2": message_with_pkeys.payload_data["field2"],
"field1": message_with_pkeys.payload_data["field1"],
"field3": message_with_pkeys.payload_data["field3"]
}
with capture_new_messages(message_with_pkeys.topic) as get_messages:
producer.publish(message_with_pkeys)
producer.flush()
offsets_and_messages = get_messages()
assert len(offsets_and_messages) == 1
dp_message = create_from_offset_and_message(
offsets_and_messages[0]
)
assert dp_message.keys == expected_keys
avro_string_writer = AvroStringWriter(
schema=expected_keys_avro_json
)
expected_encoded_keys = avro_string_writer.encode(
message_avro_representation=expected_keys
)
assert offsets_and_messages[0].message.key == expected_encoded_keys
avro_string_reader = AvroStringReader(
reader_schema=expected_keys_avro_json,
writer_schema=expected_keys_avro_json
)
decoded_keys = avro_string_reader.decode(
encoded_message=offsets_and_messages[0].message.key
)
assert decoded_keys == expected_keys
def get_writer(self, id_key, avro_schema=None):
key = id_key
avro_string_writer = self._writer_cache.get(key)
if avro_string_writer:
return avro_string_writer
avro_schema = avro_schema or self._get_avro_schema(id_key)
avro_string_writer = AvroStringWriter(schema=avro_schema)
self._writer_cache[key] = avro_string_writer
return avro_string_writer
def _avro_string_writer(self):
return AvroStringWriter(self._schema)
def previous_payload(example_schema, example_previous_payload_data):
return AvroStringWriter(
simplejson.loads(example_schema)).encode(example_previous_payload_data)
def example_payload_with_pkeys(example_schema_with_pkey,
example_payload_data_with_pkeys):
return AvroStringWriter(simplejson.loads(example_schema_with_pkey)).encode(
example_payload_data_with_pkeys)
def meta_attribute_payload(self, meta_attribute_avro_schema_json,
meta_attribute_payload_data):
writer = AvroStringWriter(schema=meta_attribute_avro_schema_json)
return writer.encode(
message_avro_representation=meta_attribute_payload_data)
def _create_payload(self, schema):
avro_schema_obj = get_avro_schema_object(
simplejson.dumps(schema.schema_json))
payload_data = generate_payload_data(avro_schema_obj)
return AvroStringWriter(avro_schema_obj).encode(payload_data)