def test_rejects_empty_avro_representation_for_writing(
self,
avro_schema_object
):
writer = AvroStringWriter(schema=avro_schema_object)
with pytest.raises(AvroTypeException):
writer.encode(message_avro_representation=None)
def test_publish_message_with_keys(self, message_with_pkeys, producer):
expected_keys_avro_json = {
"type":
"record",
"namespace":
"yelp.data_pipeline",
"name":
"primary_keys",
"doc":
"Represents primary keys present in Message payload.",
"fields": [
{
"type": "string",
"name": "field2",
"doc": "test",
"pkey": 1
},
{
"type": "int",
"name": "field1",
"doc": "test",
"pkey": 2
},
{
"type": "int",
"name": "field3",
"doc": "test",
"pkey": 3
},
]
}
expected_keys = {
"field2": message_with_pkeys.payload_data["field2"],
"field1": message_with_pkeys.payload_data["field1"],
"field3": message_with_pkeys.payload_data["field3"]
}
with capture_new_messages(message_with_pkeys.topic) as get_messages:
producer.publish(message_with_pkeys)
producer.flush()
offsets_and_messages = get_messages()
assert len(offsets_and_messages) == 1
dp_message = create_from_offset_and_message(offsets_and_messages[0])
assert dp_message.keys == expected_keys
avro_string_writer = AvroStringWriter(schema=expected_keys_avro_json)
expected_encoded_keys = avro_string_writer.encode(
message_avro_representation=expected_keys)
assert offsets_and_messages[0].message.key == expected_encoded_keys
avro_string_reader = AvroStringReader(
reader_schema=expected_keys_avro_json,
writer_schema=expected_keys_avro_json)
decoded_keys = avro_string_reader.decode(
encoded_message=offsets_and_messages[0].message.key)
assert decoded_keys == expected_keys
def test_payload_valid_for_writing_reading(self, avro_schema_object):
payload_data = generate_payload_data(avro_schema_object)
writer = AvroStringWriter(schema=avro_schema_object)
reader = AvroStringReader(
reader_schema=avro_schema_object,
writer_schema=avro_schema_object
)
encoded_payload = writer.encode(payload_data)
decoded_payload = reader.decode(encoded_payload)
assert decoded_payload == payload_data
def meta_attribute_payload(
self,
meta_attribute_avro_schema_json,
meta_attribute_payload_data
):
writer = AvroStringWriter(
schema=meta_attribute_avro_schema_json
)
return writer.encode(
message_avro_representation=meta_attribute_payload_data
)
def test_publish_message_with_keys(
self,
message_with_pkeys,
producer
):
expected_keys_avro_json = {
"type": "record",
"namespace": "yelp.data_pipeline",
"name": "primary_keys",
"doc": "Represents primary keys present in Message payload.",
"fields": [
{"type": "string", "name": "field2", "doc": "test", "pkey": 1},
{"type": "int", "name": "field1", "doc": "test", "pkey": 2},
{"type": "int", "name": "field3", "doc": "test", "pkey": 3},
]
}
expected_keys = {
"field2": message_with_pkeys.payload_data["field2"],
"field1": message_with_pkeys.payload_data["field1"],
"field3": message_with_pkeys.payload_data["field3"]
}
with capture_new_messages(message_with_pkeys.topic) as get_messages:
producer.publish(message_with_pkeys)
producer.flush()
offsets_and_messages = get_messages()
assert len(offsets_and_messages) == 1
dp_message = create_from_offset_and_message(
offsets_and_messages[0]
)
assert dp_message.keys == expected_keys
avro_string_writer = AvroStringWriter(
schema=expected_keys_avro_json
)
expected_encoded_keys = avro_string_writer.encode(
message_avro_representation=expected_keys
)
assert offsets_and_messages[0].message.key == expected_encoded_keys
avro_string_reader = AvroStringReader(
reader_schema=expected_keys_avro_json,
writer_schema=expected_keys_avro_json
)
decoded_keys = avro_string_reader.decode(
encoded_message=offsets_and_messages[0].message.key
)
assert decoded_keys == expected_keys
def meta_attribute_payload(self, meta_attribute_avro_schema_json,
meta_attribute_payload_data):
writer = AvroStringWriter(schema=meta_attribute_avro_schema_json)
return writer.encode(
message_avro_representation=meta_attribute_payload_data)