def test_rejects_empty_avro_representation_for_writing( self, avro_schema_object ): writer = AvroStringWriter(schema=avro_schema_object) with pytest.raises(AvroTypeException): writer.encode(message_avro_representation=None)
def test_publish_message_with_keys(self, message_with_pkeys, producer): expected_keys_avro_json = { "type": "record", "namespace": "yelp.data_pipeline", "name": "primary_keys", "doc": "Represents primary keys present in Message payload.", "fields": [ { "type": "string", "name": "field2", "doc": "test", "pkey": 1 }, { "type": "int", "name": "field1", "doc": "test", "pkey": 2 }, { "type": "int", "name": "field3", "doc": "test", "pkey": 3 }, ] } expected_keys = { "field2": message_with_pkeys.payload_data["field2"], "field1": message_with_pkeys.payload_data["field1"], "field3": message_with_pkeys.payload_data["field3"] } with capture_new_messages(message_with_pkeys.topic) as get_messages: producer.publish(message_with_pkeys) producer.flush() offsets_and_messages = get_messages() assert len(offsets_and_messages) == 1 dp_message = create_from_offset_and_message(offsets_and_messages[0]) assert dp_message.keys == expected_keys avro_string_writer = AvroStringWriter(schema=expected_keys_avro_json) expected_encoded_keys = avro_string_writer.encode( message_avro_representation=expected_keys) assert offsets_and_messages[0].message.key == expected_encoded_keys avro_string_reader = AvroStringReader( reader_schema=expected_keys_avro_json, writer_schema=expected_keys_avro_json) decoded_keys = avro_string_reader.decode( encoded_message=offsets_and_messages[0].message.key) assert decoded_keys == expected_keys
def test_payload_valid_for_writing_reading(self, avro_schema_object): payload_data = generate_payload_data(avro_schema_object) writer = AvroStringWriter(schema=avro_schema_object) reader = AvroStringReader( reader_schema=avro_schema_object, writer_schema=avro_schema_object ) encoded_payload = writer.encode(payload_data) decoded_payload = reader.decode(encoded_payload) assert decoded_payload == payload_data
def meta_attribute_payload( self, meta_attribute_avro_schema_json, meta_attribute_payload_data ): writer = AvroStringWriter( schema=meta_attribute_avro_schema_json ) return writer.encode( message_avro_representation=meta_attribute_payload_data )
def test_publish_message_with_keys( self, message_with_pkeys, producer ): expected_keys_avro_json = { "type": "record", "namespace": "yelp.data_pipeline", "name": "primary_keys", "doc": "Represents primary keys present in Message payload.", "fields": [ {"type": "string", "name": "field2", "doc": "test", "pkey": 1}, {"type": "int", "name": "field1", "doc": "test", "pkey": 2}, {"type": "int", "name": "field3", "doc": "test", "pkey": 3}, ] } expected_keys = { "field2": message_with_pkeys.payload_data["field2"], "field1": message_with_pkeys.payload_data["field1"], "field3": message_with_pkeys.payload_data["field3"] } with capture_new_messages(message_with_pkeys.topic) as get_messages: producer.publish(message_with_pkeys) producer.flush() offsets_and_messages = get_messages() assert len(offsets_and_messages) == 1 dp_message = create_from_offset_and_message( offsets_and_messages[0] ) assert dp_message.keys == expected_keys avro_string_writer = AvroStringWriter( schema=expected_keys_avro_json ) expected_encoded_keys = avro_string_writer.encode( message_avro_representation=expected_keys ) assert offsets_and_messages[0].message.key == expected_encoded_keys avro_string_reader = AvroStringReader( reader_schema=expected_keys_avro_json, writer_schema=expected_keys_avro_json ) decoded_keys = avro_string_reader.decode( encoded_message=offsets_and_messages[0].message.key ) assert decoded_keys == expected_keys
def meta_attribute_payload(self, meta_attribute_avro_schema_json, meta_attribute_payload_data): writer = AvroStringWriter(schema=meta_attribute_avro_schema_json) return writer.encode( message_avro_representation=meta_attribute_payload_data)