def test_rejects_empty_avro_representation_for_writing( self, avro_schema_object ): writer = AvroStringWriter(schema=avro_schema_object) with pytest.raises(AvroTypeException): writer.encode(message_avro_representation=None)
def test_publish_message_with_keys(self, message_with_pkeys, producer): expected_keys_avro_json = { "type": "record", "namespace": "yelp.data_pipeline", "name": "primary_keys", "doc": "Represents primary keys present in Message payload.", "fields": [ { "type": "string", "name": "field2", "doc": "test", "pkey": 1 }, { "type": "int", "name": "field1", "doc": "test", "pkey": 2 }, { "type": "int", "name": "field3", "doc": "test", "pkey": 3 }, ] } expected_keys = { "field2": message_with_pkeys.payload_data["field2"], "field1": message_with_pkeys.payload_data["field1"], "field3": message_with_pkeys.payload_data["field3"] } with capture_new_messages(message_with_pkeys.topic) as get_messages: producer.publish(message_with_pkeys) producer.flush() offsets_and_messages = get_messages() assert len(offsets_and_messages) == 1 dp_message = create_from_offset_and_message(offsets_and_messages[0]) assert dp_message.keys == expected_keys avro_string_writer = AvroStringWriter(schema=expected_keys_avro_json) expected_encoded_keys = avro_string_writer.encode( message_avro_representation=expected_keys) assert offsets_and_messages[0].message.key == expected_encoded_keys avro_string_reader = AvroStringReader( reader_schema=expected_keys_avro_json, writer_schema=expected_keys_avro_json) decoded_keys = avro_string_reader.decode( encoded_message=offsets_and_messages[0].message.key) assert decoded_keys == expected_keys
def test_payload_valid_for_writing_reading(self, avro_schema_object): payload_data = generate_payload_data(avro_schema_object) writer = AvroStringWriter(schema=avro_schema_object) reader = AvroStringReader( reader_schema=avro_schema_object, writer_schema=avro_schema_object ) encoded_payload = writer.encode(payload_data) decoded_payload = reader.decode(encoded_payload) assert decoded_payload == payload_data
def meta_attribute_payload( self, meta_attribute_avro_schema_json, meta_attribute_payload_data ): writer = AvroStringWriter( schema=meta_attribute_avro_schema_json ) return writer.encode( message_avro_representation=meta_attribute_payload_data )
def test_publish_message_with_keys( self, message_with_pkeys, producer ): expected_keys_avro_json = { "type": "record", "namespace": "yelp.data_pipeline", "name": "primary_keys", "doc": "Represents primary keys present in Message payload.", "fields": [ {"type": "string", "name": "field2", "doc": "test", "pkey": 1}, {"type": "int", "name": "field1", "doc": "test", "pkey": 2}, {"type": "int", "name": "field3", "doc": "test", "pkey": 3}, ] } expected_keys = { "field2": message_with_pkeys.payload_data["field2"], "field1": message_with_pkeys.payload_data["field1"], "field3": message_with_pkeys.payload_data["field3"] } with capture_new_messages(message_with_pkeys.topic) as get_messages: producer.publish(message_with_pkeys) producer.flush() offsets_and_messages = get_messages() assert len(offsets_and_messages) == 1 dp_message = create_from_offset_and_message( offsets_and_messages[0] ) assert dp_message.keys == expected_keys avro_string_writer = AvroStringWriter( schema=expected_keys_avro_json ) expected_encoded_keys = avro_string_writer.encode( message_avro_representation=expected_keys ) assert offsets_and_messages[0].message.key == expected_encoded_keys avro_string_reader = AvroStringReader( reader_schema=expected_keys_avro_json, writer_schema=expected_keys_avro_json ) decoded_keys = avro_string_reader.decode( encoded_message=offsets_and_messages[0].message.key ) assert decoded_keys == expected_keys
def get_writer(self, id_key, avro_schema=None): key = id_key avro_string_writer = self._writer_cache.get(key) if avro_string_writer: return avro_string_writer avro_schema = avro_schema or self._get_avro_schema(id_key) avro_string_writer = AvroStringWriter(schema=avro_schema) self._writer_cache[key] = avro_string_writer return avro_string_writer
def _avro_string_writer(self): return AvroStringWriter(self._schema)
def previous_payload(example_schema, example_previous_payload_data): return AvroStringWriter( simplejson.loads(example_schema)).encode(example_previous_payload_data)
def example_payload_with_pkeys(example_schema_with_pkey, example_payload_data_with_pkeys): return AvroStringWriter(simplejson.loads(example_schema_with_pkey)).encode( example_payload_data_with_pkeys)
def meta_attribute_payload(self, meta_attribute_avro_schema_json, meta_attribute_payload_data): writer = AvroStringWriter(schema=meta_attribute_avro_schema_json) return writer.encode( message_avro_representation=meta_attribute_payload_data)
def _create_payload(self, schema): avro_schema_obj = get_avro_schema_object( simplejson.dumps(schema.schema_json)) payload_data = generate_payload_data(avro_schema_obj) return AvroStringWriter(avro_schema_obj).encode(payload_data)