def _test_success_ensure_messages_published(self, topic, messages,
producer, topic_offsets,
unpublished_count):
messages_to_publish = len(messages) - unpublished_count
messages_published_first = messages[:messages_to_publish]
with setup_capture_new_messages_consumer(
topic) as consumer, mock.patch.object(data_pipeline.producer,
'logger') as mock_logger:
for message in messages_published_first:
producer.publish(message)
producer.flush()
producer.position_data_callback = mock.Mock()
producer.ensure_messages_published(messages, topic_offsets)
if unpublished_count > 0:
assert producer.position_data_callback.call_count == 1
self._assert_all_messages_published(consumer)
position_info = producer.get_checkpoint_position_data()
last_position = position_info.last_published_message_position_info
assert last_position['position'] == self.number_of_messages
self._assert_logged_info_correct(
mock_logger,
messages_already_published=len(messages_published_first),
topic=topic,
topic_offsets=topic_offsets,
message_count=len(messages))
def assert_new_topic_to_offset_map(self, producer, topic,
original_topic_to_offset_map,
published_message_count):
expected_map = dict(original_topic_to_offset_map)
original_offset = original_topic_to_offset_map.get(topic, 0)
expected_map[topic] = original_offset + published_message_count
new_pos_data = producer.get_checkpoint_position_data()
assert new_pos_data.topic_to_kafka_offset_map == expected_map
def topic_offsets(self, request, producer, random_schema, containers):
is_fresh_topic = request.param
if is_fresh_topic:
containers.create_kafka_topic(str(random_schema.topic.name))
return {}
message = CreateMessage(random_schema.schema_id, payload=str('-1'))
producer.publish(message)
producer.flush()
return producer.get_checkpoint_position_data().topic_to_kafka_offset_map
def topic_offsets(self, request, producer, random_schema, containers):
is_fresh_topic = request.param
if is_fresh_topic:
containers.create_kafka_topic(str(random_schema.topic.name))
return {}
message = CreateMessage(random_schema.schema_id, payload=str('-1'))
producer.publish(message)
producer.flush()
return producer.get_checkpoint_position_data(
).topic_to_kafka_offset_map
def test_get_position_data(self, create_message, producer):
upstream_info = {'offset': 'fake'}
message = create_message(upstream_position_info=upstream_info)
with setup_capture_new_messages_consumer(message.topic) as consumer:
producer.publish(message)
producer.flush()
position_data = producer.get_checkpoint_position_data()
self._verify_position_data(position_data, upstream_info,
message.topic)
self._verify_topic_kafka_offset(position_data, message.topic,
consumer, producer, create_message)
def assert_new_topic_to_offset_map(
self,
producer,
topic,
original_topic_to_offset_map,
published_message_count
):
expected_map = dict(original_topic_to_offset_map)
original_offset = original_topic_to_offset_map.get(topic, 0)
expected_map[topic] = original_offset + published_message_count
new_pos_data = producer.get_checkpoint_position_data()
assert new_pos_data.topic_to_kafka_offset_map == expected_map
def _verify_position_and_highwatermarks(self, topics, producer,
message_count):
topics_details = get_topics_watermarks(
kafka_client=producer._kafka_producer.kafka_client,
topics=topics,
raise_on_error=True)
position_tracker = producer._kafka_producer.position_data_tracker
for topic in topics:
actual_hwm = topics_details[topic][0][2]
expected_hwm = position_tracker.topic_to_kafka_offset_map[topic]
assert actual_hwm == expected_hwm
position_info = producer.get_checkpoint_position_data()
last_position = position_info.last_published_message_position_info
assert last_position['position'] == message_count
def test_get_position_data(self, create_message, producer):
upstream_info = {'offset': 'fake'}
message = create_message(upstream_position_info=upstream_info)
with setup_capture_new_messages_consumer(message.topic) as consumer:
producer.publish(message)
producer.flush()
position_data = producer.get_checkpoint_position_data()
self._verify_position_data(position_data, upstream_info, message.topic)
self._verify_topic_kafka_offset(
position_data,
message.topic,
consumer,
producer,
create_message
)
def _verify_position_and_highwatermarks(
self,
topics,
producer,
message_count
):
topics_details = get_topics_watermarks(
kafka_client=producer._kafka_producer.kafka_client,
topics=topics,
raise_on_error=True
)
position_tracker = producer._kafka_producer.position_data_tracker
for topic in topics:
actual_hwm = topics_details[topic][0][2]
expected_hwm = position_tracker.topic_to_kafka_offset_map[topic]
assert actual_hwm == expected_hwm
position_info = producer.get_checkpoint_position_data()
last_position = position_info.last_published_message_position_info
assert last_position['position'] == message_count
def test_multitopic_offsets(self, topic, messages, secondary_topic,
secondary_messages, producer, topic_offsets,
containers):
"""Publishes a single message on the secondary_topic, and all
messages on the primary topic, simulating the case where publishes for
one topic fail, while the other succeeds, and the one that succeeds
comes later in time. The goal is that the position data still reflects
the original message ordering, irrespective of failure.
"""
containers.create_kafka_topic(secondary_topic)
with setup_capture_new_messages_consumer(secondary_topic) as consumer:
producer.publish(secondary_messages[0])
for message in messages:
producer.publish(message)
producer.flush()
producer.ensure_messages_published(secondary_messages + messages,
topic_offsets)
position_info = producer.get_checkpoint_position_data()
last_position = position_info.last_published_message_position_info
assert last_position['position'] == self.number_of_messages
assert len(consumer.get_messages(10)) == len(secondary_messages)
def test_empty_starting_checkpoint_data(self, producer):
position_data = producer.get_checkpoint_position_data()
assert position_data.last_published_message_position_info is None
assert position_data.topic_to_last_position_info_map == {}
assert position_data.topic_to_kafka_offset_map == {}
def get_orig_topic_to_offset_map(self, producer):
pos_data = producer.get_checkpoint_position_data()
return pos_data.topic_to_kafka_offset_map
def test_empty_starting_checkpoint_data(self, producer):
position_data = producer.get_checkpoint_position_data()
assert position_data.last_published_message_position_info is None
assert position_data.topic_to_last_position_info_map == {}
assert position_data.topic_to_kafka_offset_map == {}
def get_orig_topic_to_offset_map(self, producer):
pos_data = producer.get_checkpoint_position_data()
return pos_data.topic_to_kafka_offset_map