def _build_empty_topic_to_consumer_topic_state_map(topics):
"""Builds a topic_to_consumer_topic_state_map from a list of topics where all of the interior
partition_offset_maps are empty"""
return {
topic: ConsumerTopicState({}, None)
for topic in topics
}
def _build_topic_to_consumer_topic_state_map(watermarks):
"""Builds a topic_to_consumer_topic_state_map from a kafka get_topics_watermarks response"""
return {
topic: ConsumerTopicState({
partition: int((marks.highmark + marks.lowmark) / 2)
for partition, marks in watermarks_map.items()
}, None)
for topic, watermarks_map in watermarks.items()
}
def _setup_manual_topics(self):
for topic in self.options.topics:
offset = None
# https://regex101.com/r/fL0eD9/3
match = re.match('^(.*)\|(\d+)$', topic)
if match:
topic = match.group(1)
offset = ConsumerTopicState({0: int(match.group(2))}, None)
self.topic_to_offsets_map[str(topic)] = offset
def test_consumer_initial_registration_message(self, topic):
"""
Assert that an initial RegistrationMessage is sent upon starting
the Consumer with a non-empty topic_to_consumer_topic_state_map.
"""
with attach_spy_on_func(clog, 'log_line') as func_spy:
fake_topic = ConsumerTopicState({}, 23)
with Consumer(
consumer_name='test_consumer',
team_name='bam',
expected_frequency_seconds=ExpectedFrequency.constantly,
topic_to_consumer_topic_state_map={topic: fake_topic}):
assert func_spy.call_count == 1
def test_get_messages_then_reset(self, consumer_instance, publish_messages,
message):
with consumer_instance as consumer:
publish_messages(message, count=2)
asserter = ConsumerAsserter(consumer=consumer,
expected_message=message)
# Get messages so that the topic_to_consumer_topic_state_map will
# have a ConsumerTopicState for the topic. Getting more messages
# than necessary is to verify only two published messages are consumed
messages = consumer.get_messages(count=10,
blocking=True,
timeout=TIMEOUT)
asserter.assert_messages(messages, expected_count=2)
# Set the offset to one previous so we can use reset_topics to
# receive the same two messages again
topic_map = {}
for message in messages:
topic_map[message.topic] = ConsumerTopicState(
partition_offset_map={
message.kafka_position_info.partition:
message.kafka_position_info.offset - 1
},
last_seen_schema_id=None)
with mock.patch.object(consumer,
'_get_topics_in_region_from_topic_name',
side_effect=[[x]
for x in topic_map.keys()]):
consumer.reset_topics(
topic_to_consumer_topic_state_map=topic_map)
# Verify that we do get the same two messages again
messages = consumer.get_messages(count=10,
blocking=True,
timeout=TIMEOUT)
asserter.assert_messages(messages, expected_count=2)