def test_commit_for_times_none(self, mock_kconsumer):
partition_to_offset = {
TopicPartition("topic1", 0): None,
TopicPartition("topic2", 0): None,
}
consumer_commit_for_times(mock_kconsumer, partition_to_offset)
assert mock_kconsumer.commit.call_count == 0
def test_commit_for_times_none(self, mock_kconsumer):
partition_to_offset = {
TopicPartition("topic1", 0): None,
TopicPartition("topic2", 0): None,
}
consumer_commit_for_times(mock_kconsumer, partition_to_offset)
assert mock_kconsumer.commit.call_count == 0
def test_commit_for_times_atomic(self, mock_kconsumer):
partition_to_offset = {
TopicPartition("topic1", 0): None,
TopicPartition("topic2", 0): OffsetAndTimestamp(123, 123),
}
consumer_commit_for_times(mock_kconsumer, partition_to_offset, atomic=True)
assert mock_kconsumer.commit.call_count == 0
def test_commit_for_times_atomic(self, mock_kconsumer):
partition_to_offset = {
TopicPartition("topic1", 0): None,
TopicPartition("topic2", 0): OffsetAndTimestamp(123, 123),
}
consumer_commit_for_times(mock_kconsumer, partition_to_offset, atomic=True)
assert mock_kconsumer.commit.call_count == 0
def run(cls, args, cluster_config):
if not cls.validate_args(args):
return
# Set up the KafkaConsumer
consumer = KafkaConsumer(
bootstrap_servers=cluster_config.broker_list,
client_id="kafka-consumer-manager",
group_id=args.groupid,
enable_auto_commit=False,
)
tp_timestamps = {}
# Get topics that the consumer group has committed offsets for in the past
topics = cls.get_topics_from_consumer_group_id(cluster_config,
args.groupid)
if args.timestamp and len(cls.new_offsets_dict) == 0:
# Committing for using --timestamp for all topics consumer group follows
if len(topics) == 0:
return
partition_to_offset = topic_offsets_for_timestamp(
consumer, args.timestamp, topics)
for tp in six.iterkeys(partition_to_offset):
tp_timestamps[tp] = args.timestamp
else:
# Committing via a list of topic.partition=timestamp
for topic in six.iterkeys(cls.new_offsets_dict):
for partition in six.iterkeys(cls.new_offsets_dict[topic]):
tp_timestamps[TopicPartition(
topic,
partition)] = cls.new_offsets_dict[topic][partition]
partition_to_offset = consumer.offsets_for_times(tp_timestamps)
cls.print_offsets(partition_to_offset, tp_timestamps)
# Display warning if consumer is attempting to commit offsets for a topic they
# have never committed offsets for.
for tp in six.iterkeys(partition_to_offset):
if tp.topic not in topics:
print(
"WARNING: Consumer group {cg} has never committed offsets for topic {topic}."
.format(
cg=args.groupid,
topic=tp.topic,
))
if cls.prompt_user():
consumer_commit_for_times(consumer,
partition_to_offset,
atomic=args.atomic)
else:
print("Offsets not committed")
def test_commit_for_times(self, mock_kconsumer):
timestamp = 123
topics = ["topic1", "topic2", "topic3"]
parts = [0, 1]
partition_to_offset = {
TopicPartition(topic, part): OffsetAndTimestamp(42, timestamp)
for topic in topics for part in parts
}
expected = {
TopicPartition(topic, part): OffsetAndMetadata(42, metadata=None)
for topic in topics for part in parts
}
consumer_commit_for_times(mock_kconsumer, partition_to_offset)
mock_kconsumer.commit.assert_called_once_with(expected)
def test_commit_for_times(self, mock_kconsumer):
timestamp = 123
topics = ["topic1", "topic2", "topic3"]
parts = [0, 1]
partition_to_offset = {
TopicPartition(topic, part): OffsetAndTimestamp(42, timestamp)
for topic in topics for part in parts
}
expected = {
TopicPartition(topic, part): OffsetAndMetadata(42, metadata=None)
for topic in topics for part in parts
}
consumer_commit_for_times(mock_kconsumer, partition_to_offset)
mock_kconsumer.commit.assert_called_once_with(expected)
