def test_topic_offsets_timestamp(self, mock_kconsumer):
timestamp = 123
topics = ["topic1", "topic2", "topic3"]
parts = [0, 1]
mock_kconsumer.partitions_for_topic.return_value = set(parts)
expected = {TopicPartition(topic, part): timestamp for topic in topics for part in parts}
topic_offsets_for_timestamp(mock_kconsumer, timestamp, topics)
mock_kconsumer.offsets_for_times.assert_called_once_with(expected)
def test_topic_offsets_timestamp(self, mock_kconsumer):
timestamp = 123
topics = ["topic1", "topic2", "topic3"]
parts = [0, 1]
mock_kconsumer.partitions_for_topic.return_value = set(parts)
expected = {TopicPartition(topic, part): timestamp for topic in topics for part in parts}
topic_offsets_for_timestamp(mock_kconsumer, timestamp, topics)
mock_kconsumer.offsets_for_times.assert_called_once_with(expected)
def run(cls, args, cluster_config):
if not cls.validate_args(args):
return
# Set up the KafkaConsumer
consumer = KafkaConsumer(
bootstrap_servers=cluster_config.broker_list,
client_id="kafka-consumer-manager",
group_id=args.groupid,
enable_auto_commit=False,
)
tp_timestamps = {}
# Get topics that the consumer group has committed offsets for in the past
topics = cls.get_topics_from_consumer_group_id(cluster_config,
args.groupid)
if args.timestamp and len(cls.new_offsets_dict) == 0:
# Committing for using --timestamp for all topics consumer group follows
if len(topics) == 0:
return
partition_to_offset = topic_offsets_for_timestamp(
consumer, args.timestamp, topics)
for tp in six.iterkeys(partition_to_offset):
tp_timestamps[tp] = args.timestamp
else:
# Committing via a list of topic.partition=timestamp
for topic in six.iterkeys(cls.new_offsets_dict):
for partition in six.iterkeys(cls.new_offsets_dict[topic]):
tp_timestamps[TopicPartition(
topic,
partition)] = cls.new_offsets_dict[topic][partition]
partition_to_offset = consumer.offsets_for_times(tp_timestamps)
cls.print_offsets(partition_to_offset, tp_timestamps)
# Display warning if consumer is attempting to commit offsets for a topic they
# have never committed offsets for.
for tp in six.iterkeys(partition_to_offset):
if tp.topic not in topics:
print(
"WARNING: Consumer group {cg} has never committed offsets for topic {topic}."
.format(
cg=args.groupid,
topic=tp.topic,
))
if cls.prompt_user():
consumer_commit_for_times(consumer,
partition_to_offset,
atomic=args.atomic)
else:
print("Offsets not committed")
def run(cls, args, cluster_config):
# Set up the KafkaConsumer
consumer = KafkaConsumer(
bootstrap_servers=cluster_config.broker_list,
client_id="kafka-consumer-manager",
group_id=args.groupid,
enable_auto_commit=False,
)
if args.groupid:
args.topics = cls.get_topics_from_consumer_group_id(cluster_config, args.groupid)
partition_to_offset = topic_offsets_for_timestamp(consumer, args.timestamp, topics=args.topics)
cls.print_offsets(partition_to_offset, args.timestamp)
def run(cls, args, cluster_config):
# Set up the KafkaConsumer
consumer = KafkaConsumer(
bootstrap_servers=cluster_config.broker_list,
client_id="kafka-consumer-manager",
group_id=args.groupid,
enable_auto_commit=False,
)
if args.groupid:
args.topics = cls.get_topics_from_consumer_group_id(
cluster_config, args.groupid)
partition_to_offset = topic_offsets_for_timestamp(consumer,
args.timestamp,
topics=args.topics)
cls.print_offsets(partition_to_offset, args.timestamp)