def test_offset_metadata_invalid_partition_subset(self, kafka_client_mock):
with pytest.raises(UnknownPartitions):
get_consumer_offsets_metadata(
kafka_client_mock,
self.group,
{'topic1': [1, 99]},
)
def test_offset_metadata_unknown_topic(self, kafka_client_mock):
with pytest.raises(UnknownTopic):
get_consumer_offsets_metadata(
kafka_client_mock,
"this won't even be consulted",
["something that doesn't exist"],
)
def test_offset_metadata_unknown_topic(self, kafka_client_mock):
with pytest.raises(UnknownTopic):
get_consumer_offsets_metadata(
kafka_client_mock,
"this won't even be consulted",
["something that doesn't exist"],
)
def test_offset_metadata_invalid_arguments(self, kafka_client_mock):
with pytest.raises(TypeError):
get_consumer_offsets_metadata(
kafka_client_mock,
"this won't even be consulted",
"this should be a list or dict",
)
def test_offset_metadata_invalid_partition_subset(self, kafka_client_mock):
with pytest.raises(UnknownPartitions):
get_consumer_offsets_metadata(
kafka_client_mock,
self.group,
{'topic1': [1, 99]},
)
def test_offset_metadata_invalid_arguments(self, kafka_client_mock):
with pytest.raises(TypeError):
get_consumer_offsets_metadata(
kafka_client_mock,
"this won't even be consulted",
"this should be a list or dict",
)
def test_get_metadata_kafka_error(self, kafka_client_mock):
with mock.patch.object(MyKafkaToolClient,
'load_metadata_for_topics',
side_effect=KafkaUnavailableError("Boom!"),
autospec=True) as mock_func:
with pytest.raises(KafkaUnavailableError):
get_consumer_offsets_metadata(
kafka_client_mock,
self.group,
{'topic1': [99]},
)
assert mock_func.call_count == 2
def test_get_metadata_kafka_error(self, kafka_client_mock):
with mock.patch.object(
MyKafkaToolClient,
'load_metadata_for_topics',
side_effect=KafkaUnavailableError("Boom!"),
autospec=True
) as mock_func:
with pytest.raises(KafkaUnavailableError):
get_consumer_offsets_metadata(
kafka_client_mock,
self.group,
{'topic1': [99]},
)
assert mock_func.call_count == 2
def test_offsets_kafka_error(self, kafka_client_mock):
with mock.patch.object(
MyKafkaToolClient,
'send_offset_fetch_request_kafka',
side_effect=GroupCoordinatorNotAvailableError('Boom!'),
autospec=True,
) as mock_get_kafka:
with pytest.raises(GroupCoordinatorNotAvailableError):
get_consumer_offsets_metadata(
kafka_client_mock,
self.group,
self.topics,
)
assert mock_get_kafka.call_count == 1
def test_offset_metadata_unknown_topic_no_fail(self, kafka_client_mock):
actual = get_consumer_offsets_metadata(
kafka_client_mock,
"this won't even be consulted",
["something that doesn't exist"],
raise_on_error=False
)
assert not actual
def test_offset_metadata_unknown_partitions_no_fail(self, kafka_client_mock):
actual = get_consumer_offsets_metadata(
kafka_client_mock,
self.group,
{'topic1': [99]},
raise_on_error=False
)
assert not actual
def get_offsets(cls, client, group, topics_dict):
try:
return get_consumer_offsets_metadata(
client, group, topics_dict, False,
)
except Exception:
print(
"Error: Encountered error with Kafka, please try again later.",
file=sys.stderr
)
raise
def test_offset_metadata_invalid_partition_subset_no_fail(
self, kafka_client_mock):
# Partition 99 does not exist, so we expect to have
# offset metadata ONLY for partition 1.
expected = [ConsumerPartitionOffsets('topic1', 1, 20, 30, 5)]
actual = get_consumer_offsets_metadata(kafka_client_mock,
self.group, {'topic1': [1, 99]},
raise_on_error=False)
assert 'topic1' in actual
assert actual['topic1'] == expected
def get_offsets(cls, client, group, topics_dict):
try:
return get_consumer_offsets_metadata(
client,
group,
topics_dict,
False,
)
except Exception:
print(
"Error: Encountered error with Kafka, please try again later.",
file=sys.stderr)
raise
def test_offsets_kafka_empty(self, kafka_client_mock):
with mock.patch.object(
MyKafkaToolClient,
'send_offset_fetch_request_kafka',
return_value={},
autospec=True,
) as mock_get_kafka:
actual = get_consumer_offsets_metadata(
kafka_client_mock,
self.group,
self.topics,
)
assert mock_get_kafka.call_count == 1
assert self._has_no_partitions(actual)
def restore_offsets(cls, client, parsed_consumer_offsets, storage):
"""Fetch current offsets from kafka, validate them against given
consumer-offsets data and commit the new offsets.
:param client: Kafka-client
:param parsed_consumer_offsets: Parsed consumer offset data from json file
:type parsed_consumer_offsets: dict(group: dict(topic: partition-offsets))
:param storage: String describing where to store the committed offsets.
"""
# Fetch current offsets
try:
consumer_group = parsed_consumer_offsets['groupid']
topics_offset_data = parsed_consumer_offsets['offsets']
topic_partitions = dict(
(topic, [partition for partition in offset_data.keys()])
for topic, offset_data in topics_offset_data.iteritems())
except IndexError:
print(
"Error: Given parsed consumer-offset data {consumer_offsets} "
"could not be parsed".format(
consumer_offsets=parsed_consumer_offsets),
file=sys.stderr,
)
raise
current_offsets = get_consumer_offsets_metadata(
client,
consumer_group,
topic_partitions,
offset_storage=storage,
)
# Build new offsets
new_offsets = cls.build_new_offsets(
client,
topics_offset_data,
topic_partitions,
current_offsets,
)
# Commit offsets
consumer_group = parsed_consumer_offsets['groupid']
set_consumer_offsets(
client,
consumer_group,
new_offsets,
offset_storage=storage,
)
print("Restored to new offsets {offsets}".format(
offsets=dict(new_offsets)))
def restore_offsets(cls, client, parsed_consumer_offsets, storage):
"""Fetch current offsets from kafka, validate them against given
consumer-offsets data and commit the new offsets.
:param client: Kafka-client
:param parsed_consumer_offsets: Parsed consumer offset data from json file
:type parsed_consumer_offsets: dict(group: dict(topic: partition-offsets))
:param storage: String describing where to store the committed offsets.
"""
# Fetch current offsets
try:
consumer_group = parsed_consumer_offsets['groupid']
topics_offset_data = parsed_consumer_offsets['offsets']
topic_partitions = dict(
(topic, [partition for partition in offset_data.keys()])
for topic, offset_data in topics_offset_data.iteritems()
)
except IndexError:
print(
"Error: Given parsed consumer-offset data {consumer_offsets} "
"could not be parsed".format(consumer_offsets=parsed_consumer_offsets),
file=sys.stderr,
)
raise
current_offsets = get_consumer_offsets_metadata(
client,
consumer_group,
topic_partitions,
offset_storage=storage,
)
# Build new offsets
new_offsets = cls.build_new_offsets(
client,
topics_offset_data,
topic_partitions,
current_offsets,
)
# Commit offsets
consumer_group = parsed_consumer_offsets['groupid']
set_consumer_offsets(
client,
consumer_group,
new_offsets,
offset_storage=storage,
)
print("Restored to new offsets {offsets}".format(offsets=dict(new_offsets)))
def test_offset_metadata_invalid_partition_subset_no_fail(
self,
kafka_client_mock
):
# Partition 99 does not exist, so we expect to have
# offset metadata ONLY for partition 1.
expected = [
ConsumerPartitionOffsets('topic1', 1, 20, 30, 5)
]
actual = get_consumer_offsets_metadata(
kafka_client_mock,
self.group,
{'topic1': [1, 99]},
raise_on_error=False
)
assert 'topic1' in actual
assert actual['topic1'] == expected
def run(cls, args, cluster_config):
# Setup the Kafka client
client = KafkaToolClient(cluster_config.broker_list)
client.load_metadata_for_topics()
topics_dict = cls.preprocess_args(
groupid=args.groupid,
topic=args.topic,
partitions=args.partitions,
cluster_config=cluster_config,
client=client,
storage=args.storage,
)
try:
consumer_offsets_metadata = get_consumer_offsets_metadata(
client,
args.groupid,
topics_dict,
offset_storage=args.storage,
)
except KafkaUnavailableError:
print(
"Error: Encountered error with Kafka, please try again later.",
file=sys.stderr,
)
raise
# Warn the user if a topic being subscribed to does not exist in Kafka.
for topic in topics_dict:
if topic not in consumer_offsets_metadata:
print(
"Warning: Topic {topic} does not exist in Kafka".format(
topic=topic),
file=sys.stderr,
)
cls.save_offsets(
consumer_offsets_metadata,
topics_dict,
args.json_file,
args.groupid,
)
client.close()
def run(cls, args, cluster_config):
# Setup the Kafka client
client = KafkaToolClient(cluster_config.broker_list)
client.load_metadata_for_topics()
topics_dict = cls.preprocess_args(
groupid=args.groupid,
topic=args.topic,
partitions=args.partitions,
cluster_config=cluster_config,
client=client,
)
try:
consumer_offsets_metadata = get_consumer_offsets_metadata(
client,
args.groupid,
topics_dict,
)
except KafkaUnavailableError:
print(
"Error: Encountered error with Kafka, please try again later.",
file=sys.stderr,
)
raise
# Warn the user if a topic being subscribed to does not exist in Kafka.
for topic in topics_dict:
if topic not in consumer_offsets_metadata:
print(
"Warning: Topic {topic} does not exist in Kafka"
.format(topic=topic),
file=sys.stderr,
)
cls.save_offsets(
consumer_offsets_metadata,
topics_dict,
args.json_file,
args.groupid,
)
client.close()
def test_dual_offsets_kafka_error(self, kafka_client_mock):
with mock.patch.object(
MyKafkaToolClient,
'send_offset_fetch_request',
return_value={},
autospec=True,
) as mock_get_zk, mock.patch.object(
MyKafkaToolClient,
'send_offset_fetch_request_kafka',
side_effect=ConsumerCoordinatorNotAvailableCode('Boom!'),
autospec=True,
) as mock_get_kafka:
actual = get_consumer_offsets_metadata(
kafka_client_mock,
self.group,
self.topics,
offset_storage='dual',
)
assert mock_get_zk.call_count == 1
assert mock_get_kafka.call_count == 1
assert self._has_no_partitions(actual)
