def test_get_current_consumer_offsets_unknown_topic(self, kafka_client_mock): with pytest.raises(UnknownTopic): get_current_consumer_offsets( kafka_client_mock, "this won't even be consulted", ["something that doesn't exist"], )
def test_get_current_consumer_offsets_unknown_topic(self, kafka_client_mock): with pytest.raises(UnknownTopic): get_current_consumer_offsets( kafka_client_mock, "this won't even be consulted", ["something that doesn't exist"], )
def test_get_current_consumer_offsets_invalid_partition_subset(self, kafka_client_mock): with pytest.raises(UnknownPartitions): get_current_consumer_offsets( kafka_client_mock, self.group, {'topic1': [1, 99]}, )
def test_get_current_consumer_offsets_invalid_arguments(self, kafka_client_mock): with pytest.raises(TypeError): get_current_consumer_offsets( kafka_client_mock, "this won't even be consulted", "this should be a list or dict", )
def _get_current_offsets_dual( kafka_client, group, topics, raise_on_error, ): """Get current consumer offsets from Zookeeper and from Kafka and return the higher partition offsets from the responses. """ zk_offsets = get_current_consumer_offsets( kafka_client, group, topics, False, 'zookeeper', ) try: kafka_offsets = get_current_consumer_offsets( kafka_client, group, topics, False, 'kafka', ) except GroupCoordinatorNotAvailableError: kafka_offsets = {} return merge_offsets_metadata(topics, zk_offsets, kafka_offsets)
def test_get_current_consumer_offsets_invalid_partition_subset(self, kafka_client_mock): with pytest.raises(UnknownPartitions): get_current_consumer_offsets( kafka_client_mock, self.group, {'topic1': [1, 99]}, )
def test_get_current_consumer_offsets_invalid_arguments(self, kafka_client_mock): with pytest.raises(TypeError): get_current_consumer_offsets( kafka_client_mock, "this won't even be consulted", "this should be a list or dict", )
def test_get_current_consumer_offsets_from_kafka(self, topics, kafka_client_mock): kafka_client_mock = mock.Mock(wraps=kafka_client_mock) get_current_consumer_offsets( kafka_client_mock, self.group, topics, ) assert kafka_client_mock.send_offset_fetch_request_kafka.call_count == 1
def test_get_current_consumer_offsets_from_zookeeper( self, topics, kafka_client_mock): kafka_client_mock = mock.Mock(wraps=kafka_client_mock) get_current_consumer_offsets( kafka_client_mock, self.group, topics, offset_storage='zookeeper', ) assert kafka_client_mock.send_offset_fetch_request.call_count == 1 assert kafka_client_mock.send_offset_fetch_request_kafka.call_count == 0
def step_impl4(context): old_group_offsets = get_current_consumer_offsets( context.client, context.group, [context.topic], ) new_group_offsets = get_current_consumer_offsets( context.client, NEW_GROUP, [context.topic], ) assert old_group_offsets == new_group_offsets
def step_impl4(context): old_group_offsets = get_current_consumer_offsets( context.client, context.group, [context.topic], ) new_group_offsets = get_current_consumer_offsets( context.client, NEW_GROUP, [context.topic], ) assert old_group_offsets == new_group_offsets
def test_get_current_consumer_offsets_from_kafka( self, topics, kafka_client_mock ): kafka_client_mock = mock.Mock(wraps=kafka_client_mock) get_current_consumer_offsets( kafka_client_mock, self.group, topics, ) assert kafka_client_mock.send_offset_fetch_request_kafka.call_count == 1
def test_get_current_consumer_offsets_invalid_storage( self, topics, kafka_client_mock): kafka_client_mock = mock.Mock(wraps=kafka_client_mock) with pytest.raises(InvalidOffsetStorageError): get_current_consumer_offsets( kafka_client_mock, self.group, topics, offset_storage='random_string', ) assert kafka_client_mock.send_offset_fetch_request.call_count == 0 assert kafka_client_mock.send_offset_fetch_request_kafka.call_count == 0
def test_get_current_consumer_offsets_from_zookeeper( self, topics, kafka_client_mock ): kafka_client_mock = mock.Mock(wraps=kafka_client_mock) get_current_consumer_offsets( kafka_client_mock, self.group, topics, offset_storage='zookeeper', ) assert kafka_client_mock.send_offset_fetch_request.call_count == 1 assert kafka_client_mock.send_offset_fetch_request_kafka.call_count == 0
def test_get_current_consumer_offsets_invalid_storage( self, topics, kafka_client_mock ): kafka_client_mock = mock.Mock(wraps=kafka_client_mock) with pytest.raises(InvalidOffsetStorageError): get_current_consumer_offsets( kafka_client_mock, self.group, topics, offset_storage='random_string', ) assert kafka_client_mock.send_offset_fetch_request.call_count == 0 assert kafka_client_mock.send_offset_fetch_request_kafka.call_count == 0
def test_get_current_consumer_offsets_unknown_topic_no_fail( self, kafka_client_mock): actual = get_current_consumer_offsets(kafka_client_mock, "this won't even be consulted", ["something that doesn't exist"], raise_on_error=False) assert not actual
def test_get_current_consumer_offsets(self, topics, kafka_client_mock): actual = get_current_consumer_offsets( kafka_client_mock, self.group, topics ) assert actual == {'topic1': {0: 30, 1: 20, 2: 10}}
def rename_group_with_storage_kafka( cls, client, old_groupid, new_groupid, topics, ): copied_offsets = get_current_consumer_offsets( client, old_groupid, topics, offset_storage='kafka', ) set_consumer_offsets( client, new_groupid, copied_offsets, offset_storage='kafka', ) set_consumer_offsets( client, old_groupid, nullify_offsets(topics), offset_storage='kafka', )
def test_get_current_consumer_offsets(self, topics, kafka_client_mock): actual = get_current_consumer_offsets( kafka_client_mock, self.group, topics ) assert actual == {'topic1': {0: 30, 1: 20, 2: 10}}
def test_get_current_consumer_offsets_invalid_partition_subset_no_fail( self, kafka_client_mock): actual = get_current_consumer_offsets(kafka_client_mock, self.group, {'topic1': [1, 99]}, raise_on_error=False) assert actual['topic1'][1] == 20 # Partition 99 does not exist so it shouldn't be in the result assert 99 not in actual['topic1']
def test_get_current_consumer_offsets_unknown_partitions_no_fail(self, kafka_client_mock): actual = get_current_consumer_offsets( kafka_client_mock, self.group, {'topic1': [99]}, raise_on_error=False ) assert not actual
def test_get_current_consumer_offsets_unknown_topic_no_fail(self, kafka_client_mock): actual = get_current_consumer_offsets( kafka_client_mock, "this won't even be consulted", ["something that doesn't exist"], raise_on_error=False ) assert not actual
def test_get_current_consumer_offsets_unknown_partitions_no_fail(self, kafka_client_mock): actual = get_current_consumer_offsets( kafka_client_mock, self.group, {'topic1': [99]}, raise_on_error=False ) assert not actual
def step_impl3(context): config = get_cluster_config() context.client = KafkaToolClient(config.broker_list) offsets = get_current_consumer_offsets( context.client, context.group, [context.topic], ) assert offsets[context.topic][0] == SET_OFFSET_KAFKA
def step_impl5(context): config = get_cluster_config() context.client = KafkaToolClient(config.broker_list) offsets = get_current_consumer_offsets( context.client, context.group, [context.topic], ) assert offsets[context.topic][0] == 0
def test_get_current_consumer_offsets_invalid_partition_subset_no_fail(self, kafka_client_mock): actual = get_current_consumer_offsets( kafka_client_mock, self.group, {'topic1': [1, 99]}, raise_on_error=False ) assert actual['topic1'][1] == 20 # Partition 99 does not exist so it shouldn't be in the result assert 99 not in actual['topic1']
def delete_topic(self, group, topic): offsets = get_current_consumer_offsets( self.client, group, [topic], ) set_consumer_offsets( self.client, group, nullify_offsets(offsets), )
def delete_topic(self, group, topic): offsets = get_current_consumer_offsets( self.client, group, [topic], ) set_consumer_offsets( self.client, group, nullify_offsets(offsets), )
def copy_group_kafka(cls, client, topics, source_group, destination_group): copied_offsets = get_current_consumer_offsets( client, source_group, topics, offset_storage='kafka', ) set_consumer_offsets( client, destination_group, copied_offsets, offset_storage='kafka', )
def get_consumer_offsets_metadata( kafka_client, group, topics, raise_on_error=True, ): """This method: * refreshes metadata for the kafka client * fetches group offsets * fetches watermarks :param kafka_client: KafkaToolClient instance :param group: group id :param topics: list of topics :param raise_on_error: if False the method ignores missing topics and missing partitions. It still may fail on the request send. :returns: dict <topic>: [ConsumerPartitionOffsets] """ # Refresh client metadata. We do not use the topic list, because we # don't want to accidentally create the topic if it does not exist. # If Kafka is unavailable, let's retry loading client metadata try: kafka_client.load_metadata_for_topics() except KafkaUnavailableError: kafka_client.load_metadata_for_topics() group_offsets = get_current_consumer_offsets( kafka_client, group, topics, raise_on_error ) watermarks = get_topics_watermarks( kafka_client, topics, raise_on_error ) result = {} for topic, partitions in six.iteritems(group_offsets): result[topic] = [ ConsumerPartitionOffsets( topic=topic, partition=partition, current=group_offsets[topic][partition], highmark=watermarks[topic][partition].highmark, lowmark=watermarks[topic][partition].lowmark, ) for partition in partitions ] return result
def rename_group( cls, client, old_groupid, new_groupid, topics, ): copied_offsets = get_current_consumer_offsets( client, old_groupid, topics, ) set_consumer_offsets(client, new_groupid, copied_offsets) set_consumer_offsets( client, old_groupid, nullify_offsets(topics), )
def rename_group( cls, client, old_groupid, new_groupid, topics, ): copied_offsets = get_current_consumer_offsets( client, old_groupid, topics, ) set_consumer_offsets(client, new_groupid, copied_offsets) set_consumer_offsets( client, old_groupid, nullify_offsets(topics), )
def get_consumer_offsets_metadata( kafka_client, group, topics, raise_on_error=True, ): """This method: * refreshes metadata for the kafka client * fetches group offsets * fetches watermarks :param kafka_client: KafkaToolClient instance :param group: group id :param topics: list of topics :param raise_on_error: if False the method ignores missing topics and missing partitions. It still may fail on the request send. :returns: dict <topic>: [ConsumerPartitionOffsets] """ # Refresh client metadata. We do not use the topic list, because we # don't want to accidentally create the topic if it does not exist. # If Kafka is unavailable, let's retry loading client metadata try: kafka_client.load_metadata_for_topics() except KafkaUnavailableError: kafka_client.load_metadata_for_topics() group_offsets = get_current_consumer_offsets(kafka_client, group, topics, raise_on_error) watermarks = get_topics_watermarks(kafka_client, topics, raise_on_error) result = {} for topic, partitions in six.iteritems(group_offsets): result[topic] = [ ConsumerPartitionOffsets( topic=topic, partition=partition, current=group_offsets[topic][partition], highmark=watermarks[topic][partition].highmark, lowmark=watermarks[topic][partition].lowmark, ) for partition in partitions ] return result
def get_current_offsets( kafka_client, group, topics, raise_on_error, offset_storage, ): """Get the current consumer offsets from either Zookeeper or Kafka or the combination of both. """ if offset_storage in ['zookeeper', 'kafka']: return get_current_consumer_offsets(kafka_client, group, topics, raise_on_error, offset_storage) elif offset_storage == 'dual': return _get_current_offsets_dual( kafka_client, group, topics, raise_on_error, ) else: raise InvalidOffsetStorageError(offset_storage)
def get_consumer_offset(topics, group, storage='zookeeper'): client = KafkaToolClient(KAFKA_URL) return get_current_consumer_offsets(client, group, topics, storage)
def copy_group_kafka(cls, client, topics, source_group, destination_group): copied_offsets = get_current_consumer_offsets(client, source_group, topics) set_consumer_offsets(client, destination_group, copied_offsets)