def test_send_set_offset_request(self):
offsets = TopicOffsets(self.client.cluster.topics['topic1'])
offsets.partitions[0] = 2342
offsets.partitions[1] = 8793
self.client._send_group_aware_request = MagicMock()
self.client._send_group_aware_request.return_value = 'responseobject'
val = self.client._send_set_offset_request('testgroup', [offsets])
assert val == 'responseobject'
self.client._send_group_aware_request.assert_called_once()
assert self.client._send_group_aware_request.call_args[0][
0] == 'testgroup'
req = self.client._send_group_aware_request.call_args[0][1]
assert isinstance(req, OffsetCommitV2Request)
assert req['group_id'] == 'testgroup'
assert req['group_generation_id'] == -1
assert req['member_id'] == ''
assert req['retention_time'] == -1
assert len(req['topics']) == 1
assert req['topics'][0]['topic'] == 'topic1'
assert len(req['topics'][0]['partitions']) == 2
assert req['topics'][0]['partitions'][0]['partition'] == 0
assert req['topics'][0]['partitions'][0]['offset'] == 2342
assert req['topics'][0]['partitions'][0]['metadata'] is None
assert req['topics'][0]['partitions'][1]['partition'] == 1
assert req['topics'][0]['partitions'][1]['offset'] == 8793
assert req['topics'][0]['partitions'][1]['metadata'] is None
def test_send_set_offset_request(self):
offsets = TopicOffsets(self.client.cluster.topics['topic1'])
offsets.partitions[0] = 2342
offsets.partitions[1] = 8793
self.client._send_group_aware_request = MagicMock()
self.client._send_group_aware_request.return_value = 'responseobject'
val = self.client._send_set_offset_request('testgroup', [offsets])
assert val == 'responseobject'
self.client._send_group_aware_request.assert_called_once()
assert self.client._send_group_aware_request.call_args[0][0] == 'testgroup'
req = self.client._send_group_aware_request.call_args[0][1]
assert isinstance(req, OffsetCommitV2Request)
assert req['group_id'] == 'testgroup'
assert req['group_generation_id'] == -1
assert req['member_id'] == ''
assert req['retention_time'] == -1
assert len(req['topics']) == 1
assert req['topics'][0]['topic'] == 'topic1'
assert len(req['topics'][0]['partitions']) == 2
assert req['topics'][0]['partitions'][0]['partition'] == 0
assert req['topics'][0]['partitions'][0]['offset'] == 2342
assert req['topics'][0]['partitions'][0]['metadata'] is None
assert req['topics'][0]['partitions'][1]['partition'] == 1
assert req['topics'][0]['partitions'][1]['offset'] == 8793
assert req['topics'][0]['partitions'][1]['metadata'] is None
def test_set_offsets_from_fetch(self):
topic = Topic('topic1', 2)
offsets = TopicOffsets(topic)
response = offset_fetch()
offsets.set_offsets_from_fetch(response['responses'][0]['partition_responses'])
assert offsets.partitions[0] == 4829
assert offsets.partitions[1] == 8904
def test_set_offsets_empty_from_list(self):
topic = Topic('topic1', 2)
offsets = TopicOffsets(topic)
response = list_offset_none()
offsets.set_offsets_from_list(response['responses'][0]['partition_responses'])
assert offsets.partitions[0] == 4829
assert offsets.partitions[1] == -1
def test_set_offsets_from_fetch(self):
topic = Topic('topic1', 2)
offsets = TopicOffsets(topic)
response = offset_fetch()
offsets.set_offsets_from_fetch(
response['responses'][0]['partition_responses'])
assert offsets.partitions[0] == 4829
assert offsets.partitions[1] == 8904
def test_set_offsets_empty_from_list(self):
topic = Topic('topic1', 2)
offsets = TopicOffsets(topic)
response = list_offset_none()
offsets.set_offsets_from_list(
response['responses'][0]['partition_responses'])
assert offsets.partitions[0] == 4829
assert offsets.partitions[1] == -1
def test_set_offsets_from_fetch_error(self):
topic = Topic('topic1', 2)
offsets = TopicOffsets(topic)
response = offset_fetch_error()
self.assertRaises(OffsetError, offsets.set_offsets_from_fetch,
response['responses'][0]['partition_responses'])
def test_topic_offsets_create(self):
topic = Topic('topic1', 3)
offsets = TopicOffsets(topic)
assert offsets.topic == topic
assert len(offsets.partitions) == 3
for partition in offsets.partitions:
assert partition == -1
def _send_list_offsets_to_brokers(self, request_values):
"""
Given a mapping of broker IDs to values for ListOffset requests, send the requests to all the brokers and
collate the responses into a mapping of topic names to TopicOffsets instances
Args:
request_values (dict): a mapping of broker ID (int) to value dictionaries for ListOffsets requests
Returns:
dict (string -> TopicOffsets): A dictionary mapping topic names to TopicOffsets instances that contain
offsets for all the partitions requested.
Raises:
ConnectionError: If there is a failure to send the request to a broker
OffsetError: If there is a failure retrieving offsets
"""
requests = {}
for broker_id in request_values:
requests[broker_id] = ListOffsetV0Request(
request_values[broker_id])
responses = self._send_some_brokers(requests, ignore_errors=False)
rv = {}
for broker_id in responses:
response = responses[broker_id]
for topic in response['responses']:
topic_name = topic['topic']
if topic_name not in rv:
rv[topic_name] = TopicOffsets(
self.cluster.topics[topic_name])
rv[topic_name].set_offsets_from_list(
topic['partition_responses'])
return rv
def test_set_offsets_for_group(self):
self.client.get_group = MagicMock()
self.client.get_group.return_value = self.group
self.client._send_set_offset_request = MagicMock()
self.client._send_set_offset_request.return_value = 'sendresponse'
self.client._parse_set_offset_response = MagicMock()
self.client._parse_set_offset_response.return_value = {'topic1': [0, 0]}
offsets = TopicOffsets(self.client.cluster.topics['topic1'])
offsets.partitions[0] = 2342
offsets.partitions[1] = 8793
val = self.client.set_offsets_for_group('testgroup', [offsets])
assert val == {'topic1': [0, 0]}
self.client.get_group.assert_called_once_with('testgroup')
self.client._send_set_offset_request.assert_called_once_with('testgroup', [offsets])
self.client._parse_set_offset_response.assert_called_once_with('sendresponse')
def get_offsets_for_group(self, group_name, topic_list=None):
"""
Get the latest offsets committed by the specified group. If a topic_name is specified, only the offsets for that
topic are returned. Otherwise, offsets for all topics that the group is subscribed to are returned.
Args:
group_name (string): The name of the group to fetch offsets for
topic_list (list): A list of string topic names to fetch offsets for. Defaults to None, which specifies all
topics that are subscribed to by the group.
Return:
dict (string -> TopicOffsets): A dictionary mapping topic names to TopicOffsets instances that contain
offsets for all the partitions in the topic. The dictionary will contain a single key if the topic_name
provided is not None.
Raises:
ConnectionError: If there is a failure to send requests to brokers
TopicError: If the topic does not exist or there is a problem getting information for it
GroupError: If there is a failure to get information for the specified group
OffsetError: If there is a failure retrieving offsets for the topic(s)
"""
self._raise_if_not_connected()
# Get the group we're fetching offsets for (potentially updating the group information)
group = self.get_group(group_name)
fetch_topics = self._get_topics_for_group(group, topic_list)
# Get the topic information, making sure all the leadership info is current
self._maybe_update_metadata_for_topics(fetch_topics)
request_values = {
'group_id':
group_name,
'topics': [{
'topic':
topic,
'partitions':
list(range(len(self.cluster.topics[topic].partitions)))
} for topic in fetch_topics]
}
response = self._send_group_aware_request(
group_name, OffsetFetchV1Request(request_values))
rv = {}
for topic in response['responses']:
topic_name = topic['topic']
rv[topic_name] = TopicOffsets(self.cluster.topics[topic_name])
rv[topic_name].set_offsets_from_fetch(topic['partition_responses'])
return rv
def test_send_set_offset_request_bad_topic(self):
offsets = TopicOffsets(self.client.cluster.topics['topic1'])
offsets.topic = None
self.assertRaises(TypeError, self.client._send_set_offset_request, 'testgroup', [offsets])
def test_send_set_offset_request_bad_topic(self):
offsets = TopicOffsets(self.client.cluster.topics['topic1'])
offsets.topic = None
self.assertRaises(TypeError, self.client._send_set_offset_request,
'testgroup', [offsets])
