class InterfaceOffsetsTests(unittest.TestCase):
def setUp(self):
# Dummy client for testing - we're not going to connect that bootstrap broker
self.client = Client()
self.client._connected = True
# Get the broker and topic from a metadata update
self.client._update_from_metadata(topic_metadata())
self.group = Group('testgroup')
self.offset_fetch = offset_fetch()
def test_get_offsets_for_topics(self):
self.client._maybe_update_metadata_for_topics = MagicMock()
self.client._send_list_offsets_to_brokers = MagicMock()
self.client._send_list_offsets_to_brokers.return_value = {'topic1': 'responseobj'}
val = self.client.get_offsets_for_topics(['topic1'])
self.client._maybe_update_metadata_for_topics.assert_called_once_with(['topic1'])
self.client._send_list_offsets_to_brokers.assert_called_once()
values = self.client._send_list_offsets_to_brokers.call_args[0][0]
assert len(values) == 2
for broker_id in (1, 101):
assert values[broker_id]['replica_id'] == -1
assert len(values[broker_id]['topics']) == 1
assert values[broker_id]['topics'][0]['topic'] == 'topic1'
assert len(values[broker_id]['topics'][0]['partitions']) == 1
assert values[broker_id]['topics'][0]['partitions'][0]['timestamp'] == Client.OFFSET_LATEST
assert values[1]['topics'][0]['partitions'][0]['partition'] == 0
assert values[101]['topics'][0]['partitions'][0]['partition'] == 1
assert val == {'topic1': 'responseobj'}
def test_get_offsets_for_topic(self):
self.client.get_offsets_for_topics = MagicMock()
self.client.get_offsets_for_topics.return_value = {'topic1': 'thingitreturns'}
val = self.client.get_offsets_for_topic('topic1')
self.client.get_offsets_for_topics.assert_called_once_with(['topic1'], Client.OFFSET_LATEST)
assert val == 'thingitreturns'
def test_get_offsets_for_topic_bad_timestamp(self):
self.assertRaises(TypeError, self.client.get_offsets_for_topics, ['topic1'], timestamp='notanint')
def test_get_topics_for_group_string(self):
val = self.client._get_topics_for_group(self.group, 'testtopic')
assert val == ['testtopic']
def test_get_topics_for_group_list(self):
val = self.client._get_topics_for_group(self.group, ['topica', 'topicb'])
assert val == ['topica', 'topicb']
def test_get_topics_for_group_empty(self):
self.assertRaises(GroupError, self.client._get_topics_for_group, self.group, [])
def test_get_topics_for_group_subscribed(self):
self.group.subscribed_topics = MagicMock()
self.group.subscribed_topics.return_value = ['topica']
val = self.client._get_topics_for_group(self.group, None)
self.group.subscribed_topics.assert_called_once_with()
assert val == ['topica']
def test_get_offsets_for_group(self):
self.client.get_group = MagicMock()
self.client.get_group.return_value = self.group
self.client._get_topics_for_group = MagicMock()
self.client._get_topics_for_group.return_value = ['topic1']
self.client._maybe_update_metadata_for_topics = MagicMock()
self.client._send_group_aware_request = MagicMock()
self.client._send_group_aware_request.return_value = self.offset_fetch
val = self.client.get_offsets_for_group('testgroup')
self.client.get_group.assert_called_once_with('testgroup')
self.client._get_topics_for_group.assert_called_once_with(self.group, None)
self.client._maybe_update_metadata_for_topics.assert_called_once_with(['topic1'])
self.client._send_group_aware_request.assert_called_once()
assert self.client._send_group_aware_request.call_args[0][0] == 'testgroup'
req = self.client._send_group_aware_request.call_args[0][1]
assert isinstance(req, OffsetFetchV1Request)
assert req['group_id'] == 'testgroup'
assert len(req['topics']) == 1
assert req['topics'][0]['topic'] == 'topic1'
assert len(req['topics'][0]['partitions']) == 2
assert req['topics'][0]['partitions'][0] == 0
assert req['topics'][0]['partitions'][1] == 1
assert 'topic1' in val
assert isinstance(val['topic1'], TopicOffsets)
assert val['topic1'].partitions == [4829, 8904]
def test_set_offsets_for_group_bad_offsets(self):
self.assertRaises(TypeError, self.client.set_offsets_for_group, 'testgroup', 'notalist')
def test_set_offsets_for_group_offsets_none(self):
self.assertRaises(TypeError, self.client.set_offsets_for_group, 'testgroup', None)
def test_set_offsets_for_group_not_empty(self):
self.group.state = 'Stable'
self.client.get_group = MagicMock()
self.client.get_group.return_value = self.group
self.assertRaises(GroupError, self.client.set_offsets_for_group, 'testgroup', [])
self.client.get_group.assert_called_once_with('testgroup')
def test_set_offsets_for_group(self):
self.client.get_group = MagicMock()
self.client.get_group.return_value = self.group
self.client._send_set_offset_request = MagicMock()
self.client._send_set_offset_request.return_value = 'sendresponse'
self.client._parse_set_offset_response = MagicMock()
self.client._parse_set_offset_response.return_value = {'topic1': [0, 0]}
offsets = TopicOffsets(self.client.cluster.topics['topic1'])
offsets.partitions[0] = 2342
offsets.partitions[1] = 8793
val = self.client.set_offsets_for_group('testgroup', [offsets])
assert val == {'topic1': [0, 0]}
self.client.get_group.assert_called_once_with('testgroup')
self.client._send_set_offset_request.assert_called_once_with('testgroup', [offsets])
self.client._parse_set_offset_response.assert_called_once_with('sendresponse')
class OffsetTests(unittest.TestCase):
def setUp(self):
# Dummy client for testing - we're not going to connect that bootstrap broker
self.client = Client()
# Get the broker and topic from a metadata update
self.client._update_from_metadata(topic_metadata())
self.list_offset = list_offset()
self.list_offset_error = list_offset_error()
self.list_offset_request = {1: {'replica_id': -1,
'topics': [{'topic': 'topic1',
'partitions': [{'partition': 0,
'timestamp': Client.OFFSET_LATEST,
'max_num_offsets': 1},
{'partition': 1,
'timestamp': Client.OFFSET_EARLIEST,
'max_num_offsets': 1}]}]}}
def test_send_list_offsets_to_brokers(self):
self.client.cluster.brokers[1].send = MagicMock()
self.client.cluster.brokers[1].send.return_value = (1, self.list_offset)
val = self.client._send_list_offsets_to_brokers(self.list_offset_request)
self.client.cluster.brokers[1].send.assert_called_once()
req = self.client.cluster.brokers[1].send.call_args[0][0]
assert req['replica_id'] == -1
assert len(req['topics']) == 1
assert req['topics'][0]['topic'] == 'topic1'
assert len(req['topics'][0]['partitions']) == 2
assert req['topics'][0]['partitions'][0]['partition'] == 0
assert req['topics'][0]['partitions'][0]['timestamp'] == Client.OFFSET_LATEST
assert req['topics'][0]['partitions'][0]['max_num_offsets'] == 1
assert req['topics'][0]['partitions'][1]['partition'] == 1
assert req['topics'][0]['partitions'][1]['timestamp'] == Client.OFFSET_EARLIEST
assert req['topics'][0]['partitions'][1]['max_num_offsets'] == 1
assert len(val) == 1
assert 'topic1' in val
assert isinstance(val['topic1'], TopicOffsets)
assert val['topic1'].topic == self.client.cluster.topics['topic1']
assert len(val['topic1'].partitions) == 2
assert val['topic1'].partitions[0] == 4829
assert val['topic1'].partitions[1] == 8904
def test_send_list_offsets_to_brokers_connection_error(self):
self.client.cluster.brokers[1].send = MagicMock()
self.client.cluster.brokers[1].send.side_effect = ConnectionError
self.assertRaises(ConnectionError, self.client._send_list_offsets_to_brokers, self.list_offset_request)
def test_send_list_offsets_to_brokers_offsets_error(self):
self.client.cluster.brokers[1].send = MagicMock()
self.client.cluster.brokers[1].send.return_value = (1, self.list_offset_error)
self.assertRaises(OffsetError, self.client._send_list_offsets_to_brokers, self.list_offset_request)
def test_send_set_offset_request_bad_offsets(self):
self.assertRaises(TypeError, self.client._send_set_offset_request, 'testgroup', ['notatopicoffsets'])
def test_send_set_offset_request_bad_topic(self):
offsets = TopicOffsets(self.client.cluster.topics['topic1'])
offsets.topic = None
self.assertRaises(TypeError, self.client._send_set_offset_request, 'testgroup', [offsets])
def test_send_set_offset_request(self):
offsets = TopicOffsets(self.client.cluster.topics['topic1'])
offsets.partitions[0] = 2342
offsets.partitions[1] = 8793
self.client._send_group_aware_request = MagicMock()
self.client._send_group_aware_request.return_value = 'responseobject'
val = self.client._send_set_offset_request('testgroup', [offsets])
assert val == 'responseobject'
self.client._send_group_aware_request.assert_called_once()
assert self.client._send_group_aware_request.call_args[0][0] == 'testgroup'
req = self.client._send_group_aware_request.call_args[0][1]
assert isinstance(req, OffsetCommitV2Request)
assert req['group_id'] == 'testgroup'
assert req['group_generation_id'] == -1
assert req['member_id'] == ''
assert req['retention_time'] == -1
assert len(req['topics']) == 1
assert req['topics'][0]['topic'] == 'topic1'
assert len(req['topics'][0]['partitions']) == 2
assert req['topics'][0]['partitions'][0]['partition'] == 0
assert req['topics'][0]['partitions'][0]['offset'] == 2342
assert req['topics'][0]['partitions'][0]['metadata'] is None
assert req['topics'][0]['partitions'][1]['partition'] == 1
assert req['topics'][0]['partitions'][1]['offset'] == 8793
assert req['topics'][0]['partitions'][1]['metadata'] is None
def test_parse_set_offset_response(self):
response = offset_commit_response()
val = self.client._parse_set_offset_response(response)
assert val == {'topic1': [0, 16]}
class TopicsTests(unittest.TestCase):
def setUp(self):
# Dummy client for testing - we're not going to connect that bootstrap broker
self.client = Client()
self.metadata_response = topic_metadata()
def test_maybe_update_full_metadata_expired(self):
self.client._send_any_broker = MagicMock()
self.client._send_any_broker.return_value = 'metadata_response'
self.client._update_from_metadata = MagicMock()
fake_last_time = time.time() - (self.client.configuration.metadata_refresh * 2)
self.client._last_full_metadata = fake_last_time
self.client._maybe_update_full_metadata()
assert self.client._last_full_metadata > fake_last_time
self.client._send_any_broker.assert_called_once()
arg = self.client._send_any_broker.call_args[0][0]
assert isinstance(arg, TopicMetadataV1Request)
assert arg['topics'] is None
self.client._update_from_metadata.assert_called_once_with('metadata_response', delete=True)
def test_maybe_update_full_metadata_nocache(self):
self.client._send_any_broker = MagicMock()
self.client._update_from_metadata = MagicMock()
fake_last_time = time.time() - 1
self.client._last_full_metadata = fake_last_time
self.client._maybe_update_full_metadata(cache=False)
assert self.client._last_full_metadata > fake_last_time
def test_maybe_update_full_metadata_usecache(self):
self.client._send_any_broker = MagicMock()
self.client._update_from_metadata = MagicMock()
fake_last_time = time.time() - 1
self.client._last_full_metadata = fake_last_time
self.client._maybe_update_full_metadata(cache=True)
assert self.client._last_full_metadata == fake_last_time
def test_maybe_update_metadata_for_topics_noupdate(self):
self.client._update_from_metadata(self.metadata_response)
self.client.cluster.topics['topic1']._last_updated = time.time()
self.client._update_from_metadata = MagicMock()
self.client._send_any_broker = MagicMock()
self.client._maybe_update_metadata_for_topics(['topic1'])
self.client._update_from_metadata.assert_not_called()
def test_maybe_update_metadata_for_topics_expired(self):
self.client._update_from_metadata(self.metadata_response)
self.client.cluster.topics['topic1']._last_updated = 100
self.client._update_from_metadata = MagicMock()
self.client._send_any_broker = MagicMock()
self.client._maybe_update_metadata_for_topics(['topic1'])
self.client._update_from_metadata.assert_called_once()
req = self.client._send_any_broker.call_args[0][0]
assert len(req['topics']) == 1
assert req['topics'][0] == 'topic1'
def test_maybe_update_metadata_for_topics_forced(self):
self.client._update_from_metadata(self.metadata_response)
self.client.cluster.topics['topic1']._last_updated = time.time()
self.client._update_from_metadata = MagicMock()
self.client._send_any_broker = MagicMock()
self.client._maybe_update_metadata_for_topics(['topic1'], cache=False)
self.client._update_from_metadata.assert_called_once()
req = self.client._send_any_broker.call_args[0][0]
assert len(req['topics']) == 1
assert req['topics'][0] == 'topic1'
def test_maybe_update_metadata_for_topics_nonexistent(self):
self.client._update_from_metadata(self.metadata_response)
self.client.cluster.topics['topic1']._last_updated = time.time()
self.client._update_from_metadata = MagicMock()
self.client._send_any_broker = MagicMock()
self.client._maybe_update_metadata_for_topics(['topic1', 'topic2'])
self.client._update_from_metadata.assert_called_once()
req = self.client._send_any_broker.call_args[0][0]
assert len(req['topics']) == 2
assert req['topics'][0] == 'topic1'
assert req['topics'][1] == 'topic2'
def test_update_from_metadata(self):
self.client._update_brokers_from_metadata = MagicMock()
self.client._update_topics_from_metadata = MagicMock()
self.client._update_from_metadata('fake_metadata')
self.client._update_brokers_from_metadata.assert_called_once_with('fake_metadata')
self.client._update_topics_from_metadata.assert_called_once_with('fake_metadata', delete=False)
def test_update_topics_from_metadata_create(self):
# Don't want to test the broker update code here
self.client.cluster.add_broker(Broker('host1.example.com', id=1, port=8031))
self.client.cluster.add_broker(Broker('host2.example.com', id=101, port=8032))
self.client._update_topics_from_metadata(self.metadata_response)
assert_cluster_has_topics(self.client.cluster, self.metadata_response)
def test_update_topics_from_metadata_missing_broker(self):
# Don't want to test the broker update code here
self.client.cluster.add_broker(Broker('host1.example.com', id=1, port=8031))
self.client._update_topics_from_metadata(self.metadata_response)
assert_cluster_has_topics(self.client.cluster, self.metadata_response)
assert 101 in self.client.cluster.brokers
assert self.client.cluster.brokers[101].endpoint.hostname is None
def test_maybe_delete_topics_not_in_metadata(self):
# Don't want to test the broker update code here
broker1 = Broker('host1.example.com', id=1, port=8031)
broker2 = Broker('host2.example.com', id=101, port=8032)
topic = Topic('topic1', 1)
self.client.cluster.add_broker(broker1)
self.client.cluster.add_broker(broker2)
self.client.cluster.add_topic(topic)
topic.partitions[0].add_replica(broker2)
topic.partitions[0].add_replica(broker1)
topic = Topic('topic2', 1)
self.client.cluster.add_broker(broker1)
self.client.cluster.add_broker(broker2)
self.client.cluster.add_topic(topic)
topic.partitions[0].add_replica(broker2)
topic.partitions[0].add_replica(broker1)
self.client._maybe_delete_topics_not_in_metadata(self.metadata_response, delete=True)
assert 'topic2' not in self.client.cluster.topics
assert 'topic1' in self.client.cluster.topics
def test_update_topics_from_metadata_update_replicas(self):
# Don't want to test the broker update code here
broker1 = Broker('host1.example.com', id=1, port=8031)
broker2 = Broker('host2.example.com', id=101, port=8032)
broker3 = Broker('host3.example.com', id=304, port=8033)
topic = Topic('topic1', 2)
self.client.cluster.add_broker(broker1)
self.client.cluster.add_broker(broker2)
self.client.cluster.add_broker(broker3)
self.client.cluster.add_topic(topic)
topic.partitions[0].add_replica(broker3)
topic.partitions[0].add_replica(broker1)
topic.partitions[1].add_replica(broker2)
topic.partitions[1].add_replica(broker1)
self.client._update_topics_from_metadata(self.metadata_response)
assert_cluster_has_topics(self.client.cluster, self.metadata_response)
def test_update_topics_from_metadata_delete_replicas(self):
# Don't want to test the broker update code here
broker1 = Broker('host1.example.com', id=1, port=8031)
broker2 = Broker('host2.example.com', id=101, port=8032)
broker3 = Broker('host3.example.com', id=304, port=8033)
topic = Topic('topic1', 2)
self.client.cluster.add_broker(broker1)
self.client.cluster.add_broker(broker2)
self.client.cluster.add_broker(broker3)
self.client.cluster.add_topic(topic)
topic.partitions[0].add_replica(broker2)
topic.partitions[0].add_replica(broker1)
topic.partitions[0].add_replica(broker3)
topic.partitions[1].add_replica(broker2)
topic.partitions[1].add_replica(broker1)
self.client._update_topics_from_metadata(self.metadata_response)
assert_cluster_has_topics(self.client.cluster, self.metadata_response)
def test_update_brokers_from_metadata(self):
self.client._update_brokers_from_metadata(self.metadata_response)
assert_cluster_has_brokers(self.client.cluster, self.metadata_response)
def test_update_brokers_from_metadata_update_rack(self):
broker1 = Broker('host1.example.com', id=1, port=8031)
broker1.rack = 'wrongrack'
self.client.cluster.add_broker(broker1)
self.client._update_brokers_from_metadata(self.metadata_response)
assert_cluster_has_brokers(self.client.cluster, self.metadata_response)
def test_update_brokers_from_metadata_update_host(self):
broker1 = Broker('wronghost.example.com', id=1, port=8031)
self.client.cluster.add_broker(broker1)
broker1.close = MagicMock()
self.client._update_brokers_from_metadata(self.metadata_response)
assert_cluster_has_brokers(self.client.cluster, self.metadata_response)
broker1.close.assert_called_once()
def test_map_topic_partitions_to_brokers(self):
self.client._update_from_metadata(self.metadata_response)
val = self.client._map_topic_partitions_to_brokers(['topic1'])
assert val == {1: {'topic1': [0]}, 101: {'topic1': [1]}}
def test_map_topic_partitions_to_brokers_nonexistent(self):
self.client._update_from_metadata(self.metadata_response)
self.assertRaises(TopicError, self.client._map_topic_partitions_to_brokers, ['nosuchtopic'])
class OffsetTests(unittest.TestCase):
def setUp(self):
# Dummy client for testing - we're not going to connect that bootstrap broker
self.client = Client()
# Get the broker and topic from a metadata update
self.client._update_from_metadata(topic_metadata())
self.list_offset = list_offset()
self.list_offset_error = list_offset_error()
self.list_offset_request = {
1: {
'replica_id':
-1,
'topics': [{
'topic':
'topic1',
'partitions': [{
'partition': 0,
'timestamp': Client.OFFSET_LATEST,
'max_num_offsets': 1
}, {
'partition': 1,
'timestamp': Client.OFFSET_EARLIEST,
'max_num_offsets': 1
}]
}]
}
}
def test_send_list_offsets_to_brokers(self):
self.client.cluster.brokers[1].send = MagicMock()
self.client.cluster.brokers[1].send.return_value = (1,
self.list_offset)
val = self.client._send_list_offsets_to_brokers(
self.list_offset_request)
self.client.cluster.brokers[1].send.assert_called_once()
req = self.client.cluster.brokers[1].send.call_args[0][0]
assert req['replica_id'] == -1
assert len(req['topics']) == 1
assert req['topics'][0]['topic'] == 'topic1'
assert len(req['topics'][0]['partitions']) == 2
assert req['topics'][0]['partitions'][0]['partition'] == 0
assert req['topics'][0]['partitions'][0][
'timestamp'] == Client.OFFSET_LATEST
assert req['topics'][0]['partitions'][0]['max_num_offsets'] == 1
assert req['topics'][0]['partitions'][1]['partition'] == 1
assert req['topics'][0]['partitions'][1][
'timestamp'] == Client.OFFSET_EARLIEST
assert req['topics'][0]['partitions'][1]['max_num_offsets'] == 1
assert len(val) == 1
assert 'topic1' in val
assert isinstance(val['topic1'], TopicOffsets)
assert val['topic1'].topic == self.client.cluster.topics['topic1']
assert len(val['topic1'].partitions) == 2
assert val['topic1'].partitions[0] == 4829
assert val['topic1'].partitions[1] == 8904
def test_send_list_offsets_to_brokers_connection_error(self):
self.client.cluster.brokers[1].send = MagicMock()
self.client.cluster.brokers[1].send.side_effect = ConnectionError
self.assertRaises(ConnectionError,
self.client._send_list_offsets_to_brokers,
self.list_offset_request)
def test_send_list_offsets_to_brokers_offsets_error(self):
self.client.cluster.brokers[1].send = MagicMock()
self.client.cluster.brokers[1].send.return_value = (
1, self.list_offset_error)
self.assertRaises(OffsetError,
self.client._send_list_offsets_to_brokers,
self.list_offset_request)
def test_send_set_offset_request_bad_offsets(self):
self.assertRaises(TypeError, self.client._send_set_offset_request,
'testgroup', ['notatopicoffsets'])
def test_send_set_offset_request_bad_topic(self):
offsets = TopicOffsets(self.client.cluster.topics['topic1'])
offsets.topic = None
self.assertRaises(TypeError, self.client._send_set_offset_request,
'testgroup', [offsets])
def test_send_set_offset_request(self):
offsets = TopicOffsets(self.client.cluster.topics['topic1'])
offsets.partitions[0] = 2342
offsets.partitions[1] = 8793
self.client._send_group_aware_request = MagicMock()
self.client._send_group_aware_request.return_value = 'responseobject'
val = self.client._send_set_offset_request('testgroup', [offsets])
assert val == 'responseobject'
self.client._send_group_aware_request.assert_called_once()
assert self.client._send_group_aware_request.call_args[0][
0] == 'testgroup'
req = self.client._send_group_aware_request.call_args[0][1]
assert isinstance(req, OffsetCommitV2Request)
assert req['group_id'] == 'testgroup'
assert req['group_generation_id'] == -1
assert req['member_id'] == ''
assert req['retention_time'] == -1
assert len(req['topics']) == 1
assert req['topics'][0]['topic'] == 'topic1'
assert len(req['topics'][0]['partitions']) == 2
assert req['topics'][0]['partitions'][0]['partition'] == 0
assert req['topics'][0]['partitions'][0]['offset'] == 2342
assert req['topics'][0]['partitions'][0]['metadata'] is None
assert req['topics'][0]['partitions'][1]['partition'] == 1
assert req['topics'][0]['partitions'][1]['offset'] == 8793
assert req['topics'][0]['partitions'][1]['metadata'] is None
def test_parse_set_offset_response(self):
response = offset_commit_response()
val = self.client._parse_set_offset_response(response)
assert val == {'topic1': [0, 16]}