def test_process_consumer_offset_message_invalid_message(self, parse_mock):
parse_mock.side_effect = InvalidMessageException
kafka_group_reader = KafkaGroupReader(mock.Mock())
message = mock.MagicMock(spec=ConsumerRecord)
kafka_group_reader.process_consumer_offset_message(message)
assert kafka_group_reader._kafka_groups == dict()
def test_process_consumer_offset_message_invalid_message(self, parse_mock):
parse_mock.side_effect = InvalidMessageException
kafka_group_reader = KafkaGroupReader(mock.Mock())
message = mock.MagicMock(spec=ConsumerRecord)
kafka_group_reader.process_consumer_offset_message(message)
assert kafka_group_reader._kafka_groups == dict()
def test_process_consumer_offset_message_topic_pop_no_offset(self):
kafka_config = mock.Mock()
kafka_group_reader = KafkaGroupReader(kafka_config)
kafka_group_reader.kafka_groups['test_group'] = set(['test_topic'])
assert kafka_group_reader.kafka_groups['test_group'] == set(
['test_topic'])
with mock.patch.object(
kafka_group_reader,
'parse_consumer_offset_message',
return_value=['test_group', 'test_topic', 0, None],
autospec=True):
kafka_group_reader.process_consumer_offset_message('test message')
assert kafka_group_reader.kafka_groups == {'test_group': set([])}
def test_process_consumer_offset_message_group(self, parse_mock):
parse_mock.side_effect = [('test.a', 'topic1', 0, 123),
('test.a', 'topic1', 1, 124),
('test.a', 'topic2', 0, 125),
('my_test', 'topic1', 0, 123),
('my_test', 'topic2', 0, 124),
('my_test', 'topic2', 0, None),
('my_test2', 'topic3', 0, 123), ]
kafka_group_reader = KafkaGroupReader(mock.Mock())
for _ in range(7):
message = mock.MagicMock(spec=ConsumerRecord)
kafka_group_reader.process_consumer_offset_message(message)
expected = {'test.a': {'topic1', 'topic2'}, 'my_test2': {'topic3'}, 'my_test': {'topic1'}}
assert kafka_group_reader.kafka_groups == expected
def test_process_consumer_offset_message_group(self, parse_mock):
parse_mock.side_effect = [
('test.a', 'topic1', 0, 123),
('test.a', 'topic1', 1, 124),
('test.a', 'topic2', 0, 125),
('my_test', 'topic1', 0, 123),
('my_test', 'topic2', 0, 124),
('my_test', 'topic2', 0, None),
('my_test2', 'topic3', 0, 123),
]
kafka_group_reader = KafkaGroupReader(mock.Mock())
for _ in range(7):
message = mock.MagicMock(spec=ConsumerRecord)
kafka_group_reader.process_consumer_offset_message(message)
expected = {
'test.a': {
'topic1': {
0: 123,
1: 124
},
'topic2': {
0: 125
},
},
'my_test2': {
'topic3': {
0: 123
},
},
'my_test': {
'topic1': {
0: 123
},
},
}
# Convert the defaultdict to a normal dict for comparison
actual = {}
for group, topics in six.iteritems(kafka_group_reader._kafka_groups):
actual[group] = {}
for topic, partitions in six.iteritems(topics):
actual[group][topic] = {}
for partition, offset in six.iteritems(partitions):
actual[group][topic][partition] = offset
assert actual == expected
def test_process_consumer_offset_message_topic_get(self):
kafka_config = mock.Mock()
kafka_group_reader = KafkaGroupReader(kafka_config)
assert kafka_group_reader._kafka_groups == {}
with mock.patch.object(
kafka_group_reader,
'parse_consumer_offset_message',
return_value=['test_group', 'test_topic', 0, 45],
autospec=True):
kafka_group_reader.process_consumer_offset_message('test message')
assert kafka_group_reader._kafka_groups['test_group'] == {
'test_topic': {
0: 45
}
}
def test_process_consumer_offset_message_topic_get(self):
kafka_config = mock.Mock()
kafka_group_reader = KafkaGroupReader(kafka_config)
assert kafka_group_reader._kafka_groups == {}
with mock.patch.object(
kafka_group_reader,
'parse_consumer_offset_message',
return_value=[
'test_group',
'test_topic',
0,
45
],
autospec=True
):
kafka_group_reader.process_consumer_offset_message('test message')
assert kafka_group_reader._kafka_groups['test_group'] == {'test_topic': {0: 45}}
def test_process_consumer_offset_message_group(self, parse_mock):
parse_mock.side_effect = [('test.a', 'topic1', 0, 123),
('test.a', 'topic1', 1, 124),
('test.a', 'topic2', 0, 125),
('my_test', 'topic1', 0, 123),
('my_test', 'topic2', 0, 124),
('my_test', 'topic2', 0, None),
('my_test2', 'topic3', 0, 123), ]
kafka_group_reader = KafkaGroupReader(mock.Mock())
for _ in range(7):
message = mock.MagicMock(spec=ConsumerRecord)
kafka_group_reader.process_consumer_offset_message(message)
expected = {
'test.a': {
'topic1': {0: 123, 1: 124},
'topic2': {0: 125},
},
'my_test2': {
'topic3': {0: 123},
},
'my_test': {
'topic1': {0: 123},
},
}
# Convert the defaultdict to a normal dict for comparison
actual = {}
for group, topics in six.iteritems(kafka_group_reader._kafka_groups):
actual[group] = {}
for topic, partitions in six.iteritems(topics):
actual[group][topic] = {}
for partition, offset in six.iteritems(partitions):
actual[group][topic][partition] = offset
assert actual == expected
