def test__parse_fetched_data__unknown_tp(fetcher, topic, mocker):
fetcher.config['check_crcs'] = False
tp = TopicPartition(topic, 0)
completed_fetch = CompletedFetch(
tp, 0, 0, [UnknownTopicOrPartitionError.errno, -1, None],
mocker.MagicMock())
partition_record = fetcher._parse_fetched_data(completed_fetch)
assert partition_record is None
fetcher._client.cluster.request_update.assert_called_with()
def test__parse_fetched_data__out_of_range(fetcher, topic, mocker):
fetcher.config['check_crcs'] = False
tp = TopicPartition(topic, 0)
completed_fetch = CompletedFetch(tp, 0, 0,
[OffsetOutOfRangeError.errno, -1, None],
mocker.MagicMock())
partition_record = fetcher._parse_fetched_data(completed_fetch)
assert partition_record is None
assert fetcher._subscriptions.assignment[tp].awaiting_reset is True
def test__parse_fetched_data__stale_offset(fetcher, topic, mocker):
fetcher.config['check_crcs'] = False
tp = TopicPartition(topic, 0)
msgs = []
for i in range(10):
msgs.append((None, b"foo", None))
completed_fetch = CompletedFetch(
tp, 10, 0, [0, 100, _build_record_batch(msgs)], mocker.MagicMock())
partition_record = fetcher._parse_fetched_data(completed_fetch)
assert partition_record is None
def test__parse_fetched_data__stale_offset(fetcher, topic, mocker):
fetcher.config['check_crcs'] = False
tp = TopicPartition(topic, 0)
msgs = []
for i in range(10):
msg = Message(b'foo')
msgs.append((i, -1, msg))
completed_fetch = CompletedFetch(tp, 10, 0, [0, 100, msgs],
mocker.MagicMock())
partition_record = fetcher._parse_fetched_data(completed_fetch)
assert partition_record is None
def test__parse_fetched_data(fetcher, topic, mocker):
fetcher.config['check_crcs'] = False
tp = TopicPartition(topic, 0)
msgs = []
for i in range(10):
msg = Message(b'foo')
msgs.append((i, -1, msg))
completed_fetch = CompletedFetch(tp, 0, 0, [0, 100, msgs],
mocker.MagicMock())
partition_record = fetcher._parse_fetched_data(completed_fetch)
assert isinstance(partition_record, fetcher.PartitionRecords)
assert len(partition_record) == 10
def test__message_generator(fetcher, topic, mocker):
fetcher.config['check_crcs'] = False
tp = TopicPartition(topic, 0)
msgs = []
for i in range(10):
msgs.append((None, b"foo", None))
completed_fetch = CompletedFetch(
tp, 0, 0, [0, 100, _build_record_batch(msgs)], mocker.MagicMock())
fetcher._completed_fetches.append(completed_fetch)
for i in range(10):
msg = next(fetcher)
assert isinstance(msg, ConsumerRecord)
assert msg.offset == i
assert msg.value == b'foo'
def test_fetched_records(fetcher, topic, mocker):
fetcher.config['check_crcs'] = False
tp = TopicPartition(topic, 0)
msgs = []
for i in range(10):
msgs.append((None, b"foo", None))
completed_fetch = CompletedFetch(
tp, 0, 0, [0, 100, _build_record_batch(msgs)], mocker.MagicMock())
fetcher._completed_fetches.append(completed_fetch)
records, partial = fetcher.fetched_records()
assert tp in records
assert len(records[tp]) == len(msgs)
assert all(map(lambda x: isinstance(x, ConsumerRecord), records[tp]))
assert partial is False