def test__parse_fetched_data__unknown_tp(fetcher, topic, mocker): fetcher.config['check_crcs'] = False tp = TopicPartition(topic, 0) completed_fetch = CompletedFetch( tp, 0, 0, [UnknownTopicOrPartitionError.errno, -1, None], mocker.MagicMock()) partition_record = fetcher._parse_fetched_data(completed_fetch) assert partition_record is None fetcher._client.cluster.request_update.assert_called_with()
def test__parse_fetched_data__out_of_range(fetcher, topic, mocker): fetcher.config['check_crcs'] = False tp = TopicPartition(topic, 0) completed_fetch = CompletedFetch(tp, 0, 0, [OffsetOutOfRangeError.errno, -1, None], mocker.MagicMock()) partition_record = fetcher._parse_fetched_data(completed_fetch) assert partition_record is None assert fetcher._subscriptions.assignment[tp].awaiting_reset is True
def test__parse_fetched_data__stale_offset(fetcher, topic, mocker): fetcher.config['check_crcs'] = False tp = TopicPartition(topic, 0) msgs = [] for i in range(10): msgs.append((None, b"foo", None)) completed_fetch = CompletedFetch( tp, 10, 0, [0, 100, _build_record_batch(msgs)], mocker.MagicMock()) partition_record = fetcher._parse_fetched_data(completed_fetch) assert partition_record is None
def test__parse_fetched_data__stale_offset(fetcher, topic, mocker): fetcher.config['check_crcs'] = False tp = TopicPartition(topic, 0) msgs = [] for i in range(10): msg = Message(b'foo') msgs.append((i, -1, msg)) completed_fetch = CompletedFetch(tp, 10, 0, [0, 100, msgs], mocker.MagicMock()) partition_record = fetcher._parse_fetched_data(completed_fetch) assert partition_record is None
def test__parse_fetched_data(fetcher, topic, mocker): fetcher.config['check_crcs'] = False tp = TopicPartition(topic, 0) msgs = [] for i in range(10): msg = Message(b'foo') msgs.append((i, -1, msg)) completed_fetch = CompletedFetch(tp, 0, 0, [0, 100, msgs], mocker.MagicMock()) partition_record = fetcher._parse_fetched_data(completed_fetch) assert isinstance(partition_record, fetcher.PartitionRecords) assert len(partition_record) == 10
def test__message_generator(fetcher, topic, mocker): fetcher.config['check_crcs'] = False tp = TopicPartition(topic, 0) msgs = [] for i in range(10): msgs.append((None, b"foo", None)) completed_fetch = CompletedFetch( tp, 0, 0, [0, 100, _build_record_batch(msgs)], mocker.MagicMock()) fetcher._completed_fetches.append(completed_fetch) for i in range(10): msg = next(fetcher) assert isinstance(msg, ConsumerRecord) assert msg.offset == i assert msg.value == b'foo'
def test_fetched_records(fetcher, topic, mocker): fetcher.config['check_crcs'] = False tp = TopicPartition(topic, 0) msgs = [] for i in range(10): msgs.append((None, b"foo", None)) completed_fetch = CompletedFetch( tp, 0, 0, [0, 100, _build_record_batch(msgs)], mocker.MagicMock()) fetcher._completed_fetches.append(completed_fetch) records, partial = fetcher.fetched_records() assert tp in records assert len(records[tp]) == len(msgs) assert all(map(lambda x: isinstance(x, ConsumerRecord), records[tp])) assert partial is False