async def test_send_without_response(self): """Imitate producer without acknowledge, in this case client produces messages and kafka does not send response, and we make sure that futures do not stuck in queue forever""" host, port = self.kafka_host, self.kafka_port conn = await create_conn(host, port) # prepare message builder = LegacyRecordBatchBuilder(magic=1, compression_type=0, batch_size=99999999) builder.append(offset=0, value=b"foo", key=None, timestamp=None) request = ProduceRequest(required_acks=0, timeout=10 * 1000, topics=[(b'foo', [(0, bytes(builder.build())) ])]) # produce messages without acknowledge req = [] for i in range(10): req.append(conn.send(request, expect_response=False)) # make sure futures no stuck in queue self.assertEqual(len(conn._requests), 0) for x in req: await x conn.close()
def test_read_write_serde_v0_v1_with_compression(compression_type, magic): builder = LegacyRecordBatchBuilder( magic=magic, compression_type=compression_type, batch_size=1024 * 1024) for offset in range(10): builder.append( offset, timestamp=9999999, key=b"test", value=b"Super") buffer = builder.build() # Broker will set the offset to a proper last offset value struct.pack_into(">q", buffer, 0, 9) batch = LegacyRecordBatch(buffer, magic) assert batch.validate_crc() assert batch.is_control_batch is False assert batch.is_transactional is False assert batch.producer_id is None assert batch.next_offset == 10 msgs = list(batch) for offset, msg in enumerate(msgs): assert msg.offset == offset assert msg.timestamp == (9999999 if magic else None) assert msg.timestamp_type == (0 if magic else None) assert msg.key == b"test" assert msg.value == b"Super" assert msg.checksum == (-2095076219 if magic else 278251978) & \ 0xffffffff
async def test_compacted_topic_consumption(self): # Compacted topics can have offsets skipped client = AIOKafkaClient( loop=self.loop, bootstrap_servers=[]) client.ready = mock.MagicMock() client.ready.side_effect = asyncio.coroutine(lambda a: True) client.force_metadata_update = mock.MagicMock() client.force_metadata_update.side_effect = asyncio.coroutine( lambda: False) client.send = mock.MagicMock() subscriptions = SubscriptionState(loop=self.loop) fetcher = Fetcher(client, subscriptions, loop=self.loop) tp = TopicPartition('test', 0) req = FetchRequest( -1, # replica_id 100, 100, [(tp.topic, [(tp.partition, 155, 100000)])]) builder = LegacyRecordBatchBuilder( magic=1, compression_type=0, batch_size=99999999) builder.append(160, value=b"12345", key=b"1", timestamp=None) builder.append(162, value=b"23456", key=b"2", timestamp=None) builder.append(167, value=b"34567", key=b"3", timestamp=None) batch = bytes(builder.build()) resp = FetchResponse( [('test', [( 0, 0, 3000, # partition, error_code, highwater_offset batch # Batch raw bytes )])]) subscriptions.assign_from_user({tp}) assignment = subscriptions.subscription.assignment tp_state = assignment.state_value(tp) client.send.side_effect = asyncio.coroutine(lambda n, r: resp) tp_state.seek(155) fetcher._in_flight.add(0) needs_wake_up = await fetcher._proc_fetch_request( assignment, 0, req) self.assertEqual(needs_wake_up, True) buf = fetcher._records[tp] # Test successful getone, the closest in batch offset=160 first = buf.getone() self.assertEqual(tp_state.position, 161) self.assertEqual( (first.value, first.key, first.offset), (b"12345", b"1", 160)) # Test successful getmany second, third = buf.getall() self.assertEqual(tp_state.position, 168) self.assertEqual( (second.value, second.key, second.offset), (b"23456", b"2", 162)) self.assertEqual( (third.value, third.key, third.offset), (b"34567", b"3", 167))
def test_unsupported_yet_codec(): compression_type = LegacyRecordBatch.CODEC_MASK # It doesn't exist builder = LegacyRecordBatchBuilder( magic=0, compression_type=compression_type, batch_size=1024) with pytest.raises(UnsupportedCodecError): builder.append(0, timestamp=None, key=None, value=b"M") builder.build()
def test_read_write_serde_v0_v1_no_compression(magic, key, value, checksum): builder = LegacyRecordBatchBuilder(magic=magic, compression_type=0, batch_size=1024 * 1024) builder.append(0, timestamp=9999999, key=key, value=value) buffer = builder.build() batch = LegacyRecordBatch(buffer, magic) assert batch.validate_crc() assert batch.is_control_batch is False assert batch.is_transactional is False assert batch.producer_id is None assert batch.next_offset == 1 msgs = list(batch) assert len(msgs) == 1 msg = msgs[0] assert msg.offset == 0 assert msg.timestamp == (9999999 if magic else None) assert msg.timestamp_type == (0 if magic else None) assert msg.key == key assert msg.value == value assert msg.checksum == checksum[magic] & 0xffffffff
def _make_compressed_batch(magic): builder = LegacyRecordBatchBuilder( magic=magic, compression_type=LegacyRecordBatch.CODEC_GZIP, batch_size=1024 * 1024) for offset in range(10): builder.append( offset, timestamp=9999999, key=b"test", value=b"Super") return builder.build()
def test_unavailable_codec(compression_type, name, checker_name): builder = LegacyRecordBatchBuilder( magic=0, compression_type=compression_type, batch_size=1024) builder.append(0, timestamp=None, key=None, value=b"M") correct_buffer = builder.build() with mock.patch.object(kafka.codec, checker_name) as mocked: mocked.return_value = False # Check that builder raises error builder = LegacyRecordBatchBuilder( magic=0, compression_type=compression_type, batch_size=1024) error_msg = "Libraries for {} compression codec not found".format(name) with pytest.raises(UnsupportedCodecError, match=error_msg): builder.append(0, timestamp=None, key=None, value=b"M") builder.build() # Check that reader raises same error batch = LegacyRecordBatch(bytes(correct_buffer), 0) with pytest.raises(UnsupportedCodecError, match=error_msg): list(batch)
def test_legacy_batch_size_limit(magic): # First message can be added even if it's too big builder = LegacyRecordBatchBuilder( magic=magic, compression_type=0, batch_size=1024) meta = builder.append(0, timestamp=None, key=None, value=b"M" * 2000) assert meta.size > 0 assert meta.crc is not None assert meta.offset == 0 assert meta.timestamp is not None assert len(builder.build()) > 2000 builder = LegacyRecordBatchBuilder( magic=magic, compression_type=0, batch_size=1024) meta = builder.append(0, timestamp=None, key=None, value=b"M" * 700) assert meta is not None meta = builder.append(1, timestamp=None, key=None, value=b"M" * 700) assert meta is None meta = builder.append(2, timestamp=None, key=None, value=b"M" * 700) assert meta is None assert len(builder.build()) < 1000
def test_legacy_correct_metadata_response(magic): builder = LegacyRecordBatchBuilder(magic=magic, compression_type=0, batch_size=1024 * 1024) meta = builder.append(0, timestamp=9999999, key=b"test", value=b"Super") assert meta.offset == 0 assert meta.timestamp == (9999999 if magic else -1) assert meta.crc == (-2095076219 if magic else 278251978) & 0xffffffff assert repr(meta) == ("LegacyRecordMetadata(offset=0, crc={}, size={}, " "timestamp={})".format(meta.crc, meta.size, meta.timestamp))
def test_legacy_batch_builder_validates_arguments(magic): builder = LegacyRecordBatchBuilder( magic=magic, compression_type=0, batch_size=1024 * 1024) # Key should not be str with pytest.raises(TypeError): builder.append( 0, timestamp=9999999, key="some string", value=None) # Value should not be str with pytest.raises(TypeError): builder.append( 0, timestamp=9999999, key=None, value="some string") # Timestamp should be of proper type (timestamp is ignored for magic == 0) if magic != 0: with pytest.raises(TypeError): builder.append( 0, timestamp="1243812793", key=None, value=b"some string") # Offset of invalid type with pytest.raises(TypeError): builder.append( "0", timestamp=9999999, key=None, value=b"some string") # Unknown struct errors are passed through. These are theoretical and # indicate a bug in the implementation. The C implementation locates # _encode_msg elsewhere and is less vulnerable to such bugs since it's # statically typed, so we skip the test there. if hasattr(builder, "_encode_msg"): with mock.patch.object(builder, "_encode_msg") as mocked: err = struct.error("test error") mocked.side_effect = err with pytest.raises(struct.error) as excinfo: builder.append( 0, timestamp=None, key=None, value=b"some string") assert excinfo.value == err # Ok to pass value as None builder.append( 0, timestamp=9999999, key=b"123", value=None) # Timestamp can be None builder.append( 1, timestamp=None, key=None, value=b"some string") # Ok to pass offsets in not incremental order. This should not happen thou builder.append( 5, timestamp=9999999, key=b"123", value=None) # in case error handling code fails to fix inner buffer in builder assert len(builder.build()) == 119 if magic else 95
def test_written_bytes_equals_size_in_bytes(magic): key = b"test" value = b"Super" builder = LegacyRecordBatchBuilder( magic=magic, compression_type=0, batch_size=1024 * 1024) size_in_bytes = builder.size_in_bytes( 0, timestamp=9999999, key=key, value=value) pos = builder.size() builder.append(0, timestamp=9999999, key=key, value=value) assert builder.size() - pos == size_in_bytes
def __init__(self, magic, batch_size, compression_type, *, is_transactional): if magic < 2: assert not is_transactional self._builder = LegacyRecordBatchBuilder( magic, compression_type, batch_size) else: self._builder = DefaultRecordBatchBuilder( magic, compression_type, is_transactional=is_transactional, producer_id=-1, producer_epoch=-1, base_sequence=0, batch_size=batch_size) self._relative_offset = 0 self._buffer = None self._closed = False
def prepare(magic: int): samples = [] for _ in range(BATCH_SAMPLES): batch = LegacyRecordBatchBuilder(magic, batch_size=DEFAULT_BATCH_SIZE, compression_type=0) for offset in range(MESSAGES_PER_BATCH): size = batch.append( offset, None, # random.randint(*TIMESTAMP_RANGE) random_bytes(KEY_SIZE), random_bytes(VALUE_SIZE)) assert size samples.append(bytes(batch.build())) return iter(itertools.cycle(samples))
def test_read_write_serde_v0_v1_no_compression(magic, key, value, checksum): builder = LegacyRecordBatchBuilder(magic=magic, compression_type=0, batch_size=1024 * 1024) builder.append(0, timestamp=9999999, key=key, value=value) buffer = builder.build() batch = LegacyRecordBatch(buffer, magic) msgs = list(batch) assert len(msgs) == 1 msg = msgs[0] assert msg.offset == 0 assert msg.timestamp == (9999999 if magic else None) assert msg.timestamp_type == (0 if magic else None) assert msg.key == key assert msg.value == value assert msg.checksum == checksum[magic] & 0xffffffff
def test_read_write_serde_v0_v1_with_compression(compression_type, magic): builder = LegacyRecordBatchBuilder(magic=magic, compression_type=compression_type, batch_size=1024 * 1024) for offset in range(10): builder.append(offset, timestamp=9999999, key=b"test", value=b"Super") buffer = builder.build() batch = LegacyRecordBatch(buffer, magic) msgs = list(batch) for offset, msg in enumerate(msgs): assert msg.offset == offset assert msg.timestamp == (9999999 if magic else None) assert msg.timestamp_type == (0 if magic else None) assert msg.key == b"test" assert msg.value == b"Super" assert msg.checksum == (-2095076219 if magic else 278251978) & \ 0xffffffff
def test_legacy_batch_builder_validates_arguments(magic): builder = LegacyRecordBatchBuilder(magic=magic, compression_type=0, batch_size=1024 * 1024) # Key should not be str with pytest.raises(TypeError): builder.append(0, timestamp=9999999, key="some string", value=None) # Value should not be str with pytest.raises(TypeError): builder.append(0, timestamp=9999999, key=None, value="some string") # Timestamp should be of proper type if magic != 0: with pytest.raises(TypeError): builder.append(0, timestamp="1243812793", key=None, value=b"some string") # Offset of invalid type with pytest.raises(TypeError): builder.append("0", timestamp=9999999, key=None, value=b"some string") # Ok to pass value as None builder.append(0, timestamp=9999999, key=b"123", value=None) # Timestamp can be None builder.append(1, timestamp=None, key=None, value=b"some string") # Ok to pass offsets in not incremental order. This should not happen thou builder.append(5, timestamp=9999999, key=b"123", value=None) # in case error handling code fails to fix inner buffer in builder assert len(builder.build()) == 119 if magic else 95
async def test_proc_fetch_request(self): client = AIOKafkaClient(bootstrap_servers=[]) subscriptions = SubscriptionState() fetcher = Fetcher(client, subscriptions, auto_offset_reset="latest") tp = TopicPartition('test', 0) tp_info = (tp.topic, [(tp.partition, 4, 100000)]) req = FetchRequest( -1, # replica_id 100, 100, [tp_info]) async def ready(conn): return True def force_metadata_update(): fut = create_future() fut.set_result(False) return fut client.ready = mock.MagicMock() client.ready.side_effect = ready client.force_metadata_update = mock.MagicMock() client.force_metadata_update.side_effect = force_metadata_update client.send = mock.MagicMock() builder = LegacyRecordBatchBuilder(magic=1, compression_type=0, batch_size=99999999) builder.append(offset=4, value=b"test msg", key=None, timestamp=None) raw_batch = bytes(builder.build()) fetch_response = FetchResponse([('test', [(0, 0, 9, raw_batch)])]) async def send(node, request): nonlocal fetch_response return fetch_response client.send.side_effect = send subscriptions.assign_from_user({tp}) assignment = subscriptions.subscription.assignment tp_state = assignment.state_value(tp) # The partition has no active position, so will ignore result needs_wake_up = await fetcher._proc_fetch_request(assignment, 0, req) self.assertEqual(needs_wake_up, False) self.assertEqual(fetcher._records, {}) # The partition's position does not match request's fetch offset subscriptions.seek(tp, 0) needs_wake_up = await fetcher._proc_fetch_request(assignment, 0, req) self.assertEqual(needs_wake_up, False) self.assertEqual(fetcher._records, {}) subscriptions.seek(tp, 4) needs_wake_up = await fetcher._proc_fetch_request(assignment, 0, req) self.assertEqual(needs_wake_up, True) buf = fetcher._records[tp] self.assertEqual(buf.getone().value, b"test msg") # If position changed after fetch request passed subscriptions.seek(tp, 4) needs_wake_up = await fetcher._proc_fetch_request(assignment, 0, req) subscriptions.seek(tp, 10) self.assertIsNone(buf.getone()) # If assignment is lost after fetch request passed subscriptions.seek(tp, 4) needs_wake_up = await fetcher._proc_fetch_request(assignment, 0, req) subscriptions.unsubscribe() self.assertIsNone(buf.getone()) subscriptions.assign_from_user({tp}) assignment = subscriptions.subscription.assignment tp_state = assignment.state_value(tp) # error -> no partition found (UnknownTopicOrPartitionError) subscriptions.seek(tp, 4) fetcher._records.clear() fetch_response = FetchResponse([('test', [(0, 3, 9, raw_batch)])]) cc = client.force_metadata_update.call_count needs_wake_up = await fetcher._proc_fetch_request(assignment, 0, req) self.assertEqual(needs_wake_up, False) self.assertEqual(client.force_metadata_update.call_count, cc + 1) # error -> topic auth failed (TopicAuthorizationFailedError) fetch_response = FetchResponse([('test', [(0, 29, 9, raw_batch)])]) needs_wake_up = await fetcher._proc_fetch_request(assignment, 0, req) self.assertEqual(needs_wake_up, True) with self.assertRaises(TopicAuthorizationFailedError): await fetcher.next_record([]) # error -> unknown fetch_response = FetchResponse([('test', [(0, -1, 9, raw_batch)])]) needs_wake_up = await fetcher._proc_fetch_request(assignment, 0, req) self.assertEqual(needs_wake_up, False) # error -> offset out of range with offset strategy fetch_response = FetchResponse([('test', [(0, 1, 9, raw_batch)])]) needs_wake_up = await fetcher._proc_fetch_request(assignment, 0, req) self.assertEqual(needs_wake_up, False) self.assertEqual(tp_state.has_valid_position, False) self.assertEqual(tp_state.awaiting_reset, True) self.assertEqual(tp_state.reset_strategy, OffsetResetStrategy.LATEST) # error -> offset out of range without offset strategy subscriptions.seek(tp, 4) fetcher._default_reset_strategy = OffsetResetStrategy.NONE needs_wake_up = await fetcher._proc_fetch_request(assignment, 0, req) self.assertEqual(needs_wake_up, True) with self.assertRaises(OffsetOutOfRangeError): await fetcher.next_record([]) await fetcher.close()
def test_proc_fetch_request(self): client = AIOKafkaClient( loop=self.loop, bootstrap_servers=[]) subscriptions = SubscriptionState('latest') fetcher = Fetcher(client, subscriptions, loop=self.loop) tp = TopicPartition('test', 0) tp_info = (tp.topic, [(tp.partition, 155, 100000)]) req = FetchRequest( -1, # replica_id 100, 100, [tp_info]) client.ready = mock.MagicMock() client.ready.side_effect = asyncio.coroutine(lambda a: True) client.force_metadata_update = mock.MagicMock() client.force_metadata_update.side_effect = asyncio.coroutine( lambda: False) client.send = mock.MagicMock() builder = LegacyRecordBatchBuilder( magic=1, compression_type=0, batch_size=99999999) builder.append(offset=4, value=b"test msg", key=None, timestamp=None) raw_batch = bytes(builder.build()) client.send.side_effect = asyncio.coroutine( lambda n, r: FetchResponse( [('test', [(0, 0, 9, raw_batch)])])) fetcher._in_flight.add(0) needs_wake_up = yield from fetcher._proc_fetch_request(0, req) self.assertEqual(needs_wake_up, False) state = TopicPartitionState() state.seek(0) subscriptions.assignment[tp] = state subscriptions.needs_partition_assignment = False fetcher._in_flight.add(0) needs_wake_up = yield from fetcher._proc_fetch_request(0, req) self.assertEqual(needs_wake_up, True) buf = fetcher._records[tp] self.assertEqual(buf.getone(), None) # invalid offset, msg is ignored state.seek(4) fetcher._in_flight.add(0) fetcher._records.clear() needs_wake_up = yield from fetcher._proc_fetch_request(0, req) self.assertEqual(needs_wake_up, True) buf = fetcher._records[tp] self.assertEqual(buf.getone().value, b"test msg") # error -> no partition found client.send.side_effect = asyncio.coroutine( lambda n, r: FetchResponse( [('test', [(0, 3, 9, raw_batch)])])) fetcher._in_flight.add(0) fetcher._records.clear() needs_wake_up = yield from fetcher._proc_fetch_request(0, req) self.assertEqual(needs_wake_up, False) # error -> topic auth failed client.send.side_effect = asyncio.coroutine( lambda n, r: FetchResponse( [('test', [(0, 29, 9, raw_batch)])])) fetcher._in_flight.add(0) fetcher._records.clear() needs_wake_up = yield from fetcher._proc_fetch_request(0, req) self.assertEqual(needs_wake_up, True) with self.assertRaises(TopicAuthorizationFailedError): yield from fetcher.next_record([]) # error -> unknown client.send.side_effect = asyncio.coroutine( lambda n, r: FetchResponse( [('test', [(0, -1, 9, raw_batch)])])) fetcher._in_flight.add(0) fetcher._records.clear() needs_wake_up = yield from fetcher._proc_fetch_request(0, req) self.assertEqual(needs_wake_up, False) # error -> offset out of range with offset strategy client.send.side_effect = asyncio.coroutine( lambda n, r: FetchResponse( [('test', [(0, 1, 9, raw_batch)])])) fetcher._in_flight.add(0) fetcher._records.clear() with mock.patch.object(fetcher, "update_fetch_positions") as mocked: mocked.side_effect = asyncio.coroutine(lambda o: None) needs_wake_up = yield from fetcher._proc_fetch_request(0, req) self.assertEqual(needs_wake_up, False) self.assertEqual(state.is_fetchable(), False) mocked.assert_called_with([tp]) # error -> offset out of range with strategy errors out state.seek(4) client.send.side_effect = asyncio.coroutine( lambda n, r: FetchResponse( [('test', [(0, 1, 9, [(4, 10, raw_batch)])])])) fetcher._in_flight.add(0) fetcher._records.clear() with mock.patch.object(fetcher, "update_fetch_positions") as mocked: # the exception should not fail execution here @asyncio.coroutine def mock_async_raises(offests): raise Exception() mocked.side_effect = mock_async_raises needs_wake_up = yield from fetcher._proc_fetch_request(0, req) self.assertEqual(needs_wake_up, False) self.assertEqual(state.is_fetchable(), False) mocked.assert_called_with([tp]) # error -> offset out of range without offset strategy state.seek(4) subscriptions._default_offset_reset_strategy = OffsetResetStrategy.NONE client.send.side_effect = asyncio.coroutine( lambda n, r: FetchResponse( [('test', [(0, 1, 9, raw_batch)])])) fetcher._in_flight.add(0) fetcher._records.clear() needs_wake_up = yield from fetcher._proc_fetch_request(0, req) self.assertEqual(needs_wake_up, True) with self.assertRaises(OffsetOutOfRangeError): yield from fetcher.next_record([]) yield from fetcher.close()
def __init__(self, magic, batch_size, compression_type): self._builder = LegacyRecordBatchBuilder(magic, compression_type, batch_size) self._relative_offset = 0 self._buffer = None self._closed = False