def _setup_error_after_data(self): subscriptions = SubscriptionState('latest') client = AIOKafkaClient( loop=self.loop, bootstrap_servers=[]) fetcher = Fetcher(client, subscriptions, loop=self.loop) tp1 = TopicPartition('some_topic', 0) tp2 = TopicPartition('some_topic', 1) state = TopicPartitionState() state.seek(0) subscriptions.assignment[tp1] = state state = TopicPartitionState() state.seek(0) subscriptions.assignment[tp2] = state subscriptions.needs_partition_assignment = False # Add some data messages = [ConsumerRecord( topic="some_topic", partition=1, offset=0, timestamp=0, timestamp_type=0, key=None, value=b"some", checksum=None, serialized_key_size=0, serialized_value_size=4)] fetcher._records[tp2] = FetchResult( tp2, subscriptions=subscriptions, loop=self.loop, records=iter(messages), backoff=0) # Add some error fetcher._records[tp1] = FetchError( loop=self.loop, error=OffsetOutOfRangeError({}), backoff=0) return fetcher, tp1, tp2, messages
def _setup_error_after_data(self): subscriptions = SubscriptionState(loop=self.loop) client = AIOKafkaClient( loop=self.loop, bootstrap_servers=[]) fetcher = Fetcher(client, subscriptions, loop=self.loop) tp1 = TopicPartition('some_topic', 0) tp2 = TopicPartition('some_topic', 1) subscriptions.subscribe(set(["some_topic"])) subscriptions.assign_from_subscribed({tp1, tp2}) assignment = subscriptions.subscription.assignment subscriptions.seek(tp1, 0) subscriptions.seek(tp2, 0) # Add some data messages = [ConsumerRecord( topic="some_topic", partition=1, offset=0, timestamp=0, timestamp_type=0, key=None, value=b"some", checksum=None, serialized_key_size=0, serialized_value_size=4)] fetcher._records[tp2] = FetchResult( tp2, assignment=assignment, loop=self.loop, message_iterator=iter(messages), backoff=0, fetch_offset=0) # Add some error fetcher._records[tp1] = FetchError( loop=self.loop, error=OffsetOutOfRangeError({}), backoff=0) return fetcher, tp1, tp2, messages
async def test_sender__do_txn_offset_commit_ok(self): sender = await self._setup_sender() offsets = { TopicPartition("topic", 0): OffsetAndMetadata(10, ""), TopicPartition("topic", 1): OffsetAndMetadata(11, ""), } add_handler = TxnOffsetCommitHandler(sender, offsets, "some_group") tm = sender._txn_manager tm.offset_committed = mock.Mock() # Handle response cls = TxnOffsetCommitResponse[0] resp = cls( throttle_time_ms=300, errors=[ ("topic", [ (0, NoError.errno), (1, NoError.errno) ]) ] ) backoff = add_handler.handle_response(resp) self.assertIsNone(backoff) self.assertEqual(tm.offset_committed.call_count, 2) tm.offset_committed.assert_has_calls([ mock.call(TopicPartition("topic", 0), 10, "some_group"), mock.call(TopicPartition("topic", 1), 11, "some_group"), ])
def test_txn_manager(txn_manager): assert txn_manager._pid_and_epoch.pid == NO_PRODUCER_ID assert txn_manager._pid_and_epoch.epoch == NO_PRODUCER_EPOCH assert not txn_manager._sequence_numbers assert not txn_manager.has_pid() txn_manager.set_pid_and_epoch(123, 321) assert txn_manager._pid_and_epoch.pid == 123 assert txn_manager._pid_and_epoch.epoch == 321 assert not txn_manager._sequence_numbers assert txn_manager.has_pid() tp1 = TopicPartition("topic", 1) tp2 = TopicPartition("topic", 2) assert txn_manager.sequence_number(tp1) == 0 txn_manager.increment_sequence_number(tp1, 1) assert txn_manager.sequence_number(tp1) == 1 # Changing one sequence_number does not change other assert txn_manager.sequence_number(tp2) == 0 txn_manager.increment_sequence_number(tp2, 33) assert txn_manager.sequence_number(tp2) == 33 assert txn_manager.sequence_number(tp1) == 1 # sequence number should wrap around 32 bit signed integers txn_manager.increment_sequence_number(tp1, 2**32 - 5) assert txn_manager.sequence_number(tp1) == -4
async def test_record_metric_on_rebalance(): async def coro(*arg, **kwargs): pass with patch("kafkaesk.consumer.CONSUMER_REBALANCED") as rebalance_metric: app_mock = AsyncMock() app_mock.topic_mng.list_consumer_group_offsets.return_value = { TopicPartition(topic="foobar", partition=0): OffsetAndMetadata(offset=0, metadata={}) } subscription = Subscription( "test_consumer", coro, "group", topics=["stream.foo"], ) rebalance_listener = BatchConsumer( subscription=subscription, app=app_mock, ) rebalance_listener._consumer = AsyncMock() await rebalance_listener.on_partitions_assigned( [TopicPartition(topic="foobar", partition=0)]) rebalance_metric.labels.assert_called_with( partition=0, group_id="group", event="assigned", ) rebalance_metric.labels().inc.assert_called_once()
async def test_sender__do_add_partitions_to_txn_ok(self): sender = await self._setup_sender() tps = [ TopicPartition("topic", 0), TopicPartition("topic", 1), TopicPartition("topic2", 1) ] add_handler = AddPartitionsToTxnHandler(sender, tps) tm = sender._txn_manager tm.partition_added = mock.Mock() # Handle response cls = AddPartitionsToTxnResponse[0] resp = cls( throttle_time_ms=300, errors=[ ("topic", [ (0, NoError.errno), (1, NoError.errno) ]), ("topic2", [ (1, NoError.errno) ]) ] ) backoff = add_handler.handle_response(resp) self.assertIsNone(backoff) self.assertEqual(tm.partition_added.call_count, 3) tm.partition_added.assert_has_calls([ mock.call(tp) for tp in tps ])
async def test_consumer_transactional_commit(self): producer = AIOKafkaProducer( loop=self.loop, bootstrap_servers=self.hosts, transactional_id="sobaka_producer") await producer.start() self.add_cleanup(producer.stop) producer2 = AIOKafkaProducer( loop=self.loop, bootstrap_servers=self.hosts) await producer2.start() self.add_cleanup(producer2.stop) consumer = AIOKafkaConsumer( self.topic, loop=self.loop, bootstrap_servers=self.hosts, auto_offset_reset="earliest", isolation_level="read_committed") await consumer.start() self.add_cleanup(consumer.stop) # We will produce from a transactional producer and then from a # non-transactional. This should block consumption on that partition # until transaction is committed. await producer.begin_transaction() meta = await producer.send_and_wait( self.topic, b'Hello from transaction', partition=0) meta2 = await producer2.send_and_wait( self.topic, b'Hello from non-transaction', partition=0) # The transaction blocked consumption task = self.loop.create_task(consumer.getone()) await asyncio.sleep(1, loop=self.loop) self.assertFalse(task.done()) tp = TopicPartition(self.topic, 0) self.assertEqual(consumer.last_stable_offset(tp), 0) self.assertEqual(consumer.highwater(tp), 2) await producer.commit_transaction() # Order should be preserved. We first yield the first message, although # it belongs to a committed afterwards transaction msg = await task self.assertEqual(msg.offset, meta.offset) self.assertEqual(msg.timestamp, meta.timestamp) self.assertEqual(msg.value, b"Hello from transaction") self.assertEqual(msg.key, None) msg = await consumer.getone() self.assertEqual(msg.offset, meta2.offset) self.assertEqual(msg.timestamp, meta2.timestamp) self.assertEqual(msg.value, b"Hello from non-transaction") self.assertEqual(msg.key, None) # 3, because we have a commit marker also tp = TopicPartition(self.topic, 0) self.assertEqual(consumer.last_stable_offset(tp), 3) self.assertEqual(consumer.highwater(tp), 3)
async def test_on_partitions_assigned(self, *, handler, thread): await handler.on_partitions_assigned([ TopicPartition('A', 0), TopicPartition('B', 3), ]) thread.on_partitions_assigned.assert_called_once_with({ TP('A', 0), TP('B', 3), })
async def test_is_assigned(subscription_state): tp1 = TopicPartition("topic", 0) tp2 = TopicPartition("topic", 1) assert not subscription_state.is_assigned(tp1) subscription_state.subscribe({"topic"}) assert not subscription_state.is_assigned(tp1) subscription_state.assign_from_subscribed({tp1}) assert subscription_state.is_assigned(tp1) assert not subscription_state.is_assigned(tp2)
def test_is_assigned(subscription_state): tp1 = TopicPartition("topic", 0) tp2 = TopicPartition("topic", 1) assert not subscription_state.is_assigned(tp1) subscription_state.subscribe(set(["topic"])) assert not subscription_state.is_assigned(tp1) subscription_state.assign_from_subscribed(set([tp1])) assert subscription_state.is_assigned(tp1) assert not subscription_state.is_assigned(tp2)
def test_coordinator_metadata_update_during_rebalance(self): # Race condition where client.set_topics start MetadataUpdate, but it # fails to arrive before leader performed assignment # Just ensure topics are created client = AIOKafkaClient(loop=self.loop, bootstrap_servers=self.hosts) yield from client.bootstrap() yield from self.wait_topic(client, 'topic1') yield from self.wait_topic(client, 'topic2') yield from client.close() client = AIOKafkaClient(loop=self.loop, bootstrap_servers=self.hosts) yield from client.bootstrap() self.add_cleanup(client.close) subscription = SubscriptionState('earliest') client.set_topics(["topic1"]) subscription.subscribe(topics=('topic1', )) coordinator = GroupCoordinator( client, subscription, loop=self.loop, group_id='race-rebalance-metadata-update', heartbeat_interval_ms=20000000) self.add_cleanup(coordinator.close) yield from coordinator.ensure_active_group() # Check that topic's partitions are properly assigned self.assertEqual(subscription.needs_partition_assignment, False) self.assertEqual( set(subscription.assignment.keys()), {TopicPartition("topic1", 0), TopicPartition("topic1", 1)}) _metadata_update = client._metadata_update with mock.patch.object(client, '_metadata_update') as mocked: @asyncio.coroutine def _new(*args, **kw): # Just make metadata updates a bit more slow for test # robustness yield from asyncio.sleep(0.5, loop=self.loop) res = yield from _metadata_update(*args, **kw) return res mocked.side_effect = _new subscription.subscribe(topics=('topic2', )) client.set_topics(('topic2', )) yield from coordinator.ensure_active_group() self.assertEqual(subscription.needs_partition_assignment, False) self.assertEqual( set(subscription.assignment.keys()), {TopicPartition("topic2", 0), TopicPartition("topic2", 1)})
def test_seek(subscription_state): tp = TopicPartition("topic1", 0) tp2 = TopicPartition("topic2", 0) subscription_state.assign_from_user({tp, tp2}) assignment = subscription_state.subscription.assignment assert assignment.state_value(tp) is not None assert not assignment.state_value(tp).has_valid_position assert assignment.state_value(tp)._position is None subscription_state.seek(tp, 1000) assert assignment.state_value(tp).position == 1000
def test_assigned_state(subscription_state): tp1 = TopicPartition("topic", 0) tp2 = TopicPartition("topic", 1) subscription_state.assign_from_user(set([tp1])) with pytest.raises(IllegalStateError): subscription_state._assigned_state(tp2) tp_state = subscription_state._assigned_state(tp1) assert tp_state is not None assert repr(tp_state) == \ "TopicPartitionState<Status=PartitionStatus.ASSIGNED position=None>"
async def test_sender__do_txn_offset_commit_not_ok(self): sender = await self._setup_sender() offsets = { TopicPartition("topic", 0): OffsetAndMetadata(10, ""), TopicPartition("topic", 1): OffsetAndMetadata(11, ""), } add_handler = TxnOffsetCommitHandler(sender, offsets, "some_group") tm = sender._txn_manager tm.offset_committed = mock.Mock() def create_response(error_type): cls = TxnOffsetCommitResponse[0] resp = cls(throttle_time_ms=300, errors=[("topic", [(0, error_type.errno), (1, error_type.errno)])]) return resp # Handle coordination errors for error_cls in [ CoordinatorNotAvailableError, NotCoordinatorError, RequestTimedOutError ]: with mock.patch.object(sender, "_coordinator_dead") as mocked: resp = create_response(error_cls) backoff = add_handler.handle_response(resp) self.assertEqual(backoff, 0.1) tm.offset_committed.assert_not_called() mocked.assert_called_with(CoordinationType.GROUP) # Not coordination retriable errors for error_cls in [ CoordinatorLoadInProgressError, UnknownTopicOrPartitionError ]: resp = create_response(error_cls) backoff = add_handler.handle_response(resp) self.assertEqual(backoff, 0.1) tm.offset_committed.assert_not_called() # ProducerFenced case resp = create_response(InvalidProducerEpoch) with self.assertRaises(ProducerFenced): add_handler.handle_response(resp) tm.offset_committed.assert_not_called() # Handle unknown error resp = create_response(UnknownError) with self.assertRaises(UnknownError): add_handler.handle_response(resp) tm.offset_committed.assert_not_called()
async def test_assigned_state(subscription_state): tp1 = TopicPartition("topic", 0) tp2 = TopicPartition("topic", 1) subscription_state.assign_from_user({tp1}) with pytest.raises(IllegalStateError): subscription_state._assigned_state(tp2) tp_state = subscription_state._assigned_state(tp1) assert tp_state is not None assert repr(tp_state) == ( "TopicPartitionState<Status=PartitionStatus.AWAITING_RESET" " position=None>" )
async def test_compacted_topic_consumption(self): # Compacted topics can have offsets skipped client = AIOKafkaClient( loop=self.loop, bootstrap_servers=[]) client.ready = mock.MagicMock() client.ready.side_effect = asyncio.coroutine(lambda a: True) client.force_metadata_update = mock.MagicMock() client.force_metadata_update.side_effect = asyncio.coroutine( lambda: False) client.send = mock.MagicMock() subscriptions = SubscriptionState(loop=self.loop) fetcher = Fetcher(client, subscriptions, loop=self.loop) tp = TopicPartition('test', 0) req = FetchRequest( -1, # replica_id 100, 100, [(tp.topic, [(tp.partition, 155, 100000)])]) builder = LegacyRecordBatchBuilder( magic=1, compression_type=0, batch_size=99999999) builder.append(160, value=b"12345", key=b"1", timestamp=None) builder.append(162, value=b"23456", key=b"2", timestamp=None) builder.append(167, value=b"34567", key=b"3", timestamp=None) batch = bytes(builder.build()) resp = FetchResponse( [('test', [( 0, 0, 3000, # partition, error_code, highwater_offset batch # Batch raw bytes )])]) subscriptions.assign_from_user({tp}) assignment = subscriptions.subscription.assignment tp_state = assignment.state_value(tp) client.send.side_effect = asyncio.coroutine(lambda n, r: resp) tp_state.seek(155) fetcher._in_flight.add(0) needs_wake_up = await fetcher._proc_fetch_request( assignment, 0, req) self.assertEqual(needs_wake_up, True) buf = fetcher._records[tp] # Test successful getone, the closest in batch offset=160 first = buf.getone() self.assertEqual(tp_state.position, 161) self.assertEqual( (first.value, first.key, first.offset), (b"12345", b"1", 160)) # Test successful getmany second, third = buf.getall() self.assertEqual(tp_state.position, 168) self.assertEqual( (second.value, second.key, second.offset), (b"23456", b"2", 162)) self.assertEqual( (third.value, third.key, third.offset), (b"34567", b"3", 167))
async def process_next_batch(topic_partition, count): tp = [topic_partition] if topic_partition else [] read_count = 0 if count == 1: msg = await client.getone(*tp) if topic_partition is None: topic_partition = TopicPartition(msg.topic, msg.partition) topic = topics[topic_partition.topic] decoded_msg = topic.decode(msg.value) topic.partitions[topic_partition].observer.on_next(decoded_msg) read_count += 1 else: data = await client.getmany(*tp, timeout_ms=5000, max_records=count) if len(data) > 0: msgs = data[topic_partition] topic = topics[topic_partition.topic] for msg in msgs: decoded_msg = topic.decode(msg.value) topic.partitions[topic_partition].observer.on_next( decoded_msg) read_count += 1 return read_count
def test_coordinator_ensure_active_group_on_expired_membership(self): # Do not fail ensure_active_group() if group membership has expired client = AIOKafkaClient(loop=self.loop, bootstrap_servers=self.hosts) yield from client.bootstrap() yield from self.wait_topic(client, 'topic1') subscription = SubscriptionState('earliest') subscription.subscribe(topics=('topic1', )) coordinator = GroupCoordinator(client, subscription, loop=self.loop, group_id='test-offsets-group') yield from coordinator.ensure_active_group() # during OffsetCommit, UnknownMemberIdError is raised offsets = {TopicPartition('topic1', 0): OffsetAndMetadata(1, '')} with mock.patch('aiokafka.errors.for_code') as mocked: mocked.return_value = Errors.UnknownMemberIdError with self.assertRaises(Errors.UnknownMemberIdError): yield from coordinator.commit_offsets(offsets) self.assertEqual(subscription.needs_partition_assignment, True) # same exception is raised during ensure_active_group()'s call to # commit_offsets() via _on_join_prepare() but doesn't break this method with mock.patch.object(coordinator, "commit_offsets") as mocked: @asyncio.coroutine def mock_commit_offsets(*args, **kwargs): raise Errors.UnknownMemberIdError() mocked.side_effect = mock_commit_offsets yield from coordinator.ensure_active_group() yield from coordinator.close() yield from client.close()
async def test_sender__produce_request_not_ok(self): sender = await self._setup_sender() tp = TopicPartition("my_topic", 0) batch_mock = mock.Mock() send_handler = SendProduceReqHandler(sender, {tp: batch_mock}) def create_response(error_type): cls = ProduceResponse[4] resp = cls(throttle_time_ms=300, topics=[("my_topic", [(0, error_type.errno, 0, -1)])]) return resp # Special case for DuplicateSequenceNumber resp = create_response(DuplicateSequenceNumber) send_handler.handle_response(resp) batch_mock.done.assert_called_with(0, -1, None) batch_mock.failure.assert_not_called() self.assertEqual(send_handler._to_reenqueue, []) batch_mock.reset_mock() # Special case for InvalidProducerEpoch resp = create_response(InvalidProducerEpoch) send_handler.handle_response(resp) batch_mock.done.assert_not_called() self.assertNotEqual(batch_mock.failure.call_count, 0) self.assertEqual(send_handler._to_reenqueue, [])
async def test_sender__do_txn_offset_commit_create(self): sender = await self._setup_sender() offsets = { TopicPartition("topic", 0): OffsetAndMetadata(10, ""), TopicPartition("topic", 1): OffsetAndMetadata(11, ""), } add_handler = TxnOffsetCommitHandler(sender, offsets, "some_group") req = add_handler.create_request() self.assertEqual(req.API_KEY, TxnOffsetCommitRequest[0].API_KEY) self.assertEqual(req.API_VERSION, 0) self.assertEqual(req.transactional_id, "test_tid") self.assertEqual(req.group_id, "some_group") self.assertEqual(req.producer_id, 120) self.assertEqual(req.producer_epoch, 22) self.assertEqual(req.topics, [("topic", [(0, 10, ""), (1, 11, "")])])
async def send_batch(self, batch, topic, *, partition): """Submit a BatchBuilder for publication. Arguments: batch (BatchBuilder): batch object to be published. topic (str): topic where the batch will be published. partition (int): partition where this batch will be published. Returns: asyncio.Future: object that will be set when the batch is delivered. """ # first make sure the metadata for the topic is available await self.client._wait_on_metadata(topic) # We only validate we have the partition in the metadata here partition = self._partition(topic, partition, None, None, None, None) # Ensure transaction is started and not committing if self._txn_manager is not None: txn_manager = self._txn_manager if txn_manager.transactional_id is not None and \ not self._txn_manager.is_in_transaction(): raise IllegalOperation( "Can't send messages while not in transaction") tp = TopicPartition(topic, partition) log.debug("Sending batch to %s", tp) future = await self._message_accumulator.add_batch( batch, tp, self._request_timeout_ms / 1000) return future
def handle_response(self, resp): txn_manager = self._sender._txn_manager group_id = self._group_id for topic, partitions in resp.errors: for partition, error_code in partitions: tp = TopicPartition(topic, partition) error_type = Errors.for_code(error_code) if error_type is Errors.NoError: offset = self._offsets[tp].offset log.debug( "Offset %s for partition %s committed to group %s", offset, tp, group_id) txn_manager.offset_committed(tp, offset, group_id) elif (error_type is CoordinatorNotAvailableError or error_type is NotCoordinatorError or # Copied from Java. Not sure why it's only in this case error_type is RequestTimedOutError): self._sender._coordinator_dead(CoordinationType.GROUP) return self._default_backoff elif (error_type is CoordinatorLoadInProgressError or error_type is UnknownTopicOrPartitionError): # We will just retry after backoff return self._default_backoff elif error_type is InvalidProducerEpoch: raise ProducerFenced() else: log.error( "Could not commit offset for partition %s due to " "unexpected error: %s", partition, error_type) raise error_type()
def test_subscribe_topic(subscription_state): mock_listener = MockListener() subscription_state.subscribe({"tp1", "tp2"}, listener=mock_listener) assert subscription_state.subscription is not None assert subscription_state.subscription.topics == {"tp1", "tp2"} assert subscription_state.subscription.assignment is None assert subscription_state.subscription.active is True assert subscription_state.subscription.unsubscribe_future.done() is False # After subscription to topic we can't change the subscription to pattern # or user assignment with pytest.raises(IllegalStateError): subscription_state.subscribe_pattern( pattern=re.compile("^tests-.*$"), listener=mock_listener) with pytest.raises(IllegalStateError): subscription_state.assign_from_user([TopicPartition("topic", 0)]) # Subsciption of the same type can be applied old_subsciption = subscription_state.subscription subscription_state.subscribe( {"tp1", "tp2", "tp3"}, listener=mock_listener) assert subscription_state.subscription is not None assert subscription_state.subscription.topics == {"tp1", "tp2", "tp3"} assert old_subsciption is not subscription_state.subscription assert old_subsciption.active is False assert old_subsciption.unsubscribe_future.done() is True
def test_assigned_partitions(subscription_state): assert subscription_state.assigned_partitions() == set([]) subscription_state.subscribe(topics=set(["tp1"])) assert subscription_state.assigned_partitions() == set([]) assignment = {TopicPartition("tp1", 0)} subscription_state.assign_from_subscribed(assignment) assert subscription_state.assigned_partitions() == assignment
def test_fetch_result_and_error(loop): # Add some data messages = [ ConsumerRecord(topic="some_topic", partition=1, offset=0, timestamp=0, timestamp_type=0, key=None, value=b"some", checksum=None, serialized_key_size=0, serialized_value_size=4) ] result = FetchResult(TopicPartition("test", 0), assignment=mock.Mock(), loop=loop, message_iterator=iter(messages), backoff=0, fetch_offset=0) assert repr(result) == "<FetchResult position=0>" error = FetchError(loop=loop, error=OffsetOutOfRangeError({}), backoff=0) # Python3.7 got rid of trailing comma in exceptions, which makes the line # diffrent between 3.6 and 3.7. assert repr(error) in [ "<FetchError error=OffsetOutOfRangeError({},)>", "<FetchError error=OffsetOutOfRangeError({})>" ]
async def send(self, topic, value=None, key=None, partition=None, timestamp_ms=None): """Publish a message to a topic. Arguments: topic (str): topic where the message will be published value (optional): message value. Must be type bytes, or be serializable to bytes via configured value_serializer. If value is None, key is required and message acts as a 'delete'. See kafka compaction documentation for more details: http://kafka.apache.org/documentation.html#compaction (compaction requires kafka >= 0.8.1) partition (int, optional): optionally specify a partition. If not set, the partition will be selected using the configured 'partitioner'. key (optional): a key to associate with the message. Can be used to determine which partition to send the message to. If partition is None (and producer's partitioner config is left as default), then messages with the same key will be delivered to the same partition (but if key is None, partition is chosen randomly). Must be type bytes, or be serializable to bytes via configured key_serializer. timestamp_ms (int, optional): epoch milliseconds (from Jan 1 1970 UTC) to use as the message timestamp. Defaults to current time. Returns: asyncio.Future: object that will be set when message is processed Raises: kafka.KafkaTimeoutError: if we can't schedule this record ( pending buffer is full) in up to `request_timeout_ms` milliseconds. Note: The returned future will wait based on `request_timeout_ms` setting. Cancelling the returned future **will not** stop event from being sent, but cancelling the ``send`` coroutine itself **will**. """ assert value is not None or self.client.api_version >= (0, 8, 1), ( 'Null messages require kafka >= 0.8.1') assert not (value is None and key is None), \ 'Need at least one: key or value' # first make sure the metadata for the topic is available await self.client._wait_on_metadata(topic) key_bytes, value_bytes = self._serialize(topic, key, value) partition = self._partition(topic, partition, key, value, key_bytes, value_bytes) tp = TopicPartition(topic, partition) log.debug("Sending (key=%s value=%s) to %s", key, value, tp) fut = await self._message_accumulator.add_message( tp, key_bytes, value_bytes, self._request_timeout_ms / 1000, timestamp_ms=timestamp_ms) return fut
async def test_sender__do_add_partitions_to_txn_create(self): sender = await self._setup_sender() tps = [ TopicPartition("topic", 0), TopicPartition("topic", 1), TopicPartition("topic2", 1) ] add_handler = AddPartitionsToTxnHandler(sender, tps) req = add_handler.create_request() self.assertEqual(req.API_KEY, AddPartitionsToTxnRequest[0].API_KEY) self.assertEqual(req.API_VERSION, 0) self.assertEqual(req.transactional_id, "test_tid") self.assertEqual(req.producer_id, 120) self.assertEqual(req.producer_epoch, 22) self.assertEqual(list(sorted(req.topics)), list(sorted([("topic", [0, 1]), ("topic2", [1])])))
def test_assignment(self, *, cthread, _consumer): cthread._consumer = _consumer _consumer.assignment.return_value = { TopicPartition(TP1.topic, TP1.partition), } assignment = cthread.assignment() assert assignment == {TP1} assert all(isinstance(x, TP) for x in assignment)
def handle_response(self, response): for topic, partitions in response.topics: for partition_info in partitions: global_error = None log_start_offset = None if response.API_VERSION < 2: partition, error_code, offset = partition_info # Mimic CREATE_TIME to take user provided timestamp timestamp = -1 elif 2 <= response.API_VERSION <= 4: partition, error_code, offset, timestamp = partition_info elif 5 <= response.API_VERSION <= 7: ( partition, error_code, offset, timestamp, log_start_offset ) = partition_info else: # the ignored parameter is record_error of type # list[(batch_index: int, error_message: str)] ( partition, error_code, offset, timestamp, log_start_offset, _, global_error ) = partition_info tp = TopicPartition(topic, partition) error = Errors.for_code(error_code) batch = self._batches.get(tp) if batch is None: continue if error is Errors.NoError: batch.done(offset, timestamp, log_start_offset) elif error is DuplicateSequenceNumber: # If we have received a duplicate sequence error, # it means that the sequence number has advanced # beyond the sequence of the current batch, and we # haven't retained batch metadata on the broker to # return the correct offset and timestamp. # # The only thing we can do is to return success to # the user and not return a valid offset and # timestamp. batch.done(offset, timestamp, log_start_offset) elif not self._can_retry(error(), batch): if error is InvalidProducerEpoch: exc = ProducerFenced() elif error is TopicAuthorizationFailedError: exc = error(topic) else: exc = error() batch.failure(exception=exc) else: log.warning( "Got error produce response on topic-partition" " %s, retrying. Error: %s", tp, global_error or error) # Ok, we can retry this batch if getattr(error, "invalid_metadata", False): self._client.force_metadata_update() self._to_reenqueue.append(batch)
def test_offsets_failed_scenarios(self): client = AIOKafkaClient(loop=self.loop, bootstrap_servers=self.hosts) yield from client.bootstrap() yield from self.wait_topic(client, 'topic1') subscription = SubscriptionState('earliest') subscription.subscribe(topics=('topic1', )) coordinator = GroupCoordinator(client, subscription, loop=self.loop, group_id='test-offsets-group') yield from coordinator.ensure_active_group() offsets = {TopicPartition('topic1', 0): OffsetAndMetadata(1, '')} yield from coordinator.commit_offsets(offsets) with mock.patch('aiokafka.errors.for_code') as mocked: mocked.return_value = Errors.GroupAuthorizationFailedError with self.assertRaises(Errors.GroupAuthorizationFailedError): yield from coordinator.commit_offsets(offsets) mocked.return_value = Errors.TopicAuthorizationFailedError with self.assertRaises(Errors.TopicAuthorizationFailedError): yield from coordinator.commit_offsets(offsets) mocked.return_value = Errors.InvalidCommitOffsetSizeError with self.assertRaises(Errors.InvalidCommitOffsetSizeError): yield from coordinator.commit_offsets(offsets) mocked.return_value = Errors.GroupLoadInProgressError with self.assertRaises(Errors.GroupLoadInProgressError): yield from coordinator.commit_offsets(offsets) mocked.return_value = Errors.RebalanceInProgressError with self.assertRaises(Errors.RebalanceInProgressError): yield from coordinator.commit_offsets(offsets) self.assertEqual(subscription.needs_partition_assignment, True) subscription.needs_partition_assignment = False mocked.return_value = Errors.UnknownMemberIdError with self.assertRaises(Errors.UnknownMemberIdError): yield from coordinator.commit_offsets(offsets) self.assertEqual(subscription.needs_partition_assignment, True) mocked.return_value = KafkaError with self.assertRaises(KafkaError): yield from coordinator.commit_offsets(offsets) mocked.return_value = Errors.NotCoordinatorForGroupError with self.assertRaises(Errors.NotCoordinatorForGroupError): yield from coordinator.commit_offsets(offsets) self.assertEqual(coordinator.coordinator_id, None) with self.assertRaises(Errors.GroupCoordinatorNotAvailableError): yield from coordinator.commit_offsets(offsets) yield from coordinator.close() yield from client.close()