def test_fetcher__update_fetch_positions(self):
client = AIOKafkaClient(loop=self.loop, bootstrap_servers=[])
subscriptions = SubscriptionState(loop=self.loop)
fetcher = Fetcher(client, subscriptions, loop=self.loop)
self.add_cleanup(fetcher.close)
# Disable backgroud task
fetcher._fetch_task.cancel()
try:
yield from fetcher._fetch_task
except asyncio.CancelledError:
pass
fetcher._fetch_task = ensure_future(asyncio.sleep(1000000,
loop=self.loop),
loop=self.loop)
partition = TopicPartition('test', 0)
offsets = {partition: OffsetAndTimestamp(12, -1)}
@asyncio.coroutine
def _proc_offset_request(node_id, topic_data):
return offsets
fetcher._proc_offset_request = mock.Mock()
fetcher._proc_offset_request.side_effect = _proc_offset_request
def reset_assignment():
subscriptions.assign_from_user({partition})
assignment = subscriptions.subscription.assignment
tp_state = assignment.state_value(partition)
return assignment, tp_state
assignment, tp_state = reset_assignment()
self.assertIsNone(tp_state._position)
# CASE: reset from committed
# In basic case we will need to wait for committed
update_task = ensure_future(fetcher._update_fetch_positions(
assignment, 0, [partition]),
loop=self.loop)
yield from asyncio.sleep(0.1, loop=self.loop)
self.assertFalse(update_task.done())
# Will continue only after committed is resolved
tp_state.reset_committed(OffsetAndMetadata(4, ""))
needs_wakeup = yield from update_task
self.assertFalse(needs_wakeup)
self.assertEqual(tp_state._position, 4)
self.assertEqual(fetcher._proc_offset_request.call_count, 0)
# CASE: reset for already valid position will have no effect
tp_state.begin_commit() # reset commit, to make sure we don't wait it
yield from fetcher._update_fetch_positions(assignment, 0, [partition])
self.assertEqual(tp_state._position, 4)
self.assertEqual(fetcher._proc_offset_request.call_count, 0)
# CASE: awaiting_reset for the partition
tp_state.await_reset(OffsetResetStrategy.LATEST)
self.assertIsNone(tp_state._position)
yield from fetcher._update_fetch_positions(assignment, 0, [partition])
self.assertEqual(tp_state._position, 12)
self.assertEqual(fetcher._proc_offset_request.call_count, 1)
# CASE: seeked while waiting for committed to be resolved
assignment, tp_state = reset_assignment()
update_task = ensure_future(fetcher._update_fetch_positions(
assignment, 0, [partition]),
loop=self.loop)
yield from asyncio.sleep(0.1, loop=self.loop)
self.assertFalse(update_task.done())
tp_state.seek(8)
tp_state.reset_committed(OffsetAndMetadata(4, ""))
yield from update_task
self.assertEqual(tp_state._position, 8)
self.assertEqual(fetcher._proc_offset_request.call_count, 1)
# CASE: awaiting_reset during waiting
assignment, tp_state = reset_assignment()
update_task = ensure_future(fetcher._update_fetch_positions(
assignment, 0, [partition]),
loop=self.loop)
yield from asyncio.sleep(0.1, loop=self.loop)
self.assertFalse(update_task.done())
tp_state.await_reset(OffsetResetStrategy.LATEST)
tp_state.reset_committed(OffsetAndMetadata(4, ""))
yield from update_task
self.assertEqual(tp_state._position, 12)
self.assertEqual(fetcher._proc_offset_request.call_count, 2)
# CASE: reset using default strategy if committed offset undefined
assignment, tp_state = reset_assignment()
tp_state.reset_committed(OffsetAndMetadata(-1, ""))
yield from fetcher._update_fetch_positions(assignment, 0, [partition])
self.assertEqual(tp_state._position, 12)
self.assertEqual(fetcher._records, {})
# CASE: set error if _default_reset_strategy = OffsetResetStrategy.NONE
assignment, tp_state = reset_assignment()
tp_state.reset_committed(OffsetAndMetadata(-1, ""))
fetcher._default_reset_strategy = OffsetResetStrategy.NONE
needs_wakeup = yield from fetcher._update_fetch_positions(
assignment, 0, [partition])
self.assertTrue(needs_wakeup)
self.assertIsNone(tp_state._position)
self.assertIsInstance(fetcher._records[partition], FetchError)
fetcher._records.clear()
# CASE: if _proc_offset_request errored, we will retry on another spin
fetcher._proc_offset_request.side_effect = UnknownError()
assignment, tp_state = reset_assignment()
tp_state.await_reset(OffsetResetStrategy.LATEST)
yield from fetcher._update_fetch_positions(assignment, 0, [partition])
self.assertIsNone(tp_state._position)
self.assertTrue(tp_state.awaiting_reset)
# CASE: reset 2 partitions separately, 1 will rese, 1 will get
# committed
fetcher._proc_offset_request.side_effect = _proc_offset_request
partition2 = TopicPartition('test', 1)
subscriptions.assign_from_user({partition, partition2})
assignment = subscriptions.subscription.assignment
tp_state = assignment.state_value(partition)
tp_state2 = assignment.state_value(partition2)
tp_state.await_reset(OffsetResetStrategy.LATEST)
tp_state2.reset_committed(OffsetAndMetadata(5, ""))
yield from fetcher._update_fetch_positions(assignment, 0,
[partition, partition2])
self.assertEqual(tp_state.position, 12)
self.assertEqual(tp_state2.position, 5)
def commit(self, offsets=None):
""" Commit offsets to Kafka.
This commits offsets only to Kafka. The offsets committed using this
API will be used on the first fetch after every rebalance and also on
startup. As such, if you need to store offsets in anything other than
Kafka, this API should not be used.
Currently only supports kafka-topic offset storage (not zookeeper)
When explicitly passing ``offsets`` use either offset of next record,
or tuple of offset and metadata::
tp = TopicPartition(msg.topic, msg.partition)
metadata = "Some utf-8 metadata"
# Either
await consumer.commit({tp: msg.offset + 1})
# Or position directly
await consumer.commit({tp: (msg.offset + 1, metadata)})
.. note:: If you want `fire and forget` commit, like ``commit_async()``
in *kafka-python*, just run it in a task. Something like::
fut = loop.create_task(consumer.commit())
fut.add_done_callback(on_commit_done)
Arguments:
offsets (dict, optional): {TopicPartition: (offset, metadata)} dict
to commit with the configured ``group_id``. Defaults to current
consumed offsets for all subscribed partitions.
Raises:
IllegalOperation: If used with ``group_id == None``.
IllegalStateError: If partitions not assigned.
ValueError: If offsets is of wrong format.
CommitFailedError: If membership already changed on broker.
KafkaError: If commit failed on broker side. This could be due to
invalid offset, too long metadata, authorization failure, etc.
.. versionchanged:: 0.4.0
Changed ``AssertionError`` to ``IllegalStateError`` in case of
unassigned partition.
.. versionchanged:: 0.4.0
Will now raise ``CommitFailedError`` in case membership changed,
as (posibly) this partition is handled by another consumer.
"""
if self._group_id is None:
raise IllegalOperation("Requires group_id")
subscription = self._subscription.subscription
if subscription is None:
raise IllegalStateError("Not subscribed to any topics")
assignment = subscription.assignment
if assignment is None:
raise IllegalStateError("No partitions assigned")
if offsets is None:
offsets = assignment.all_consumed_offsets()
else:
# validate `offsets` structure
if not offsets or not isinstance(offsets, dict):
raise ValueError(offsets)
formatted_offsets = {}
for tp, offset_and_metadata in offsets.items():
if not isinstance(tp, TopicPartition):
raise ValueError("Key should be TopicPartition instance")
if tp not in assignment.tps:
raise IllegalStateError(
"Partition {} is not assigned".format(tp))
if isinstance(offset_and_metadata, int):
offset, metadata = offset_and_metadata, ""
else:
try:
offset, metadata = offset_and_metadata
except Exception:
raise ValueError(offsets)
if not isinstance(metadata, str):
raise ValueError("Metadata should be a string")
formatted_offsets[tp] = OffsetAndMetadata(offset, metadata)
offsets = formatted_offsets
yield from self._coordinator.commit_offsets(assignment, offsets)
def _new_offsetandmetadata(self, offset: int, meta: str) -> Any:
return OffsetAndMetadata(offset, meta)
async def test_sender__do_txn_offset_commit_not_ok(self):
sender = await self._setup_sender()
offsets = {
TopicPartition("topic", 0): OffsetAndMetadata(10, ""),
TopicPartition("topic", 1): OffsetAndMetadata(11, ""),
}
add_handler = TxnOffsetCommitHandler(sender, offsets, "some_group")
tm = sender._txn_manager
tm.begin_transaction()
tm.offset_committed = mock.Mock()
def create_response(error_type):
cls = TxnOffsetCommitResponse[0]
resp = cls(
throttle_time_ms=300,
errors=[
("topic", [
(0, error_type.errno),
(1, error_type.errno)
])
]
)
return resp
# Handle coordination errors
for error_cls in [
CoordinatorNotAvailableError, NotCoordinatorError,
RequestTimedOutError]:
with mock.patch.object(sender, "_coordinator_dead") as mocked:
resp = create_response(error_cls)
backoff = add_handler.handle_response(resp)
self.assertEqual(backoff, 0.1)
tm.offset_committed.assert_not_called()
mocked.assert_called_with(CoordinationType.GROUP)
# Not coordination retriable errors
for error_cls in [
CoordinatorLoadInProgressError, UnknownTopicOrPartitionError]:
resp = create_response(error_cls)
backoff = add_handler.handle_response(resp)
self.assertEqual(backoff, 0.1)
tm.offset_committed.assert_not_called()
# ProducerFenced case
resp = create_response(InvalidProducerEpoch)
with self.assertRaises(ProducerFenced):
add_handler.handle_response(resp)
tm.offset_committed.assert_not_called()
# TransactionalIdAuthorizationFailed case
resp = create_response(TransactionalIdAuthorizationFailed)
with self.assertRaises(TransactionalIdAuthorizationFailed) as cm:
add_handler.handle_response(resp)
tm.offset_committed.assert_not_called()
self.assertEqual(cm.exception.args[0], "test_tid")
# Handle unknown error
resp = create_response(UnknownError)
with self.assertRaises(UnknownError):
add_handler.handle_response(resp)
tm.offset_committed.assert_not_called()
# GroupAuthorizationFailedError case
self.assertNotEqual(tm.state, TransactionState.ABORTABLE_ERROR)
resp = create_response(GroupAuthorizationFailedError)
backoff = add_handler.handle_response(resp)
self.assertIsNone(backoff)
tm.offset_committed.assert_not_called()
with self.assertRaises(GroupAuthorizationFailedError) as cm:
tm.committing_transaction()
self.assertEqual(cm.exception.args[0], "some_group")
self.assertEqual(tm.state, TransactionState.ABORTABLE_ERROR)
