async def send_offsets_to_transaction(self, offsets, group_id):
self._ensure_transactional()
if not self._txn_manager.is_in_transaction():
raise IllegalOperation("Not in the middle of a transaction")
if not group_id or not isinstance(group_id, str):
raise ValueError(group_id)
# validate `offsets` structure
formatted_offsets = commit_structure_validate(offsets)
log.debug(
"Begin adding offsets %s for consumer group %s to transaction",
formatted_offsets, group_id)
fut = self._txn_manager.add_offsets_to_txn(formatted_offsets, group_id)
await asyncio.shield(fut, loop=self._loop)
def send_offsets_to_transaction(self, offsets, group_id):
self._ensure_transactional()
if not self._txn_manager.is_in_transaction():
raise IllegalOperation("Not in the middle of a transaction")
if not group_id or not isinstance(group_id, str):
raise ValueError(group_id)
# validate `offsets` structure
formatted_offsets = commit_structure_validate(offsets)
log.debug(
"Begin adding offsets %s for consumer group %s to transaction",
formatted_offsets, group_id)
fut = self._txn_manager.add_offsets_to_txn(formatted_offsets, group_id)
yield from asyncio.shield(fut, loop=self._loop)
def send_offsets_to_transaction(self, transactional_id, offsets, group_id):
txn_manager = self._transactions[transactional_id]
if not txn_manager.is_in_transaction():
raise IllegalOperation("Not in the middle of a transaction")
if not group_id or not isinstance(group_id, str):
raise ValueError(group_id)
# validate `offsets` structure
formatted_offsets = commit_structure_validate(offsets)
log.debug(
"Begin adding offsets %s for consumer group %s to transaction",
formatted_offsets, group_id)
fut = txn_manager.add_offsets_to_txn(formatted_offsets, group_id)
log.debug('+WAIT FOR RESPONSE OR ERROR %r' % (fut,))
yield from asyncio.shield(fut, loop=self._loop)
log.debug('-WAIT FOR RESPONSE OR ERROR %r' % (fut,))
def commit(self, offsets=None):
""" Commit offsets to Kafka.
This commits offsets only to Kafka. The offsets committed using this
API will be used on the first fetch after every rebalance and also on
startup. As such, if you need to store offsets in anything other than
Kafka, this API should not be used.
Currently only supports kafka-topic offset storage (not zookeeper)
When explicitly passing ``offsets`` use either offset of next record,
or tuple of offset and metadata::
tp = TopicPartition(msg.topic, msg.partition)
metadata = "Some utf-8 metadata"
# Either
await consumer.commit({tp: msg.offset + 1})
# Or position directly
await consumer.commit({tp: (msg.offset + 1, metadata)})
.. note:: If you want `fire and forget` commit, like ``commit_async()``
in *kafka-python*, just run it in a task. Something like::
fut = loop.create_task(consumer.commit())
fut.add_done_callback(on_commit_done)
Arguments:
offsets (dict, optional): {TopicPartition: (offset, metadata)} dict
to commit with the configured ``group_id``. Defaults to current
consumed offsets for all subscribed partitions.
Raises:
IllegalOperation: If used with ``group_id == None``.
IllegalStateError: If partitions not assigned.
ValueError: If offsets is of wrong format.
CommitFailedError: If membership already changed on broker.
KafkaError: If commit failed on broker side. This could be due to
invalid offset, too long metadata, authorization failure, etc.
.. versionchanged:: 0.4.0
Changed ``AssertionError`` to ``IllegalStateError`` in case of
unassigned partition.
.. versionchanged:: 0.4.0
Will now raise ``CommitFailedError`` in case membership changed,
as (posibly) this partition is handled by another consumer.
"""
if self._group_id is None:
raise IllegalOperation("Requires group_id")
subscription = self._subscription.subscription
if subscription is None:
raise IllegalStateError("Not subscribed to any topics")
assignment = subscription.assignment
if assignment is None:
raise IllegalStateError("No partitions assigned")
if offsets is None:
offsets = assignment.all_consumed_offsets()
else:
offsets = commit_structure_validate(offsets)
for tp in offsets:
if tp not in assignment.tps:
raise IllegalStateError(
"Partition {} is not assigned".format(tp))
yield from self._coordinator.commit_offsets(assignment, offsets)