async def send_offsets_to_transaction(self, offsets, group_id): self._ensure_transactional() if not self._txn_manager.is_in_transaction(): raise IllegalOperation("Not in the middle of a transaction") if not group_id or not isinstance(group_id, str): raise ValueError(group_id) # validate `offsets` structure formatted_offsets = commit_structure_validate(offsets) log.debug( "Begin adding offsets %s for consumer group %s to transaction", formatted_offsets, group_id) fut = self._txn_manager.add_offsets_to_txn(formatted_offsets, group_id) await asyncio.shield(fut, loop=self._loop)
def send_offsets_to_transaction(self, offsets, group_id): self._ensure_transactional() if not self._txn_manager.is_in_transaction(): raise IllegalOperation("Not in the middle of a transaction") if not group_id or not isinstance(group_id, str): raise ValueError(group_id) # validate `offsets` structure formatted_offsets = commit_structure_validate(offsets) log.debug( "Begin adding offsets %s for consumer group %s to transaction", formatted_offsets, group_id) fut = self._txn_manager.add_offsets_to_txn(formatted_offsets, group_id) yield from asyncio.shield(fut, loop=self._loop)
def send_offsets_to_transaction(self, transactional_id, offsets, group_id): txn_manager = self._transactions[transactional_id] if not txn_manager.is_in_transaction(): raise IllegalOperation("Not in the middle of a transaction") if not group_id or not isinstance(group_id, str): raise ValueError(group_id) # validate `offsets` structure formatted_offsets = commit_structure_validate(offsets) log.debug( "Begin adding offsets %s for consumer group %s to transaction", formatted_offsets, group_id) fut = txn_manager.add_offsets_to_txn(formatted_offsets, group_id) log.debug('+WAIT FOR RESPONSE OR ERROR %r' % (fut,)) yield from asyncio.shield(fut, loop=self._loop) log.debug('-WAIT FOR RESPONSE OR ERROR %r' % (fut,))
def commit(self, offsets=None): """ Commit offsets to Kafka. This commits offsets only to Kafka. The offsets committed using this API will be used on the first fetch after every rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API should not be used. Currently only supports kafka-topic offset storage (not zookeeper) When explicitly passing ``offsets`` use either offset of next record, or tuple of offset and metadata:: tp = TopicPartition(msg.topic, msg.partition) metadata = "Some utf-8 metadata" # Either await consumer.commit({tp: msg.offset + 1}) # Or position directly await consumer.commit({tp: (msg.offset + 1, metadata)}) .. note:: If you want `fire and forget` commit, like ``commit_async()`` in *kafka-python*, just run it in a task. Something like:: fut = loop.create_task(consumer.commit()) fut.add_done_callback(on_commit_done) Arguments: offsets (dict, optional): {TopicPartition: (offset, metadata)} dict to commit with the configured ``group_id``. Defaults to current consumed offsets for all subscribed partitions. Raises: IllegalOperation: If used with ``group_id == None``. IllegalStateError: If partitions not assigned. ValueError: If offsets is of wrong format. CommitFailedError: If membership already changed on broker. KafkaError: If commit failed on broker side. This could be due to invalid offset, too long metadata, authorization failure, etc. .. versionchanged:: 0.4.0 Changed ``AssertionError`` to ``IllegalStateError`` in case of unassigned partition. .. versionchanged:: 0.4.0 Will now raise ``CommitFailedError`` in case membership changed, as (posibly) this partition is handled by another consumer. """ if self._group_id is None: raise IllegalOperation("Requires group_id") subscription = self._subscription.subscription if subscription is None: raise IllegalStateError("Not subscribed to any topics") assignment = subscription.assignment if assignment is None: raise IllegalStateError("No partitions assigned") if offsets is None: offsets = assignment.all_consumed_offsets() else: offsets = commit_structure_validate(offsets) for tp in offsets: if tp not in assignment.tps: raise IllegalStateError( "Partition {} is not assigned".format(tp)) yield from self._coordinator.commit_offsets(assignment, offsets)