async def _commit(self, offsets: Mapping[TP, Tuple[int, str]]) -> bool: table = terminal.logtable( [(str(tp), str(offset), meta) for tp, (offset, meta) in offsets.items()], title='Commit Offsets', headers=['TP', 'Offset', 'Metadata'], ) self.log.dev('COMMITTING OFFSETS:\n%s', table) try: assignment = self.assignment() commitable: Dict[TP, OffsetAndMetadata] = {} revoked: Dict[TP, OffsetAndMetadata] = {} commitable_offsets: Dict[TP, int] = {} for tp, (offset, meta) in offsets.items(): offset_and_metadata = self._new_offsetandmetadata(offset, meta) if tp in assignment: commitable_offsets[tp] = offset commitable[tp] = offset_and_metadata else: revoked[tp] = offset_and_metadata if revoked: self.log.info( 'Discarded commit for revoked partitions that ' 'will be eventually processed again: %r', revoked, ) if not commitable: return False with flight_recorder(self.log, timeout=300.0) as on_timeout: on_timeout.info('+aiokafka_consumer.commit()') await self._consumer.commit(commitable) on_timeout.info('-aiokafka._consumer.commit()') self._committed_offset.update(commitable_offsets) self.app.monitor.on_tp_commit(commitable_offsets) self._last_batch = None return True except CommitFailedError as exc: if 'already rebalanced' in str(exc): return False self.log.exception(f'Committing raised exception: %r', exc) await self.crash(exc) return False except IllegalStateError as exc: self.log.exception(f'Got exception: {exc}\n' f'Current assignment: {self.assignment()}') await self.crash(exc) return False
async def _commit_offsets(self, offsets: Mapping[TP, int], start_new_transaction: bool = True) -> bool: table = terminal.logtable( [(str(tp), str(offset)) for tp, offset in offsets.items()], title='Commit Offsets', headers=['TP', 'Offset'], ) self.log.dev('COMMITTING OFFSETS:\n%s', table) assignment = self.assignment() committable_offsets: Dict[TP, int] = {} revoked: Dict[TP, int] = {} for tp, offset in offsets.items(): if tp in assignment: committable_offsets[tp] = offset else: revoked[tp] = offset if revoked: self.log.info( 'Discarded commit for revoked partitions that ' 'will be eventually processed again: %r', revoked, ) if not committable_offsets: return False with flight_recorder(self.log, timeout=300.0) as on_timeout: did_commit = False on_timeout.info('+consumer.commit()') if self.in_transaction: did_commit = await self.transactions.commit( committable_offsets, start_new_transaction=start_new_transaction, ) else: did_commit = await self._commit(committable_offsets) on_timeout.info('-consumer.commit()') if did_commit: on_timeout.info('+tables.on_commit') self.app.tables.on_commit(committable_offsets) on_timeout.info('-tables.on_commit') self._committed_offset.update(committable_offsets) self.app.monitor.on_tp_commit(committable_offsets) for tp in offsets: self._last_batch.pop(tp, None) return did_commit