def _stats_to_logtable(self, title: str,
stats: RecoveryStatsMapping) -> str:
table_data = [
list(
map(
str,
[
tp.topic,
tp.partition,
s.highwater,
s.offset,
s.remaining,
],
)) for tp, s in sorted(stats.items())
]
return terminal.logtable(
list(self._consolidate_table_keys(table_data)),
title=title,
headers=[
"topic",
"partition",
"need offset",
"have offset",
"remaining",
],
)
async def _build_offsets(self,
consumer: ConsumerT,
tps: Set[TP],
destination: Counter[TP],
title: str) -> None:
# -- Update offsets
# Offsets may have been compacted, need to get to the recent ones
earliest = await consumer.earliest_offsets(*tps)
# FIXME To be consistent with the offset -1 logic
earliest = {tp: offset - 1 for tp, offset in earliest.items()}
for tp in tps:
last_value = destination[tp]
new_value = earliest[tp]
if last_value is None:
destination[tp] = new_value
elif new_value is None:
destination[tp] = last_value
else:
destination[tp] = max(last_value, new_value)
table = terminal.logtable(
[(k.topic, k.partition, v) for k, v in destination.items()],
title=f'Reading Starts At - {title.capitalize()}',
headers=['topic', 'partition', 'offset'],
)
self.log.info('%s offsets at start of reading:\n%s', title, table)
def _start_offsets_logtable(self, offsets: Mapping[TP, int], *,
title: str) -> str:
table_data = [[k.topic, str(k.partition),
str(v)] for k, v in sorted(offsets.items())]
return terminal.logtable(
list(self._consolidate_table_keys(table_data)),
title=f"Reading Starts At - {title.capitalize()}",
headers=["topic", "partition", "offset"],
)
def _highwater_logtable(self, highwaters: Mapping[TP, int], *,
title: str) -> str:
table_data = [[k.topic, str(k.partition),
str(v)] for k, v in sorted(highwaters.items())]
return terminal.logtable(
list(self._consolidate_table_keys(table_data)),
title=f"Highwater - {title.capitalize()}",
headers=["topic", "partition", "highwater"],
)
def _sync_offsets(self, reader: ChangelogReaderT) -> None:
table = terminal.logtable(
[(k.topic, k.partition, v) for k, v in reader.offsets.items()],
title='Sync Offset',
headers=['topic', 'partition', 'offset'],
)
self.log.info('Syncing offsets:\n%s', table)
for tp, offset in reader.offsets.items():
if offset >= 0:
table_offset = self._table_offsets.get(tp, -1)
self._table_offsets[tp] = max(table_offset, offset)
table = terminal.logtable(
[(k.topic, k.partition, v)
for k, v in self._table_offsets.items()],
title='Table Offsets',
headers=['topic', 'partition', 'offset'],
)
self.log.info('After syncing:\n%s', table)
async def _update_offsets(self) -> None:
# Offsets may have been compacted, need to get to the recent ones
consumer = self.app.consumer
earliest = await consumer.earliest_offsets(*self.tps)
# FIXME: To be consistent with the offset -1 logic
earliest = {tp: offset - 1 for tp, offset in earliest.items()}
for tp in self.tps:
self.offsets[tp] = max(self.offsets[tp], earliest[tp])
table = terminal.logtable(
[(k.topic, k.partition, v) for k, v in self.offsets.items()],
title='Reading Starts At',
headers=['topic', 'partition', 'offset'],
)
self.log.info('Updated offsets at start of reading:\n%s', table)
async def _commit(self, offsets: Mapping[TP, Tuple[int, str]]) -> bool:
table = terminal.logtable(
[(str(tp), str(offset), meta)
for tp, (offset, meta) in offsets.items()],
title='Commit Offsets',
headers=['TP', 'Offset', 'Metadata'],
)
self.log.dev('COMMITTING OFFSETS:\n%s', table)
try:
assignment = self.assignment()
commitable: Dict[TP, OffsetAndMetadata] = {}
revoked: Dict[TP, OffsetAndMetadata] = {}
commitable_offsets: Dict[TP, int] = {}
for tp, (offset, meta) in offsets.items():
offset_and_metadata = self._new_offsetandmetadata(offset, meta)
if tp in assignment:
commitable_offsets[tp] = offset
commitable[tp] = offset_and_metadata
else:
revoked[tp] = offset_and_metadata
if revoked:
self.log.info(
'Discarded commit for revoked partitions that '
'will be eventually processed again: %r',
revoked,
)
if not commitable:
return False
with flight_recorder(self.log, timeout=300.0) as on_timeout:
on_timeout.info('+aiokafka_consumer.commit()')
await self._consumer.commit(commitable)
on_timeout.info('-aiokafka._consumer.commit()')
self._committed_offset.update(commitable_offsets)
self.app.monitor.on_tp_commit(commitable_offsets)
self._last_batch = None
return True
except CommitFailedError as exc:
if 'already rebalanced' in str(exc):
return False
self.log.exception(f'Committing raised exception: %r', exc)
await self.crash(exc)
return False
except IllegalStateError as exc:
self.log.exception(f'Got exception: {exc}\n'
f'Current assignment: {self.assignment()}')
await self.crash(exc)
return False
async def _build_highwaters(self) -> None:
consumer = self.app.consumer
tps = self.tps
highwaters = await consumer.highwaters(*tps)
self._highwaters.clear()
self._highwaters.update({
# FIXME the -1 here is because of the way we commit offsets
tp: highwaters[tp] - 1
for tp in tps
})
table = terminal.logtable(
[[k.topic, str(k.partition), str(v)]
for k, v in self._highwaters.items()],
title='Highwater',
headers=['topic', 'partition', 'highwater'],
)
self.log.info('Highwater for changelog partitions:\n%s', table)
async def _commit_offsets(self,
offsets: Mapping[TP, int],
start_new_transaction: bool = True) -> bool:
table = terminal.logtable(
[(str(tp), str(offset)) for tp, offset in offsets.items()],
title='Commit Offsets',
headers=['TP', 'Offset'],
)
self.log.dev('COMMITTING OFFSETS:\n%s', table)
assignment = self.assignment()
committable_offsets: Dict[TP, int] = {}
revoked: Dict[TP, int] = {}
for tp, offset in offsets.items():
if tp in assignment:
committable_offsets[tp] = offset
else:
revoked[tp] = offset
if revoked:
self.log.info(
'Discarded commit for revoked partitions that '
'will be eventually processed again: %r',
revoked,
)
if not committable_offsets:
return False
with flight_recorder(self.log, timeout=300.0) as on_timeout:
did_commit = False
on_timeout.info('+consumer.commit()')
if self.in_transaction:
did_commit = await self.transactions.commit(
committable_offsets,
start_new_transaction=start_new_transaction,
)
else:
did_commit = await self._commit(committable_offsets)
on_timeout.info('-consumer.commit()')
if did_commit:
on_timeout.info('+tables.on_commit')
self.app.tables.on_commit(committable_offsets)
on_timeout.info('-tables.on_commit')
self._committed_offset.update(committable_offsets)
self.app.monitor.on_tp_commit(committable_offsets)
for tp in offsets:
self._last_batch.pop(tp, None)
return did_commit
