async def _publish_stats(self) -> None:
"""Emit stats (remaining to fetch) while in active recovery."""
interval = self.stats_interval
await self.sleep(interval)
async for sleep_time in self.itertimer(
interval, name='Recovery.stats'):
if self.in_recovery:
now = monotonic()
stats = self.active_stats()
num_samples = len(self._processing_times)
if stats and \
num_samples >= self.num_samples_required_for_estimate:
remaining_total = self.active_remaining_total()
self.log.info(
'Still fetching changelog topics for recovery, '
'estimated time remaining %s '
'(total remaining=%r):\n%s',
self.active_remaining_seconds(remaining_total),
remaining_total,
self._stats_to_logtable(
'Remaining for active recovery', stats),
)
elif stats:
await self._verify_remaining(now, stats)
else:
recovery_started_at = self._recovery_started_at
if recovery_started_at is None:
self.log.error(
'POSSIBLE INTERNAL ERROR: '
'Recovery marked as started but missing '
'self._recovery_started_at timestamp.')
else:
secs_since_started = now - recovery_started_at
if secs_since_started >= 30.0:
# This shouldn't happen, but we want to
# log an error in case it does.
self.log.error(
'POSSIBLE INTERNAL ERROR: '
'Recovery has no remaining offsets to fetch, '
'but we have spent %s waiting for the worker '
'to transition out of recovery state...',
humanize_seconds(secs_since_started),
)
def active_remaining_seconds(self, remaining: float) -> str:
s = self._estimated_active_remaining_secs(remaining)
return humanize_seconds(s, now="none") if s else "???"
def test_humanize_seconds__prefix():
assert humanize_seconds(4, prefix='about ') == 'about 4.00 seconds'
def test_humanize_seconds(seconds, expected, now, microseconds):
now = now or 'now'
secs = humanize_seconds(seconds, microseconds=microseconds, now=now)
assert secs == expected
def test_humanize_seconds__prefix():
assert humanize_seconds(4, prefix="about ") == "about 4.00 seconds"
def test_humanize_seconds(seconds, expected):
assert humanize_seconds(seconds) == expected
