async def _publish_stats(self) -> None: """Emit stats (remaining to fetch) while in active recovery.""" interval = self.stats_interval await self.sleep(interval) async for sleep_time in self.itertimer( interval, name='Recovery.stats'): if self.in_recovery: now = monotonic() stats = self.active_stats() num_samples = len(self._processing_times) if stats and \ num_samples >= self.num_samples_required_for_estimate: remaining_total = self.active_remaining_total() self.log.info( 'Still fetching changelog topics for recovery, ' 'estimated time remaining %s ' '(total remaining=%r):\n%s', self.active_remaining_seconds(remaining_total), remaining_total, self._stats_to_logtable( 'Remaining for active recovery', stats), ) elif stats: await self._verify_remaining(now, stats) else: recovery_started_at = self._recovery_started_at if recovery_started_at is None: self.log.error( 'POSSIBLE INTERNAL ERROR: ' 'Recovery marked as started but missing ' 'self._recovery_started_at timestamp.') else: secs_since_started = now - recovery_started_at if secs_since_started >= 30.0: # This shouldn't happen, but we want to # log an error in case it does. self.log.error( 'POSSIBLE INTERNAL ERROR: ' 'Recovery has no remaining offsets to fetch, ' 'but we have spent %s waiting for the worker ' 'to transition out of recovery state...', humanize_seconds(secs_since_started), )
def active_remaining_seconds(self, remaining: float) -> str: s = self._estimated_active_remaining_secs(remaining) return humanize_seconds(s, now="none") if s else "???"
def test_humanize_seconds__prefix(): assert humanize_seconds(4, prefix='about ') == 'about 4.00 seconds'
def test_humanize_seconds(seconds, expected, now, microseconds): now = now or 'now' secs = humanize_seconds(seconds, microseconds=microseconds, now=now) assert secs == expected
def test_humanize_seconds__prefix(): assert humanize_seconds(4, prefix="about ") == "about 4.00 seconds"
def test_humanize_seconds(seconds, expected): assert humanize_seconds(seconds) == expected