def catch_up_aggregation(self):
logger.log(logger.CLASS_AGGREGATOR, 'Daily aggregation started')
time_from = self._get_start_time_of_non_aggregated_data()
if time_from is None:
logger.log(logger.CLASS_AGGREGATOR,
'Daily aggregation is up-to-date')
return
logger.log(
logger.CLASS_AGGREGATOR,
'Starting daily aggregation from {time}'.format(time=time_from))
now = timeutil.aggregator_now()
current_from = time_from
while True:
current_to = self._day_end(current_from)
if current_to > now:
break
self.aggregate(current_from, current_to)
current_from = self._next_day_start(current_from)
logger.log(logger.CLASS_AGGREGATOR, 'Daily aggregation complete')
def _timer_func(self):
aggregation_started_at = timeutil.aggregator_now()
self.hourly_aggregator.catch_up_aggregation()
self.daily_aggregator.catch_up_aggregation()
next_hour_start = timeutil.start_of_next_hour(aggregation_started_at)
timeout = next_hour_start - aggregation_started_at
self._schedule_next_call(timeout.total_seconds())
def _schedule_next_call(self, timeout):
if timeout < 0:
timeout = 0
with self._lock:
now = timeutil.aggregator_now()
logger.log(
logger.CLASS_AGGREGATOR,
'Next aggregation scheduled in {timeout} seconds, at {time}'.
format(timeout=timeout, time=now + timedelta(seconds=timeout)))
self._timer = threading.Timer(timeout, self._timer_func)
self._timer.start()
def _get_start_time_of_non_aggregated_data():
latest = db.select_latest('days')
if latest is None:
logger.log(logger.CLASS_AGGREGATOR, 'No data in days table')
earliest = db.select_earliest('hours')
if earliest is None:
logger.log(logger.CLASS_AGGREGATOR, 'No data in hours table')
return None
else:
logger.log(
logger.CLASS_AGGREGATOR,
'Earliest record in hours table dated {time}'.format(
time=earliest[0]))
dt = timeutil.parse_db_time(earliest[0])
return timeutil.start_of_day(dt)
else:
latest = timeutil.parse_db_time(latest[0])
logger.log(logger.CLASS_AGGREGATOR,
'Latest aggregated day is {time}'.format(time=latest))
now = timeutil.aggregator_now()
if timeutil.start_of_day(latest) == timeutil.start_of_day(now):
return None
else:
return timeutil.start_of_next_day(latest)