def _periodic_due(self, now):
# get start of current hour, allowing for odd time zones
threshold = timezone.local_replace(now, minute=0, second=0)
# make list of due sections
sections = []
# hourly
last_update = self.status.get_datetime('last update', 'hourly')
if not last_update or last_update < threshold:
sections.append('hourly')
# daily
threshold = timezone.local_replace(threshold,
use_dst=self.use_dst,
hour=self.day_end_hour)
last_update = self.status.get_datetime('last update', 'daily')
if not last_update or last_update < threshold:
sections.append('daily')
# 12 hourly == daily
last_update = self.status.get_datetime('last update', '12 hourly')
if not last_update or last_update < threshold:
sections.append('12 hourly')
return sections
# 12 hourly == daily +- 12 hours
threshold = timezone.local_replace(now,
use_dst=self.use_dst,
hour=((self.day_end_hour + 12) %
24),
minute=0,
second=0)
if last_update < threshold:
sections.append('12 hourly')
return sections
def _periodic_due(self, now):
# get start of current hour, allowing for odd time zones
threshold = timezone.local_replace(now, minute=0, second=0)
# make list of due sections
sections = []
# hourly
last_update = self.status.get_datetime('last update', 'hourly')
if not last_update or last_update < threshold:
sections.append('hourly')
# daily
threshold = timezone.local_replace(
threshold, use_dst=self.use_dst, hour=self.day_end_hour)
last_update = self.status.get_datetime('last update', 'daily')
if not last_update or last_update < threshold:
sections.append('daily')
# 12 hourly == daily
last_update = self.status.get_datetime('last update', '12 hourly')
if not last_update or last_update < threshold:
sections.append('12 hourly')
return sections
# 12 hourly == daily +- 12 hours
threshold = timezone.local_replace(
now, use_dst=self.use_dst,
hour=((self.day_end_hour + 12) % 24), minute=0, second=0)
if last_update < threshold:
sections.append('12 hourly')
return sections
def generate_monthly(rain_day_threshold, day_end_hour, use_dst, daily_data,
monthly_data, process_from):
"""Generate monthly summaries from daily data."""
start = monthly_data.before(datetime.max)
if start is None:
start = datetime.min
start = daily_data.after(start + SECOND)
if process_from:
if start:
start = min(start, process_from)
else:
start = process_from
if start is None:
return start
# set start to start of first day of month (local time)
start = timezone.local_replace(start,
use_dst=use_dst,
day=1,
hour=day_end_hour,
minute=0,
second=0)
if day_end_hour >= 12:
# month actually starts on the last day of previous month
start -= DAY
del monthly_data[start:]
stop = daily_data.before(datetime.max)
if stop is None:
return None
month_start = start
acc = MonthAcc(rain_day_threshold)
count = 0
while month_start <= stop:
count += 1
if count % 12 == 0:
logger.info("monthly: %s", month_start.isoformat(' '))
else:
logger.debug("monthly: %s", month_start.isoformat(' '))
month_end = month_start + WEEK
if month_end.month < 12:
month_end = month_end.replace(month=month_end.month + 1)
else:
month_end = month_end.replace(month=1, year=month_end.year + 1)
month_end = month_end - WEEK
if use_dst:
# month might straddle summer time start or end
month_end = timezone.local_replace(month_end + HOURx3,
use_dst=use_dst,
hour=day_end_hour)
acc.reset()
for data in daily_data[month_start:month_end]:
acc.add_daily(data)
new_data = acc.result()
if new_data:
new_data['start'] = month_start
monthly_data[new_data['idx']] = new_data
month_start = month_end
return start
def generate_monthly(rain_day_threshold, day_end_hour, use_dst,
daily_data, monthly_data, process_from):
"""Generate monthly summaries from daily data."""
start = monthly_data.before(datetime.max)
if start is None:
start = datetime.min
start = daily_data.after(start + SECOND)
if process_from:
if start:
start = min(start, process_from)
else:
start = process_from
if start is None:
return start
# set start to start of first day of month (local time)
start = timezone.local_replace(
start, use_dst=use_dst, day=1, hour=day_end_hour, minute=0, second=0)
if day_end_hour >= 12:
# month actually starts on the last day of previous month
start -= DAY
del monthly_data[start:]
stop = daily_data.before(datetime.max)
month_start = start
acc = MonthAcc(rain_day_threshold)
count = 0
while month_start <= stop:
count += 1
if count % 12 == 0:
logger.info("monthly: %s", month_start.isoformat(' '))
else:
logger.debug("monthly: %s", month_start.isoformat(' '))
month_end = month_start + WEEK
if month_end.month < 12:
month_end = month_end.replace(month=month_end.month+1)
else:
month_end = month_end.replace(month=1, year=month_end.year+1)
month_end = month_end - WEEK
if use_dst:
# month might straddle summer time start or end
month_end = timezone.local_replace(
month_end + HOURx3, use_dst=use_dst, hour=day_end_hour)
acc.reset()
for data in daily_data[month_start:month_end]:
acc.add_daily(data)
new_data = acc.result()
if new_data:
new_data['start'] = month_start
monthly_data[new_data['idx']] = new_data
month_start = month_end
return start
def generate_daily(day_end_hour, use_dst, calib_data, hourly_data, daily_data,
process_from):
"""Generate daily summaries from calibrated and hourly data."""
start = daily_data.before(datetime.max)
if start is None:
start = datetime.min
start = calib_data.after(start + SECOND)
if process_from:
if start:
start = min(start, process_from)
else:
start = process_from
if start is None:
return start
# round to start of this day, in local time
start = timezone.local_replace(start,
use_dst=use_dst,
hour=day_end_hour,
minute=0,
second=0)
del daily_data[start:]
stop = calib_data.before(datetime.max)
day_start = start
acc = DayAcc()
count = 0
while day_start <= stop:
count += 1
if count % 30 == 0:
logger.info("daily: %s", day_start.isoformat(' '))
else:
logger.debug("daily: %s", day_start.isoformat(' '))
day_end = day_start + DAY
if use_dst:
# day might be 23 or 25 hours long
day_end = timezone.local_replace(day_end + HOURx3,
use_dst=use_dst,
hour=day_end_hour)
acc.reset()
for data in calib_data[day_start:day_end]:
acc.add_raw(data)
for data in hourly_data[day_start:day_end]:
acc.add_hourly(data)
new_data = acc.result()
if new_data:
new_data['start'] = day_start
daily_data[new_data['idx']] = new_data
day_start = day_end
return start
def monthlygen(inputdata):
"""Internal generator function"""
month_start = start
count = 0
while month_start <= stop:
count += 1
if count % 12 == 0:
logger.info("monthly: %s", month_start.isoformat(' '))
else:
logger.debug("monthly: %s", month_start.isoformat(' '))
month_end = month_start + WEEK
if month_end.month < 12:
month_end = month_end.replace(month=month_end.month + 1)
else:
month_end = month_end.replace(month=1, year=month_end.year + 1)
month_end = month_end - WEEK
if use_dst:
# month might straddle summer time start or end
month_end = timezone.local_replace(month_end + HOURx3,
use_dst=use_dst,
hour=day_end_hour)
acc.reset()
for data in inputdata[month_start:month_end]:
acc.add_daily(data)
new_data = acc.result()
if new_data:
new_data['start'] = month_start
yield new_data
month_start = month_end
def dailygen(inputdata):
"""Internal generator function"""
day_start = start
count = 0
while day_start <= stop:
count += 1
if count % 30 == 0:
logger.info("daily: %s", day_start.isoformat(' '))
else:
logger.debug("daily: %s", day_start.isoformat(' '))
day_end = day_start + DAY
if use_dst:
# day might be 23 or 25 hours long
day_end = timezone.local_replace(day_end + HOURx3,
use_dst=use_dst,
hour=day_end_hour)
acc.reset()
for data in inputdata[day_start:day_end]:
acc.add_raw(data)
for data in hourly_data[day_start:day_end]:
acc.add_hourly(data)
new_data = acc.result()
if new_data:
new_data['start'] = day_start
yield new_data
day_start = day_end
def monthlygen(inputdata):
"""Internal generator function"""
month_start = start
count = 0
while month_start <= stop:
count += 1
if count % 12 == 0:
logger.info("monthly: %s", month_start.isoformat(' '))
else:
logger.debug("monthly: %s", month_start.isoformat(' '))
month_end = month_start + WEEK
if month_end.month < 12:
month_end = month_end.replace(month=month_end.month+1)
else:
month_end = month_end.replace(month=1, year=month_end.year+1)
month_end = month_end - WEEK
if use_dst:
# month might straddle summer time start or end
month_end = timezone.local_replace(
month_end + HOURx3, use_dst=use_dst, hour=day_end_hour)
acc.reset()
for data in inputdata[month_start:month_end]:
acc.add_daily(data)
new_data = acc.result()
if new_data:
new_data['start'] = month_start
yield new_data
month_start = month_end
def dailygen(inputdata):
"""Internal generator function"""
day_start = start
count = 0
while day_start <= stop:
count += 1
if count % 30 == 0:
logger.info("daily: %s", day_start.isoformat(' '))
else:
logger.debug("daily: %s", day_start.isoformat(' '))
day_end = day_start + DAY
if use_dst:
# day might be 23 or 25 hours long
day_end = timezone.local_replace(
day_end + HOURx3, use_dst=use_dst, hour=day_end_hour)
acc.reset()
for data in inputdata[day_start:day_end]:
acc.add_raw(data)
for data in hourly_data[day_start:day_end]:
acc.add_hourly(data)
new_data = acc.result()
if new_data:
new_data['start'] = day_start
yield new_data
day_start = day_end
def rain_day_local(self, data):
# compute rain since day start
day_end_hour, use_dst = get_day_end_hour(self.context.params)
day_start = timezone.local_replace(
data['idx'], use_dst=use_dst, hour=day_end_hour, minute=0, second=0)
day_start = self.context.calib_data.nearest(day_start)
return max(
data['rain'] - self.context.calib_data[day_start]['rain'], 0.0)
def generate_daily(day_end_hour, use_dst,
calib_data, hourly_data, daily_data, process_from):
"""Generate daily summaries from calibrated and hourly data."""
start = daily_data.before(datetime.max)
if start is None:
start = datetime.min
start = calib_data.after(start + SECOND)
if process_from:
if start:
start = min(start, process_from)
else:
start = process_from
if start is None:
return start
# round to start of this day, in local time
start = timezone.local_replace(
start, use_dst=use_dst, hour=day_end_hour, minute=0, second=0)
del daily_data[start:]
stop = calib_data.before(datetime.max)
day_start = start
acc = DayAcc()
count = 0
while day_start <= stop:
count += 1
if count % 30 == 0:
logger.info("daily: %s", day_start.isoformat(' '))
else:
logger.debug("daily: %s", day_start.isoformat(' '))
day_end = day_start + DAY
if use_dst:
# day might be 23 or 25 hours long
day_end = timezone.local_replace(
day_end + HOURx3, use_dst=use_dst, hour=day_end_hour)
acc.reset()
for data in calib_data[day_start:day_end]:
acc.add_raw(data)
for data in hourly_data[day_start:day_end]:
acc.add_hourly(data)
new_data = acc.result()
if new_data:
new_data['start'] = day_start
daily_data[new_data['idx']] = new_data
day_start = day_end
return start
