def dailygen(inputdata):
"""Internal generator function"""
day_start = start
count = 0
while day_start <= stop:
count += 1
if count % 30 == 0:
logger.info("daily: %s", day_start.isoformat(' '))
else:
logger.debug("daily: %s", day_start.isoformat(' '))
day_end = day_start + DAY
if use_dst:
# day might be 23 or 25 hours long
day_end = timezone.local_replace(
day_end + HOURx3, use_dst=use_dst, hour=day_end_hour)
acc.reset()
for data in inputdata[day_start:day_end]:
acc.add_raw(data)
for data in hourly_data[day_start:day_end]:
acc.add_hourly(data)
new_data = acc.result()
if new_data:
new_data['start'] = day_start
yield new_data
day_start = day_end
def reprocess(data_dir, update):
if update:
logger.warning("Updating status to detect invalid wind_dir")
count = 0
raw_data = pywws.storage.RawStore(data_dir)
for data in raw_data[:]:
count += 1
idx = data['idx']
if count % 10000 == 0:
logger.info("update: %s", idx.isoformat(' '))
elif count % 500 == 0:
logger.debug("update: %s", idx.isoformat(' '))
if data['wind_dir'] is not None and (data['wind_dir'] & 0x80):
data['wind_dir'] = None
raw_data[idx] = data
raw_data.flush()
# delete old format summary files
logger.warning('Deleting old summaries')
for summary in ['calib', 'hourly', 'daily', 'monthly']:
for root, dirs, files in os.walk(
os.path.join(data_dir, summary), topdown=False):
logger.info(root)
for file in files:
os.unlink(os.path.join(root, file))
os.rmdir(root)
# create data summaries
logger.warning('Generating hourly and daily summaries')
with pywws.storage.pywws_context(data_dir) as context:
pywws.process.process_data(context)
return 0
def monthlygen(inputdata, start, local_start):
"""Internal generator function"""
acc = MonthAcc(rain_day_threshold)
month_start = start
count = 0
while month_start <= stop:
count += 1
if count % 12 == 0:
logger.info("monthly: %s", month_start.isoformat(' '))
else:
logger.debug("monthly: %s", month_start.isoformat(' '))
if local_start.month < 12:
local_start = local_start.replace(month=local_start.month+1)
else:
local_start = local_start.replace(
month=1, year=local_start.year+1)
month_end = time_zone.local_to_utc(local_start)
month_end = time_zone.day_start(
month_end, day_end_hour, use_dst=use_dst)
acc.reset()
for data in inputdata[month_start:month_end]:
acc.add_daily(data)
new_data = acc.result()
if new_data:
new_data['start'] = month_start
yield new_data
month_start = month_end
def reprocess(data_dir, update):
with pywws.storage.pywws_context(data_dir) as context:
if update:
logger.warning("Updating status to detect invalid wind_dir")
count = 0
raw_data = context.raw_data
for data in raw_data[:]:
count += 1
idx = data['idx']
if count % 10000 == 0:
logger.info("update: %s", idx.isoformat(' '))
elif count % 500 == 0:
logger.debug("update: %s", idx.isoformat(' '))
if data['wind_dir'] is not None and (data['wind_dir'] & 0x80):
data['wind_dir'] = None
raw_data[idx] = data
raw_data.flush()
# delete old format summary files
logger.warning('Deleting old summaries')
context.calib_data.clear()
context.hourly_data.clear()
context.daily_data.clear()
context.monthly_data.clear()
# create data summaries
logger.warning('Generating hourly and daily summaries')
pywws.process.process_data(context)
return 0
def dailygen(inputdata):
"""Internal generator function"""
day_start = start
count = 0
while day_start <= stop:
count += 1
if count % 30 == 0:
logger.info("daily: %s", day_start.isoformat(' '))
else:
logger.debug("daily: %s", day_start.isoformat(' '))
day_end = day_start + DAY
if use_dst:
# day might be 23 or 25 hours long
day_end = timezone.local_replace(day_end + HOURx3,
use_dst=use_dst,
hour=day_end_hour)
acc.reset()
for data in inputdata[day_start:day_end]:
acc.add_raw(data)
for data in hourly_data[day_start:day_end]:
acc.add_hourly(data)
new_data = acc.result()
if new_data:
new_data['start'] = day_start
yield new_data
day_start = day_end
def reprocess(data_dir, update):
with pywws.storage.pywws_context(data_dir) as context:
if update:
logger.warning("Updating status to detect invalid wind_dir")
count = 0
raw_data = context.raw_data
for data in raw_data[:]:
count += 1
idx = data['idx']
if count % 10000 == 0:
logger.info("update: %s", idx.isoformat(' '))
elif count % 500 == 0:
logger.debug("update: %s", idx.isoformat(' '))
if data['wind_dir'] is not None and (data['wind_dir'] & 0x80):
data['wind_dir'] = None
raw_data[idx] = data
raw_data.flush()
# delete old format summary files
logger.warning('Deleting old summaries')
context.calib_data.clear()
context.hourly_data.clear()
context.daily_data.clear()
context.monthly_data.clear()
# create data summaries
logger.warning('Generating hourly and daily summaries')
pywws.process.process_data(context)
return 0
def monthlygen(inputdata):
"""Internal generator function"""
month_start = start
count = 0
while month_start <= stop:
count += 1
if count % 12 == 0:
logger.info("monthly: %s", month_start.isoformat(' '))
else:
logger.debug("monthly: %s", month_start.isoformat(' '))
month_end = month_start + WEEK
if month_end.month < 12:
month_end = month_end.replace(month=month_end.month + 1)
else:
month_end = month_end.replace(month=1, year=month_end.year + 1)
month_end = month_end - WEEK
if use_dst:
# month might straddle summer time start or end
month_end = timezone.local_replace(month_end + HOURx3,
use_dst=use_dst,
hour=day_end_hour)
acc.reset()
for data in inputdata[month_start:month_end]:
acc.add_daily(data)
new_data = acc.result()
if new_data:
new_data['start'] = month_start
yield new_data
month_start = month_end
def calibrate_data(params, raw_data, calib_data):
"""'Calibrate' raw data, using a user-supplied function."""
start = calib_data.before(datetime.max)
if start is None:
start = datetime.min
start = raw_data.after(start + SECOND)
if start is None:
return start
del calib_data[start:]
calibrator = Calib(params, raw_data)
count = 0
for data in raw_data[start:]:
idx = data['idx']
count += 1
if count % 10000 == 0:
logger.info("calib: %s", idx.isoformat(' '))
elif count % 500 == 0:
logger.debug("calib: %s", idx.isoformat(' '))
for key in ('rain', 'abs_pressure', 'temp_in'):
if data[key] is None:
logger.error('Ignoring invalid data at %s', idx.isoformat(' '))
break
else:
calib_data[idx] = calibrator.calib(data)
return start
def calibrate_data(params, raw_data, calib_data):
"""'Calibrate' raw data, using a user-supplied function."""
start = calib_data.before(datetime.max)
if start is None:
start = datetime.min
start = raw_data.after(start + SECOND)
if start is None:
return start
del calib_data[start:]
calibrator = Calib(params, raw_data)
count = 0
for data in raw_data[start:]:
idx = data['idx']
count += 1
if count % 10000 == 0:
logger.info("calib: %s", idx.isoformat(' '))
elif count % 500 == 0:
logger.debug("calib: %s", idx.isoformat(' '))
for key in ('rain', 'abs_pressure', 'temp_in'):
if data[key] is None:
logger.error('Ignoring invalid data at %s', idx.isoformat(' '))
break
else:
calib_data[idx] = calibrator.calib(data)
return start
def monthlygen(inputdata):
"""Internal generator function"""
month_start = start
count = 0
while month_start <= stop:
count += 1
if count % 12 == 0:
logger.info("monthly: %s", month_start.isoformat(' '))
else:
logger.debug("monthly: %s", month_start.isoformat(' '))
month_end = month_start + WEEK
if month_end.month < 12:
month_end = month_end.replace(month=month_end.month+1)
else:
month_end = month_end.replace(month=1, year=month_end.year+1)
month_end = month_end - WEEK
if use_dst:
# month might straddle summer time start or end
month_end = timezone.local_replace(
month_end + HOURx3, use_dst=use_dst, hour=day_end_hour)
acc.reset()
for data in inputdata[month_start:month_end]:
acc.add_daily(data)
new_data = acc.result()
if new_data:
new_data['start'] = month_start
yield new_data
month_start = month_end
def generate_monthly(rain_day_threshold, day_end_hour, use_dst, daily_data,
monthly_data, process_from):
"""Generate monthly summaries from daily data."""
start = monthly_data.before(datetime.max)
if start is None:
start = datetime.min
start = daily_data.after(start + SECOND)
if process_from:
if start:
start = min(start, process_from)
else:
start = process_from
if start is None:
return start
# set start to start of first day of month (local time)
start = timezone.local_replace(start,
use_dst=use_dst,
day=1,
hour=day_end_hour,
minute=0,
second=0)
if day_end_hour >= 12:
# month actually starts on the last day of previous month
start -= DAY
del monthly_data[start:]
stop = daily_data.before(datetime.max)
if stop is None:
return None
month_start = start
acc = MonthAcc(rain_day_threshold)
count = 0
while month_start <= stop:
count += 1
if count % 12 == 0:
logger.info("monthly: %s", month_start.isoformat(' '))
else:
logger.debug("monthly: %s", month_start.isoformat(' '))
month_end = month_start + WEEK
if month_end.month < 12:
month_end = month_end.replace(month=month_end.month + 1)
else:
month_end = month_end.replace(month=1, year=month_end.year + 1)
month_end = month_end - WEEK
if use_dst:
# month might straddle summer time start or end
month_end = timezone.local_replace(month_end + HOURx3,
use_dst=use_dst,
hour=day_end_hour)
acc.reset()
for data in daily_data[month_start:month_end]:
acc.add_daily(data)
new_data = acc.result()
if new_data:
new_data['start'] = month_start
monthly_data[new_data['idx']] = new_data
month_start = month_end
return start
def session(self):
logger.info("Uploading to web site with FTP")
self.ftp = ftplib.FTP()
self.ftp.connect(self.site, self.port)
logger.debug('welcome message\n' + self.ftp.getwelcome())
self.ftp.login(self.user, self.password)
self.ftp.cwd(self.directory)
try:
yield None
finally:
self.ftp.close()
def session(self):
logger.info("Uploading to web site with FTP")
self.ftp = ftplib.FTP()
self.ftp.connect(self.site, self.port)
logger.debug('welcome message\n' + self.ftp.getwelcome())
self.ftp.login(self.user, self.password)
self.ftp.cwd(self.directory)
try:
yield None
finally:
self.ftp.close()
def generate_monthly(rain_day_threshold, day_end_hour, use_dst,
daily_data, monthly_data, process_from):
"""Generate monthly summaries from daily data."""
start = monthly_data.before(datetime.max)
if start is None:
start = datetime.min
start = daily_data.after(start + SECOND)
if process_from:
if start:
start = min(start, process_from)
else:
start = process_from
if start is None:
return start
# set start to start of first day of month (local time)
start = timezone.local_replace(
start, use_dst=use_dst, day=1, hour=day_end_hour, minute=0, second=0)
if day_end_hour >= 12:
# month actually starts on the last day of previous month
start -= DAY
del monthly_data[start:]
stop = daily_data.before(datetime.max)
month_start = start
acc = MonthAcc(rain_day_threshold)
count = 0
while month_start <= stop:
count += 1
if count % 12 == 0:
logger.info("monthly: %s", month_start.isoformat(' '))
else:
logger.debug("monthly: %s", month_start.isoformat(' '))
month_end = month_start + WEEK
if month_end.month < 12:
month_end = month_end.replace(month=month_end.month+1)
else:
month_end = month_end.replace(month=1, year=month_end.year+1)
month_end = month_end - WEEK
if use_dst:
# month might straddle summer time start or end
month_end = timezone.local_replace(
month_end + HOURx3, use_dst=use_dst, hour=day_end_hour)
acc.reset()
for data in daily_data[month_start:month_end]:
acc.add_daily(data)
new_data = acc.result()
if new_data:
new_data['start'] = month_start
monthly_data[new_data['idx']] = new_data
month_start = month_end
return start
def generate_daily(day_end_hour, use_dst, calib_data, hourly_data, daily_data,
process_from):
"""Generate daily summaries from calibrated and hourly data."""
start = daily_data.before(datetime.max)
if start is None:
start = datetime.min
start = calib_data.after(start + SECOND)
if process_from:
if start:
start = min(start, process_from)
else:
start = process_from
if start is None:
return start
# round to start of this day, in local time
start = timezone.local_replace(start,
use_dst=use_dst,
hour=day_end_hour,
minute=0,
second=0)
del daily_data[start:]
stop = calib_data.before(datetime.max)
day_start = start
acc = DayAcc()
count = 0
while day_start <= stop:
count += 1
if count % 30 == 0:
logger.info("daily: %s", day_start.isoformat(' '))
else:
logger.debug("daily: %s", day_start.isoformat(' '))
day_end = day_start + DAY
if use_dst:
# day might be 23 or 25 hours long
day_end = timezone.local_replace(day_end + HOURx3,
use_dst=use_dst,
hour=day_end_hour)
acc.reset()
for data in calib_data[day_start:day_end]:
acc.add_raw(data)
for data in hourly_data[day_start:day_end]:
acc.add_hourly(data)
new_data = acc.result()
if new_data:
new_data['start'] = day_start
daily_data[new_data['idx']] = new_data
day_start = day_end
return start
def calibgen(inputdata):
"""Internal generator function"""
count = 0
for data in inputdata:
idx = data['idx']
count += 1
if count % 10000 == 0:
logger.info("calib: %s", idx.isoformat(' '))
elif count % 500 == 0:
logger.debug("calib: %s", idx.isoformat(' '))
for key in ('rain', 'abs_pressure', 'temp_in'):
if data[key] is None:
logger.error('Ignoring invalid data at %s', idx.isoformat(' '))
break
else:
yield calibrator.calib(data)
def calibgen(inputdata):
"""Internal generator function"""
count = 0
for data in inputdata:
idx = data['idx']
count += 1
if count % 10000 == 0:
logger.info("calib: %s", idx.isoformat(' '))
elif count % 500 == 0:
logger.debug("calib: %s", idx.isoformat(' '))
for key in ('rain', 'abs_pressure', 'temp_in'):
if data[key] is None:
logger.error('Ignoring invalid data at %s', idx.isoformat(' '))
break
else:
yield calibrator.calib(data)
def generate_daily(day_end_hour, use_dst,
calib_data, hourly_data, daily_data, process_from):
"""Generate daily summaries from calibrated and hourly data."""
start = daily_data.before(datetime.max)
if start is None:
start = datetime.min
start = calib_data.after(start + SECOND)
if process_from:
if start:
start = min(start, process_from)
else:
start = process_from
if start is None:
return start
# round to start of this day, in local time
start = timezone.local_replace(
start, use_dst=use_dst, hour=day_end_hour, minute=0, second=0)
del daily_data[start:]
stop = calib_data.before(datetime.max)
day_start = start
acc = DayAcc()
count = 0
while day_start <= stop:
count += 1
if count % 30 == 0:
logger.info("daily: %s", day_start.isoformat(' '))
else:
logger.debug("daily: %s", day_start.isoformat(' '))
day_end = day_start + DAY
if use_dst:
# day might be 23 or 25 hours long
day_end = timezone.local_replace(
day_end + HOURx3, use_dst=use_dst, hour=day_end_hour)
acc.reset()
for data in calib_data[day_start:day_end]:
acc.add_raw(data)
for data in hourly_data[day_start:day_end]:
acc.add_hourly(data)
new_data = acc.result()
if new_data:
new_data['start'] = day_start
daily_data[new_data['idx']] = new_data
day_start = day_end
return start
def mqtt_send_data(self, timestamp, prepared_data, ignore_last_update=False):
import paho.mqtt.client as mosquitto
import time
import json
topic = prepared_data['topic']
hostname = prepared_data['hostname']
port = prepared_data['port']
client_id = prepared_data['client_id']
retain = prepared_data['retain'] == 'True'
auth = prepared_data['auth'] == 'True'
multi_topic = prepared_data['multi_topic'] == 'True'
# clean up the object
del prepared_data['topic']
del prepared_data['hostname']
del prepared_data['port']
del prepared_data['client_id']
del prepared_data['retain']
del prepared_data['auth']
del prepared_data['multi_topic']
mosquitto_client = mosquitto.Client(client_id, protocol=mosquitto.MQTTv31)
if auth:
logger.debug(
"%s:Username and password configured", self.service_name)
if(self.password == "unknown"):
mosquitto_client.username_pw_set(self.user)
else:
mosquitto_client.username_pw_set(self.user, self.password)
else:
logger.debug(
"%s:Username and password unconfigured, ignoring", self.service_name)
logger.debug(
"%s:timestamp: %s. publishing on topic [%s] to hostname [%s] and " +
"port [%s] with a client_id [%s] and retain is %s",
self.service_name, timestamp.isoformat(' '), topic, hostname, port,
client_id, retain)
mosquitto_client.connect(hostname, int(port))
mosquitto_client.publish(topic, json.dumps(prepared_data), retain=retain)
if multi_topic:
#Publish a messages, one for each item in prepared_data to separate Subtopics.
for item in prepared_data:
if prepared_data[item] == '':
prepared_data[item] = 'None'
mosquitto_client.publish(topic + "/" + item, prepared_data[item], retain=retain)
#Need to make sure the messages have been flushed to the server.
mosquitto_client.loop(timeout=0.5)
logger.debug("%s:published data: %s", self.service_name, prepared_data)
mosquitto_client.disconnect()
return True
def hourlygen(inputdata, prev):
"""Internal generator function"""
hour_start = start
count = 0
while hour_start <= stop:
count += 1
if count % 1008 == 0:
logger.info("hourly: %s", hour_start.isoformat(' '))
elif count % 24 == 0:
logger.debug("hourly: %s", hour_start.isoformat(' '))
hour_end = hour_start + HOUR
acc.reset()
for data in inputdata[hour_start:hour_end]:
if data['rel_pressure']:
pressure_history.append(
(data['idx'], data['rel_pressure']))
if prev:
err = data['idx'] - prev['idx']
if abs(err - timedelta(minutes=data['delay'])) > TIME_ERR:
logger.info('unexpected data interval %s %s',
data['idx'].isoformat(' '), str(err))
acc.add_raw(data)
prev = data
new_data = acc.result()
if new_data and (new_data['idx'] -
hour_start) >= timedelta(minutes=9):
# compute pressure trend
new_data['pressure_trend'] = None
if new_data['rel_pressure']:
target = new_data['idx'] - HOURx3
while (len(pressure_history) >= 2
and abs(pressure_history[0][0] - target) >
abs(pressure_history[1][0] - target)):
pressure_history.popleft()
if (pressure_history
and abs(pressure_history[0][0] - target) < HOUR):
new_data['pressure_trend'] = (
new_data['rel_pressure'] - pressure_history[0][1])
# store new hourly data
yield new_data
hour_start = hour_end
def log_data(self, sync=None, clear=False):
# get sync config value
if sync is None:
if self.fixed_block['read_period'] <= 5:
sync = int(self.params.get('config', 'logdata sync', '1'))
else:
sync = int(self.params.get('config', 'logdata sync', '0'))
# get address and date-time of last complete logged data
logger.info('Synchronising to weather station')
range_hi = datetime.max
range_lo = datetime.min
last_delay = self.ws.get_data(self.ws.current_pos())['delay']
if last_delay == 0:
prev_date = datetime.min
else:
prev_date = datetime.utcnow()
for data, last_ptr, logged in self.ws.live_data(
logged_only=(sync > 1)):
last_date = data['idx']
logger.debug('Reading time %s', last_date.strftime('%H:%M:%S'))
if logged:
break
if sync < 2 and self.ws._station_clock.clock:
err = last_date - datetime.fromtimestamp(
self.ws._station_clock.clock)
last_date -= timedelta(minutes=data['delay'],
seconds=err.seconds % 60)
logger.debug('log time %s', last_date.strftime('%H:%M:%S'))
last_ptr = self.ws.dec_ptr(last_ptr)
break
if sync < 1:
hi = last_date - timedelta(minutes=data['delay'])
if last_date - prev_date > timedelta(seconds=50):
lo = hi - timedelta(seconds=60)
elif data['delay'] == last_delay:
lo = hi - timedelta(seconds=60)
hi = hi - timedelta(seconds=48)
else:
lo = hi - timedelta(seconds=48)
last_delay = data['delay']
prev_date = last_date
range_hi = min(range_hi, hi)
range_lo = max(range_lo, lo)
err = (range_hi - range_lo) / 2
last_date = range_lo + err
logger.debug('est log time %s +- %ds (%s..%s)',
last_date.strftime('%H:%M:%S'), err.seconds,
lo.strftime('%H:%M:%S'), hi.strftime('%H:%M:%S'))
if err < timedelta(seconds=15):
last_ptr = self.ws.dec_ptr(last_ptr)
break
# go back through stored data, until we catch up with what we've already got
logger.info('Fetching data')
self.status.set('data', 'ptr',
'%06x,%s' % (last_ptr, last_date.isoformat(' ')))
self.fetch_logged(last_date, last_ptr)
if clear:
logger.info('Clearing weather station memory')
ptr = self.ws.fixed_format['data_count'][0]
self.ws.write_data([(ptr, 1), (ptr + 1, 0)])
def log_data(self, sync=None, clear=False):
# get sync config value
if sync is None:
if self.fixed_block['read_period'] <= 5:
sync = int(self.params.get('config', 'logdata sync', '1'))
else:
sync = int(self.params.get('config', 'logdata sync', '0'))
# get address and date-time of last complete logged data
logger.info('Synchronising to weather station')
range_hi = datetime.max
range_lo = datetime.min
last_delay = self.ws.get_data(self.ws.current_pos())['delay']
if last_delay == 0:
prev_date = datetime.min
else:
prev_date = datetime.utcnow()
for data, last_ptr, logged in self.ws.live_data(logged_only=(sync > 1)):
last_date = data['idx']
logger.debug('Reading time %s', last_date.strftime('%H:%M:%S'))
if logged:
break
if sync < 2 and self.ws._station_clock.clock:
err = last_date - datetime.fromtimestamp(
self.ws._station_clock.clock)
last_date -= timedelta(
minutes=data['delay'], seconds=err.seconds % 60)
logger.debug('log time %s', last_date.strftime('%H:%M:%S'))
last_ptr = self.ws.dec_ptr(last_ptr)
break
if sync < 1:
hi = last_date - timedelta(minutes=data['delay'])
if last_date - prev_date > timedelta(seconds=50):
lo = hi - timedelta(seconds=60)
elif data['delay'] == last_delay:
lo = hi - timedelta(seconds=60)
hi = hi - timedelta(seconds=48)
else:
lo = hi - timedelta(seconds=48)
last_delay = data['delay']
prev_date = last_date
range_hi = min(range_hi, hi)
range_lo = max(range_lo, lo)
err = (range_hi - range_lo) / 2
last_date = range_lo + err
logger.debug('est log time %s +- %ds (%s..%s)',
last_date.strftime('%H:%M:%S'), err.seconds,
lo.strftime('%H:%M:%S'), hi.strftime('%H:%M:%S'))
if err < timedelta(seconds=15):
last_ptr = self.ws.dec_ptr(last_ptr)
break
# go back through stored data, until we catch up with what we've already got
logger.info('Fetching data')
self.status.set(
'data', 'ptr', '%06x,%s' % (last_ptr, last_date.isoformat(' ')))
self.fetch_logged(last_date, last_ptr)
if clear:
logger.info('Clearing weather station memory')
ptr = self.ws.fixed_format['data_count'][0]
self.ws.write_data([(ptr, 1), (ptr+1, 0)])
def hourlygen(inputdata, prev):
"""Internal generator function"""
hour_start = start
count = 0
while hour_start <= stop:
count += 1
if count % 1008 == 0:
logger.info("hourly: %s", hour_start.isoformat(' '))
elif count % 24 == 0:
logger.debug("hourly: %s", hour_start.isoformat(' '))
hour_end = hour_start + HOUR
acc.reset()
for data in inputdata[hour_start:hour_end]:
if data['rel_pressure']:
pressure_history.append((data['idx'], data['rel_pressure']))
if prev:
err = data['idx'] - prev['idx']
if abs(err - timedelta(minutes=data['delay'])) > TIME_ERR:
logger.info('unexpected data interval %s %s',
data['idx'].isoformat(' '), str(err))
acc.add_raw(data)
prev = data
new_data = acc.result()
if new_data and (new_data['idx'] - hour_start) >= timedelta(minutes=9):
# compute pressure trend
new_data['pressure_trend'] = None
if new_data['rel_pressure']:
target = new_data['idx'] - HOURx3
while (len(pressure_history) >= 2 and
abs(pressure_history[0][0] - target) >
abs(pressure_history[1][0] - target)):
pressure_history.popleft()
if (pressure_history and
abs(pressure_history[0][0] - target) < HOUR):
new_data['pressure_trend'] = (
new_data['rel_pressure'] - pressure_history[0][1])
# store new hourly data
yield new_data
hour_start = hour_end
def aprs_send_data(self,
timestamp,
prepared_data,
ignore_last_update=False):
"""Upload a weather data record using APRS.
The :obj:`prepared_data` parameter contains the data to be uploaded.
It should be a dictionary of string keys and string values.
:param timestamp: the timestamp of the data to upload.
:type timestamp: datetime
:param prepared_data: the data to upload.
:type prepared_data: dict
:param ignore_last_update: don't get or set the 'last update'
status.ini entry.
:type ignore_last_update: bool
:return: success status
:rtype: bool
"""
login = '******' % (
prepared_data['designator'], prepared_data['passcode'],
__version__)
packet = '%s>APRS,TCPIP*:@%sz%s/%s_%s/%sg%st%sr%sP%sb%sh%s.pywws-%s\n' % (
prepared_data['designator'], prepared_data['idx'],
prepared_data['latitude'], prepared_data['longitude'],
prepared_data['wind_dir'], prepared_data['wind_ave'],
prepared_data['wind_gust'], prepared_data['temp_out'],
prepared_data['rain_hour'], prepared_data['rain_day'],
prepared_data['rel_pressure'], prepared_data['hum_out'],
__version__)
logger.debug('%s:packet: "%s"', self.service_name, packet)
login = login.encode('ASCII')
packet = packet.encode('ASCII')
sock = socket.socket()
try:
sock.connect(self.server)
try:
response = sock.recv(4096)
logger.debug('%s:server software: %s', self.service_name,
response.strip())
sock.sendall(login)
response = sock.recv(4096)
logger.debug('%s:server login ack: %s', self.service_name,
response.strip())
sock.sendall(packet)
sock.shutdown(socket.SHUT_RDWR)
finally:
sock.close()
except Exception as ex:
new_ex = str(ex)
if new_ex == self.old_ex:
log = logger.debug
else:
log = logger.error
self.old_ex = new_ex
log('%s:exc: %s', self.service_name, new_ex)
return False
if not ignore_last_update:
self.set_last_update(timestamp)
return True
def generate_hourly(calib_data, hourly_data, process_from):
"""Generate hourly summaries from calibrated data."""
start = hourly_data.before(datetime.max)
if start is None:
start = datetime.min
start = calib_data.after(start + SECOND)
if process_from:
if start:
start = min(start, process_from)
else:
start = process_from
if start is None:
return start
# set start of hour in local time (not all time offsets are integer hours)
start += timezone.standard_offset
start = start.replace(minute=0, second=0)
start -= timezone.standard_offset
del hourly_data[start:]
# preload pressure history, and find last valid rain
prev = None
pressure_history = deque()
last_rain = None
for data in calib_data[start - HOURx3:start]:
if data['rel_pressure']:
pressure_history.append((data['idx'], data['rel_pressure']))
if data['rain'] is not None:
last_rain = data['rain']
prev = data
# iterate over data in one hour chunks
stop = calib_data.before(datetime.max)
hour_start = start
acc = HourAcc(last_rain)
count = 0
while hour_start <= stop:
count += 1
if count % 1008 == 0:
logger.info("hourly: %s", hour_start.isoformat(' '))
elif count % 24 == 0:
logger.debug("hourly: %s", hour_start.isoformat(' '))
hour_end = hour_start + HOUR
acc.reset()
for data in calib_data[hour_start:hour_end]:
if data['rel_pressure']:
pressure_history.append((data['idx'], data['rel_pressure']))
if prev:
err = data['idx'] - prev['idx']
if abs(err - timedelta(minutes=data['delay'])) > TIME_ERR:
logger.info('unexpected data interval %s %s',
data['idx'].isoformat(' '), str(err))
acc.add_raw(data)
prev = data
new_data = acc.result()
if new_data and (new_data['idx'] - hour_start) >= timedelta(minutes=9):
# compute pressure trend
new_data['pressure_trend'] = None
if new_data['rel_pressure']:
target = new_data['idx'] - HOURx3
while (len(pressure_history) >= 2
and abs(pressure_history[0][0] - target) >
abs(pressure_history[1][0] - target)):
pressure_history.popleft()
if (pressure_history
and abs(pressure_history[0][0] - target) < HOUR):
new_data['pressure_trend'] = (new_data['rel_pressure'] -
pressure_history[0][1])
# store new hourly data
hourly_data[new_data['idx']] = new_data
hour_start = hour_end
return start
def generate_hourly(calib_data, hourly_data, process_from):
"""Generate hourly summaries from calibrated data."""
start = hourly_data.before(datetime.max)
if start is None:
start = datetime.min
start = calib_data.after(start + SECOND)
if process_from:
if start:
start = min(start, process_from)
else:
start = process_from
if start is None:
return start
# set start of hour in local time (not all time offsets are integer hours)
start += timezone.standard_offset
start = start.replace(minute=0, second=0)
start -= timezone.standard_offset
del hourly_data[start:]
# preload pressure history, and find last valid rain
prev = None
pressure_history = deque()
last_rain = None
for data in calib_data[start - HOURx3:start]:
if data['rel_pressure']:
pressure_history.append((data['idx'], data['rel_pressure']))
if data['rain'] is not None:
last_rain = data['rain']
prev = data
# iterate over data in one hour chunks
stop = calib_data.before(datetime.max)
hour_start = start
acc = HourAcc(last_rain)
count = 0
while hour_start <= stop:
count += 1
if count % 1008 == 0:
logger.info("hourly: %s", hour_start.isoformat(' '))
elif count % 24 == 0:
logger.debug("hourly: %s", hour_start.isoformat(' '))
hour_end = hour_start + HOUR
acc.reset()
for data in calib_data[hour_start:hour_end]:
if data['rel_pressure']:
pressure_history.append((data['idx'], data['rel_pressure']))
if prev:
err = data['idx'] - prev['idx']
if abs(err - timedelta(minutes=data['delay'])) > TIME_ERR:
logger.info('unexpected data interval %s %s',
data['idx'].isoformat(' '), str(err))
acc.add_raw(data)
prev = data
new_data = acc.result()
if new_data and (new_data['idx'] - hour_start) >= timedelta(minutes=9):
# compute pressure trend
new_data['pressure_trend'] = None
if new_data['rel_pressure']:
target = new_data['idx'] - HOURx3
while (len(pressure_history) >= 2 and
abs(pressure_history[0][0] - target) >
abs(pressure_history[1][0] - target)):
pressure_history.popleft()
if (pressure_history and
abs(pressure_history[0][0] - target) < HOUR):
new_data['pressure_trend'] = (
new_data['rel_pressure'] - pressure_history[0][1])
# store new hourly data
hourly_data[new_data['idx']] = new_data
hour_start = hour_end
return start
def aprs_send_data(self, timestamp, prepared_data, ignore_last_update=False):
"""Upload a weather data record using APRS.
The :obj:`prepared_data` parameter contains the data to be uploaded.
It should be a dictionary of string keys and string values.
:param timestamp: the timestamp of the data to upload.
:type timestamp: datetime
:param prepared_data: the data to upload.
:type prepared_data: dict
:param ignore_last_update: don't get or set the 'last update'
status.ini entry.
:type ignore_last_update: bool
:return: success status
:rtype: bool
"""
login = '******' % (
prepared_data['designator'], prepared_data['passcode'], __version__)
packet = '%s>APRS,TCPIP*:@%sz%s/%s_%s/%sg%st%sr%sP%sb%sh%s.pywws-%s\n' % (
prepared_data['designator'], prepared_data['idx'],
prepared_data['latitude'], prepared_data['longitude'],
prepared_data['wind_dir'], prepared_data['wind_ave'],
prepared_data['wind_gust'], prepared_data['temp_out'],
prepared_data['rain_hour'], prepared_data['rain_day'],
prepared_data['rel_pressure'], prepared_data['hum_out'],
__version__
)
logger.debug('%s:packet: "%s"', self.service_name, packet)
login = login.encode('ASCII')
packet = packet.encode('ASCII')
sock = socket.socket()
try:
sock.connect(self.server)
try:
response = sock.recv(4096)
logger.debug('%s:server software: %s',
self.service_name, response.strip())
sock.sendall(login)
response = sock.recv(4096)
logger.debug('%s:server login ack: %s',
self.service_name, response.strip())
sock.sendall(packet)
sock.shutdown(socket.SHUT_RDWR)
finally:
sock.close()
except Exception as ex:
new_ex = str(ex)
if new_ex == self.old_ex:
log = logger.debug
else:
log = logger.error
self.old_ex = new_ex
log('%s:exc: %s', self.service_name, new_ex)
return False
if not ignore_last_update:
self.set_last_update(timestamp)
return True
def http_send_data(self, timestamp, prepared_data, ignore_last_update=False):
"""Upload a weather data record using HTTP.
The :obj:`prepared_data` parameter contains the data to be uploaded.
It should be a dictionary of string keys and string values.
:param timestamp: the timestamp of the data to upload.
:type timestamp: datetime
:param prepared_data: the data to upload.
:type prepared_data: dict
:param ignore_last_update: don't get or set the 'last update'
status.ini entry.
:type ignore_last_update: bool
:return: success status
:rtype: bool
"""
coded_data = urlencode(prepared_data)
logger.debug('%s:%s', self.service_name, coded_data)
new_ex = self.old_ex
ex_info = []
success = False
try:
if self.use_get:
request = Request(self.server + '?' + coded_data)
else:
request = Request(self.server, coded_data.encode('ASCII'))
if self.auth_type == 'basic':
request.add_header('Authorization', self.auth)
if self.http_headers is not None:
for header in self.http_headers:
request.add_header(header[0], header[1])
rsp = urlopen(request)
response = rsp.readlines()
rsp.close()
if response == self.old_response:
log = logger.debug
else:
log = logger.error
self.old_response = response
for line in response:
log('%s:rsp: %s', self.service_name, line.strip())
for n, expected in enumerate(self.expected_result):
if n < len(response):
actual = response[n].decode('utf-8')
if not re.match(expected, actual):
break
else:
self.old_response = response
if not ignore_last_update:
self.set_last_update(timestamp)
return True
return False
except HTTPError as ex:
if ex.code == 429 and self.service_name == 'metoffice':
# UK Met Office server uses 429 to signal duplicate data
success = True
if sys.version_info >= (2, 7):
new_ex = '[%d]%s' % (ex.code, ex.reason)
else:
new_ex = str(ex)
ex_info = str(ex.info()).split('\n')
try:
for line in ex.readlines():
line = line.decode('utf-8')
ex_info.append(re.sub('<.+?>', '', line))
except Exception:
pass
except URLError as ex:
new_ex = str(ex.reason)
except Exception as ex:
new_ex = str(ex)
if new_ex == self.old_ex:
log = logger.debug
else:
log = logger.error
self.old_ex = new_ex
log('%s:exc: %s', self.service_name, new_ex)
for extra in ex_info:
extra = extra.strip()
if extra:
log('info: %s', extra)
if success and not ignore_last_update:
self.set_last_update(timestamp)
return success
def mqtt_send_data(self,
timestamp,
prepared_data,
ignore_last_update=False):
import paho.mqtt.client as mosquitto
import time
import json
topic = prepared_data['topic']
hostname = prepared_data['hostname']
port = prepared_data['port']
client_id = prepared_data['client_id']
retain = prepared_data['retain'] == 'True'
auth = prepared_data['auth'] == 'True'
multi_topic = prepared_data['multi_topic'] == 'True'
# clean up the object
del prepared_data['topic']
del prepared_data['hostname']
del prepared_data['port']
del prepared_data['client_id']
del prepared_data['retain']
del prepared_data['auth']
del prepared_data['multi_topic']
mosquitto_client = mosquitto.Client(client_id,
protocol=mosquitto.MQTTv31)
if auth:
logger.debug("%s:Username and password configured",
self.service_name)
if (self.password == "unknown"):
mosquitto_client.username_pw_set(self.user)
else:
mosquitto_client.username_pw_set(self.user, self.password)
else:
logger.debug("%s:Username and password unconfigured, ignoring",
self.service_name)
logger.debug(
"%s:timestamp: %s. publishing on topic [%s] to hostname [%s] and "
+ "port [%s] with a client_id [%s] and retain is %s",
self.service_name, timestamp.isoformat(' '), topic, hostname, port,
client_id, retain)
mosquitto_client.connect(hostname, int(port))
mosquitto_client.publish(topic,
json.dumps(prepared_data),
retain=retain)
if multi_topic:
#Publish a messages, one for each item in prepared_data to separate Subtopics.
for item in prepared_data:
if prepared_data[item] == '':
prepared_data[item] = 'None'
mosquitto_client.publish(topic + "/" + item,
prepared_data[item],
retain=retain)
#Need to make sure the messages have been flushed to the server.
mosquitto_client.loop(timeout=0.5)
logger.debug("%s:published data: %s", self.service_name, prepared_data)
mosquitto_client.disconnect()
return True
def http_send_data(self,
timestamp,
prepared_data,
ignore_last_update=False):
"""Upload a weather data record using HTTP.
The :obj:`prepared_data` parameter contains the data to be uploaded.
It should be a dictionary of string keys and string values.
:param timestamp: the timestamp of the data to upload.
:type timestamp: datetime
:param prepared_data: the data to upload.
:type prepared_data: dict
:param ignore_last_update: don't get or set the 'last update'
status.ini entry.
:type ignore_last_update: bool
:return: success status
:rtype: bool
"""
coded_data = urlencode(prepared_data)
logger.debug('%s:%s', self.service_name, coded_data)
new_ex = self.old_ex
ex_info = []
success = False
try:
if self.use_get:
request = Request(self.server + '?' + coded_data)
else:
request = Request(self.server, coded_data.encode('ASCII'))
if self.auth_type == 'basic':
request.add_header('Authorization', self.auth)
if self.http_headers is not None:
for header in self.http_headers:
request.add_header(header[0], header[1])
rsp = urlopen(request)
response = rsp.readlines()
rsp.close()
if response == self.old_response:
log = logger.debug
else:
log = logger.error
self.old_response = response
for line in response:
log('%s:rsp: %s', self.service_name, line.strip())
for n, expected in enumerate(self.expected_result):
if n < len(response):
actual = response[n].decode('utf-8')
if not re.match(expected, actual):
break
else:
self.old_response = response
if not ignore_last_update:
self.set_last_update(timestamp)
return True
return False
except HTTPError as ex:
if ex.code == 429 and self.service_name == 'metoffice':
# UK Met Office server uses 429 to signal duplicate data
success = True
if sys.version_info >= (2, 7):
new_ex = '[%d]%s' % (ex.code, ex.reason)
else:
new_ex = str(ex)
ex_info = str(ex.info()).split('\n')
try:
for line in ex.readlines():
line = line.decode('utf-8')
ex_info.append(re.sub('<.+?>', '', line))
except Exception:
pass
except URLError as ex:
new_ex = str(ex.reason)
except Exception as ex:
new_ex = str(ex)
if new_ex == self.old_ex:
log = logger.debug
else:
log = logger.error
self.old_ex = new_ex
log('%s:exc: %s', self.service_name, new_ex)
for extra in ex_info:
extra = extra.strip()
if extra:
log('info: %s', extra)
if success and not ignore_last_update:
self.set_last_update(timestamp)
return success
