def post(self, status, media):
if len(media) > 1:
logger.error('Tweepy library cannot post multiple media')
if media:
self.api.update_with_media(media[0], status[:257], **self.kwargs)
else:
self.api.update_status(status[:280], **self.kwargs)
def calibrate_data(params, raw_data, calib_data):
"""'Calibrate' raw data, using a user-supplied function."""
start = calib_data.before(datetime.max)
if start is None:
start = datetime.min
start = raw_data.after(start + SECOND)
if start is None:
return start
del calib_data[start:]
calibrator = Calib(params, raw_data)
count = 0
for data in raw_data[start:]:
idx = data['idx']
count += 1
if count % 10000 == 0:
logger.info("calib: %s", idx.isoformat(' '))
elif count % 500 == 0:
logger.debug("calib: %s", idx.isoformat(' '))
for key in ('rain', 'abs_pressure', 'temp_in'):
if data[key] is None:
logger.error('Ignoring invalid data at %s', idx.isoformat(' '))
break
else:
calib_data[idx] = calibrator.calib(data)
return start
def calibrate_data(params, raw_data, calib_data):
"""'Calibrate' raw data, using a user-supplied function."""
start = calib_data.before(datetime.max)
if start is None:
start = datetime.min
start = raw_data.after(start + SECOND)
if start is None:
return start
del calib_data[start:]
calibrator = Calib(params, raw_data)
count = 0
for data in raw_data[start:]:
idx = data['idx']
count += 1
if count % 10000 == 0:
logger.info("calib: %s", idx.isoformat(' '))
elif count % 500 == 0:
logger.debug("calib: %s", idx.isoformat(' '))
for key in ('rain', 'abs_pressure', 'temp_in'):
if data[key] is None:
logger.error('Ignoring invalid data at %s', idx.isoformat(' '))
break
else:
calib_data[idx] = calibrator.calib(data)
return start
def post(self, status, media):
if len(media) > 1:
logger.error('Tweepy library cannot post multiple media')
if media:
self.api.update_with_media(media[0], status[:257], **self.kwargs)
else:
self.api.update_status(status[:280], **self.kwargs)
def fetch_logged(self, last_date, last_ptr):
# offset last stored time by half logging interval
last_stored = self.last_stored_time + timedelta(
seconds=self.fixed_block['read_period'] * 30)
if last_date <= last_stored:
# nothing to do
return
# data_count includes record currently being updated every 48 seconds
max_count = self.fixed_block['data_count'] - 1
count = 0
# initialise detection of data left after a station reboot
saved_date = self.last_stored_time
saved_ptr = self.last_stored_ptr
self.last_stored_ptr = None
duplicates = []
while last_date > last_stored and count < max_count:
data = self.ws.get_data(last_ptr)
if last_ptr == saved_ptr:
if any(data[key] != self.raw_data[saved_date][key]
for key in ('hum_in', 'temp_in', 'hum_out', 'temp_out',
'abs_pressure', 'wind_ave', 'wind_gust',
'wind_dir', 'rain', 'status')):
# pointer matches but data is different, so no duplicates
duplicates = None
saved_ptr = None
else:
# potential duplicate data
duplicates.append(last_date)
saved_date = self.raw_data.before(saved_date)
saved_ptr = self.ws.dec_ptr(saved_ptr)
if (data['delay'] is None or data['delay'] > max(
self.fixed_block['read_period'] * 2, 35)):
logger.error('invalid data at %04x, %s', last_ptr,
last_date.isoformat(' '))
last_date -= timedelta(minutes=self.fixed_block['read_period'])
else:
self.raw_data[last_date] = data
count += 1
last_date -= timedelta(minutes=data['delay'])
last_ptr = self.ws.dec_ptr(last_ptr)
if duplicates:
for d in duplicates:
del self.raw_data[d]
count -= len(duplicates)
last_date = self.raw_data.nearest(last_date) or datetime.max
next_date = self.raw_data.after(last_date + SECOND)
if next_date:
gap = (next_date - last_date).seconds // 60
gap -= self.fixed_block['read_period']
if gap > 0:
logger.critical("%d minutes gap in data detected", gap)
logger.info("%d catchup records", count)
def fetch_logged(self, last_date, last_ptr):
# offset last stored time by half logging interval
last_stored = self.last_stored_time + timedelta(
seconds=self.fixed_block['read_period'] * 30)
if last_date <= last_stored:
# nothing to do
return
# data_count includes record currently being updated every 48 seconds
max_count = self.fixed_block['data_count'] - 1
count = 0
# initialise detection of data left after a station reboot
saved_date = self.last_stored_time
saved_ptr = self.last_stored_ptr
self.last_stored_ptr = None
duplicates = []
while last_date > last_stored and count < max_count:
data = self.ws.get_data(last_ptr)
if last_ptr == saved_ptr:
if any(data[key] != self.raw_data[saved_date][key] for key in (
'hum_in', 'temp_in', 'hum_out', 'temp_out',
'abs_pressure', 'wind_ave', 'wind_gust', 'wind_dir',
'rain', 'status')):
# pointer matches but data is different, so no duplicates
duplicates = None
saved_ptr = None
else:
# potential duplicate data
duplicates.append(last_date)
saved_date = self.raw_data.before(saved_date)
saved_ptr = self.ws.dec_ptr(saved_ptr)
if (data['delay'] is None or
data['delay'] > max(self.fixed_block['read_period'] * 2, 35)):
logger.error('invalid data at %04x, %s',
last_ptr, last_date.isoformat(' '))
last_date -= timedelta(minutes=self.fixed_block['read_period'])
else:
self.raw_data[last_date] = data
count += 1
last_date -= timedelta(minutes=data['delay'])
last_ptr = self.ws.dec_ptr(last_ptr)
if duplicates:
for d in duplicates:
del self.raw_data[d]
count -= len(duplicates)
last_date = self.raw_data.nearest(last_date)
next_date = self.raw_data.after(last_date + SECOND)
if next_date:
gap = (next_date - last_date).seconds // 60
gap -= self.fixed_block['read_period']
if gap > 0:
logger.critical("%d minutes gap in data detected", gap)
logger.info("%d catchup records", count)
def calibgen(inputdata):
"""Internal generator function"""
count = 0
for data in inputdata:
idx = data['idx']
count += 1
if count % 10000 == 0:
logger.info("calib: %s", idx.isoformat(' '))
elif count % 500 == 0:
logger.debug("calib: %s", idx.isoformat(' '))
for key in ('rain', 'abs_pressure', 'temp_in'):
if data[key] is None:
logger.error('Ignoring invalid data at %s', idx.isoformat(' '))
break
else:
yield calibrator.calib(data)
def calibgen(inputdata):
"""Internal generator function"""
count = 0
for data in inputdata:
idx = data['idx']
count += 1
if count % 10000 == 0:
logger.info("calib: %s", idx.isoformat(' '))
elif count % 500 == 0:
logger.debug("calib: %s", idx.isoformat(' '))
for key in ('rain', 'abs_pressure', 'temp_in'):
if data[key] is None:
logger.error('Ignoring invalid data at %s', idx.isoformat(' '))
break
else:
yield calibrator.calib(data)
def process(self, live_data, template_file):
def jump(idx, count):
while count > 0:
new_idx = data_set.after(idx + SECOND)
if new_idx == None:
break
idx = new_idx
count -= 1
while count < 0:
new_idx = data_set.before(idx)
if new_idx == None:
break
idx = new_idx
count += 1
return idx, count == 0
params = self.params
if not live_data:
idx = self.calib_data.before(datetime.max)
if not idx:
logger.error("No calib data - run pywws.process first")
return
live_data = self.calib_data[idx]
# get default character encoding of template input & output files
self.encoding = params.get('config', 'template encoding', 'iso-8859-1')
file_encoding = self.encoding
if file_encoding == 'html':
file_encoding = 'ascii'
# get conversions module to create its 'private' wind dir text
# array, then copy it to deprecated wind_dir_text variable
winddir_text(0)
wind_dir_text = conversions._winddir_text_array
hour_diff = self.computations.hour_diff
rain_hour = self.computations.rain_hour
rain_day = self.computations.rain_day
rain_24hr = self.computations.rain_24hr
pressure_offset = float(self.params.get('config', 'pressure offset'))
fixed_block = literal_eval(self.status.get('fixed', 'fixed block'))
# start off with no time rounding
round_time = None
# start off in hourly data mode
data_set = self.hourly_data
# start off in utc
local_time = False
# start off with default use_locale setting
use_locale = self.use_locale
# jump to last item
idx, valid_data = jump(datetime.max, -1)
if not valid_data:
logger.error("No summary data - run pywws.process first")
return
data = data_set[idx]
# open template file, if not already a file(like) object
if hasattr(template_file, 'readline'):
tmplt = template_file
else:
tmplt = open(template_file, 'rb')
# do the text processing
line = ''
while True:
new_line = tmplt.readline()
if not new_line:
break
if isinstance(new_line, bytes) or sys.version_info[0] < 3:
new_line = new_line.decode(file_encoding)
line += new_line
parts = line.split('#')
if len(parts) % 2 == 0:
# odd number of '#'
line = line.rstrip('\r\n')
continue
for i, part in enumerate(parts):
if i % 2 == 0:
# not a processing directive
if i == 0 or part != '\n':
yield part
continue
if part and part[0] == '!':
# comment
continue
# Python 2 shlex can't handle unicode
if sys.version_info[0] < 3:
part = part.encode(file_encoding)
command = shlex.split(part)
if sys.version_info[0] < 3:
command = map(lambda x: x.decode(file_encoding), command)
if command == []:
# empty command == print a single '#'
yield u'#'
elif command[0] in list(data.keys()) + ['calc']:
# output a value
if not valid_data:
continue
# format is: key fmt_string no_value_string conversion
# get value
if command[0] == 'calc':
x = eval(command[1])
del command[1]
else:
x = data[command[0]]
# adjust time
if isinstance(x, datetime):
if round_time:
x += round_time
if local_time:
x = timezone.to_local(x)
else:
x = timezone.to_utc(x)
# convert data
if x is not None and len(command) > 3:
x = eval(command[3])
# get format
fmt = u'%s'
if len(command) > 1:
fmt = command[1]
# write output
if x is None:
if len(command) > 2:
yield command[2]
elif isinstance(x, datetime):
if sys.version_info[0] < 3:
fmt = fmt.encode(file_encoding)
x = x.strftime(fmt)
if sys.version_info[0] < 3:
if self.encoding == 'html':
x = x.decode('ascii', errors='xmlcharrefreplace')
else:
x = x.decode(file_encoding)
yield x
elif not use_locale:
yield fmt % (x)
elif sys.version_info >= (2, 7) or '%%' not in fmt:
yield locale.format_string(fmt, x)
else:
yield locale.format_string(
fmt.replace('%%', '##'), x).replace('##', '%')
elif command[0] == 'monthly':
data_set = self.monthly_data
idx, valid_data = jump(datetime.max, -1)
data = data_set[idx]
elif command[0] == 'daily':
data_set = self.daily_data
idx, valid_data = jump(datetime.max, -1)
data = data_set[idx]
elif command[0] == 'hourly':
data_set = self.hourly_data
idx, valid_data = jump(datetime.max, -1)
data = data_set[idx]
elif command[0] == 'raw':
data_set = self.calib_data
idx, valid_data = jump(datetime.max, -1)
data = data_set[idx]
elif command[0] == 'live':
data_set = self.calib_data
idx = live_data['idx']
valid_data = True
data = live_data
elif command[0] == 'timezone':
if command[1] == 'utc':
local_time = False
elif command[1] == 'local':
local_time = True
else:
logger.error("Unknown time zone: %s", command[1])
return
elif command[0] == 'locale':
use_locale = eval(command[1])
elif command[0] == 'encoding':
self.encoding = command[1]
file_encoding = self.encoding
if file_encoding == 'html':
file_encoding = 'ascii'
elif command[0] == 'roundtime':
if eval(command[1]):
round_time = timedelta(seconds=30)
else:
round_time = None
elif command[0] == 'jump':
prevdata = data
idx, valid_data = jump(idx, int(command[1]))
data = data_set[idx]
elif command[0] == 'goto':
prevdata = data
time_str = command[1]
if '%' in time_str:
if local_time:
lcl = timezone.to_local(idx)
else:
lcl = timezone.to_utc(idx)
time_str = lcl.strftime(time_str)
new_idx = pywws.weatherstation.WSDateTime.from_csv(time_str)
if local_time:
new_idx = timezone.to_naive(timezone.localize(new_idx))
new_idx = data_set.after(new_idx)
if new_idx:
idx = new_idx
data = data_set[idx]
valid_data = True
else:
valid_data = False
elif command[0] == 'loop':
loop_count = int(command[1])
loop_start = tmplt.tell()
elif command[0] == 'endloop':
loop_count -= 1
if valid_data and loop_count > 0:
tmplt.seek(loop_start, 0)
else:
logger.error("Unknown processing directive: #%s#", part)
return
line = ''