def rain_day(self, data):
calib_data = self.context.calib_data
midnight = time_zone.utc_to_local(data['idx'])
midnight = midnight.replace(hour=0, minute=0, second=0)
midnight = time_zone.local_to_utc(midnight)
midnight_data = calib_data[calib_data.nearest(midnight)]
return max(0.0, data['rain'] - midnight_data['rain'])
def generate_monthly(rain_day_threshold, day_end_hour, use_dst,
daily_data, monthly_data, process_from):
"""Generate monthly summaries from daily data."""
start = monthly_data.before(datetime.max)
if start is None:
start = datetime.min
start = daily_data.after(start + SECOND)
if process_from:
if start:
start = min(start, process_from)
else:
start = process_from
if start is None:
return start
# set start to noon on start of first day of month (local time)
local_start = time_zone.utc_to_local(start).replace(tzinfo=None)
local_start = local_start.replace(day=1, hour=12, minute=0, second=0)
# go back to UTC and get start of day (which might be previous day)
start = time_zone.local_to_utc(local_start)
start = time_zone.day_start(start, day_end_hour, use_dst=use_dst)
del monthly_data[start:]
stop = daily_data.before(datetime.max)
if stop is None:
return None
def monthlygen(inputdata, start, local_start):
"""Internal generator function"""
acc = MonthAcc(rain_day_threshold)
month_start = start
count = 0
while month_start <= stop:
count += 1
if count % 12 == 0:
logger.info("monthly: %s", month_start.isoformat(' '))
else:
logger.debug("monthly: %s", month_start.isoformat(' '))
if local_start.month < 12:
local_start = local_start.replace(month=local_start.month+1)
else:
local_start = local_start.replace(
month=1, year=local_start.year+1)
month_end = time_zone.local_to_utc(local_start)
month_end = time_zone.day_start(
month_end, day_end_hour, use_dst=use_dst)
acc.reset()
for data in inputdata[month_start:month_end]:
acc.add_daily(data)
new_data = acc.result()
if new_data:
new_data['start'] = month_start
yield new_data
month_start = month_end
monthly_data.update(monthlygen(daily_data, start, local_start))
return start
def _eval_time(self, time_str):
# get timestamp of last data item
result = self.hourly_data.before(datetime.max)
if not result:
result = datetime.utcnow() # only if no hourly data
# convert to local time
result = time_zone.utc_to_local(result)
# set to start of the day
result = result.replace(hour=0, minute=0, second=0, microsecond=0)
# apply time string
result = eval('result.replace(%s)' % time_str)
# convert back to UTC
return time_zone.local_to_utc(result)
def main(argv=None):
from pywws.timezone import time_zone
if argv is None:
argv = sys.argv
try:
opts, args = getopt.getopt(argv[1:], "h", ['help'])
except getopt.error as msg:
print('Error: %s\n' % msg, file=sys.stderr)
print(__usage__.strip(), file=sys.stderr)
return 1
# process options
for o, a in opts:
if o in ('-h', '--help'):
print(__usage__.strip())
return 0
# check arguments
if len(args) != 1:
print("Error: 1 argument required", file=sys.stderr)
print(__usage__.strip(), file=sys.stderr)
return 2
data_dir = args[0]
with pywws.storage.pywws_context(data_dir) as context:
params = context.params
pywws.localisation.set_application_language(params)
hourly_data = context.hourly_data
idx = hourly_data.before(datetime.max)
print('Zambretti (current):', zambretti(params, hourly_data[idx]))
idx = time_zone.utc_to_local(idx)
if idx.hour < 8 or (idx.hour == 8 and idx.minute < 30):
idx -= timedelta(hours=24)
idx = idx.replace(hour=9, minute=0, second=0)
idx = time_zone.local_to_utc(idx)
idx = hourly_data.nearest(idx)
lcl = time_zone.utc_to_local(idx)
print('Zambretti (at %s):' % lcl.strftime('%H:%M %Z'), zambretti(
params, hourly_data[idx]))
return 0
def plot_data(self, plot_no, plot, source):
class Record(object):
pass
_ = pywws.localisation.translation.ugettext
subplot_list = plot.get_children('subplot')
subplot_count = len(subplot_list)
if subplot_count < 1:
return u''
result = u''
pressure_offset = self.pressure_offset
# add some useful functions
hour_diff = self.computations.hour_diff
rain_hour = self.computations.rain_hour
rain_day = self.computations.rain_day
rain_24hr = self.computations.rain_24hr
# label x axis of last plot
if plot_no == self.plot_count - 1:
x_lo = time_zone.utc_to_local(self.x_lo)
x_hi = time_zone.utc_to_local(self.x_hi)
if self.duration <= DAY:
# TX_NOTE Keep the "(%Z)" formatting string
xlabel = _('Time (%Z)')
elif self.duration <= timedelta(days=7):
xlabel = _('Day')
else:
xlabel = _('Date')
xlabel = self.graph.get_value('xlabel', xlabel)
if sys.version_info[0] < 3:
xlabel = xlabel.encode(self.encoding[0])
xlabel = x_hi.strftime(xlabel)
if sys.version_info[0] < 3:
xlabel = xlabel.decode(self.encoding[0])
result += u'set xlabel "%s"\n' % xlabel
dateformat = '%Y/%m/%d'
dateformat = self.graph.get_value('dateformat', dateformat)
if sys.version_info[0] < 3:
dateformat = dateformat.encode(self.encoding[0])
ldat = x_lo.strftime(dateformat)
rdat = x_hi.strftime(dateformat)
if sys.version_info[0] < 3:
ldat = ldat.decode(self.encoding[0])
rdat = rdat.decode(self.encoding[0])
if ldat:
result += u'set label "%s" at "%s", graph -0.3 left\n' % (
ldat, (self.x_lo + self.utcoffset).isoformat())
if rdat != ldat:
result += u'set label "%s" at "%s", graph -0.3 right\n' % (
rdat, (self.x_hi + self.utcoffset).isoformat())
# set bottom margin
bmargin = eval(plot.get_value('bmargin', '-1'))
result += u'set bmargin %g\n' % (bmargin)
# set y ranges and tics
yrange = plot.get_value('yrange', None)
y2range = plot.get_value('y2range', None)
ytics = plot.get_value('ytics', 'autofreq')
y2tics = plot.get_value('y2tics', '')
if y2tics and not y2range:
y2range = yrange
elif y2range and not y2tics:
y2tics = 'autofreq'
if yrange:
result += u'set yrange [%s]\n' % (yrange.replace(',', ':'))
else:
result += u'set yrange [*:*]\n'
if y2range:
result += u'set y2range [%s]\n' % (y2range.replace(',', ':'))
if y2tics:
result += u'set ytics nomirror %s; set y2tics %s\n' % (ytics,
y2tics)
else:
result += u'unset y2tics; set ytics mirror %s\n' % (ytics)
# set grid
result += u'unset grid\n'
grid = plot.get_value('grid', None)
if grid is not None:
result += u'set grid %s\n' % grid
start = self.x_lo
stop = self.x_hi
cumu_start = start
if source == self.calib_data:
boxwidth = 240 # assume 5 minute data interval
start = source.before(start)
elif source == self.hourly_data:
boxwidth = 2800
start = source.before(start)
interval = timedelta(minutes=90)
elif source == self.monthly_data:
boxwidth = 2800 * 24 * 30
interval = timedelta(days=46)
else:
interval = timedelta(hours=36)
boxwidth = 2800 * 24
boxwidth = eval(plot.get_value('boxwidth', str(boxwidth)))
result += u'set boxwidth %d\n' % boxwidth
for command in plot.get_values('command'):
result += u'%s\n' % command
stop = source.after(stop)
if stop:
stop = stop + timedelta(minutes=1)
# write data files
subplots = []
for subplot_no in range(subplot_count):
subplot = Record()
subplot.subplot = subplot_list[subplot_no]
subplot.dat_file = os.path.join(
self.work_dir, 'plot_%d_%d.dat' % (plot_no, subplot_no))
self.tmp_files.append(subplot.dat_file)
subplot.dat = open(subplot.dat_file, 'w')
subplot.xcalc = subplot.subplot.get_value('xcalc', None)
subplot.ycalc = subplot.subplot.get_value('ycalc', None)
subplot.cummulative = 'last_ycalc' in subplot.ycalc
if subplot.xcalc:
subplot.xcalc = compile(subplot.xcalc, '<string>', 'eval')
subplot.ycalc = compile(subplot.ycalc, '<string>', 'eval')
subplot.last_ycalcs = 0.0
subplot.last_idx = None
subplot.using = '($2)'
subplots.append(subplot)
for data in source[start:stop]:
for subplot in subplots:
if subplot.xcalc:
idx = eval(subplot.xcalc)
if idx is None:
continue
else:
idx = data['idx']
idx += self.utcoffset
if not subplot.cummulative and subplot.last_idx:
if source == self.calib_data:
interval = timedelta(
minutes=((data['delay'] * 3) + 1) // 2)
if idx - subplot.last_idx > interval:
# missing data
subplot.dat.write('%s ?\n' % (idx.isoformat()))
subplot.last_idx = idx
try:
if subplot.cummulative and data['idx'] <= cumu_start:
value = 0.0
else:
last_ycalc = subplot.last_ycalcs
value = eval(subplot.ycalc)
if not isinstance(value, tuple):
value = (value, )
values = (idx.isoformat(), ) + value
vformat = '%s' + (' %g' * len(value)) + '\n'
subplot.dat.write(vformat % values)
subplot.using = ':'.join(
'($%d)' % x for x in range(2,
len(values) + 1))
subplot.last_ycalcs = value[0]
except TypeError:
if not subplot.cummulative:
subplot.dat.write('%s ?\n' % (idx.isoformat()))
subplot.last_ycalcs = 0.0
for subplot in subplots:
# ensure the data file isn't empty
idx = self.x_hi + self.duration
subplot.dat.write('%s ?\n' % (idx.isoformat()))
subplot.dat.close()
# plot data
result += u'plot '
colour_idx = 0
for subplot_no in range(subplot_count):
subplot = subplots[subplot_no]
colour_idx += 1
colour = subplot.subplot.get_value('colour', str(colour_idx))
style = subplot.subplot.get_value(
'style', 'smooth unique lc %s lw 1' % (colour))
words = style.split()
if len(words) > 1 and words[0] in ('+', 'x', 'line',
'candlesticks',
'candlesticksw'):
width = float(words[1])
else:
width = 1
if len(words) > 2 and words[0] in ('candlesticksw'):
whiskerwidth = float(words[2])
else:
whiskerwidth = 1
whiskerbars = ''
if style == 'box':
style = 'lc %s lw 0 with boxes' % (colour)
elif words[0] == 'candlesticks':
style = 'lc %s lw %g with candlesticks' % (colour, width)
elif words[0] == 'candlesticksw':
style = 'lc %s lw %g with candlesticks' % (colour, width)
whiskerbars = ' whiskerbars %g' % (whiskerwidth)
elif words[0] == '+':
style = 'lc %s lw %g pt 1 with points' % (colour, width)
elif words[0] == 'x':
style = 'lc %s lw %g pt 2 with points' % (colour, width)
elif words[0] == 'line':
style = 'smooth unique lc %s lw %g' % (colour, width)
axes = subplot.subplot.get_value('axes', 'x1y1')
title = subplot.subplot.get_value('title', '')
result += u' "%s" using 1:%s axes %s %s title "%s"%s' % (
subplot.dat_file, subplot.using, axes, style, title,
whiskerbars)
if subplot_no != subplot_count - 1:
result += u', \\'
result += u'\n'
return result
def do_plot(self, input_file, output_file):
if isinstance(input_file, GraphFileReader):
self.graph = input_file
else:
# read XML graph description
self.graph = GraphFileReader(input_file)
# get list of plots
plot_list = self.graph.get_children(self.plot_name)
self.plot_count = len(plot_list)
if self.plot_count < 1:
# nothing to plot
logger.info('%s has no %s nodes', self.graph.input_file,
self.plot_name)
self.graph.close()
return 1
# get start and end datetimes
self.x_lo = self.graph.get_value('start', None)
self.x_hi = self.graph.get_value('stop', None)
self.duration = self.graph.get_value('duration', None)
if self.duration:
self.duration = eval('timedelta(%s)' % self.duration)
else:
self.duration = DAY
if self.x_lo:
self.x_lo = self._eval_time(self.x_lo)
if self.x_hi:
self.x_hi = self._eval_time(self.x_hi)
self.duration = self.x_hi - self.x_lo
else:
self.x_hi = self.x_lo + self.duration
elif self.x_hi:
self.x_hi = self._eval_time(self.x_hi)
self.x_lo = self.x_hi - self.duration
else:
self.x_hi = self.hourly_data.before(datetime.max)
if not self.x_hi:
self.x_hi = datetime.utcnow() # only if no hourly data
if self.duration < HOUR * 6:
# set end of graph to start of the next minute after last item
self.x_hi += timedelta(seconds=55)
self.x_hi = self.x_hi.replace(second=0)
else:
# set end of graph to start of the next hour after last item
self.x_hi += timedelta(minutes=55)
self.x_hi = self.x_hi.replace(minute=0, second=0)
self.x_lo = self.x_hi - self.duration
# use a fixed offset to convert UTC to X axis values
self.utcoffset = time_zone.utc_to_local(
self.x_hi).replace(tzinfo=None) - self.x_hi
# open gnuplot command file
self.tmp_files = []
cmd_file = os.path.join(self.work_dir, 'plot.cmd')
self.tmp_files.append(cmd_file)
of = codecs.open(cmd_file, 'w', encoding=self.encoding[0])
# write gnuplot set up
of.write('set encoding %s\n' % (self.encoding[1]))
lcl = locale.getlocale()
if lcl[0]:
of.write('set locale "%s.%s"\n' % lcl)
self.rows = self.get_default_rows()
self.cols = (self.plot_count + self.rows - 1) // self.rows
self.rows, self.cols = eval(
self.graph.get_value('layout', '%d, %d' % (self.rows, self.cols)))
w, h = self.get_default_plot_size()
w = w * self.cols
h = h * self.rows
w, h = eval(self.graph.get_value('size', '(%d, %d)' % (w, h)))
fileformat = self.graph.get_value('fileformat', 'png')
if fileformat == 'svg':
terminal = 'svg enhanced font "arial,9" dynamic rounded'
elif u' ' not in fileformat:
terminal = '%s large' % (fileformat)
else:
terminal = fileformat
if u'size' not in terminal:
terminal += u' size %d,%d' % (w, h)
terminal = self.graph.get_value('terminal', terminal)
of.write('set terminal %s\n' % (terminal))
of.write('set output "%s"\n' % (output_file))
# set overall title
title = self.graph.get_value('title', '')
if title:
if '%' in title:
x_hi = time_zone.utc_to_local(self.x_hi)
if sys.version_info[0] < 3:
title = title.encode(self.encoding[0])
title = x_hi.strftime(title)
if sys.version_info[0] < 3:
title = title.decode(self.encoding[0])
title = 'title "%s"' % title
of.write('set multiplot layout %d, %d %s\n' %
(self.rows, self.cols, title))
# do actual plots
of.write(self.get_preamble())
for plot_no in range(self.plot_count):
plot = plot_list[plot_no]
# set key / title location
title = plot.get_value('title', '')
of.write('set key horizontal title "%s"\n' % title)
# optional yaxis labels
ylabel = plot.get_value('ylabel', '')
if ylabel:
ylabelangle = plot.get_value('ylabelangle', '')
if ylabelangle:
ylabelangle = ' rotate by %s' % (ylabelangle)
of.write('set ylabel "%s"%s\n' % (ylabel, ylabelangle))
else:
of.write('set ylabel\n')
y2label = plot.get_value('y2label', '')
if y2label:
y2labelangle = plot.get_value('y2labelangle', '')
if y2labelangle:
y2labelangle = ' rotate by %s' % (y2labelangle)
of.write('set y2label "%s"%s\n' % (y2label, y2labelangle))
else:
of.write('set y2label\n')
# set data source
source = plot.get_value('source', 'raw')
if source == 'raw':
source = self.calib_data
elif source == 'hourly':
source = self.hourly_data
elif source == 'monthly':
source = self.monthly_data
else:
source = self.daily_data
# do the plot
of.write(self.plot_data(plot_no, plot, source))
of.close()
self.graph.close()
# run gnuplot on file
subprocess.check_call(['gnuplot', cmd_file])
for file in self.tmp_files:
os.unlink(file)
return 0
def __init__(self, context):
self.context = context
self.params = context.params
self.status = context.status
self.raw_data = context.raw_data
self.calib_data = context.calib_data
self.hourly_data = context.hourly_data
self.daily_data = context.daily_data
self.monthly_data = context.monthly_data
self.flush = literal_eval(
self.params.get('config', 'frequent writes', 'False'))
# get directories
self.template_dir = self.params.get(
'paths', 'templates', os.path.expanduser('~/weather/templates/'))
self.graph_template_dir = self.params.get(
'paths', 'graph_templates',
os.path.expanduser('~/weather/graph_templates/'))
self.module_dir = self.params.get(
'paths', 'modules', os.path.expanduser('~/weather/modules/'))
# create calibration object
self.calibrator = Calib(self.params, self.raw_data)
# create templater object
self.templater = pywws.template.Template(context)
# create plotter objects
self.plotter = pywws.plot.GraphPlotter(context, self.context.work_dir)
self.roseplotter = pywws.windrose.RosePlotter(context,
self.context.work_dir)
# get daytime end hour
self.day_end_hour, self.use_dst = pywws.process.get_day_end_hour(
self.params)
# parse "cron" sections
self.cron = {}
for section in self.params._config.sections():
if section.split()[0] != 'cron':
continue
import croniter
last_update = self.status.get_datetime('last update', section)
last_update = last_update or datetime.utcnow()
self.cron[section] = croniter.croniter(
self.params.get(section, 'format', ''),
start_time=time_zone.utc_to_local(last_update))
self.cron[section].get_next()
# create service uploader objects
self.services = {}
for section in list(self.cron.keys()) + [
'live', 'logged', 'hourly', '12 hourly', 'daily'
]:
for name, options in self._parse_templates(section, 'services'):
if name in self.services:
continue
if os.path.exists(os.path.join(self.module_dir, name + '.py')):
sys.path.insert(0, self.module_dir)
mod = importlib.import_module(name)
del sys.path[0]
else:
mod = importlib.import_module('pywws.service.' + name)
self.services[name] = mod.ToService(context)
# check for obsolete entries
if self.params.get(section, 'twitter'):
logger.error('Obsolete twitter entry in weather.ini [%s]',
section)
if self.params.get(section, 'yowindow'):
logger.error('Obsolete yowindow entry in weather.ini [%s]',
section)
def process(self, live_data, template_file):
def jump(idx, count):
while count > 0:
new_idx = data_set.after(idx + SECOND)
if new_idx == None:
break
idx = new_idx
count -= 1
while count < 0:
new_idx = data_set.before(idx)
if new_idx == None:
break
idx = new_idx
count += 1
return idx, count == 0
params = self.params
if not live_data:
idx = self.calib_data.before(datetime.max)
if not idx:
logger.error("No calib data - run pywws.process first")
return
live_data = self.calib_data[idx]
# get default character encoding of template input & output files
self.encoding = params.get('config', 'template encoding', 'iso-8859-1')
file_encoding = self.encoding
if file_encoding == 'html':
file_encoding = 'ascii'
# get conversions module to create its 'private' wind dir text
# array, then copy it to deprecated wind_dir_text variable
winddir_text(0)
wind_dir_text = conversions._winddir_text_array
hour_diff = self.computations.hour_diff
rain_hour = self.computations.rain_hour
rain_day = self.computations.rain_day
rain_24hr = self.computations.rain_24hr
pressure_offset = float(self.params.get('config', 'pressure offset'))
fixed_block = literal_eval(self.status.get('fixed', 'fixed block'))
# start off with no time rounding
round_time = None
# start off in hourly data mode
data_set = self.hourly_data
# start off in utc
local_time = False
# start off with default use_locale setting
use_locale = self.use_locale
# jump to last item
idx, valid_data = jump(datetime.max, -1)
if not valid_data:
logger.error("No summary data - run pywws.process first")
return
data = data_set[idx]
# open template file, if not already a file(like) object
if hasattr(template_file, 'readline'):
tmplt = template_file
else:
tmplt = open(template_file, 'rb')
# do the text processing
line = ''
while True:
new_line = tmplt.readline()
if not new_line:
break
if isinstance(new_line, bytes) or sys.version_info[0] < 3:
new_line = new_line.decode(file_encoding)
line += new_line
parts = line.split('#')
if len(parts) % 2 == 0:
# odd number of '#'
line = line.rstrip('\r\n')
continue
for i, part in enumerate(parts):
if i % 2 == 0:
# not a processing directive
if i == 0 or part != '\n':
yield part
continue
if part and part[0] == '!':
# comment
continue
# Python 2 shlex can't handle unicode
if sys.version_info[0] < 3:
part = part.encode(file_encoding)
command = shlex.split(part)
if sys.version_info[0] < 3:
command = map(lambda x: x.decode(file_encoding), command)
if command == []:
# empty command == print a single '#'
yield u'#'
elif command[0] in list(data.keys()) + ['calc']:
# output a value
if not valid_data:
continue
# format is: key fmt_string no_value_string conversion
# get value
if command[0] == 'calc':
x = eval(command[1])
del command[1]
else:
x = data[command[0]]
# adjust time
if isinstance(x, datetime):
if round_time:
x += round_time
if local_time:
x = time_zone.utc_to_local(x)
else:
x = x.replace(tzinfo=time_zone.utc)
# convert data
if x is not None and len(command) > 3:
x = eval(command[3])
# get format
fmt = u'%s'
if len(command) > 1:
fmt = command[1]
# write output
if x is None:
if len(command) > 2:
yield command[2]
elif isinstance(x, datetime):
if sys.version_info[0] < 3:
fmt = fmt.encode(file_encoding)
x = x.strftime(fmt)
if sys.version_info[0] < 3:
if self.encoding == 'html':
x = x.decode('ascii',
errors='xmlcharrefreplace')
else:
x = x.decode(file_encoding)
yield x
elif not use_locale:
yield fmt % (x)
elif sys.version_info >= (2, 7) or '%%' not in fmt:
yield locale.format_string(fmt, x)
else:
yield locale.format_string(fmt.replace('%%', '##'),
x).replace('##', '%')
elif command[0] == 'monthly':
data_set = self.monthly_data
idx, valid_data = jump(datetime.max, -1)
data = data_set[idx]
elif command[0] == 'daily':
data_set = self.daily_data
idx, valid_data = jump(datetime.max, -1)
data = data_set[idx]
elif command[0] == 'hourly':
data_set = self.hourly_data
idx, valid_data = jump(datetime.max, -1)
data = data_set[idx]
elif command[0] == 'raw':
data_set = self.calib_data
idx, valid_data = jump(datetime.max, -1)
data = data_set[idx]
elif command[0] == 'live':
data_set = self.calib_data
idx = live_data['idx']
valid_data = True
data = live_data
elif command[0] == 'timezone':
if command[1] == 'utc':
local_time = False
elif command[1] == 'local':
local_time = True
else:
logger.error("Unknown time zone: %s", command[1])
return
elif command[0] == 'locale':
use_locale = eval(command[1])
elif command[0] == 'encoding':
self.encoding = command[1]
file_encoding = self.encoding
if file_encoding == 'html':
file_encoding = 'ascii'
elif command[0] == 'roundtime':
if eval(command[1]):
round_time = timedelta(seconds=30)
else:
round_time = None
elif command[0] == 'jump':
prevdata = data
idx, valid_data = jump(idx, int(command[1]))
data = data_set[idx]
elif command[0] == 'goto':
prevdata = data
time_str = command[1]
if '%' in time_str:
if local_time:
lcl = time_zone.utc_to_local(idx)
else:
lcl = idx.replace(tzinfo=time_zone.utc)
time_str = lcl.strftime(time_str)
new_idx = pywws.weatherstation.WSDateTime.from_csv(
time_str)
if local_time:
new_idx = time_zone.local_to_utc(new_idx)
new_idx = data_set.after(new_idx)
if new_idx:
idx = new_idx
data = data_set[idx]
valid_data = True
else:
valid_data = False
elif command[0] == 'loop':
loop_count = int(command[1])
loop_start = tmplt.tell()
elif command[0] == 'endloop':
loop_count -= 1
if valid_data and loop_count > 0:
tmplt.seek(loop_start, 0)
else:
logger.error("Unknown processing directive: #%s#", part)
return
line = ''