def _do_cron(self, live_data=None):
if not self.cron:
return
# get timestamp of latest data
if live_data:
now = live_data['idx']
else:
now = self.calib_data.before(datetime.max)
if not now:
now = datetime.utcnow()
# convert to local time
local_now = now + local_utc_offset(now)
# get list of due sections
sections = []
for section in self.cron:
if self.cron[section].get_current(datetime) > local_now:
continue
sections.append(section)
while self.cron[section].get_current(datetime) <= local_now:
self.cron[section].get_next()
if not sections:
return
# do it!
self._do_common(sections, live_data)
for section in sections:
self.status.set('last update', section, now.isoformat(' '))
def _do_cron(self, live_data=None):
if not self.cron:
return
# get timestamp of latest data
if live_data:
now = live_data['idx']
else:
now = self.calib_data.before(datetime.max)
if not now:
now = datetime.utcnow()
# convert to local time
local_now = now + local_utc_offset(now)
# get list of due sections
sections = []
for section in self.cron:
if self.cron[section].get_current(datetime) > local_now:
continue
sections.append(section)
while self.cron[section].get_current(datetime) <= local_now:
self.cron[section].get_next()
if not sections:
return
# do it!
self._do_common(sections, live_data)
for section in sections:
self.status.set('last update', section, now.isoformat(' '))
def _eval_time(self, time_str):
# get timestamp of last data item
result = self.hourly_data.before(datetime.max)
if not result:
result = datetime.utcnow() # only if no hourly data
result += local_utc_offset(result)
# set to start of the day
result = result.replace(hour=0, minute=0, second=0, microsecond=0)
# apply time string
return eval('result.replace(%s)' % time_str)
def __init__(self,
params,
status,
raw_data,
calib_data,
hourly_data,
daily_data,
monthly_data,
asynch=False):
self.logger = logging.getLogger('pywws.Tasks.RegularTasks')
self.params = params
self.status = status
self.raw_data = raw_data
self.calib_data = calib_data
self.hourly_data = hourly_data
self.daily_data = daily_data
self.monthly_data = monthly_data
self.asynch = asynch
self.flush = eval(self.params.get('config', 'frequent writes',
'False'))
# get directories
self.work_dir = self.params.get('paths', 'work', '/tmp/weather')
if not os.path.isdir(self.work_dir):
raise RuntimeError('Directory "' + self.work_dir +
'" does not exist.')
self.template_dir = self.params.get(
'paths', 'templates', os.path.expanduser('~/weather/templates/'))
self.graph_template_dir = self.params.get(
'paths', 'graph_templates',
os.path.expanduser('~/weather/graph_templates/'))
self.local_dir = self.params.get(
'paths', 'local_files', os.path.expanduser('~/weather/results/'))
# create calibration object
self.calibrator = Calib(self.params, self.raw_data)
# create templater object
self.templater = Template.Template(self.params, self.status,
self.calib_data, self.hourly_data,
self.daily_data, self.monthly_data)
# create plotter objects
self.plotter = Plot.GraphPlotter(self.params, self.status,
self.calib_data, self.hourly_data,
self.daily_data, self.monthly_data,
self.work_dir)
self.roseplotter = WindRose.RosePlotter(
self.params, self.status, self.calib_data, self.hourly_data,
self.daily_data, self.monthly_data, self.work_dir)
# create FTP uploader object
self.uploader = Upload.Upload(self.params)
self.uploads_directory = os.path.join(self.work_dir, 'uploads')
if not os.path.isdir(self.uploads_directory):
os.mkdir(self.uploads_directory)
# delay creation of a Twitter object until we know it's needed
self.twitter = None
# create a YoWindow object
self.yowindow = YoWindow.YoWindow(self.calib_data)
# get daytime end hour, in UTC
self.day_end_hour = eval(params.get('config', 'day end hour', '21'))
self.day_end_hour = (self.day_end_hour -
(STDOFFSET.seconds // 3600)) % 24
# parse "cron" sections
self.cron = {}
for section in self.params._config.sections():
if section.split()[0] != 'cron':
continue
import croniter
self.cron[section] = croniter.croniter(
self.params.get(section, 'format', ''))
self.cron[section].get_prev()
last_update = self.status.get_datetime('last update', section)
if last_update:
last_update = last_update + local_utc_offset(last_update)
while self.cron[section].get_current(datetime) <= last_update:
self.cron[section].get_next()
# create service uploader objects
self.services = {}
for section in self.cron.keys() + [
'live', 'logged', 'hourly', '12 hourly', 'daily'
]:
for name in eval(self.params.get(section, 'services', '[]')):
if name not in self.services:
self.services[name] = ToService(self.params,
self.status,
self.calib_data,
service_name=name)
# check for deprecated syntax
if self.params.get(section, 'twitter') not in (None, '[]'):
self.logger.warning('Deprecated twitter entry in [%s]',
section)
if self.params.get(section, 'yowindow'):
self.logger.warning('Deprecated yowindow entry in [%s]',
section)
# create queues for things to upload / send
self.tweet_queue = deque()
self.service_queue = {}
for name in self.services:
self.service_queue[name] = deque()
self.uploads_queue = deque()
# start asynchronous thread to do uploads
if self.asynch:
self.logger.info('Starting asynchronous thread')
self.shutdown_thread = threading.Event()
self.wake_thread = threading.Event()
self.thread = threading.Thread(target=self._asynch_thread)
self.thread.start()
def DoPlot(self, input_file, output_file):
if isinstance(input_file, GraphFileReader):
self.graph = input_file
else:
# read XML graph description
self.graph = GraphFileReader(input_file)
# get list of plots
plot_list = self.graph.get_children(self.plot_name)
self.plot_count = len(plot_list)
if self.plot_count < 1:
# nothing to plot
self.logger.info('%s has no %s nodes', self.graph.input_file, self.plot_name)
self.graph.close()
return 1
# get start and end datetimes
self.x_lo = self.graph.get_value('start', None)
self.x_hi = self.graph.get_value('stop', None)
self.duration = self.graph.get_value('duration', None)
if self.duration:
self.duration = eval('timedelta(%s)' % self.duration)
else:
self.duration = timedelta(hours=24)
if self.x_lo:
self.x_lo = self._eval_time(self.x_lo)
if self.x_hi:
self.x_hi = self._eval_time(self.x_hi)
self.duration = self.x_hi - self.x_lo
else:
self.x_hi = self.x_lo + self.duration
elif self.x_hi:
self.x_hi = self._eval_time(self.x_hi)
self.x_lo = self.x_hi - self.duration
else:
self.x_hi = self.hourly_data.before(datetime.max)
if not self.x_hi:
self.x_hi = datetime.utcnow() # only if no hourly data
self.x_hi += local_utc_offset(self.x_hi)
if self.duration < timedelta(hours=6):
# set end of graph to start of the next minute after last item
self.x_hi += timedelta(seconds=55)
self.x_hi = self.x_hi.replace(second=0)
else:
# set end of graph to start of the next hour after last item
self.x_hi += timedelta(minutes=55)
self.x_hi = self.x_hi.replace(minute=0, second=0)
self.x_lo = self.x_hi - self.duration
self.utcoffset = local_utc_offset(self.x_hi)
# open gnuplot command file
self.tmp_files = []
cmd_file = os.path.join(self.work_dir, 'plot.cmd')
self.tmp_files.append(cmd_file)
of = codecs.open(cmd_file, 'w', encoding=self.encoding[0])
# write gnuplot set up
of.write('set encoding %s\n' % (self.encoding[1]))
lcl = locale.getlocale()
if lcl[0]:
of.write('set locale "%s.%s"\n' % lcl)
self.rows = self.GetDefaultRows()
self.cols = (self.plot_count + self.rows - 1) // self.rows
self.rows, self.cols = eval(self.graph.get_value(
'layout', '%d, %d' % (self.rows, self.cols)))
w, h = self.GetDefaultPlotSize()
w = w * self.cols
h = h * self.rows
w, h = eval(self.graph.get_value('size', '(%d, %d)' % (w, h)))
fileformat = self.graph.get_value('fileformat', 'png')
if fileformat == 'svg':
terminal = 'svg enhanced font "arial,9" dynamic rounded'
elif u' ' not in fileformat:
terminal = '%s large' % (fileformat)
else:
terminal = fileformat
if u'size' not in terminal:
terminal += u' size %d,%d' % (w, h)
terminal = self.graph.get_value('terminal', terminal)
of.write('set terminal %s\n' % (terminal))
of.write('set output "%s"\n' % (output_file))
# set overall title
title = self.graph.get_value('title', '')
if title:
if '%' in title:
x_hi = (self.x_hi -
self.utcoffset).replace(tzinfo=utc).astimezone(Local)
if sys.version_info[0] < 3:
title = title.encode(self.encoding[0])
title = x_hi.strftime(title)
if sys.version_info[0] < 3:
title = title.decode(self.encoding[0])
title = 'title "%s"' % title
of.write('set multiplot layout %d, %d %s\n' % (self.rows, self.cols, title))
# do actual plots
of.write(self.GetPreamble())
for plot_no in range(self.plot_count):
plot = plot_list[plot_no]
# set key / title location
title = plot.get_value('title', '')
of.write('set key horizontal title "%s"\n' % title)
# optional yaxis labels
ylabel = plot.get_value('ylabel', '')
if ylabel:
ylabelangle = plot.get_value('ylabelangle', '')
if ylabelangle:
ylabelangle = ' rotate by %s' % (ylabelangle)
of.write('set ylabel "%s"%s\n' % (ylabel, ylabelangle))
else:
of.write('set ylabel\n')
y2label = plot.get_value('y2label', '')
if y2label:
y2labelangle = plot.get_value('y2labelangle', '')
if y2labelangle:
y2labelangle = ' rotate by %s' % (y2labelangle)
of.write('set y2label "%s"%s\n' % (y2label, y2labelangle))
else:
of.write('set y2label\n')
# set data source
source = plot.get_value('source', 'raw')
if source == 'raw':
source = self.raw_data
elif source == 'hourly':
source = self.hourly_data
elif source == 'monthly':
source = self.monthly_data
else:
source = self.daily_data
# do the plot
of.write(self.PlotData(plot_no, plot, source))
of.close()
self.graph.close()
# run gnuplot on file
subprocess.check_call(['gnuplot', cmd_file])
for file in self.tmp_files:
os.unlink(file)
return 0
def __init__(self, params, status,
raw_data, calib_data, hourly_data, daily_data, monthly_data,
asynch=False):
self.logger = logging.getLogger('pywws.Tasks.RegularTasks')
self.params = params
self.status = status
self.raw_data = raw_data
self.calib_data = calib_data
self.hourly_data = hourly_data
self.daily_data = daily_data
self.monthly_data = monthly_data
self.asynch = asynch
self.flush = eval(self.params.get('config', 'frequent writes', 'False'))
# get directories
self.work_dir = self.params.get('paths', 'work', '/tmp/weather')
if not os.path.isdir(self.work_dir):
raise RuntimeError(
'Directory "' + self.work_dir + '" does not exist.')
self.template_dir = self.params.get(
'paths', 'templates', os.path.expanduser('~/weather/templates/'))
self.graph_template_dir = self.params.get(
'paths', 'graph_templates', os.path.expanduser('~/weather/graph_templates/'))
self.local_dir = self.params.get(
'paths', 'local_files', os.path.expanduser('~/weather/results/'))
# create calibration object
self.calibrator = Calib(self.params, self.raw_data)
# create templater object
self.templater = Template.Template(
self.params, self.status, self.calib_data, self.hourly_data,
self.daily_data, self.monthly_data)
# create plotter objects
self.plotter = Plot.GraphPlotter(
self.params, self.status, self.calib_data, self.hourly_data,
self.daily_data, self.monthly_data, self.work_dir)
self.roseplotter = WindRose.RosePlotter(
self.params, self.status, self.calib_data, self.hourly_data,
self.daily_data, self.monthly_data, self.work_dir)
# create FTP uploader object
self.uploader = Upload.Upload(self.params)
self.uploads_directory = os.path.join(self.work_dir, 'uploads')
if not os.path.isdir(self.uploads_directory):
os.mkdir(self.uploads_directory)
# delay creation of a Twitter object until we know it's needed
self.twitter = None
# create a YoWindow object
self.yowindow = YoWindow.YoWindow(self.calib_data)
# get daytime end hour, in UTC
self.day_end_hour = eval(params.get('config', 'day end hour', '21'))
self.day_end_hour = (self.day_end_hour - (STDOFFSET.seconds // 3600)) % 24
# parse "cron" sections
self.cron = {}
for section in self.params._config.sections():
if section.split()[0] != 'cron':
continue
import croniter
self.cron[section] = croniter.croniter(
self.params.get(section, 'format', ''))
self.cron[section].get_prev()
last_update = self.status.get_datetime('last update', section)
if last_update:
last_update = last_update + local_utc_offset(last_update)
while self.cron[section].get_current(datetime) <= last_update:
self.cron[section].get_next()
# create service uploader objects
self.services = {}
for section in self.cron.keys() + [
'live', 'logged', 'hourly', '12 hourly', 'daily']:
for name in eval(self.params.get(section, 'services', '[]')):
if name not in self.services:
self.services[name] = ToService(
self.params, self.status, self.calib_data,
service_name=name)
# check for deprecated syntax
if self.params.get(section, 'twitter') not in (None, '[]'):
self.logger.warning(
'Deprecated twitter entry in [%s]', section)
if self.params.get(section, 'yowindow'):
self.logger.warning(
'Deprecated yowindow entry in [%s]', section)
# create queues for things to upload / send
self.tweet_queue = deque()
self.service_queue = {}
for name in self.services:
self.service_queue[name] = deque()
self.uploads_queue = deque()
# start asynchronous thread to do uploads
if self.asynch:
self.logger.info('Starting asynchronous thread')
self.shutdown_thread = threading.Event()
self.wake_thread = threading.Event()
self.thread = threading.Thread(target=self._asynch_thread)
self.thread.start()
