def calibrate_data(params, raw_data, calib_data):
"""'Calibrate' raw data, using a user-supplied function."""
start = calib_data.before(datetime.max)
if start is None:
start = datetime.min
start = raw_data.after(start + SECOND)
if start is None:
return start
del calib_data[start:]
calibrator = Calib(params, raw_data)
count = 0
for data in raw_data[start:]:
idx = data['idx']
count += 1
if count % 10000 == 0:
logger.info("calib: %s", idx.isoformat(' '))
elif count % 500 == 0:
logger.debug("calib: %s", idx.isoformat(' '))
for key in ('rain', 'abs_pressure', 'temp_in'):
if data[key] is None:
logger.error('Ignoring invalid data at %s', idx.isoformat(' '))
break
else:
calib_data[idx] = calibrator.calib(data)
return start
def calibrate_data(logger, params, raw_data, calib_data):
"""'Calibrate' raw data, using a user-supplied function."""
start = calib_data.before(datetime.max)
if start is None:
start = datetime.min
start = raw_data.after(start + SECOND)
if start is None:
return start
del calib_data[start:]
calibrator = Calib(params, raw_data)
count = 0
for data in raw_data[start:]:
idx = data['idx']
count += 1
if count % 10000 == 0:
logger.info("calib: %s", idx.isoformat(' '))
elif count % 500 == 0:
logger.debug("calib: %s", idx.isoformat(' '))
for key in ('rain', 'abs_pressure', 'temp_in'):
if data[key] is None:
logger.error('Ignoring invalid data at %s', idx.isoformat(' '))
break
else:
calib_data[idx] = calibrator.calib(data)
return start
def calibrate_data(logger, params, raw_data, calib_data):
"""'Calibrate' raw data, using a user-supplied function."""
start = calib_data.before(datetime.max)
if start is None:
start = datetime.min
start = raw_data.after(start + SECOND)
if start is None:
return start
del calib_data[start:]
calibrator = Calib(params, raw_data)
count = 0
for data in raw_data[start:]:
idx = data['idx']
count += 1
if count % 10000 == 0:
logger.info("calib: %s", idx.isoformat(' '))
elif count % 500 == 0:
logger.debug("calib: %s", idx.isoformat(' '))
calib_data[idx] = calibrator.calib(data)
return start
def __init__(self, params, status,
calib_data, hourly_data, daily_data, monthly_data):
self.logger = logging.getLogger('pywws.Tasks.RegularTasks')
self.params = params
self.status = status
self.calib_data = calib_data
self.hourly_data = hourly_data
self.daily_data = daily_data
self.monthly_data = monthly_data
# get directories
self.work_dir = self.params.get('paths', 'work', '/tmp/weather')
self.template_dir = self.params.get(
'paths', 'templates', os.path.expanduser('~/weather/templates/'))
self.graph_template_dir = self.params.get(
'paths', 'graph_templates', os.path.expanduser('~/weather/graph_templates/'))
self.local_dir = self.params.get(
'paths', 'local_files', os.path.expanduser('~/weather/results/'))
# create calibration object
self.calibrator = Calib(self.params, self.status)
# create templater object
self.templater = Template.Template(
self.params, self.status, self.calib_data, self.hourly_data,
self.daily_data, self.monthly_data)
# create plotter objects
self.plotter = Plot.GraphPlotter(
self.params, self.status, self.calib_data, self.hourly_data,
self.daily_data, self.monthly_data, self.work_dir)
self.roseplotter = WindRose.RosePlotter(
self.params, self.status, self.calib_data, self.hourly_data,
self.daily_data, self.monthly_data, self.work_dir)
# directory of service uploaders
self.services = dict()
# create a YoWindow object
self.yowindow = YoWindow.YoWindow(self.calib_data)
# get local time's offset from UTC, without DST
now = self.calib_data.before(datetime.max)
if not now:
now = datetime.utcnow()
time_offset = Local.utcoffset(now) - Local.dst(now)
# get daytime end hour, in UTC
self.day_end_hour = eval(params.get('config', 'day end hour', '21'))
self.day_end_hour = (self.day_end_hour - (time_offset.seconds // 3600)) % 24
# create service uploader objects
for section in ('live', 'logged', 'hourly', '12 hourly', 'daily'):
for service in eval(self.params.get(section, 'services', '[]')):
if service not in self.services:
self.services[service] = ToService(
self.params, self.status, self.calib_data,
service_name=service)
def __init__(self, params, status,
raw_data, calib_data, hourly_data, daily_data, monthly_data,
asynch=False):
self.logger = logging.getLogger('pywws.Tasks.RegularTasks')
self.params = params
self.status = status
self.raw_data = raw_data
self.calib_data = calib_data
self.hourly_data = hourly_data
self.daily_data = daily_data
self.monthly_data = monthly_data
self.asynch = asynch
self.flush = eval(self.params.get('config', 'frequent writes', 'False'))
# get directories
self.work_dir = self.params.get('paths', 'work', '/tmp/weather')
if not os.path.isdir(self.work_dir):
raise RuntimeError(
'Directory "' + self.work_dir + '" does not exist.')
self.template_dir = self.params.get(
'paths', 'templates', os.path.expanduser('~/weather/templates/'))
self.graph_template_dir = self.params.get(
'paths', 'graph_templates', os.path.expanduser('~/weather/graph_templates/'))
self.local_dir = self.params.get(
'paths', 'local_files', os.path.expanduser('~/weather/results/'))
# create calibration object
self.calibrator = Calib(self.params, self.raw_data)
# create templater object
self.templater = Template.Template(
self.params, self.status, self.calib_data, self.hourly_data,
self.daily_data, self.monthly_data)
# create plotter objects
self.plotter = Plot.GraphPlotter(
self.params, self.status, self.calib_data, self.hourly_data,
self.daily_data, self.monthly_data, self.work_dir)
self.roseplotter = WindRose.RosePlotter(
self.params, self.status, self.calib_data, self.hourly_data,
self.daily_data, self.monthly_data, self.work_dir)
# create FTP uploader object
self.uploader = Upload.Upload(self.params)
self.uploads_directory = os.path.join(self.work_dir, 'uploads')
if not os.path.isdir(self.uploads_directory):
os.mkdir(self.uploads_directory)
# delay creation of a Twitter object until we know it's needed
self.twitter = None
# create a YoWindow object
self.yowindow = YoWindow.YoWindow(self.calib_data)
# get daytime end hour, in UTC
self.day_end_hour = eval(params.get('config', 'day end hour', '21'))
self.day_end_hour = (self.day_end_hour - (STDOFFSET.seconds // 3600)) % 24
# parse "cron" sections
self.cron = {}
for section in self.params._config.sections():
if section.split()[0] != 'cron':
continue
import croniter
self.cron[section] = croniter.croniter(
self.params.get(section, 'format', ''))
self.cron[section].get_prev()
last_update = self.status.get_datetime('last update', section)
if last_update:
last_update = last_update + Local.utcoffset(last_update)
while self.cron[section].get_current(datetime) <= last_update:
self.cron[section].get_next()
# create service uploader objects
self.services = {}
for section in self.cron.keys() + [
'live', 'logged', 'hourly', '12 hourly', 'daily']:
for name in eval(self.params.get(section, 'services', '[]')):
if name not in self.services:
self.services[name] = ToService(
self.params, self.status, self.calib_data,
service_name=name)
# check for deprecated syntax
if self.params.get(section, 'twitter') not in (None, '[]'):
self.logger.warning(
'Deprecated twitter entry in [%s]', section)
if self.params.get(section, 'yowindow'):
self.logger.warning(
'Deprecated yowindow entry in [%s]', section)
# create queues for things to upload / send
self.tweet_queue = deque()
self.service_queue = {}
for name in self.services:
self.service_queue[name] = deque()
self.uploads_queue = deque()
# start asynchronous thread to do uploads
if self.asynch:
self.logger.info('Starting asynchronous thread')
self.shutdown_thread = threading.Event()
self.wake_thread = threading.Event()
self.thread = threading.Thread(target=self._asynch_thread)
self.thread.start()
def __init__(self, context):
self.context = context
self.params = context.params
self.status = context.status
self.raw_data = context.raw_data
self.calib_data = context.calib_data
self.hourly_data = context.hourly_data
self.daily_data = context.daily_data
self.monthly_data = context.monthly_data
self.flush = eval(self.params.get('config', 'frequent writes', 'False'))
# get directories
self.work_dir = self.params.get('paths', 'work', '/tmp/pywws')
if not os.path.isdir(self.work_dir):
os.makedirs(self.work_dir)
self.template_dir = self.params.get(
'paths', 'templates', os.path.expanduser('~/weather/templates/'))
self.graph_template_dir = self.params.get(
'paths', 'graph_templates', os.path.expanduser('~/weather/graph_templates/'))
self.local_dir = self.params.get(
'paths', 'local_files', os.path.expanduser('~/weather/results/'))
self.module_dir = self.params.get(
'paths', 'modules', os.path.expanduser('~/weather/modules/'))
# create calibration object
self.calibrator = Calib(self.params, self.raw_data)
# create templater object
self.templater = pywws.template.Template(context)
# create plotter objects
self.plotter = pywws.plot.GraphPlotter(context, self.work_dir)
self.roseplotter = pywws.windrose.RosePlotter(context, self.work_dir)
# create FTP uploads directory
self.uploads_directory = os.path.join(self.work_dir, 'uploads')
if not os.path.isdir(self.uploads_directory):
os.makedirs(self.uploads_directory)
# delay creation of uploader object until we know it's needed
self.uploader = None
# delay creation of a Twitter object until we know it's needed
self.twitter = None
# get daytime end hour
self.day_end_hour, self.use_dst = pywws.process.get_day_end_hour(
self.params)
# parse "cron" sections
self.cron = {}
for section in self.params._config.sections():
if section.split()[0] != 'cron':
continue
import croniter
self.cron[section] = croniter.croniter(
self.params.get(section, 'format', ''))
self.cron[section].get_prev()
last_update = self.status.get_datetime('last update', section)
if last_update:
last_update += timezone.utcoffset(last_update)
while self.cron[section].get_current(datetime) <= last_update:
self.cron[section].get_next()
# create service uploader objects
self.services = {}
for section in list(self.cron.keys()) + [
'live', 'logged', 'hourly', '12 hourly', 'daily']:
for name in eval(self.params.get(section, 'services', '[]')):
if name in self.services:
continue
if os.path.exists(os.path.join(self.module_dir, name + '.py')):
sys.path.insert(0, self.module_dir)
mod = importlib.import_module(name)
del sys.path[0]
else:
mod = importlib.import_module('pywws.service.' + name)
self.services[name] = mod.ToService(context)
# check for obsolete entries
if self.params.get(section, 'twitter'):
logger.error(
'Obsolete twitter entry in weather.ini [%s]', section)
if self.params.get(section, 'yowindow'):
logger.error(
'Obsolete yowindow entry in weather.ini [%s]', section)
# check for 'local' template results
if os.path.isdir(self.local_dir):
return
has_local = False
for section in list(self.cron.keys()) + [
'live', 'logged', 'hourly', '12 hourly', 'daily']:
for template, flags in self._parse_templates(section, 'text'):
if 'L' in flags:
has_local = True
for template, flags in self._parse_templates(section, 'plot'):
if 'L' in flags:
has_local = True
if has_local:
os.makedirs(self.local_dir)
break
class RegularTasks(object):
def __init__(self, context):
self.context = context
self.params = context.params
self.status = context.status
self.raw_data = context.raw_data
self.calib_data = context.calib_data
self.hourly_data = context.hourly_data
self.daily_data = context.daily_data
self.monthly_data = context.monthly_data
self.flush = eval(self.params.get('config', 'frequent writes', 'False'))
# get directories
self.work_dir = self.params.get('paths', 'work', '/tmp/pywws')
if not os.path.isdir(self.work_dir):
os.makedirs(self.work_dir)
self.template_dir = self.params.get(
'paths', 'templates', os.path.expanduser('~/weather/templates/'))
self.graph_template_dir = self.params.get(
'paths', 'graph_templates', os.path.expanduser('~/weather/graph_templates/'))
self.local_dir = self.params.get(
'paths', 'local_files', os.path.expanduser('~/weather/results/'))
self.module_dir = self.params.get(
'paths', 'modules', os.path.expanduser('~/weather/modules/'))
# create calibration object
self.calibrator = Calib(self.params, self.raw_data)
# create templater object
self.templater = pywws.template.Template(context)
# create plotter objects
self.plotter = pywws.plot.GraphPlotter(context, self.work_dir)
self.roseplotter = pywws.windrose.RosePlotter(context, self.work_dir)
# create FTP uploads directory
self.uploads_directory = os.path.join(self.work_dir, 'uploads')
if not os.path.isdir(self.uploads_directory):
os.makedirs(self.uploads_directory)
# delay creation of uploader object until we know it's needed
self.uploader = None
# delay creation of a Twitter object until we know it's needed
self.twitter = None
# get daytime end hour
self.day_end_hour, self.use_dst = pywws.process.get_day_end_hour(
self.params)
# parse "cron" sections
self.cron = {}
for section in self.params._config.sections():
if section.split()[0] != 'cron':
continue
import croniter
self.cron[section] = croniter.croniter(
self.params.get(section, 'format', ''))
self.cron[section].get_prev()
last_update = self.status.get_datetime('last update', section)
if last_update:
last_update += timezone.utcoffset(last_update)
while self.cron[section].get_current(datetime) <= last_update:
self.cron[section].get_next()
# create service uploader objects
self.services = {}
for section in list(self.cron.keys()) + [
'live', 'logged', 'hourly', '12 hourly', 'daily']:
for name in eval(self.params.get(section, 'services', '[]')):
if name in self.services:
continue
if os.path.exists(os.path.join(self.module_dir, name + '.py')):
sys.path.insert(0, self.module_dir)
mod = importlib.import_module(name)
del sys.path[0]
else:
mod = importlib.import_module('pywws.service.' + name)
self.services[name] = mod.ToService(context)
# check for obsolete entries
if self.params.get(section, 'twitter'):
logger.error(
'Obsolete twitter entry in weather.ini [%s]', section)
if self.params.get(section, 'yowindow'):
logger.error(
'Obsolete yowindow entry in weather.ini [%s]', section)
# check for 'local' template results
if os.path.isdir(self.local_dir):
return
has_local = False
for section in list(self.cron.keys()) + [
'live', 'logged', 'hourly', '12 hourly', 'daily']:
for template, flags in self._parse_templates(section, 'text'):
if 'L' in flags:
has_local = True
for template, flags in self._parse_templates(section, 'plot'):
if 'L' in flags:
has_local = True
if has_local:
os.makedirs(self.local_dir)
break
def has_live_tasks(self):
if self.cron:
return True
for name in eval(self.params.get('live', 'services', '[]')):
return True
for template in eval(self.params.get('live', 'plot', '[]')):
return True
for template in eval(self.params.get('live', 'text', '[]')):
return True
return False
def _parse_templates(self, section, option):
for template in eval(self.params.get(section, option, '[]')):
if isinstance(template, (list, tuple)):
yield template
else:
yield template, ''
def _do_common(self, now, sections, live_data=None):
logger.info('doing task sections {!r}'.format(sections))
# make lists of tasks from all sections, avoiding repeats
service_tasks = []
text_tasks = []
plot_tasks = []
for section in sections:
for name in eval(self.params.get(section, 'services', '[]')):
if name not in service_tasks:
service_tasks.append(name)
for task in self._parse_templates(section, 'text'):
if task not in text_tasks:
text_tasks.append(task)
for task in self._parse_templates(section, 'plot'):
if task not in plot_tasks:
plot_tasks.append(task)
# do service tasks
for name in service_tasks:
if name in self.services:
self.services[name].upload(live_data=live_data)
# do plot templates
for template, flags in plot_tasks:
self.do_plot(template, local='L' in flags)
# do text templates
for template, flags in text_tasks:
if 'T' in flags:
self.do_twitter(template, live_data)
continue
self.do_template(template, data=live_data, local='L' in flags)
# upload non local files
upload_files = []
for name in os.listdir(self.uploads_directory):
path = os.path.join(self.uploads_directory, name)
if os.path.isfile(path):
upload_files.append(path)
if upload_files:
if not self.uploader:
self.uploader = pywws.towebsite.ToWebSite(self.context)
self.uploader.upload(upload_files, delete=True)
# update status
for section in sections:
self.status.set('last update', section, now.isoformat(' '))
# save any unsaved data
if self.flush or 'hourly' in sections:
self.context.flush()
def _cron_due(self, now):
if not self.cron:
return []
# convert now to local time
local_now = now + timezone.utcoffset(now)
# make list of due sections
sections = []
for section in self.cron:
if self.cron[section].get_current(datetime) > local_now:
continue
sections.append(section)
while self.cron[section].get_current(datetime) <= local_now:
self.cron[section].get_next()
return sections
def _periodic_due(self, now):
# get start of current hour, allowing for odd time zones
threshold = timezone.local_replace(now, minute=0, second=0)
# make list of due sections
sections = []
# hourly
last_update = self.status.get_datetime('last update', 'hourly')
if not last_update or last_update < threshold:
sections.append('hourly')
# daily
threshold = timezone.local_replace(
threshold, use_dst=self.use_dst, hour=self.day_end_hour)
last_update = self.status.get_datetime('last update', 'daily')
if not last_update or last_update < threshold:
sections.append('daily')
# 12 hourly == daily
last_update = self.status.get_datetime('last update', '12 hourly')
if not last_update or last_update < threshold:
sections.append('12 hourly')
return sections
# 12 hourly == daily +- 12 hours
threshold = timezone.local_replace(
now, use_dst=self.use_dst,
hour=((self.day_end_hour + 12) % 24), minute=0, second=0)
if last_update < threshold:
sections.append('12 hourly')
return sections
def do_live(self, data):
calib_data = self.calibrator.calib(data)
now = calib_data['idx']
sections = ['live'] + self._cron_due(now) + self._periodic_due(now)
self._do_common(now, sections, live_data=calib_data)
def do_tasks(self):
now = self.calib_data.before(datetime.max)
if not now:
raise RuntimeError('No processed data available')
if not self.context.live_logging:
# do periodic tasks if they would be due by next logging time
now += timedelta(minutes=self.calib_data[now]['delay'])
sections = ['logged'] + self._cron_due(now) + self._periodic_due(now)
self._do_common(now, sections)
if not self.context.live_logging:
# cleanly shut down upload threads
for name in self.services:
self.services[name].stop()
if self.twitter:
self.twitter.stop()
if self.uploader:
self.uploader.stop()
def do_twitter(self, template, data=None):
if not self.twitter:
import pywws.totwitter
self.twitter = pywws.totwitter.ToTwitter(self.context)
logger.info("Templating %s", template)
input_file = os.path.join(self.template_dir, template)
tweet = self.templater.make_text(input_file, live_data=data)
logger.info("Tweeting")
self.twitter.upload(tweet)
def do_plot(self, template, local=False):
logger.info("Graphing %s", template)
input_file = os.path.join(self.graph_template_dir, template)
output_file = os.path.splitext(template)[0]
if local:
output_file = os.path.join(self.local_dir, output_file)
else:
output_file = os.path.join(self.uploads_directory, output_file)
input_xml = pywws.plot.GraphFileReader(input_file)
if (input_xml.get_children(self.plotter.plot_name) and
self.plotter.do_plot(input_xml, output_file) == 0):
return output_file
if (input_xml.get_children(self.roseplotter.plot_name) and
self.roseplotter.do_plot(input_xml, output_file) == 0):
return output_file
logger.warning('nothing to graph in %s', input_file)
return None
def do_template(self, template, data=None, local=False):
logger.info("Templating %s", template)
input_file = os.path.join(self.template_dir, template)
if local:
output_file = os.path.join(self.local_dir, template)
else:
output_file = os.path.join(self.uploads_directory, template)
self.templater.make_file(input_file, output_file, live_data=data)
return output_file
def __init__(self, context):
self.context = context
self.params = context.params
self.status = context.status
self.raw_data = context.raw_data
self.calib_data = context.calib_data
self.hourly_data = context.hourly_data
self.daily_data = context.daily_data
self.monthly_data = context.monthly_data
self.flush = eval(self.params.get('config', 'frequent writes',
'False'))
# get directories
self.work_dir = self.params.get('paths', 'work', '/tmp/pywws')
if not os.path.isdir(self.work_dir):
os.makedirs(self.work_dir)
self.template_dir = self.params.get(
'paths', 'templates', os.path.expanduser('~/weather/templates/'))
self.graph_template_dir = self.params.get(
'paths', 'graph_templates',
os.path.expanduser('~/weather/graph_templates/'))
self.local_dir = self.params.get(
'paths', 'local_files', os.path.expanduser('~/weather/results/'))
self.module_dir = self.params.get(
'paths', 'modules', os.path.expanduser('~/weather/modules/'))
# create calibration object
self.calibrator = Calib(self.params, self.raw_data)
# create templater object
self.templater = pywws.template.Template(context)
# create plotter objects
self.plotter = pywws.plot.GraphPlotter(context, self.work_dir)
self.roseplotter = pywws.windrose.RosePlotter(context, self.work_dir)
# create FTP uploads directory
self.uploads_directory = os.path.join(self.work_dir, 'uploads')
if not os.path.isdir(self.uploads_directory):
os.makedirs(self.uploads_directory)
# delay creation of uploader object until we know it's needed
self.uploader = None
# delay creation of a Twitter object until we know it's needed
self.twitter = None
# get daytime end hour
self.day_end_hour, self.use_dst = pywws.process.get_day_end_hour(
self.params)
# parse "cron" sections
self.cron = {}
for section in self.params._config.sections():
if section.split()[0] != 'cron':
continue
import croniter
self.cron[section] = croniter.croniter(
self.params.get(section, 'format', ''))
self.cron[section].get_prev()
last_update = self.status.get_datetime('last update', section)
if last_update:
last_update += timezone.utcoffset(last_update)
while self.cron[section].get_current(datetime) <= last_update:
self.cron[section].get_next()
# create service uploader objects
self.services = {}
for section in list(self.cron.keys()) + [
'live', 'logged', 'hourly', '12 hourly', 'daily'
]:
for name in eval(self.params.get(section, 'services', '[]')):
if name in self.services:
continue
if os.path.exists(os.path.join(self.module_dir, name + '.py')):
sys.path.insert(0, self.module_dir)
mod = importlib.import_module(name)
del sys.path[0]
else:
mod = importlib.import_module('pywws.service.' + name)
self.services[name] = mod.ToService(context)
# check for obsolete entries
if self.params.get(section, 'twitter'):
logger.error('Obsolete twitter entry in weather.ini [%s]',
section)
if self.params.get(section, 'yowindow'):
logger.error('Obsolete yowindow entry in weather.ini [%s]',
section)
# check for 'local' template results
if os.path.isdir(self.local_dir):
return
has_local = False
for section in list(self.cron.keys()) + [
'live', 'logged', 'hourly', '12 hourly', 'daily'
]:
for template, flags in self._parse_templates(section, 'text'):
if 'L' in flags:
has_local = True
for template, flags in self._parse_templates(section, 'plot'):
if 'L' in flags:
has_local = True
if has_local:
os.makedirs(self.local_dir)
break
class RegularTasks(object):
def __init__(self, context):
self.context = context
self.params = context.params
self.status = context.status
self.raw_data = context.raw_data
self.calib_data = context.calib_data
self.hourly_data = context.hourly_data
self.daily_data = context.daily_data
self.monthly_data = context.monthly_data
self.flush = eval(self.params.get('config', 'frequent writes',
'False'))
# get directories
self.work_dir = self.params.get('paths', 'work', '/tmp/pywws')
if not os.path.isdir(self.work_dir):
os.makedirs(self.work_dir)
self.template_dir = self.params.get(
'paths', 'templates', os.path.expanduser('~/weather/templates/'))
self.graph_template_dir = self.params.get(
'paths', 'graph_templates',
os.path.expanduser('~/weather/graph_templates/'))
self.local_dir = self.params.get(
'paths', 'local_files', os.path.expanduser('~/weather/results/'))
self.module_dir = self.params.get(
'paths', 'modules', os.path.expanduser('~/weather/modules/'))
# create calibration object
self.calibrator = Calib(self.params, self.raw_data)
# create templater object
self.templater = pywws.template.Template(context)
# create plotter objects
self.plotter = pywws.plot.GraphPlotter(context, self.work_dir)
self.roseplotter = pywws.windrose.RosePlotter(context, self.work_dir)
# create FTP uploads directory
self.uploads_directory = os.path.join(self.work_dir, 'uploads')
if not os.path.isdir(self.uploads_directory):
os.makedirs(self.uploads_directory)
# delay creation of uploader object until we know it's needed
self.uploader = None
# delay creation of a Twitter object until we know it's needed
self.twitter = None
# get daytime end hour
self.day_end_hour, self.use_dst = pywws.process.get_day_end_hour(
self.params)
# parse "cron" sections
self.cron = {}
for section in self.params._config.sections():
if section.split()[0] != 'cron':
continue
import croniter
self.cron[section] = croniter.croniter(
self.params.get(section, 'format', ''))
self.cron[section].get_prev()
last_update = self.status.get_datetime('last update', section)
if last_update:
last_update += timezone.utcoffset(last_update)
while self.cron[section].get_current(datetime) <= last_update:
self.cron[section].get_next()
# create service uploader objects
self.services = {}
for section in list(self.cron.keys()) + [
'live', 'logged', 'hourly', '12 hourly', 'daily'
]:
for name in eval(self.params.get(section, 'services', '[]')):
if name in self.services:
continue
if os.path.exists(os.path.join(self.module_dir, name + '.py')):
sys.path.insert(0, self.module_dir)
mod = importlib.import_module(name)
del sys.path[0]
else:
mod = importlib.import_module('pywws.service.' + name)
self.services[name] = mod.ToService(context)
# check for obsolete entries
if self.params.get(section, 'twitter'):
logger.error('Obsolete twitter entry in weather.ini [%s]',
section)
if self.params.get(section, 'yowindow'):
logger.error('Obsolete yowindow entry in weather.ini [%s]',
section)
# check for 'local' template results
if os.path.isdir(self.local_dir):
return
has_local = False
for section in list(self.cron.keys()) + [
'live', 'logged', 'hourly', '12 hourly', 'daily'
]:
for template, flags in self._parse_templates(section, 'text'):
if 'L' in flags:
has_local = True
for template, flags in self._parse_templates(section, 'plot'):
if 'L' in flags:
has_local = True
if has_local:
os.makedirs(self.local_dir)
break
def has_live_tasks(self):
if self.cron:
return True
for name in eval(self.params.get('live', 'services', '[]')):
return True
for template in eval(self.params.get('live', 'plot', '[]')):
return True
for template in eval(self.params.get('live', 'text', '[]')):
return True
return False
def _parse_templates(self, section, option):
for template in eval(self.params.get(section, option, '[]')):
if isinstance(template, (list, tuple)):
yield template
else:
yield template, ''
def _do_common(self, now, sections, live_data=None):
logger.info('doing task sections {!r}'.format(sections))
# make lists of tasks from all sections, avoiding repeats
service_tasks = []
text_tasks = []
plot_tasks = []
for section in sections:
for name in eval(self.params.get(section, 'services', '[]')):
if name not in service_tasks:
service_tasks.append(name)
for task in self._parse_templates(section, 'text'):
if task not in text_tasks:
text_tasks.append(task)
for task in self._parse_templates(section, 'plot'):
if task not in plot_tasks:
plot_tasks.append(task)
# do service tasks
for name in service_tasks:
if name in self.services:
self.services[name].upload(live_data=live_data)
# do plot templates
for template, flags in plot_tasks:
self.do_plot(template, local='L' in flags)
# do text templates
for template, flags in text_tasks:
if 'T' in flags:
self.do_twitter(template, live_data)
continue
self.do_template(template, data=live_data, local='L' in flags)
# upload non local files
upload_files = []
for name in os.listdir(self.uploads_directory):
path = os.path.join(self.uploads_directory, name)
if os.path.isfile(path):
upload_files.append(path)
if upload_files:
if not self.uploader:
self.uploader = pywws.towebsite.ToWebSite(self.context)
self.uploader.upload(upload_files, delete=True)
# update status
for section in sections:
self.status.set('last update', section, now.isoformat(' '))
# save any unsaved data
if self.flush or 'hourly' in sections:
self.context.flush()
def _cron_due(self, now):
if not self.cron:
return []
# convert now to local time
local_now = now + timezone.utcoffset(now)
# make list of due sections
sections = []
for section in self.cron:
if self.cron[section].get_current(datetime) > local_now:
continue
sections.append(section)
while self.cron[section].get_current(datetime) <= local_now:
self.cron[section].get_next()
return sections
def _periodic_due(self, now):
# get start of current hour, allowing for odd time zones
threshold = timezone.local_replace(now, minute=0, second=0)
# make list of due sections
sections = []
# hourly
last_update = self.status.get_datetime('last update', 'hourly')
if not last_update or last_update < threshold:
sections.append('hourly')
# daily
threshold = timezone.local_replace(threshold,
use_dst=self.use_dst,
hour=self.day_end_hour)
last_update = self.status.get_datetime('last update', 'daily')
if not last_update or last_update < threshold:
sections.append('daily')
# 12 hourly == daily
last_update = self.status.get_datetime('last update', '12 hourly')
if not last_update or last_update < threshold:
sections.append('12 hourly')
return sections
# 12 hourly == daily +- 12 hours
threshold = timezone.local_replace(now,
use_dst=self.use_dst,
hour=((self.day_end_hour + 12) %
24),
minute=0,
second=0)
if last_update < threshold:
sections.append('12 hourly')
return sections
def do_live(self, data):
calib_data = self.calibrator.calib(data)
now = calib_data['idx']
sections = ['live'] + self._cron_due(now) + self._periodic_due(now)
self._do_common(now, sections, live_data=calib_data)
def do_tasks(self):
now = self.calib_data.before(datetime.max)
if not now:
raise RuntimeError('No processed data available')
if not self.context.live_logging:
# do periodic tasks if they would be due by next logging time
now += timedelta(minutes=self.calib_data[now]['delay'])
sections = ['logged'] + self._cron_due(now) + self._periodic_due(now)
self._do_common(now, sections)
if not self.context.live_logging:
# cleanly shut down upload threads
for name in self.services:
self.services[name].stop()
if self.twitter:
self.twitter.stop()
if self.uploader:
self.uploader.stop()
def do_twitter(self, template, data=None):
if not self.twitter:
import pywws.totwitter
self.twitter = pywws.totwitter.ToTwitter(self.context)
logger.info("Templating %s", template)
input_file = os.path.join(self.template_dir, template)
tweet = self.templater.make_text(input_file, live_data=data)
logger.info("Tweeting")
self.twitter.upload(tweet)
def do_plot(self, template, local=False):
logger.info("Graphing %s", template)
input_file = os.path.join(self.graph_template_dir, template)
output_file = os.path.splitext(template)[0]
if local:
output_file = os.path.join(self.local_dir, output_file)
else:
output_file = os.path.join(self.uploads_directory, output_file)
input_xml = pywws.plot.GraphFileReader(input_file)
if (input_xml.get_children(self.plotter.plot_name)
and self.plotter.do_plot(input_xml, output_file) == 0):
return output_file
if (input_xml.get_children(self.roseplotter.plot_name)
and self.roseplotter.do_plot(input_xml, output_file) == 0):
return output_file
logger.warning('nothing to graph in %s', input_file)
return None
def do_template(self, template, data=None, local=False):
logger.info("Templating %s", template)
input_file = os.path.join(self.template_dir, template)
if local:
output_file = os.path.join(self.local_dir, template)
else:
output_file = os.path.join(self.uploads_directory, template)
self.templater.make_file(input_file, output_file, live_data=data)
return output_file
def __init__(self, context):
self.context = context
self.params = context.params
self.status = context.status
self.raw_data = context.raw_data
self.calib_data = context.calib_data
self.hourly_data = context.hourly_data
self.daily_data = context.daily_data
self.monthly_data = context.monthly_data
self.flush = literal_eval(
self.params.get('config', 'frequent writes', 'False'))
# get directories
self.template_dir = self.params.get(
'paths', 'templates', os.path.expanduser('~/weather/templates/'))
self.graph_template_dir = self.params.get(
'paths', 'graph_templates',
os.path.expanduser('~/weather/graph_templates/'))
self.module_dir = self.params.get(
'paths', 'modules', os.path.expanduser('~/weather/modules/'))
# create calibration object
self.calibrator = Calib(self.params, self.raw_data)
# create templater object
self.templater = pywws.template.Template(context)
# create plotter objects
self.plotter = pywws.plot.GraphPlotter(context, self.context.work_dir)
self.roseplotter = pywws.windrose.RosePlotter(context,
self.context.work_dir)
# get daytime end hour
self.day_end_hour, self.use_dst = pywws.process.get_day_end_hour(
self.params)
# parse "cron" sections
self.cron = {}
for section in self.params._config.sections():
if section.split()[0] != 'cron':
continue
import croniter
last_update = self.status.get_datetime('last update', section)
last_update = last_update or datetime.utcnow()
self.cron[section] = croniter.croniter(
self.params.get(section, 'format', ''),
start_time=time_zone.utc_to_local(last_update))
self.cron[section].get_next()
# create service uploader objects
self.services = {}
for section in list(self.cron.keys()) + [
'live', 'logged', 'hourly', '12 hourly', 'daily'
]:
for name, options in self._parse_templates(section, 'services'):
if name in self.services:
continue
if os.path.exists(os.path.join(self.module_dir, name + '.py')):
sys.path.insert(0, self.module_dir)
mod = importlib.import_module(name)
del sys.path[0]
else:
mod = importlib.import_module('pywws.service.' + name)
self.services[name] = mod.ToService(context)
# check for obsolete entries
if self.params.get(section, 'twitter'):
logger.error('Obsolete twitter entry in weather.ini [%s]',
section)
if self.params.get(section, 'yowindow'):
logger.error('Obsolete yowindow entry in weather.ini [%s]',
section)
class RegularTasks(object):
def __init__(self, context):
self.context = context
self.params = context.params
self.status = context.status
self.raw_data = context.raw_data
self.calib_data = context.calib_data
self.hourly_data = context.hourly_data
self.daily_data = context.daily_data
self.monthly_data = context.monthly_data
self.flush = literal_eval(
self.params.get('config', 'frequent writes', 'False'))
# get directories
self.template_dir = self.params.get(
'paths', 'templates', os.path.expanduser('~/weather/templates/'))
self.graph_template_dir = self.params.get(
'paths', 'graph_templates',
os.path.expanduser('~/weather/graph_templates/'))
self.module_dir = self.params.get(
'paths', 'modules', os.path.expanduser('~/weather/modules/'))
# create calibration object
self.calibrator = Calib(self.params, self.raw_data)
# create templater object
self.templater = pywws.template.Template(context)
# create plotter objects
self.plotter = pywws.plot.GraphPlotter(context, self.context.work_dir)
self.roseplotter = pywws.windrose.RosePlotter(context,
self.context.work_dir)
# get daytime end hour
self.day_end_hour, self.use_dst = pywws.process.get_day_end_hour(
self.params)
# parse "cron" sections
self.cron = {}
for section in self.params._config.sections():
if section.split()[0] != 'cron':
continue
import croniter
last_update = self.status.get_datetime('last update', section)
last_update = last_update or datetime.utcnow()
self.cron[section] = croniter.croniter(
self.params.get(section, 'format', ''),
start_time=time_zone.utc_to_local(last_update))
self.cron[section].get_next()
# create service uploader objects
self.services = {}
for section in list(self.cron.keys()) + [
'live', 'logged', 'hourly', '12 hourly', 'daily'
]:
for name, options in self._parse_templates(section, 'services'):
if name in self.services:
continue
if os.path.exists(os.path.join(self.module_dir, name + '.py')):
sys.path.insert(0, self.module_dir)
mod = importlib.import_module(name)
del sys.path[0]
else:
mod = importlib.import_module('pywws.service.' + name)
self.services[name] = mod.ToService(context)
# check for obsolete entries
if self.params.get(section, 'twitter'):
logger.error('Obsolete twitter entry in weather.ini [%s]',
section)
if self.params.get(section, 'yowindow'):
logger.error('Obsolete yowindow entry in weather.ini [%s]',
section)
def has_live_tasks(self):
if self.cron:
return True
for name in literal_eval(self.params.get('live', 'services', '[]')):
return True
for template in literal_eval(self.params.get('live', 'plot', '[]')):
return True
for template in literal_eval(self.params.get('live', 'text', '[]')):
return True
return False
def _parse_templates(self, section, option):
for template in literal_eval(self.params.get(section, option, '[]')):
if isinstance(template, (list, tuple)):
yield template[0], template[1:]
else:
yield template, ()
def _do_common(self, now, sections, live_data=None):
logger.info('doing task sections {!r}'.format(sections))
# make lists of tasks from all sections, avoiding repeats
service_tasks = []
text_tasks = []
plot_tasks = []
for section in sections:
for task in self._parse_templates(section, 'services'):
if task not in service_tasks:
service_tasks.append(task)
for task in self._parse_templates(section, 'text'):
if task not in text_tasks:
text_tasks.append(task)
for task in self._parse_templates(section, 'plot'):
if task not in plot_tasks:
plot_tasks.append(task)
# do plot templates
for template, flags in plot_tasks:
self.do_plot(template)
# do text templates
for template, flags in text_tasks:
self.do_template(template, data=live_data)
# do service tasks
for name, options in service_tasks:
self.services[name].upload(live_data=live_data, options=options)
# allow all services to sent some catchup records
catchup = list(self.services.keys())
stop = time.time() + 20.0
while catchup and time.time() < stop:
for name in list(catchup):
if self.services[name].do_catchup():
catchup.remove(name)
# update status
for section in sections:
self.status.set('last update', section, now.isoformat(' '))
# save any unsaved data
if self.flush or 'hourly' in sections:
self.context.flush()
def _cron_due(self, now):
if not self.cron:
return []
# make list of due sections
sections = []
for section in self.cron:
if time_zone.local_to_utc(
self.cron[section].get_current(datetime)) > now:
continue
sections.append(section)
while time_zone.local_to_utc(
self.cron[section].get_next(datetime)) <= now:
pass
return sections
def _periodic_due(self, now):
# make list of due sections
sections = []
# hourly
threshold = time_zone.hour_start(now)
last_update = self.status.get_datetime('last update', 'hourly')
if not last_update or last_update < threshold:
sections.append('hourly')
# daily
threshold = time_zone.day_start(now,
self.day_end_hour,
use_dst=self.use_dst)
last_update = self.status.get_datetime('last update', 'daily')
if not last_update or last_update < threshold:
sections.append('daily')
# 12 hourly
threshold = max(
threshold,
time_zone.day_start(now, (self.day_end_hour + 12) % 24,
use_dst=self.use_dst))
last_update = self.status.get_datetime('last update', '12 hourly')
if not last_update or last_update < threshold:
sections.append('12 hourly')
return sections
def do_live(self, data):
calib_data = self.calibrator.calib(data)
now = calib_data['idx']
sections = ['live'] + self._cron_due(now) + self._periodic_due(now)
self._do_common(now, sections, live_data=calib_data)
def do_tasks(self):
now = self.calib_data.before(datetime.max)
if not now:
raise RuntimeError('No processed data available')
if not self.context.live_logging:
# do periodic tasks if they would be due by next logging time
now += timedelta(minutes=self.calib_data[now]['delay'])
sections = ['logged'] + self._cron_due(now) + self._periodic_due(now)
self._do_common(now, sections)
if not self.context.live_logging:
# cleanly shut down upload threads
for name in self.services:
self.services[name].stop()
def do_plot(self, template):
logger.info("Graphing %s", template)
input_file = os.path.join(self.graph_template_dir, template)
output_file = os.path.join(self.context.output_dir,
os.path.splitext(template)[0])
input_xml = pywws.plot.GraphFileReader(input_file)
if (input_xml.get_children(self.plotter.plot_name)
and self.plotter.do_plot(input_xml, output_file) == 0):
return output_file
if (input_xml.get_children(self.roseplotter.plot_name)
and self.roseplotter.do_plot(input_xml, output_file) == 0):
return output_file
logger.warning('nothing to graph in %s', input_file)
return None
def do_template(self, template, data=None):
logger.info("Templating %s", template)
input_file = os.path.join(self.template_dir, template)
output_file = os.path.join(self.context.output_dir, template)
self.templater.make_file(input_file, output_file, live_data=data)
return output_file
class RegularTasks(object):
def __init__(self, params, status,
calib_data, hourly_data, daily_data, monthly_data):
self.logger = logging.getLogger('pywws.Tasks.RegularTasks')
self.params = params
self.status = status
self.calib_data = calib_data
self.hourly_data = hourly_data
self.daily_data = daily_data
self.monthly_data = monthly_data
# get directories
self.work_dir = self.params.get('paths', 'work', '/tmp/weather')
self.template_dir = self.params.get(
'paths', 'templates', os.path.expanduser('~/weather/templates/'))
self.graph_template_dir = self.params.get(
'paths', 'graph_templates', os.path.expanduser('~/weather/graph_templates/'))
self.local_dir = self.params.get(
'paths', 'local_files', os.path.expanduser('~/weather/results/'))
# create calibration object
self.calibrator = Calib(self.params, self.status)
# create templater object
self.templater = Template.Template(
self.params, self.status, self.calib_data, self.hourly_data,
self.daily_data, self.monthly_data)
# create plotter objects
self.plotter = Plot.GraphPlotter(
self.params, self.status, self.calib_data, self.hourly_data,
self.daily_data, self.monthly_data, self.work_dir)
self.roseplotter = WindRose.RosePlotter(
self.params, self.status, self.calib_data, self.hourly_data,
self.daily_data, self.monthly_data, self.work_dir)
# directory of service uploaders
self.services = dict()
# create a YoWindow object
self.yowindow = YoWindow.YoWindow(self.calib_data)
# get local time's offset from UTC, without DST
now = self.calib_data.before(datetime.max)
if not now:
now = datetime.utcnow()
time_offset = Local.utcoffset(now) - Local.dst(now)
# get daytime end hour, in UTC
self.day_end_hour = eval(params.get('config', 'day end hour', '21'))
self.day_end_hour = (self.day_end_hour - (time_offset.seconds // 3600)) % 24
# create service uploader objects
for section in ('live', 'logged', 'hourly', '12 hourly', 'daily'):
for service in eval(self.params.get(section, 'services', '[]')):
if service not in self.services:
self.services[service] = ToService(
self.params, self.status, self.calib_data,
service_name=service)
def has_live_tasks(self):
yowindow_file = self.params.get('live', 'yowindow', '')
if yowindow_file:
return True
for template in eval(self.params.get('live', 'twitter', '[]')):
return True
for service in eval(self.params.get('live', 'services', '[]')):
return True
for template in eval(self.params.get('live', 'plot', '[]')):
return True
for template in eval(self.params.get('live', 'text', '[]')):
return True
return False
def _parse_templates(self, section, option):
for template in eval(self.params.get(section, option, '[]')):
if isinstance(template, (list, tuple)):
yield template
else:
yield template, ''
def _do_common(self, sections, live_data=None):
OK = True
for section in sections:
yowindow_file = self.params.get(section, 'yowindow', '')
if yowindow_file:
self.yowindow.write_file(yowindow_file)
break
for section in sections:
for template, flags in self._parse_templates(section, 'twitter'):
if not self.do_twitter(template):
OK = False
all_services = []
for section in sections:
for service in eval(self.params.get(section, 'services', '[]')):
if service not in all_services:
all_services.append(service)
for service in all_services:
self.services[service].Upload(live_data is None, live_data)
uploads = []
local_files = []
for section in sections:
for template, flags in self._parse_templates(section, 'text'):
if 'T' in flags:
if not self.do_twitter(template, live_data):
OK = False
continue
upload = self.do_template(template, live_data)
if 'L' in flags:
local_files.append(upload)
else:
uploads.append(upload)
for template, flags in self._parse_templates(section, 'plot'):
upload = self.do_plot(template)
if not upload:
continue
if 'L' in flags:
local_files.append(upload)
else:
uploads.append(upload)
if local_files:
if not os.path.isdir(self.local_dir):
os.makedirs(self.local_dir)
for file in local_files:
shutil.move(file, self.local_dir)
if uploads:
if not Upload.Upload(self.params, uploads):
OK = False
for file in uploads:
os.unlink(file)
return OK
def do_live(self, data):
return self._do_common(['live'], self.calibrator.calib(data))
def do_tasks(self):
sections = ['logged']
self.params.unset('logged', 'last update')
now = self.calib_data.before(datetime.max)
if now:
now += timedelta(minutes=self.calib_data[now]['delay'])
else:
now = datetime.utcnow()
threshold = now.replace(minute=0, second=0, microsecond=0)
last_update = self.params.get_datetime('hourly', 'last update')
if last_update:
self.params.unset('hourly', 'last update')
self.status.set('last update', 'hourly', last_update.isoformat(' '))
last_update = self.status.get_datetime('last update', 'hourly')
if (not last_update) or (last_update < threshold):
# time to do hourly tasks
sections.append('hourly')
# set 12 hourly threshold
threshold -= timedelta(hours=(threshold.hour - self.day_end_hour) % 12)
last_update = self.params.get_datetime('12 hourly', 'last update')
if last_update:
self.params.unset('12 hourly', 'last update')
self.status.set('last update', '12 hourly', last_update.isoformat(' '))
last_update = self.status.get_datetime('last update', '12 hourly')
if (not last_update) or (last_update < threshold):
# time to do 12 hourly tasks
sections.append('12 hourly')
# set daily threshold
threshold -= timedelta(hours=(threshold.hour - self.day_end_hour) % 24)
last_update = self.params.get_datetime('daily', 'last update')
if last_update:
self.params.unset('daily', 'last update')
self.status.set('last update', 'daily', last_update.isoformat(' '))
last_update = self.status.get_datetime('last update', 'daily')
if (not last_update) or (last_update < threshold):
# time to do daily tasks
sections.append('daily')
OK = self._do_common(sections)
if OK:
for section in sections:
self.status.set('last update', section, now.isoformat(' '))
if 'hourly' in sections:
# save any unsaved data
self.params.flush()
self.status.flush()
self.calib_data.flush()
self.hourly_data.flush()
self.daily_data.flush()
self.monthly_data.flush()
return OK
def do_twitter(self, template, data=None):
from pywws import ToTwitter
twitter = ToTwitter.ToTwitter(self.params)
self.logger.info("Templating %s", template)
input_file = os.path.join(self.template_dir, template)
tweet = self.templater.make_text(input_file, live_data=data)
self.logger.info("Tweeting")
return twitter.Upload(tweet[:140])
def do_plot(self, template):
self.logger.info("Graphing %s", template)
input_file = os.path.join(self.graph_template_dir, template)
output_file = os.path.join(self.work_dir, os.path.splitext(template)[0])
if self.plotter.DoPlot(input_file, output_file) == 0:
return output_file
elif self.roseplotter.DoPlot(input_file, output_file) == 0:
return output_file
return None
def do_template(self, template, data=None):
self.logger.info("Templating %s", template)
input_file = os.path.join(self.template_dir, template)
output_file = os.path.join(self.work_dir, template)
self.templater.make_file(input_file, output_file, live_data=data)
return output_file
