def calibrate_data(logger, params, raw_data, calib_data): """'Calibrate' raw data, using a user-supplied function.""" start = calib_data.before(datetime.max) if start is None: start = datetime.min start = raw_data.after(start + SECOND) if start is None: return start del calib_data[start:] calibrator = Calib(params, raw_data) count = 0 for data in raw_data[start:]: idx = data['idx'] count += 1 if count % 10000 == 0: logger.info("calib: %s", idx.isoformat(' ')) elif count % 500 == 0: logger.debug("calib: %s", idx.isoformat(' ')) for key in ('rain', 'abs_pressure', 'temp_in'): if data[key] is None: logger.error('Ignoring invalid data at %s', idx.isoformat(' ')) break else: calib_data[idx] = calibrator.calib(data) return start
def calibrate_data(params, raw_data, calib_data): """'Calibrate' raw data, using a user-supplied function.""" start = calib_data.before(datetime.max) if start is None: start = datetime.min start = raw_data.after(start + SECOND) if start is None: return start del calib_data[start:] calibrator = Calib(params, raw_data) count = 0 for data in raw_data[start:]: idx = data['idx'] count += 1 if count % 10000 == 0: logger.info("calib: %s", idx.isoformat(' ')) elif count % 500 == 0: logger.debug("calib: %s", idx.isoformat(' ')) for key in ('rain', 'abs_pressure', 'temp_in'): if data[key] is None: logger.error('Ignoring invalid data at %s', idx.isoformat(' ')) break else: calib_data[idx] = calibrator.calib(data) return start
def calibrate_data(logger, params, raw_data, calib_data): """'Calibrate' raw data, using a user-supplied function.""" start = calib_data.before(datetime.max) if start is None: start = datetime.min start = raw_data.after(start + SECOND) if start is None: return start del calib_data[start:] calibrator = Calib(params, raw_data) count = 0 for data in raw_data[start:]: idx = data['idx'] count += 1 if count % 10000 == 0: logger.info("calib: %s", idx.isoformat(' ')) elif count % 500 == 0: logger.debug("calib: %s", idx.isoformat(' ')) calib_data[idx] = calibrator.calib(data) return start
class RegularTasks(object): def __init__(self, context): self.context = context self.params = context.params self.status = context.status self.raw_data = context.raw_data self.calib_data = context.calib_data self.hourly_data = context.hourly_data self.daily_data = context.daily_data self.monthly_data = context.monthly_data self.flush = eval(self.params.get('config', 'frequent writes', 'False')) # get directories self.work_dir = self.params.get('paths', 'work', '/tmp/pywws') if not os.path.isdir(self.work_dir): os.makedirs(self.work_dir) self.template_dir = self.params.get( 'paths', 'templates', os.path.expanduser('~/weather/templates/')) self.graph_template_dir = self.params.get( 'paths', 'graph_templates', os.path.expanduser('~/weather/graph_templates/')) self.local_dir = self.params.get( 'paths', 'local_files', os.path.expanduser('~/weather/results/')) self.module_dir = self.params.get( 'paths', 'modules', os.path.expanduser('~/weather/modules/')) # create calibration object self.calibrator = Calib(self.params, self.raw_data) # create templater object self.templater = pywws.template.Template(context) # create plotter objects self.plotter = pywws.plot.GraphPlotter(context, self.work_dir) self.roseplotter = pywws.windrose.RosePlotter(context, self.work_dir) # create FTP uploads directory self.uploads_directory = os.path.join(self.work_dir, 'uploads') if not os.path.isdir(self.uploads_directory): os.makedirs(self.uploads_directory) # delay creation of uploader object until we know it's needed self.uploader = None # delay creation of a Twitter object until we know it's needed self.twitter = None # get daytime end hour self.day_end_hour, self.use_dst = pywws.process.get_day_end_hour( self.params) # parse "cron" sections self.cron = {} for section in self.params._config.sections(): if section.split()[0] != 'cron': continue import croniter self.cron[section] = croniter.croniter( self.params.get(section, 'format', '')) self.cron[section].get_prev() last_update = self.status.get_datetime('last update', section) if last_update: last_update += timezone.utcoffset(last_update) while self.cron[section].get_current(datetime) <= last_update: self.cron[section].get_next() # create service uploader objects self.services = {} for section in list(self.cron.keys()) + [ 'live', 'logged', 'hourly', '12 hourly', 'daily']: for name in eval(self.params.get(section, 'services', '[]')): if name in self.services: continue if os.path.exists(os.path.join(self.module_dir, name + '.py')): sys.path.insert(0, self.module_dir) mod = importlib.import_module(name) del sys.path[0] else: mod = importlib.import_module('pywws.service.' + name) self.services[name] = mod.ToService(context) # check for obsolete entries if self.params.get(section, 'twitter'): logger.error( 'Obsolete twitter entry in weather.ini [%s]', section) if self.params.get(section, 'yowindow'): logger.error( 'Obsolete yowindow entry in weather.ini [%s]', section) # check for 'local' template results if os.path.isdir(self.local_dir): return has_local = False for section in list(self.cron.keys()) + [ 'live', 'logged', 'hourly', '12 hourly', 'daily']: for template, flags in self._parse_templates(section, 'text'): if 'L' in flags: has_local = True for template, flags in self._parse_templates(section, 'plot'): if 'L' in flags: has_local = True if has_local: os.makedirs(self.local_dir) break def has_live_tasks(self): if self.cron: return True for name in eval(self.params.get('live', 'services', '[]')): return True for template in eval(self.params.get('live', 'plot', '[]')): return True for template in eval(self.params.get('live', 'text', '[]')): return True return False def _parse_templates(self, section, option): for template in eval(self.params.get(section, option, '[]')): if isinstance(template, (list, tuple)): yield template else: yield template, '' def _do_common(self, now, sections, live_data=None): logger.info('doing task sections {!r}'.format(sections)) # make lists of tasks from all sections, avoiding repeats service_tasks = [] text_tasks = [] plot_tasks = [] for section in sections: for name in eval(self.params.get(section, 'services', '[]')): if name not in service_tasks: service_tasks.append(name) for task in self._parse_templates(section, 'text'): if task not in text_tasks: text_tasks.append(task) for task in self._parse_templates(section, 'plot'): if task not in plot_tasks: plot_tasks.append(task) # do service tasks for name in service_tasks: if name in self.services: self.services[name].upload(live_data=live_data) # do plot templates for template, flags in plot_tasks: self.do_plot(template, local='L' in flags) # do text templates for template, flags in text_tasks: if 'T' in flags: self.do_twitter(template, live_data) continue self.do_template(template, data=live_data, local='L' in flags) # upload non local files upload_files = [] for name in os.listdir(self.uploads_directory): path = os.path.join(self.uploads_directory, name) if os.path.isfile(path): upload_files.append(path) if upload_files: if not self.uploader: self.uploader = pywws.towebsite.ToWebSite(self.context) self.uploader.upload(upload_files, delete=True) # update status for section in sections: self.status.set('last update', section, now.isoformat(' ')) # save any unsaved data if self.flush or 'hourly' in sections: self.context.flush() def _cron_due(self, now): if not self.cron: return [] # convert now to local time local_now = now + timezone.utcoffset(now) # make list of due sections sections = [] for section in self.cron: if self.cron[section].get_current(datetime) > local_now: continue sections.append(section) while self.cron[section].get_current(datetime) <= local_now: self.cron[section].get_next() return sections def _periodic_due(self, now): # get start of current hour, allowing for odd time zones threshold = timezone.local_replace(now, minute=0, second=0) # make list of due sections sections = [] # hourly last_update = self.status.get_datetime('last update', 'hourly') if not last_update or last_update < threshold: sections.append('hourly') # daily threshold = timezone.local_replace( threshold, use_dst=self.use_dst, hour=self.day_end_hour) last_update = self.status.get_datetime('last update', 'daily') if not last_update or last_update < threshold: sections.append('daily') # 12 hourly == daily last_update = self.status.get_datetime('last update', '12 hourly') if not last_update or last_update < threshold: sections.append('12 hourly') return sections # 12 hourly == daily +- 12 hours threshold = timezone.local_replace( now, use_dst=self.use_dst, hour=((self.day_end_hour + 12) % 24), minute=0, second=0) if last_update < threshold: sections.append('12 hourly') return sections def do_live(self, data): calib_data = self.calibrator.calib(data) now = calib_data['idx'] sections = ['live'] + self._cron_due(now) + self._periodic_due(now) self._do_common(now, sections, live_data=calib_data) def do_tasks(self): now = self.calib_data.before(datetime.max) if not now: raise RuntimeError('No processed data available') if not self.context.live_logging: # do periodic tasks if they would be due by next logging time now += timedelta(minutes=self.calib_data[now]['delay']) sections = ['logged'] + self._cron_due(now) + self._periodic_due(now) self._do_common(now, sections) if not self.context.live_logging: # cleanly shut down upload threads for name in self.services: self.services[name].stop() if self.twitter: self.twitter.stop() if self.uploader: self.uploader.stop() def do_twitter(self, template, data=None): if not self.twitter: import pywws.totwitter self.twitter = pywws.totwitter.ToTwitter(self.context) logger.info("Templating %s", template) input_file = os.path.join(self.template_dir, template) tweet = self.templater.make_text(input_file, live_data=data) logger.info("Tweeting") self.twitter.upload(tweet) def do_plot(self, template, local=False): logger.info("Graphing %s", template) input_file = os.path.join(self.graph_template_dir, template) output_file = os.path.splitext(template)[0] if local: output_file = os.path.join(self.local_dir, output_file) else: output_file = os.path.join(self.uploads_directory, output_file) input_xml = pywws.plot.GraphFileReader(input_file) if (input_xml.get_children(self.plotter.plot_name) and self.plotter.do_plot(input_xml, output_file) == 0): return output_file if (input_xml.get_children(self.roseplotter.plot_name) and self.roseplotter.do_plot(input_xml, output_file) == 0): return output_file logger.warning('nothing to graph in %s', input_file) return None def do_template(self, template, data=None, local=False): logger.info("Templating %s", template) input_file = os.path.join(self.template_dir, template) if local: output_file = os.path.join(self.local_dir, template) else: output_file = os.path.join(self.uploads_directory, template) self.templater.make_file(input_file, output_file, live_data=data) return output_file
class RegularTasks(object): def __init__(self, context): self.context = context self.params = context.params self.status = context.status self.raw_data = context.raw_data self.calib_data = context.calib_data self.hourly_data = context.hourly_data self.daily_data = context.daily_data self.monthly_data = context.monthly_data self.flush = eval(self.params.get('config', 'frequent writes', 'False')) # get directories self.work_dir = self.params.get('paths', 'work', '/tmp/pywws') if not os.path.isdir(self.work_dir): os.makedirs(self.work_dir) self.template_dir = self.params.get( 'paths', 'templates', os.path.expanduser('~/weather/templates/')) self.graph_template_dir = self.params.get( 'paths', 'graph_templates', os.path.expanduser('~/weather/graph_templates/')) self.local_dir = self.params.get( 'paths', 'local_files', os.path.expanduser('~/weather/results/')) self.module_dir = self.params.get( 'paths', 'modules', os.path.expanduser('~/weather/modules/')) # create calibration object self.calibrator = Calib(self.params, self.raw_data) # create templater object self.templater = pywws.template.Template(context) # create plotter objects self.plotter = pywws.plot.GraphPlotter(context, self.work_dir) self.roseplotter = pywws.windrose.RosePlotter(context, self.work_dir) # create FTP uploads directory self.uploads_directory = os.path.join(self.work_dir, 'uploads') if not os.path.isdir(self.uploads_directory): os.makedirs(self.uploads_directory) # delay creation of uploader object until we know it's needed self.uploader = None # delay creation of a Twitter object until we know it's needed self.twitter = None # get daytime end hour self.day_end_hour, self.use_dst = pywws.process.get_day_end_hour( self.params) # parse "cron" sections self.cron = {} for section in self.params._config.sections(): if section.split()[0] != 'cron': continue import croniter self.cron[section] = croniter.croniter( self.params.get(section, 'format', '')) self.cron[section].get_prev() last_update = self.status.get_datetime('last update', section) if last_update: last_update += timezone.utcoffset(last_update) while self.cron[section].get_current(datetime) <= last_update: self.cron[section].get_next() # create service uploader objects self.services = {} for section in list(self.cron.keys()) + [ 'live', 'logged', 'hourly', '12 hourly', 'daily' ]: for name in eval(self.params.get(section, 'services', '[]')): if name in self.services: continue if os.path.exists(os.path.join(self.module_dir, name + '.py')): sys.path.insert(0, self.module_dir) mod = importlib.import_module(name) del sys.path[0] else: mod = importlib.import_module('pywws.service.' + name) self.services[name] = mod.ToService(context) # check for obsolete entries if self.params.get(section, 'twitter'): logger.error('Obsolete twitter entry in weather.ini [%s]', section) if self.params.get(section, 'yowindow'): logger.error('Obsolete yowindow entry in weather.ini [%s]', section) # check for 'local' template results if os.path.isdir(self.local_dir): return has_local = False for section in list(self.cron.keys()) + [ 'live', 'logged', 'hourly', '12 hourly', 'daily' ]: for template, flags in self._parse_templates(section, 'text'): if 'L' in flags: has_local = True for template, flags in self._parse_templates(section, 'plot'): if 'L' in flags: has_local = True if has_local: os.makedirs(self.local_dir) break def has_live_tasks(self): if self.cron: return True for name in eval(self.params.get('live', 'services', '[]')): return True for template in eval(self.params.get('live', 'plot', '[]')): return True for template in eval(self.params.get('live', 'text', '[]')): return True return False def _parse_templates(self, section, option): for template in eval(self.params.get(section, option, '[]')): if isinstance(template, (list, tuple)): yield template else: yield template, '' def _do_common(self, now, sections, live_data=None): logger.info('doing task sections {!r}'.format(sections)) # make lists of tasks from all sections, avoiding repeats service_tasks = [] text_tasks = [] plot_tasks = [] for section in sections: for name in eval(self.params.get(section, 'services', '[]')): if name not in service_tasks: service_tasks.append(name) for task in self._parse_templates(section, 'text'): if task not in text_tasks: text_tasks.append(task) for task in self._parse_templates(section, 'plot'): if task not in plot_tasks: plot_tasks.append(task) # do service tasks for name in service_tasks: if name in self.services: self.services[name].upload(live_data=live_data) # do plot templates for template, flags in plot_tasks: self.do_plot(template, local='L' in flags) # do text templates for template, flags in text_tasks: if 'T' in flags: self.do_twitter(template, live_data) continue self.do_template(template, data=live_data, local='L' in flags) # upload non local files upload_files = [] for name in os.listdir(self.uploads_directory): path = os.path.join(self.uploads_directory, name) if os.path.isfile(path): upload_files.append(path) if upload_files: if not self.uploader: self.uploader = pywws.towebsite.ToWebSite(self.context) self.uploader.upload(upload_files, delete=True) # update status for section in sections: self.status.set('last update', section, now.isoformat(' ')) # save any unsaved data if self.flush or 'hourly' in sections: self.context.flush() def _cron_due(self, now): if not self.cron: return [] # convert now to local time local_now = now + timezone.utcoffset(now) # make list of due sections sections = [] for section in self.cron: if self.cron[section].get_current(datetime) > local_now: continue sections.append(section) while self.cron[section].get_current(datetime) <= local_now: self.cron[section].get_next() return sections def _periodic_due(self, now): # get start of current hour, allowing for odd time zones threshold = timezone.local_replace(now, minute=0, second=0) # make list of due sections sections = [] # hourly last_update = self.status.get_datetime('last update', 'hourly') if not last_update or last_update < threshold: sections.append('hourly') # daily threshold = timezone.local_replace(threshold, use_dst=self.use_dst, hour=self.day_end_hour) last_update = self.status.get_datetime('last update', 'daily') if not last_update or last_update < threshold: sections.append('daily') # 12 hourly == daily last_update = self.status.get_datetime('last update', '12 hourly') if not last_update or last_update < threshold: sections.append('12 hourly') return sections # 12 hourly == daily +- 12 hours threshold = timezone.local_replace(now, use_dst=self.use_dst, hour=((self.day_end_hour + 12) % 24), minute=0, second=0) if last_update < threshold: sections.append('12 hourly') return sections def do_live(self, data): calib_data = self.calibrator.calib(data) now = calib_data['idx'] sections = ['live'] + self._cron_due(now) + self._periodic_due(now) self._do_common(now, sections, live_data=calib_data) def do_tasks(self): now = self.calib_data.before(datetime.max) if not now: raise RuntimeError('No processed data available') if not self.context.live_logging: # do periodic tasks if they would be due by next logging time now += timedelta(minutes=self.calib_data[now]['delay']) sections = ['logged'] + self._cron_due(now) + self._periodic_due(now) self._do_common(now, sections) if not self.context.live_logging: # cleanly shut down upload threads for name in self.services: self.services[name].stop() if self.twitter: self.twitter.stop() if self.uploader: self.uploader.stop() def do_twitter(self, template, data=None): if not self.twitter: import pywws.totwitter self.twitter = pywws.totwitter.ToTwitter(self.context) logger.info("Templating %s", template) input_file = os.path.join(self.template_dir, template) tweet = self.templater.make_text(input_file, live_data=data) logger.info("Tweeting") self.twitter.upload(tweet) def do_plot(self, template, local=False): logger.info("Graphing %s", template) input_file = os.path.join(self.graph_template_dir, template) output_file = os.path.splitext(template)[0] if local: output_file = os.path.join(self.local_dir, output_file) else: output_file = os.path.join(self.uploads_directory, output_file) input_xml = pywws.plot.GraphFileReader(input_file) if (input_xml.get_children(self.plotter.plot_name) and self.plotter.do_plot(input_xml, output_file) == 0): return output_file if (input_xml.get_children(self.roseplotter.plot_name) and self.roseplotter.do_plot(input_xml, output_file) == 0): return output_file logger.warning('nothing to graph in %s', input_file) return None def do_template(self, template, data=None, local=False): logger.info("Templating %s", template) input_file = os.path.join(self.template_dir, template) if local: output_file = os.path.join(self.local_dir, template) else: output_file = os.path.join(self.uploads_directory, template) self.templater.make_file(input_file, output_file, live_data=data) return output_file
class RegularTasks(object): def __init__(self, context): self.context = context self.params = context.params self.status = context.status self.raw_data = context.raw_data self.calib_data = context.calib_data self.hourly_data = context.hourly_data self.daily_data = context.daily_data self.monthly_data = context.monthly_data self.flush = literal_eval( self.params.get('config', 'frequent writes', 'False')) # get directories self.template_dir = self.params.get( 'paths', 'templates', os.path.expanduser('~/weather/templates/')) self.graph_template_dir = self.params.get( 'paths', 'graph_templates', os.path.expanduser('~/weather/graph_templates/')) self.module_dir = self.params.get( 'paths', 'modules', os.path.expanduser('~/weather/modules/')) # create calibration object self.calibrator = Calib(self.params, self.raw_data) # create templater object self.templater = pywws.template.Template(context) # create plotter objects self.plotter = pywws.plot.GraphPlotter(context, self.context.work_dir) self.roseplotter = pywws.windrose.RosePlotter(context, self.context.work_dir) # get daytime end hour self.day_end_hour, self.use_dst = pywws.process.get_day_end_hour( self.params) # parse "cron" sections self.cron = {} for section in self.params._config.sections(): if section.split()[0] != 'cron': continue import croniter last_update = self.status.get_datetime('last update', section) last_update = last_update or datetime.utcnow() self.cron[section] = croniter.croniter( self.params.get(section, 'format', ''), start_time=time_zone.utc_to_local(last_update)) self.cron[section].get_next() # create service uploader objects self.services = {} for section in list(self.cron.keys()) + [ 'live', 'logged', 'hourly', '12 hourly', 'daily' ]: for name, options in self._parse_templates(section, 'services'): if name in self.services: continue if os.path.exists(os.path.join(self.module_dir, name + '.py')): sys.path.insert(0, self.module_dir) mod = importlib.import_module(name) del sys.path[0] else: mod = importlib.import_module('pywws.service.' + name) self.services[name] = mod.ToService(context) # check for obsolete entries if self.params.get(section, 'twitter'): logger.error('Obsolete twitter entry in weather.ini [%s]', section) if self.params.get(section, 'yowindow'): logger.error('Obsolete yowindow entry in weather.ini [%s]', section) def has_live_tasks(self): if self.cron: return True for name in literal_eval(self.params.get('live', 'services', '[]')): return True for template in literal_eval(self.params.get('live', 'plot', '[]')): return True for template in literal_eval(self.params.get('live', 'text', '[]')): return True return False def _parse_templates(self, section, option): for template in literal_eval(self.params.get(section, option, '[]')): if isinstance(template, (list, tuple)): yield template[0], template[1:] else: yield template, () def _do_common(self, now, sections, live_data=None): logger.info('doing task sections {!r}'.format(sections)) # make lists of tasks from all sections, avoiding repeats service_tasks = [] text_tasks = [] plot_tasks = [] for section in sections: for task in self._parse_templates(section, 'services'): if task not in service_tasks: service_tasks.append(task) for task in self._parse_templates(section, 'text'): if task not in text_tasks: text_tasks.append(task) for task in self._parse_templates(section, 'plot'): if task not in plot_tasks: plot_tasks.append(task) # do plot templates for template, flags in plot_tasks: self.do_plot(template) # do text templates for template, flags in text_tasks: self.do_template(template, data=live_data) # do service tasks for name, options in service_tasks: self.services[name].upload(live_data=live_data, options=options) # allow all services to sent some catchup records catchup = list(self.services.keys()) stop = time.time() + 20.0 while catchup and time.time() < stop: for name in list(catchup): if self.services[name].do_catchup(): catchup.remove(name) # update status for section in sections: self.status.set('last update', section, now.isoformat(' ')) # save any unsaved data if self.flush or 'hourly' in sections: self.context.flush() def _cron_due(self, now): if not self.cron: return [] # make list of due sections sections = [] for section in self.cron: if time_zone.local_to_utc( self.cron[section].get_current(datetime)) > now: continue sections.append(section) while time_zone.local_to_utc( self.cron[section].get_next(datetime)) <= now: pass return sections def _periodic_due(self, now): # make list of due sections sections = [] # hourly threshold = time_zone.hour_start(now) last_update = self.status.get_datetime('last update', 'hourly') if not last_update or last_update < threshold: sections.append('hourly') # daily threshold = time_zone.day_start(now, self.day_end_hour, use_dst=self.use_dst) last_update = self.status.get_datetime('last update', 'daily') if not last_update or last_update < threshold: sections.append('daily') # 12 hourly threshold = max( threshold, time_zone.day_start(now, (self.day_end_hour + 12) % 24, use_dst=self.use_dst)) last_update = self.status.get_datetime('last update', '12 hourly') if not last_update or last_update < threshold: sections.append('12 hourly') return sections def do_live(self, data): calib_data = self.calibrator.calib(data) now = calib_data['idx'] sections = ['live'] + self._cron_due(now) + self._periodic_due(now) self._do_common(now, sections, live_data=calib_data) def do_tasks(self): now = self.calib_data.before(datetime.max) if not now: raise RuntimeError('No processed data available') if not self.context.live_logging: # do periodic tasks if they would be due by next logging time now += timedelta(minutes=self.calib_data[now]['delay']) sections = ['logged'] + self._cron_due(now) + self._periodic_due(now) self._do_common(now, sections) if not self.context.live_logging: # cleanly shut down upload threads for name in self.services: self.services[name].stop() def do_plot(self, template): logger.info("Graphing %s", template) input_file = os.path.join(self.graph_template_dir, template) output_file = os.path.join(self.context.output_dir, os.path.splitext(template)[0]) input_xml = pywws.plot.GraphFileReader(input_file) if (input_xml.get_children(self.plotter.plot_name) and self.plotter.do_plot(input_xml, output_file) == 0): return output_file if (input_xml.get_children(self.roseplotter.plot_name) and self.roseplotter.do_plot(input_xml, output_file) == 0): return output_file logger.warning('nothing to graph in %s', input_file) return None def do_template(self, template, data=None): logger.info("Templating %s", template) input_file = os.path.join(self.template_dir, template) output_file = os.path.join(self.context.output_dir, template) self.templater.make_file(input_file, output_file, live_data=data) return output_file
class RegularTasks(object): def __init__(self, params, status, calib_data, hourly_data, daily_data, monthly_data): self.logger = logging.getLogger('pywws.Tasks.RegularTasks') self.params = params self.status = status self.calib_data = calib_data self.hourly_data = hourly_data self.daily_data = daily_data self.monthly_data = monthly_data # get directories self.work_dir = self.params.get('paths', 'work', '/tmp/weather') self.template_dir = self.params.get( 'paths', 'templates', os.path.expanduser('~/weather/templates/')) self.graph_template_dir = self.params.get( 'paths', 'graph_templates', os.path.expanduser('~/weather/graph_templates/')) self.local_dir = self.params.get( 'paths', 'local_files', os.path.expanduser('~/weather/results/')) # create calibration object self.calibrator = Calib(self.params, self.status) # create templater object self.templater = Template.Template( self.params, self.status, self.calib_data, self.hourly_data, self.daily_data, self.monthly_data) # create plotter objects self.plotter = Plot.GraphPlotter( self.params, self.status, self.calib_data, self.hourly_data, self.daily_data, self.monthly_data, self.work_dir) self.roseplotter = WindRose.RosePlotter( self.params, self.status, self.calib_data, self.hourly_data, self.daily_data, self.monthly_data, self.work_dir) # directory of service uploaders self.services = dict() # create a YoWindow object self.yowindow = YoWindow.YoWindow(self.calib_data) # get local time's offset from UTC, without DST now = self.calib_data.before(datetime.max) if not now: now = datetime.utcnow() time_offset = Local.utcoffset(now) - Local.dst(now) # get daytime end hour, in UTC self.day_end_hour = eval(params.get('config', 'day end hour', '21')) self.day_end_hour = (self.day_end_hour - (time_offset.seconds // 3600)) % 24 # create service uploader objects for section in ('live', 'logged', 'hourly', '12 hourly', 'daily'): for service in eval(self.params.get(section, 'services', '[]')): if service not in self.services: self.services[service] = ToService( self.params, self.status, self.calib_data, service_name=service) def has_live_tasks(self): yowindow_file = self.params.get('live', 'yowindow', '') if yowindow_file: return True for template in eval(self.params.get('live', 'twitter', '[]')): return True for service in eval(self.params.get('live', 'services', '[]')): return True for template in eval(self.params.get('live', 'plot', '[]')): return True for template in eval(self.params.get('live', 'text', '[]')): return True return False def _parse_templates(self, section, option): for template in eval(self.params.get(section, option, '[]')): if isinstance(template, (list, tuple)): yield template else: yield template, '' def _do_common(self, sections, live_data=None): OK = True for section in sections: yowindow_file = self.params.get(section, 'yowindow', '') if yowindow_file: self.yowindow.write_file(yowindow_file) break for section in sections: for template, flags in self._parse_templates(section, 'twitter'): if not self.do_twitter(template): OK = False all_services = [] for section in sections: for service in eval(self.params.get(section, 'services', '[]')): if service not in all_services: all_services.append(service) for service in all_services: self.services[service].Upload(live_data is None, live_data) uploads = [] local_files = [] for section in sections: for template, flags in self._parse_templates(section, 'text'): if 'T' in flags: if not self.do_twitter(template, live_data): OK = False continue upload = self.do_template(template, live_data) if 'L' in flags: local_files.append(upload) else: uploads.append(upload) for template, flags in self._parse_templates(section, 'plot'): upload = self.do_plot(template) if not upload: continue if 'L' in flags: local_files.append(upload) else: uploads.append(upload) if local_files: if not os.path.isdir(self.local_dir): os.makedirs(self.local_dir) for file in local_files: shutil.move(file, self.local_dir) if uploads: if not Upload.Upload(self.params, uploads): OK = False for file in uploads: os.unlink(file) return OK def do_live(self, data): return self._do_common(['live'], self.calibrator.calib(data)) def do_tasks(self): sections = ['logged'] self.params.unset('logged', 'last update') now = self.calib_data.before(datetime.max) if now: now += timedelta(minutes=self.calib_data[now]['delay']) else: now = datetime.utcnow() threshold = now.replace(minute=0, second=0, microsecond=0) last_update = self.params.get_datetime('hourly', 'last update') if last_update: self.params.unset('hourly', 'last update') self.status.set('last update', 'hourly', last_update.isoformat(' ')) last_update = self.status.get_datetime('last update', 'hourly') if (not last_update) or (last_update < threshold): # time to do hourly tasks sections.append('hourly') # set 12 hourly threshold threshold -= timedelta(hours=(threshold.hour - self.day_end_hour) % 12) last_update = self.params.get_datetime('12 hourly', 'last update') if last_update: self.params.unset('12 hourly', 'last update') self.status.set('last update', '12 hourly', last_update.isoformat(' ')) last_update = self.status.get_datetime('last update', '12 hourly') if (not last_update) or (last_update < threshold): # time to do 12 hourly tasks sections.append('12 hourly') # set daily threshold threshold -= timedelta(hours=(threshold.hour - self.day_end_hour) % 24) last_update = self.params.get_datetime('daily', 'last update') if last_update: self.params.unset('daily', 'last update') self.status.set('last update', 'daily', last_update.isoformat(' ')) last_update = self.status.get_datetime('last update', 'daily') if (not last_update) or (last_update < threshold): # time to do daily tasks sections.append('daily') OK = self._do_common(sections) if OK: for section in sections: self.status.set('last update', section, now.isoformat(' ')) if 'hourly' in sections: # save any unsaved data self.params.flush() self.status.flush() self.calib_data.flush() self.hourly_data.flush() self.daily_data.flush() self.monthly_data.flush() return OK def do_twitter(self, template, data=None): from pywws import ToTwitter twitter = ToTwitter.ToTwitter(self.params) self.logger.info("Templating %s", template) input_file = os.path.join(self.template_dir, template) tweet = self.templater.make_text(input_file, live_data=data) self.logger.info("Tweeting") return twitter.Upload(tweet[:140]) def do_plot(self, template): self.logger.info("Graphing %s", template) input_file = os.path.join(self.graph_template_dir, template) output_file = os.path.join(self.work_dir, os.path.splitext(template)[0]) if self.plotter.DoPlot(input_file, output_file) == 0: return output_file elif self.roseplotter.DoPlot(input_file, output_file) == 0: return output_file return None def do_template(self, template, data=None): self.logger.info("Templating %s", template) input_file = os.path.join(self.template_dir, template) output_file = os.path.join(self.work_dir, template) self.templater.make_file(input_file, output_file, live_data=data) return output_file