def _do_cron(self, live_data=None): if not self.cron: return # get timestamp of latest data if live_data: now = live_data['idx'] else: now = self.calib_data.before(datetime.max) if not now: now = datetime.utcnow() # convert to local time local_now = now + local_utc_offset(now) # get list of due sections sections = [] for section in self.cron: if self.cron[section].get_current(datetime) > local_now: continue sections.append(section) while self.cron[section].get_current(datetime) <= local_now: self.cron[section].get_next() if not sections: return # do it! self._do_common(sections, live_data) for section in sections: self.status.set('last update', section, now.isoformat(' '))
def _eval_time(self, time_str): # get timestamp of last data item result = self.hourly_data.before(datetime.max) if not result: result = datetime.utcnow() # only if no hourly data result += local_utc_offset(result) # set to start of the day result = result.replace(hour=0, minute=0, second=0, microsecond=0) # apply time string return eval('result.replace(%s)' % time_str)
def __init__(self, params, status, raw_data, calib_data, hourly_data, daily_data, monthly_data, asynch=False): self.logger = logging.getLogger('pywws.Tasks.RegularTasks') self.params = params self.status = status self.raw_data = raw_data self.calib_data = calib_data self.hourly_data = hourly_data self.daily_data = daily_data self.monthly_data = monthly_data self.asynch = asynch self.flush = eval(self.params.get('config', 'frequent writes', 'False')) # get directories self.work_dir = self.params.get('paths', 'work', '/tmp/weather') if not os.path.isdir(self.work_dir): raise RuntimeError('Directory "' + self.work_dir + '" does not exist.') self.template_dir = self.params.get( 'paths', 'templates', os.path.expanduser('~/weather/templates/')) self.graph_template_dir = self.params.get( 'paths', 'graph_templates', os.path.expanduser('~/weather/graph_templates/')) self.local_dir = self.params.get( 'paths', 'local_files', os.path.expanduser('~/weather/results/')) # create calibration object self.calibrator = Calib(self.params, self.raw_data) # create templater object self.templater = Template.Template(self.params, self.status, self.calib_data, self.hourly_data, self.daily_data, self.monthly_data) # create plotter objects self.plotter = Plot.GraphPlotter(self.params, self.status, self.calib_data, self.hourly_data, self.daily_data, self.monthly_data, self.work_dir) self.roseplotter = WindRose.RosePlotter( self.params, self.status, self.calib_data, self.hourly_data, self.daily_data, self.monthly_data, self.work_dir) # create FTP uploader object self.uploader = Upload.Upload(self.params) self.uploads_directory = os.path.join(self.work_dir, 'uploads') if not os.path.isdir(self.uploads_directory): os.mkdir(self.uploads_directory) # delay creation of a Twitter object until we know it's needed self.twitter = None # create a YoWindow object self.yowindow = YoWindow.YoWindow(self.calib_data) # get daytime end hour, in UTC self.day_end_hour = eval(params.get('config', 'day end hour', '21')) self.day_end_hour = (self.day_end_hour - (STDOFFSET.seconds // 3600)) % 24 # parse "cron" sections self.cron = {} for section in self.params._config.sections(): if section.split()[0] != 'cron': continue import croniter self.cron[section] = croniter.croniter( self.params.get(section, 'format', '')) self.cron[section].get_prev() last_update = self.status.get_datetime('last update', section) if last_update: last_update = last_update + local_utc_offset(last_update) while self.cron[section].get_current(datetime) <= last_update: self.cron[section].get_next() # create service uploader objects self.services = {} for section in self.cron.keys() + [ 'live', 'logged', 'hourly', '12 hourly', 'daily' ]: for name in eval(self.params.get(section, 'services', '[]')): if name not in self.services: self.services[name] = ToService(self.params, self.status, self.calib_data, service_name=name) # check for deprecated syntax if self.params.get(section, 'twitter') not in (None, '[]'): self.logger.warning('Deprecated twitter entry in [%s]', section) if self.params.get(section, 'yowindow'): self.logger.warning('Deprecated yowindow entry in [%s]', section) # create queues for things to upload / send self.tweet_queue = deque() self.service_queue = {} for name in self.services: self.service_queue[name] = deque() self.uploads_queue = deque() # start asynchronous thread to do uploads if self.asynch: self.logger.info('Starting asynchronous thread') self.shutdown_thread = threading.Event() self.wake_thread = threading.Event() self.thread = threading.Thread(target=self._asynch_thread) self.thread.start()
def DoPlot(self, input_file, output_file): if isinstance(input_file, GraphFileReader): self.graph = input_file else: # read XML graph description self.graph = GraphFileReader(input_file) # get list of plots plot_list = self.graph.get_children(self.plot_name) self.plot_count = len(plot_list) if self.plot_count < 1: # nothing to plot self.logger.info('%s has no %s nodes', self.graph.input_file, self.plot_name) self.graph.close() return 1 # get start and end datetimes self.x_lo = self.graph.get_value('start', None) self.x_hi = self.graph.get_value('stop', None) self.duration = self.graph.get_value('duration', None) if self.duration: self.duration = eval('timedelta(%s)' % self.duration) else: self.duration = timedelta(hours=24) if self.x_lo: self.x_lo = self._eval_time(self.x_lo) if self.x_hi: self.x_hi = self._eval_time(self.x_hi) self.duration = self.x_hi - self.x_lo else: self.x_hi = self.x_lo + self.duration elif self.x_hi: self.x_hi = self._eval_time(self.x_hi) self.x_lo = self.x_hi - self.duration else: self.x_hi = self.hourly_data.before(datetime.max) if not self.x_hi: self.x_hi = datetime.utcnow() # only if no hourly data self.x_hi += local_utc_offset(self.x_hi) if self.duration < timedelta(hours=6): # set end of graph to start of the next minute after last item self.x_hi += timedelta(seconds=55) self.x_hi = self.x_hi.replace(second=0) else: # set end of graph to start of the next hour after last item self.x_hi += timedelta(minutes=55) self.x_hi = self.x_hi.replace(minute=0, second=0) self.x_lo = self.x_hi - self.duration self.utcoffset = local_utc_offset(self.x_hi) # open gnuplot command file self.tmp_files = [] cmd_file = os.path.join(self.work_dir, 'plot.cmd') self.tmp_files.append(cmd_file) of = codecs.open(cmd_file, 'w', encoding=self.encoding[0]) # write gnuplot set up of.write('set encoding %s\n' % (self.encoding[1])) lcl = locale.getlocale() if lcl[0]: of.write('set locale "%s.%s"\n' % lcl) self.rows = self.GetDefaultRows() self.cols = (self.plot_count + self.rows - 1) // self.rows self.rows, self.cols = eval(self.graph.get_value( 'layout', '%d, %d' % (self.rows, self.cols))) w, h = self.GetDefaultPlotSize() w = w * self.cols h = h * self.rows w, h = eval(self.graph.get_value('size', '(%d, %d)' % (w, h))) fileformat = self.graph.get_value('fileformat', 'png') if fileformat == 'svg': terminal = 'svg enhanced font "arial,9" dynamic rounded' elif u' ' not in fileformat: terminal = '%s large' % (fileformat) else: terminal = fileformat if u'size' not in terminal: terminal += u' size %d,%d' % (w, h) terminal = self.graph.get_value('terminal', terminal) of.write('set terminal %s\n' % (terminal)) of.write('set output "%s"\n' % (output_file)) # set overall title title = self.graph.get_value('title', '') if title: if '%' in title: x_hi = (self.x_hi - self.utcoffset).replace(tzinfo=utc).astimezone(Local) if sys.version_info[0] < 3: title = title.encode(self.encoding[0]) title = x_hi.strftime(title) if sys.version_info[0] < 3: title = title.decode(self.encoding[0]) title = 'title "%s"' % title of.write('set multiplot layout %d, %d %s\n' % (self.rows, self.cols, title)) # do actual plots of.write(self.GetPreamble()) for plot_no in range(self.plot_count): plot = plot_list[plot_no] # set key / title location title = plot.get_value('title', '') of.write('set key horizontal title "%s"\n' % title) # optional yaxis labels ylabel = plot.get_value('ylabel', '') if ylabel: ylabelangle = plot.get_value('ylabelangle', '') if ylabelangle: ylabelangle = ' rotate by %s' % (ylabelangle) of.write('set ylabel "%s"%s\n' % (ylabel, ylabelangle)) else: of.write('set ylabel\n') y2label = plot.get_value('y2label', '') if y2label: y2labelangle = plot.get_value('y2labelangle', '') if y2labelangle: y2labelangle = ' rotate by %s' % (y2labelangle) of.write('set y2label "%s"%s\n' % (y2label, y2labelangle)) else: of.write('set y2label\n') # set data source source = plot.get_value('source', 'raw') if source == 'raw': source = self.raw_data elif source == 'hourly': source = self.hourly_data elif source == 'monthly': source = self.monthly_data else: source = self.daily_data # do the plot of.write(self.PlotData(plot_no, plot, source)) of.close() self.graph.close() # run gnuplot on file subprocess.check_call(['gnuplot', cmd_file]) for file in self.tmp_files: os.unlink(file) return 0
def __init__(self, params, status, raw_data, calib_data, hourly_data, daily_data, monthly_data, asynch=False): self.logger = logging.getLogger('pywws.Tasks.RegularTasks') self.params = params self.status = status self.raw_data = raw_data self.calib_data = calib_data self.hourly_data = hourly_data self.daily_data = daily_data self.monthly_data = monthly_data self.asynch = asynch self.flush = eval(self.params.get('config', 'frequent writes', 'False')) # get directories self.work_dir = self.params.get('paths', 'work', '/tmp/weather') if not os.path.isdir(self.work_dir): raise RuntimeError( 'Directory "' + self.work_dir + '" does not exist.') self.template_dir = self.params.get( 'paths', 'templates', os.path.expanduser('~/weather/templates/')) self.graph_template_dir = self.params.get( 'paths', 'graph_templates', os.path.expanduser('~/weather/graph_templates/')) self.local_dir = self.params.get( 'paths', 'local_files', os.path.expanduser('~/weather/results/')) # create calibration object self.calibrator = Calib(self.params, self.raw_data) # create templater object self.templater = Template.Template( self.params, self.status, self.calib_data, self.hourly_data, self.daily_data, self.monthly_data) # create plotter objects self.plotter = Plot.GraphPlotter( self.params, self.status, self.calib_data, self.hourly_data, self.daily_data, self.monthly_data, self.work_dir) self.roseplotter = WindRose.RosePlotter( self.params, self.status, self.calib_data, self.hourly_data, self.daily_data, self.monthly_data, self.work_dir) # create FTP uploader object self.uploader = Upload.Upload(self.params) self.uploads_directory = os.path.join(self.work_dir, 'uploads') if not os.path.isdir(self.uploads_directory): os.mkdir(self.uploads_directory) # delay creation of a Twitter object until we know it's needed self.twitter = None # create a YoWindow object self.yowindow = YoWindow.YoWindow(self.calib_data) # get daytime end hour, in UTC self.day_end_hour = eval(params.get('config', 'day end hour', '21')) self.day_end_hour = (self.day_end_hour - (STDOFFSET.seconds // 3600)) % 24 # parse "cron" sections self.cron = {} for section in self.params._config.sections(): if section.split()[0] != 'cron': continue import croniter self.cron[section] = croniter.croniter( self.params.get(section, 'format', '')) self.cron[section].get_prev() last_update = self.status.get_datetime('last update', section) if last_update: last_update = last_update + local_utc_offset(last_update) while self.cron[section].get_current(datetime) <= last_update: self.cron[section].get_next() # create service uploader objects self.services = {} for section in self.cron.keys() + [ 'live', 'logged', 'hourly', '12 hourly', 'daily']: for name in eval(self.params.get(section, 'services', '[]')): if name not in self.services: self.services[name] = ToService( self.params, self.status, self.calib_data, service_name=name) # check for deprecated syntax if self.params.get(section, 'twitter') not in (None, '[]'): self.logger.warning( 'Deprecated twitter entry in [%s]', section) if self.params.get(section, 'yowindow'): self.logger.warning( 'Deprecated yowindow entry in [%s]', section) # create queues for things to upload / send self.tweet_queue = deque() self.service_queue = {} for name in self.services: self.service_queue[name] = deque() self.uploads_queue = deque() # start asynchronous thread to do uploads if self.asynch: self.logger.info('Starting asynchronous thread') self.shutdown_thread = threading.Event() self.wake_thread = threading.Event() self.thread = threading.Thread(target=self._asynch_thread) self.thread.start()