def run(self): import weeutil.rsyncupload # We don't try to collect performance statistics about rsync, because # rsync will report them for us. Check the debug log messages. try: local_root = os.path.join( self.config_dict['WEEWX_ROOT'], self.skin_dict.get('HTML_ROOT', self.config_dict['StdReport']['HTML_ROOT'])) rsync_data = weeutil.rsyncupload.RsyncUpload( local_root=local_root, remote_root=self.skin_dict['path'], server=self.skin_dict['server'], user=self.skin_dict.get('user'), port=self.skin_dict.get('port'), ssh_options=self.skin_dict.get('ssh_options'), compress=to_bool(self.skin_dict.get('compress', False)), delete=to_bool(self.skin_dict.get('delete', False)), log_success=to_bool( search_up(self.skin_dict, 'log_success', True))) except KeyError: syslog.syslog( syslog.LOG_DEBUG, "rsyncgenerator: rsync upload not requested. Skipped.") return try: rsync_data.run() except IOError as e: (cl, unused_ob, unused_tr) = sys.exc_info() syslog.syslog( syslog.LOG_ERR, "rsyncgenerator: " "Caught exception %s: %s" % (cl, e))
def run(self): """Main entry point for file generation using Cheetah Templates.""" t1 = time.time() self.setup() # Make a deep copy of the skin dictionary (we will be modifying it): gen_dict = copy.deepcopy(self.skin_dict) # Look for options in [CheetahGenerator], section_name = "CheetahGenerator" # but accept options from [FileGenerator] for backward compatibility. if "FileGenerator" in gen_dict and "CheetahGenerator" not in gen_dict: section_name = "FileGenerator" # The default summary time span is 'None'. gen_dict[section_name]['summarize_by'] = 'None' # determine how much logging is desired log_success = to_bool( search_up(gen_dict[section_name], 'log_success', True)) # configure the search list extensions self.initExtensions(gen_dict[section_name]) # Generate any templates in the given dictionary: ngen = self.generate(gen_dict[section_name], self.gen_ts) self.teardown() elapsed_time = time.time() - t1 if log_success: log.info("Generated %d files for report %s in %.2f seconds", ngen, self.skin_dict['REPORT_NAME'], elapsed_time)
def run(self): copy_dict = self.skin_dict['CopyGenerator'] # determine how much logging is desired log_success = to_bool(search_up(copy_dict, 'log_success', True)) copy_list = [] if self.first_run: # Get the list of files to be copied only once, at the first # invocation of the generator. Wrap in a try block in case the # list does not exist. try: copy_list += weeutil.weeutil.option_as_list( copy_dict['copy_once']) except KeyError: pass # Get the list of files to be copied everytime. Again, wrap in a # try block. try: copy_list += weeutil.weeutil.option_as_list( copy_dict['copy_always']) except KeyError: pass # Change directory to the skin subdirectory: os.chdir( os.path.join(self.config_dict['WEEWX_ROOT'], self.skin_dict['SKIN_ROOT'], self.skin_dict['skin'])) # Figure out the destination of the files html_dest_dir = os.path.join(self.config_dict['WEEWX_ROOT'], self.skin_dict['HTML_ROOT']) # The copy list can contain wildcard characters. Go through the # list globbing any character expansions ncopy = 0 for pattern in copy_list: # Glob this pattern; then go through each resultant filename: for _file in glob.glob(pattern): # Final destination is the join of the html destination # directory and any relative subdirectory on the filename: dest_dir = os.path.join(html_dest_dir, os.path.dirname(_file)) # Make the destination directory, wrapping it in a try block in # case it already exists: try: os.makedirs(dest_dir) except OSError: pass # This version of copy does not copy over modification time, # so it will look like a new file, causing it to be (for # example) ftp'd to the server: shutil.copy(_file, dest_dir) ncopy += 1 if log_success: syslog.syslog( syslog.LOG_INFO, "copygenerator: " "copied %d files to %s" % (ncopy, html_dest_dir))
def run(self): import weeutil.ftpupload # determine how much logging is desired log_success = to_bool(search_up(self.skin_dict, 'log_success', True)) t1 = time.time() try: local_root = os.path.join( self.config_dict['WEEWX_ROOT'], self.skin_dict.get('HTML_ROOT', self.config_dict['StdReport']['HTML_ROOT'])) ftp_data = weeutil.ftpupload.FtpUpload( server=self.skin_dict['server'], user=self.skin_dict['user'], password=self.skin_dict['password'], local_root=local_root, remote_root=self.skin_dict['path'], port=int(self.skin_dict.get('port', 21)), name=self.skin_dict['REPORT_NAME'], passive=to_bool(self.skin_dict.get('passive', True)), max_tries=int(self.skin_dict.get('max_tries', 3)), secure=to_bool(self.skin_dict.get('secure_ftp', False)), debug=int(self.skin_dict.get('debug', 0)), secure_data=to_bool(self.skin_dict.get('secure_data', True))) except KeyError: syslog.syslog(syslog.LOG_DEBUG, "ftpgenerator: FTP upload not requested. Skipped.") return try: n = ftp_data.run() except (socket.timeout, socket.gaierror, ftplib.all_errors, IOError) as e: (cl, unused_ob, unused_tr) = sys.exc_info() syslog.syslog(syslog.LOG_ERR, "ftpgenerator: " "Caught exception %s: %s" % (cl, e)) weeutil.weeutil.log_traceback(" **** ") return if log_success: t2 = time.time() syslog.syslog( syslog.LOG_INFO, "ftpgenerator: ftp'd %d files in %0.2f seconds" % (n, (t2 - t1)))
def run(self): copy_dict = self.skin_dict['CopyGenerator'] # determine how much logging is desired log_success = to_bool(search_up(copy_dict, 'log_success', True)) copy_list = [] if self.first_run: # Get the list of files to be copied only once, at the first # invocation of the generator. Wrap in a try block in case the # list does not exist. try: copy_list += weeutil.weeutil.option_as_list(copy_dict['copy_once']) except KeyError: pass # Get the list of files to be copied everytime. Again, wrap in a # try block. try: copy_list += weeutil.weeutil.option_as_list(copy_dict['copy_always']) except KeyError: pass # Change directory to the skin subdirectory: os.chdir(os.path.join(self.config_dict['WEEWX_ROOT'], self.skin_dict['SKIN_ROOT'], self.skin_dict['skin'])) # Figure out the destination of the files html_dest_dir = os.path.join(self.config_dict['WEEWX_ROOT'], self.skin_dict['HTML_ROOT']) # The copy list can contain wildcard characters. Go through the # list globbing any character expansions ncopy = 0 for pattern in copy_list: # Glob this pattern; then go through each resultant path: for path in glob.glob(pattern): ncopy += weeutil.weeutil.deep_copy_path(path, html_dest_dir) if log_success: syslog.syslog(syslog.LOG_INFO, "copygenerator: " "copied %d files to %s" % (ncopy, html_dest_dir))
def genImages(self, gen_ts): """Generate the images. The time scales will be chosen to include the given timestamp, with nice beginning and ending times. gen_ts: The time around which plots are to be generated. This will also be used as the bottom label in the plots. [optional. Default is to use the time of the last record in the database.] """ t1 = time.time() ngen = 0 # determine how much logging is desired log_success = to_bool(search_up(self.image_dict, 'log_success', True)) # Loop over each time span class (day, week, month, etc.): for timespan in self.image_dict.sections: # Now, loop over all plot names in this time span class: for plotname in self.image_dict[timespan].sections: # Accumulate all options from parent nodes: plot_options = weeutil.weeutil.accumulateLeaves( self.image_dict[timespan][plotname]) plotgen_ts = gen_ts if not plotgen_ts: binding = plot_options['data_binding'] archive = self.db_binder.get_manager(binding) plotgen_ts = archive.lastGoodStamp() if not plotgen_ts: plotgen_ts = time.time() image_root = os.path.join(self.config_dict['WEEWX_ROOT'], plot_options['HTML_ROOT']) # Get the path that the image is going to be saved to: img_file = os.path.join(image_root, '%s.png' % plotname) ai = to_int(plot_options.get('aggregate_interval')) # Check whether this plot needs to be done at all: if skipThisPlot(plotgen_ts, ai, img_file): continue # skip image files that are fresh, but only if staleness is defined stale = to_int(plot_options.get('stale_age')) if stale is not None: t_now = time.time() try: last_mod = os.path.getmtime(img_file) if t_now - last_mod < stale: syslog.syslog( syslog.LOG_DEBUG, "imagegenerator: Skip '%s': last_mod=%s age=%s stale=%s" % (img_file, last_mod, t_now - last_mod, stale)) continue except os.error: pass # Create the subdirectory that the image is to be put in. # Wrap in a try block in case it already exists. try: os.makedirs(os.path.dirname(img_file)) except OSError: pass # Create a new instance of a time plot and start adding to it plot = weeplot.genplot.TimePlot(plot_options) # Calculate a suitable min, max time for the requested time. (minstamp, maxstamp, timeinc) = weeplot.utilities.scaletime( plotgen_ts - int(plot_options.get('time_length', 86400)), plotgen_ts) # Override the x interval if the user has given an explicit interval: timeinc_user = to_int(plot_options.get('x_interval')) if timeinc_user is not None: timeinc = timeinc_user plot.setXScaling((minstamp, maxstamp, timeinc)) # Set the y-scaling, using any user-supplied hints: plot.setYScaling( weeutil.weeutil.convertToFloat( plot_options.get('yscale', ['None', 'None', 'None']))) # Get a suitable bottom label: bottom_label_format = plot_options.get('bottom_label_format', '%m/%d/%y %H:%M') bottom_label = time.strftime(bottom_label_format, time.localtime(plotgen_ts)) plot.setBottomLabel(bottom_label) # Set day/night display plot.setLocation(self.stn_info.latitude_f, self.stn_info.longitude_f) plot.setDayNight( to_bool(plot_options.get('show_daynight', False)), weeplot.utilities.tobgr( plot_options.get('daynight_day_color', '0xffffff')), weeplot.utilities.tobgr( plot_options.get('daynight_night_color', '0xf0f0f0')), weeplot.utilities.tobgr( plot_options.get('daynight_edge_color', '0xefefef'))) # Loop over each line to be added to the plot. for line_name in self.image_dict[timespan][plotname].sections: # Accumulate options from parent nodes. line_options = weeutil.weeutil.accumulateLeaves( self.image_dict[timespan][plotname][line_name]) # See what SQL variable type to use for this line. By # default, use the section name. var_type = line_options.get('data_type', line_name) # Look for aggregation type: aggregate_type = line_options.get('aggregate_type') if aggregate_type in (None, '', 'None', 'none'): # No aggregation specified. aggregate_type = aggregate_interval = None else: try: # Aggregation specified. Get the interval. aggregate_interval = line_options.as_int( 'aggregate_interval') except KeyError: syslog.syslog( syslog.LOG_ERR, "imagegenerator: aggregate interval required for aggregate type %s" % aggregate_type) syslog.syslog( syslog.LOG_ERR, "imagegenerator: line type %s skipped" % var_type) continue # Now its time to find and hit the database: binding = line_options['data_binding'] archive = self.db_binder.get_manager(binding) (start_vec_t, stop_vec_t, data_vec_t) = \ archive.getSqlVectors((minstamp, maxstamp), var_type, aggregate_type=aggregate_type, aggregate_interval=aggregate_interval) if weewx.debug: assert (len(start_vec_t) == len(stop_vec_t)) # Get the type of plot ("bar', 'line', or 'vector') plot_type = line_options.get('plot_type', 'line') if aggregate_type and aggregate_type.lower() in ( 'avg', 'max', 'min') and plot_type != 'bar': # Put the point in the middle of the aggregate_interval for these aggregation types start_vec_t = ValueTuple([ x - aggregate_interval / 2.0 for x in start_vec_t[0] ], start_vec_t[1], start_vec_t[2]) stop_vec_t = ValueTuple([ x - aggregate_interval / 2.0 for x in stop_vec_t[0] ], stop_vec_t[1], stop_vec_t[2]) # Do any necessary unit conversions: new_start_vec_t = self.converter.convert(start_vec_t) new_stop_vec_t = self.converter.convert(stop_vec_t) new_data_vec_t = self.converter.convert(data_vec_t) # Add a unit label. NB: all will get overwritten except the # last. Get the label from the configuration dictionary. unit_label = line_options.get( 'y_label', weewx.units.get_label_string(self.formatter, self.converter, var_type)) # Strip off any leading and trailing whitespace so it's # easy to center plot.setUnitLabel(unit_label.strip()) # See if a line label has been explicitly requested: label = line_options.get('label') if not label: # No explicit label. Look up a generic one. NB: title_dict is a KeyDict which # will substitute the key if the value is not in the dictionary. label = self.title_dict[var_type] # See if a color has been explicitly requested. color = line_options.get('color') if color is not None: color = weeplot.utilities.tobgr(color) fill_color = line_options.get('fill_color') if fill_color is not None: fill_color = weeplot.utilities.tobgr(fill_color) # Get the line width, if explicitly requested. width = to_int(line_options.get('width')) interval_vec = None gap_fraction = None # Some plot types require special treatments: if plot_type == 'vector': vector_rotate_str = line_options.get('vector_rotate') vector_rotate = -float( vector_rotate_str ) if vector_rotate_str is not None else None else: vector_rotate = None if plot_type == 'bar': interval_vec = [ x[1] - x[0] for x in zip(new_start_vec_t.value, new_stop_vec_t.value) ] elif plot_type == 'line': gap_fraction = to_float( line_options.get('line_gap_fraction')) if gap_fraction is not None: if not 0 < gap_fraction < 1: syslog.syslog( syslog.LOG_ERR, "imagegenerator: Gap fraction %5.3f outside range 0 to 1. Ignored." % gap_fraction) gap_fraction = None # Get the type of line (only 'solid' or 'none' for now) line_type = line_options.get('line_type', 'solid') if line_type.strip().lower() in ['', 'none']: line_type = None marker_type = line_options.get('marker_type') marker_size = to_int(line_options.get('marker_size', 8)) # Add the line to the emerging plot: plot.addLine( weeplot.genplot.PlotLine(new_stop_vec_t[0], new_data_vec_t[0], label=label, color=color, fill_color=fill_color, width=width, plot_type=plot_type, line_type=line_type, marker_type=marker_type, marker_size=marker_size, bar_width=interval_vec, vector_rotate=vector_rotate, gap_fraction=gap_fraction)) # OK, the plot is ready. Render it onto an image image = plot.render() try: # Now save the image image.save(img_file) ngen += 1 except IOError as e: syslog.syslog( syslog.LOG_CRIT, "imagegenerator: Unable to save to file '%s' %s:" % (img_file, e)) t2 = time.time() if log_success: syslog.syslog( syslog.LOG_INFO, "imagegenerator: Generated %d images for %s in %.2f seconds" % (ngen, self.skin_dict['REPORT_NAME'], t2 - t1))
def gen_images(self, gen_ts): """Generate the images. The time scales will be chosen to include the given timestamp, with nice beginning and ending times. Args: gen_ts (int): The time around which plots are to be generated. This will also be used as the bottom label in the plots. [optional. Default is to use the time of the last record in the database.] """ t1 = time.time() ngen = 0 # determine how much logging is desired log_success = to_bool(search_up(self.image_dict, 'log_success', True)) # Loop over each time span class (day, week, month, etc.): for timespan in self.image_dict.sections: # Now, loop over all plot names in this time span class: for plotname in self.image_dict[timespan].sections: # Accumulate all options from parent nodes: plot_options = accumulateLeaves( self.image_dict[timespan][plotname]) plotgen_ts = gen_ts if not plotgen_ts: binding = plot_options['data_binding'] db_manager = self.db_binder.get_manager(binding) plotgen_ts = db_manager.lastGoodStamp() if not plotgen_ts: plotgen_ts = time.time() image_root = os.path.join(self.config_dict['WEEWX_ROOT'], plot_options['HTML_ROOT']) # Get the path that the image is going to be saved to: img_file = os.path.join(image_root, '%s.png' % plotname) # Check whether this plot needs to be done at all: if _skip_this_plot(plotgen_ts, plot_options, img_file): continue # Generate the plot. plot = self.gen_plot(plotgen_ts, plot_options, self.image_dict[timespan][plotname]) # 'plot' will be None if skip_if_empty was truthy, and the plot contains no data if plot: # We have a valid plot. Render it onto an image image = plot.render() # Create the subdirectory that the image is to be put in. Wrap in a try block # in case it already exists. try: os.makedirs(os.path.dirname(img_file)) except OSError: pass try: # Now save the image image.save(img_file) ngen += 1 except IOError as e: log.error("Unable to save to file '%s' %s:", img_file, e) t2 = time.time() if log_success: log.info("Generated %d images for report %s in %.2f seconds", ngen, self.skin_dict['REPORT_NAME'], t2 - t1)
def __init__(self, config_dict, skin_dict, gen_ts, first_run, stn_info, record=None): # initialize my superclass super(StackedWindRoseImageGenerator, self).__init__(config_dict, skin_dict, gen_ts, first_run, stn_info, record) # do we log on success self.log_success = weeutil.weeutil.to_bool( search_up(self.skin_dict, 'log_success', True)) # do we log on failure self.log_failure = weeutil.weeutil.to_bool( search_up(self.skin_dict, 'log_failure', True)) # get the data binding to use self.data_binding = config_dict['StdArchive'].get( 'data_binding', 'wx_binding') self.image_dict = skin_dict['StackedWindRoseImageGenerator'] self.title_dict = skin_dict['Labels']['Generic'] self.converter = weewx.units.Converter.fromSkinDict(skin_dict) self.formatter = weewx.units.Formatter.fromSkinDict(skin_dict) self.unit_helper = weewx.units.UnitInfoHelper(self.formatter, self.converter) # set image attributes self.image_width = int(self.image_dict['image_width']) self.image_height = int(self.image_dict['image_height']) self.image_background_box_color = int( self.image_dict['image_background_box_color'], 0) self.image_background_circle_color = int( self.image_dict['image_background_circle_color'], 0) self.image_background_range_ring_color = int( self.image_dict['image_background_range_ring_color'], 0) self.image_background_image = self.image_dict['image_background_image'] # set wind rose attributes self.windrose_plot_border = int( self.image_dict['windrose_plot_border']) self.windrose_legend_bar_width = int( self.image_dict['windrose_legend_bar_width']) self.windrose_font_path = self.image_dict['windrose_font_path'] self.windrose_plot_font_size = int( self.image_dict['windrose_plot_font_size']) self.windrose_plot_font_color = int( self.image_dict['windrose_plot_font_color'], 0) self.windrose_legend_font_size = int( self.image_dict['windrose_legend_font_size']) self.windrose_legend_font_color = int( self.image_dict['windrose_legend_font_color'], 0) self.windrose_label_font_size = int( self.image_dict['windrose_label_font_size']) self.windrose_label_font_color = int( self.image_dict['windrose_label_font_color'], 0) # set the petal colours # first get any petal colours specified in the config, if not defined # then use some sensible defaults _colors = weeutil.weeutil.option_as_list( self.image_dict.get('windrose_plot_petal_colors', DEFAULT_PETAL_COLORS)) # verify our colors are valid _petal_colors = [] # iterate over the colors we have and if they are valid keep them # otherwise discard them for _color in _colors: # parse the color, we will get bck a tuple representing the RGB # values or None if the color is invalid _col = parse_color(_color) # if we have a non None response it is valid if _col is not None: # valid color, append it to the petal color list _petal_colors.append(_col) # we have a list of valid colors but do we have enough, we need 7 if len(_petal_colors) < 7: # if we don't have 7 augment the colors list with unused colors # from the defaults until we have 7 _required = 7 - len(_petal_colors) for _color in DEFAULT_PETAL_COLORS: if _required > 0: _parsed = parse_color(_color) if _parsed not in _petal_colors: _petal_colors.append(_parsed) _required -= 1 # save the final ist of petal colors self.petal_colors = list(_petal_colors) # get petal width, if not defined then set default to 16 (degrees) try: self.windrose_plot_petal_width = int( self.image_dict['windrose_plot_petal_width']) except KeyError: self.windrose_plot_petal_width = 16 # Boundaries for speed range bands, these mark the colour boundaries # on the stacked bar in the legend. 7 elements only (ie 0, 10% of max, # 20% of max ... 100% of max) self.speed_factor = [0.0, 0.1, 0.2, 0.3, 0.5, 0.7, 1.0] # initialise some properties for later use self.plotgen_ts = None self.label = None self.time_stamp = None self.time_stamp_location = None self.units = None self.unit_label = None self.obs = None self.dir_name = None self.max_ring_value = None self.label_dir = None self.plot_font = None self.legend_font = None self.label_font = None self.rose_max_dia = None self.origin_x = None self.origin_y = None self.draw = None