def __init__(self, config_dict, config_path, wd_config_dict, import_config_path, options): # call our parents __init__ super(WDSource, self).__init__(config_dict, wd_config_dict, options) # save the import config path self.import_config_path = import_config_path # save the import config dict self.wd_config_dict = wd_config_dict # our parent uses 'derive' as the default interval setting, for WD the # default should be 1 (minute) so redo the interval setting with our # default self.interval = wd_config_dict.get('interval', 1) # wind dir bounds self.wind_dir = [0, 360] # How the WeeWX field 'rain' is populated depends on the source rain # data. If the only data available is cumulative then the WeeWX rain # field is calculated as the difference between successive cumulative # values. WD provides a rain per interval field so that data can be # used to map directly to the WeeWX rain field. If rain is to be # calculated from a cumulative value then self.rain must be set to # 'cumulative', to map directly to the WeeWX rain field self.rain must # be set to None. self.rain = None # field delimiter used in text format monthly log files, default to # space self.txt_delimiter = str(wd_config_dict.get('txt_delimiter', ' ')) # field delimiter used in csv format monthly log files, default to # comma self.csv_delimiter = str(wd_config_dict.get('csv_delimiter', ',')) # decimal separator used in monthly log files, default to decimal point self.decimal = wd_config_dict.get('decimal', '.') # ignore extreme > 255.0 values for temperature and humidity fields self.ignore_extreme_temp_hum = weeutil.weeutil.tobool( wd_config_dict.get('ignore_extreme_temp_hum', True)) # initialise the import field-to-WeeWX archive field map self.map = None # property holding the current log file name being processed self.file_name = None # WD logs use either US or Metric units. The units used in each case # are: # Metric units: C, knots, hPa, mm # US units: F, mph, inHg, inch # # The user must specify the units to be used in the import config file. # This can be by either by specifying the log units as Metric or US # using the 'units' config option. Alternatively temperature, pressure, # rainfall and speed units can be specified individually under the # [Units] stanza. First check for a valid 'units' config option then # check for individual group units. Do some basic error checking and # validation, if one of the fields is missing or invalid then we need # to catch the error and raise it as we can't go on. log_unit_config = wd_config_dict.get('Units') if log_unit_config is not None: # get the units config option log_unit_sys = wd_config_dict['Units'].get('units') # accept any capitalization of USA as == US log_unit_sys = log_unit_sys if log_unit_sys.upper( ) != 'USA' else 'US' # does the units config option specify a valid log unit system if log_unit_sys is None or log_unit_sys.upper() not in [ 'METRIC', 'US' ]: # log unit system not specified look for individual entries # temperature temp_u = wd_config_dict['Units'].get('temperature') if temp_u is not None: if temp_u in weewx.units.default_unit_format_dict: self._header_map['temperature']['units'] = temp_u self._header_map['dewpoint']['units'] = temp_u self._header_map['heatindex']['units'] = temp_u self._header_map['soiltemp']['units'] = temp_u self._header_map['temp1']['units'] = temp_u self._header_map['temp2']['units'] = temp_u self._header_map['temp3']['units'] = temp_u self._header_map['temp4']['units'] = temp_u self._header_map['temp5']['units'] = temp_u self._header_map['temp6']['units'] = temp_u self._header_map['temp7']['units'] = temp_u else: _msg = "Unknown units '%s' specified for Weather Display " \ "temperature fields in %s." % (temp_u, self.import_config_path) raise weewx.UnitError(_msg) else: _msg = "No units specified for Weather Display temperature " \ "fields in %s." % (self.import_config_path,) raise weewx.UnitError(_msg) # pressure press_u = wd_config_dict['Units'].get('pressure') if press_u is not None: if press_u in ['inHg', 'hPa']: self._header_map['barometer']['units'] = press_u else: _msg = "Unknown units '%s' specified for Weather Display " \ "pressure fields in %s." % (press_u, self.import_config_path) raise weewx.UnitError(_msg) else: _msg = "No units specified for Weather Display pressure " \ "fields in %s." % (self.import_config_path,) raise weewx.UnitError(_msg) # rain rain_u = wd_config_dict['Units'].get('rain') if rain_u is not None: if rain_u in ['inch', 'mm']: self._header_map['rainlastmin']['units'] = rain_u self._header_map['dailyrain']['units'] = rain_u self._header_map['monthlyrain']['units'] = rain_u self._header_map['yearlyrain']['units'] = rain_u self._header_map['dailyet']['units'] = rain_u else: _msg = "Unknown units '%s' specified for Weather Display " \ "rain fields in %s." % (rain_u, self.import_config_path) raise weewx.UnitError(_msg) else: _msg = "No units specified for Weather Display rain fields " \ "in %s." % (self.import_config_path,) raise weewx.UnitError(_msg) # speed speed_u = wd_config_dict['Units'].get('speed') if speed_u is not None: if speed_u in ['inch', 'mm']: self._header_map['windspeed']['units'] = speed_u self._header_map['gustspeed']['units'] = speed_u else: _msg = "Unknown units '%s' specified for Weather Display " \ "speed fields in %s." % (speed_u, self.import_config_path) raise weewx.UnitError(_msg) else: _msg = "No units specified for Weather Display speed fields " \ "in %s." % (self.import_config_path,) raise weewx.UnitError(_msg) else: # log unit system specified _unit_sys = log_unit_sys.upper() # do we have a valid log unit system if _unit_sys in ['METRIC', 'US']: # valid log unit system so assign units as applicable self._header_map['temperature'][ 'units'] = self.wd_unit_sys['temperature'][_unit_sys] self._header_map['dewpoint']['units'] = self.wd_unit_sys[ 'temperature'][_unit_sys] self._header_map['heatindex']['units'] = self.wd_unit_sys[ 'temperature'][_unit_sys] self._header_map['barometer']['units'] = self.wd_unit_sys[ 'barometer'][_unit_sys] self._header_map['windspeed']['units'] = self.wd_unit_sys[ 'windspeed'][_unit_sys] self._header_map['gustspeed']['units'] = self.wd_unit_sys[ 'gustspeed'][_unit_sys] self._header_map['rainlastmin'][ 'units'] = self.wd_unit_sys['rainlastmin'][_unit_sys] self._header_map['soiltemp']['units'] = self.wd_unit_sys[ 'soiltemp'][_unit_sys] for _num in range(1, 8): _temp = 'temp%s' % _num self._header_map[_temp]['units'] = self.wd_unit_sys[ _temp][_unit_sys] else: # no valid Units config found, we can't go on so raise an error raise weewx.UnitError( "Invalid setting for 'units' config option.") else: # there is no Units config, we can't go on so raise an error raise weewx.UnitError("No Weather Display units config found.") # obtain a list of logs files to be processed _to_process = wd_config_dict.get('logs_to_process', list(self.logs.keys())) self.logs_to_process = weeutil.weeutil.option_as_list(_to_process) # can missing log files be ignored self.ignore_missing_log = weeutil.weeutil.to_bool( wd_config_dict.get('ignore_missing_log', True)) # get our source file path try: self.source = wd_config_dict['directory'] except KeyError: _msg = "Weather Display monthly logs directory not specified in '%s'." % import_config_path raise weewx.ViolatedPrecondition(_msg) # get the source file encoding, default to utf-8-sig self.source_encoding = self.wd_config_dict.get('source_encoding', 'utf-8-sig') # Now get a list on monthly log files sorted from oldest to newest. # This is complicated by the log file naming convention used by WD. # first the 1 digit months _lg_5_list = glob.glob(self.source + '/' + '[0-9]' * 5 + 'lg.txt') # and the 2 digit months _lg_6_list = glob.glob(self.source + '/' + '[0-9]' * 6 + 'lg.txt') # concatenate the two lists to get the complete list month_lg_list = _lg_5_list + _lg_6_list # create a list of log files in chronological order (month, year) _temp = [] # create a list of log files, adding year and month fields for sorting for p in month_lg_list: # obtain the file name fn = os.path.split(p)[1] # obtain the numeric part of the file name _digits = ''.join(c for c in fn if c.isdigit()) # append a list of format [path+file name, month, year] _temp.append([p, int(_digits[:-4]), int(_digits[-4:])]) # now sort the list keeping just the log file path and name self.log_list = [ a[0] for a in sorted(_temp, key=lambda el: (el[2], el[1])) ] # if there are no log files then there is nothing to be done if len(self.log_list) == 0: raise weeimport.WeeImportIOError( "No Weather Display monthly logs found in directory '%s'." % self.source) # Some log files have entries that belong in a different month. # Initialise a list to hold these extra records for processing during # the appropriate month self.extras = {} for l in self.logs_to_process: self.extras[l] = [] # tell the user/log what we intend to do _msg = "Weather Display monthly log files in the '%s' directory will be imported" % self.source print(_msg) log.info(_msg) _msg = "The following options will be used:" if self.verbose: print(_msg) log.debug(_msg) _msg = " config=%s, import-config=%s" % (config_path, self.import_config_path) if self.verbose: print(_msg) log.debug(_msg) if options.date: _msg = " date=%s" % options.date else: # we must have --from and --to _msg = " from=%s, to=%s" % (options.date_from, options.date_to) if self.verbose: print(_msg) log.debug(_msg) _msg = " dry-run=%s, calc_missing=%s, ignore_invalid_data=%s" % ( self.dry_run, self.calc_missing, self.ignore_invalid_data) if self.verbose: print(_msg) log.debug(_msg) if log_unit_sys is not None and log_unit_sys.upper() in [ 'METRIC', 'US' ]: # valid unit system specified _msg = " monthly logs are in %s units" % log_unit_sys.upper() if self.verbose: print(_msg) log.debug(_msg) else: # group units specified _msg = " monthly logs use the following units:" if self.verbose: print(_msg) log.debug(_msg) _msg = " temperature=%s pressure=%s" % (temp_u, press_u) if self.verbose: print(_msg) log.debug(_msg) _msg = " rain=%s speed=%s" % (rain_u, speed_u) if self.verbose: print(_msg) log.debug(_msg) _msg = " tranche=%s, interval=%s" % (self.tranche, self.interval) if self.verbose: print(_msg) log.debug(_msg) _msg = " UV=%s, radiation=%s ignore extreme temperature and humidity=%s" % ( self.UV_sensor, self.solar_sensor, self.ignore_extreme_temp_hum) if self.verbose: print(_msg) log.debug(_msg) _msg = "Using database binding '%s', which is bound to database '%s'" % ( self.db_binding_wx, self.dbm.database_name) print(_msg) log.info(_msg) _msg = "Destination table '%s' unit system is '%#04x' (%s)." % ( self.dbm.table_name, self.archive_unit_sys, unit_nicknames[self.archive_unit_sys]) print(_msg) log.info(_msg) if self.calc_missing: print("Missing derived observations will be calculated.") if not self.UV_sensor: print("All WeeWX UV fields will be set to None.") if not self.solar_sensor: print("All WeeWX radiation fields will be set to None.") if options.date or options.date_from: print("Observations timestamped after %s and up to and" % timestamp_to_string(self.first_ts)) print("including %s will be imported." % timestamp_to_string(self.last_ts)) if self.dry_run: print( "This is a dry run, imported data will not be saved to archive." )
def __init__(self, config_dict, config_path, cumulus_config_dict, import_config_path, options): # call our parents __init__ super(CumulusSource, self).__init__(config_dict, cumulus_config_dict, options) # save our import config path self.import_config_path = import_config_path # save our import config dict self.cumulus_config_dict = cumulus_config_dict # wind dir bounds self.wind_dir = [0, 360] # field delimiter used in monthly log files, default to comma self.delimiter = str(cumulus_config_dict.get('delimiter', ',')) # decimal separator used in monthly log files, default to decimal point self.decimal = cumulus_config_dict.get('decimal', '.') # date separator used in monthly log files, default to solidus separator = cumulus_config_dict.get('separator', '/') # we combine Cumulus date and time fields to give a fixed format # date-time string self.raw_datetime_format = separator.join(('%d', '%m', '%y %H:%M')) # Cumulus log files provide a number of cumulative rainfall fields. We # cannot use the daily rainfall as this may reset at some time of day # other than midnight (as required by WeeWX). So we use field 26, total # rainfall since midnight and treat it as a cumulative value. self.rain = 'cumulative' # initialise our import field-to-WeeWX archive field map self.map = None # Cumulus log files have a number of 'rain' fields that can be used to # derive the WeeWX rain field. Which one is available depends on the # Cumulus version that created the logs. The preferred field is field # 26(AA) - total rainfall since midnight but it is only available in # Cumulus v1.9.4 or later. If that field is not available then the # preferred field in field 09(J) - total rainfall today then field # 11(L) - total rainfall counter. Initialise the rain_source_confirmed # property now and we will deal with it later when we have some source # data. self.rain_source_confirmed = None # Units of measure for some obs (eg temperatures) cannot be derived from # the Cumulus monthly log files. These units must be specified by the # user in the import config file. Read these units and fill in the # missing unit data in the header map. Do some basic error checking and # validation, if one of the fields is missing or invalid then we need # to catch the error and raise it as we can't go on. # Temperature try: temp_u = cumulus_config_dict['Units'].get('temperature') except KeyError: _msg = "No units specified for Cumulus temperature " \ "fields in %s." % (self.import_config_path, ) raise weewx.UnitError(_msg) else: if temp_u in weewx.units.default_unit_format_dict: self._header_map['cur_out_temp']['units'] = temp_u self._header_map['curr_in_temp']['units'] = temp_u self._header_map['cur_dewpoint']['units'] = temp_u self._header_map['cur_heatindex']['units'] = temp_u self._header_map['cur_windchill']['units'] = temp_u self._header_map['cur_app_temp']['units'] = temp_u else: _msg = "Unknown units '%s' specified for Cumulus " \ "temperature fields in %s." % (temp_u, self.import_config_path) raise weewx.UnitError(_msg) # Pressure try: press_u = cumulus_config_dict['Units'].get('pressure') except KeyError: _msg = "No units specified for Cumulus pressure " \ "fields in %s." % (self.import_config_path, ) raise weewx.UnitError(_msg) else: if press_u in ['inHg', 'mbar', 'hPa']: self._header_map['cur_slp']['units'] = press_u else: _msg = "Unknown units '%s' specified for Cumulus " \ "pressure fields in %s." % (press_u, self.import_config_path) raise weewx.UnitError(_msg) # Rain try: rain_u = cumulus_config_dict['Units'].get('rain') except KeyError: _msg = "No units specified for Cumulus " \ "rain fields in %s." % (self.import_config_path, ) raise weewx.UnitError(_msg) else: if rain_u in rain_units_dict: self._header_map['midnight_rain']['units'] = rain_u self._header_map['cur_rain_rate']['units'] = rain_units_dict[ rain_u] else: _msg = "Unknown units '%s' specified for Cumulus " \ "rain fields in %s." % (rain_u, self.import_config_path) raise weewx.UnitError(_msg) # Speed try: speed_u = cumulus_config_dict['Units'].get('speed') except KeyError: _msg = "No units specified for Cumulus " \ "speed fields in %s." % (self.import_config_path, ) raise weewx.UnitError(_msg) else: if speed_u in weewx.units.default_unit_format_dict: self._header_map['avg_wind_speed']['units'] = speed_u self._header_map['gust_wind_speed']['units'] = speed_u else: _msg = "Unknown units '%s' specified for Cumulus " \ "speed fields in %s." % (speed_u, self.import_config_path) raise weewx.UnitError(_msg) # get our source file path try: self.source = cumulus_config_dict['directory'] except KeyError: _msg = "Cumulus monthly logs directory not specified in '%s'." % import_config_path raise weewx.ViolatedPrecondition(_msg) # get the source file encoding, default to utf-8-sig self.source_encoding = self.cumulus_config_dict.get( 'source_encoding', 'utf-8-sig') # property holding the current log file name being processed self.file_name = None # Now get a list on monthly log files sorted from oldest to newest month_log_list = glob.glob(self.source + '/?????log.txt') _temp = [(fn, fn[-9:-7], time.strptime(fn[-12:-9], '%b').tm_mon) for fn in month_log_list] self.log_list = [ a[0] for a in sorted(_temp, key=lambda el: (el[1], el[2])) ] if len(self.log_list) == 0: raise weeimport.WeeImportIOError( "No Cumulus monthly logs found in directory '%s'." % self.source) # tell the user/log what we intend to do _msg = "Cumulus monthly log files in the '%s' directory will be imported" % self.source print(_msg) log.info(_msg) _msg = "The following options will be used:" if self.verbose: print(_msg) log.debug(_msg) _msg = " config=%s, import-config=%s" % (config_path, self.import_config_path) if self.verbose: print(_msg) log.debug(_msg) if options.date: _msg = " date=%s" % options.date else: # we must have --from and --to _msg = " from=%s, to=%s" % (options.date_from, options.date_to) if self.verbose: print(_msg) log.debug(_msg) _msg = " dry-run=%s, calc_missing=%s, " \ "ignore_invalid_data=%s" % (self.dry_run, self.calc_missing, self.ignore_invalid_data) if self.verbose: print(_msg) log.debug(_msg) _msg = " tranche=%s, interval=%s" % (self.tranche, self.interval) if self.verbose: print(_msg) log.debug(_msg) _msg = " UV=%s, radiation=%s" % (self.UV_sensor, self.solar_sensor) if self.verbose: print(_msg) log.debug(_msg) _msg = "Using database binding '%s', which is bound " \ "to database '%s'" % (self.db_binding_wx, self.dbm.database_name) print(_msg) log.info(_msg) _msg = "Destination table '%s' unit system " \ "is '%#04x' (%s)." % (self.dbm.table_name, self.archive_unit_sys, unit_nicknames[self.archive_unit_sys]) print(_msg) log.info(_msg) if self.calc_missing: print("Missing derived observations will be calculated.") if not self.UV_sensor: print("All WeeWX UV fields will be set to None.") if not self.solar_sensor: print("All WeeWX radiation fields will be set to None.") if options.date or options.date_from: print("Observations timestamped after %s and " "up to and" % timestamp_to_string(self.first_ts)) print("including %s will be imported." % timestamp_to_string(self.last_ts)) if self.dry_run: print( "This is a dry run, imported data will not be saved to archive." )
def __init__(self, config_dict, config_path, cumulus_config_dict, import_config_path, options, log): # call our parents __init__ super(CumulusSource, self).__init__(config_dict, cumulus_config_dict, options, log) # save our import config path self.import_config_path = import_config_path # save our import config dict self.cumulus_config_dict = cumulus_config_dict # wind dir bounds self.wind_dir = [0, 360] # Decimal separator used in monthly log files, default to decimal point self.decimal = cumulus_config_dict.get('decimal', '.') # Field delimiter used in monthly log files, default to comma self.delimiter = cumulus_config_dict.get('delimiter', ',') # We combine Cumulus date and time fields to give a fixed format # date-time string self.raw_datetime_format = '%d/%m/%y %H:%M' # Cumulus log files provide a number of cumulative rainfall fields. We # cannot use the daily rainfall as this may reset at some time of day # other than midnight (as required by weewx). So we use field 26, total # rainfall since midnight and treat it as a cumulative value. self.rain = 'cumulative' # initialise our import field-to-weewx archive field map self.map = None # Units of measure for some obs (eg temperatures) cannot be derived from # the Cumulus monthly log files. These units must be specified by the # user in the import config file. Read these units and fill in the # missing unit data in the header map. Do some basic error checking and # validation, if one of the fields is missing or invalid then we need # to catch the error and raise it as we can't go on. # Temperature try: temp_u = cumulus_config_dict['Units'].get('temperature') except: _msg = "No units specified for Cumulus temperature fields in %s." % (self.import_config_path, ) raise weewx.UnitError(_msg) else: if temp_u in weewx.units.default_unit_format_dict: self._header_map['cur_out_temp']['units'] = temp_u self._header_map['curr_in_temp']['units'] = temp_u self._header_map['cur_dewpoint']['units'] = temp_u self._header_map['cur_heatindex']['units'] = temp_u self._header_map['cur_windchill']['units'] = temp_u self._header_map['cur_app_temp']['units'] = temp_u else: _msg = "Unknown units '%s' specified for Cumulus temperature fields in %s." % (temp_u, self.import_config_path) raise weewx.UnitError(_msg) # Pressure try: press_u = cumulus_config_dict['Units'].get('pressure') except: _msg = "No units specified for Cumulus pressure fields in %s." % (self.import_config_path, ) raise weewx.UnitError(_msg) else: if press_u in ['inHg', 'mbar', 'hPa']: self._header_map['cur_slp']['units'] = press_u else: _msg = "Unknown units '%s' specified for Cumulus pressure fields in %s." % (press_u, self.import_config_path) raise weewx.UnitError(_msg) # Rain try: rain_u = cumulus_config_dict['Units'].get('rain') except: _msg = "No units specified for Cumulus rain fields in %s." % (self.import_config_path, ) raise weewx.UnitError(_msg) else: if rain_u in rain_units_dict: self._header_map['midnight_rain']['units'] = rain_u self._header_map['cur_rain_rate']['units'] = rain_units_dict[rain_u] else: _msg = "Unknown units '%s' specified for Cumulus rain fields in %s." % (rain_u, self.import_config_path) raise weewx.UnitError(_msg) # Speed try: speed_u = cumulus_config_dict['Units'].get('speed') except: _msg = "No units specified for Cumulus speed fields in %s." % (self.import_config_path, ) raise weewx.UnitError(_msg) else: if speed_u in weewx.units.default_unit_format_dict: self._header_map['avg_wind_speed']['units'] = speed_u self._header_map['gust_wind_speed']['units'] = speed_u else: _msg = "Unknown units '%s' specified for Cumulus speed fields in %s." % (speed_u, self.import_config_path) raise weewx.UnitError(_msg) # get our source file path try: self.source = cumulus_config_dict['directory'] except KeyError: raise weewx.ViolatedPrecondition("Cumulus monthly logs directory not specified in '%s'." % import_config_path) # Now get a list on monthly log files sorted from oldest to newest month_log_list = glob.glob(self.source + '/?????log.txt') _temp = [(fn, fn[-9:-7], time.strptime(fn[-12:-9],'%b').tm_mon) for fn in month_log_list] self.log_list = [a[0] for a in sorted(_temp, key = lambda el : (el[1], el[2]))] if len(self.log_list) == 0: raise weeimport.WeeImportIOError( "No Cumulus monthly logs found in directory '%s'." % self.source) # tell the user/log what we intend to do _msg = "Cumulus monthly log files in the '%s' directory will be imported" % self.source self.wlog.printlog(syslog.LOG_INFO, _msg) _msg = "The following options will be used:" self.wlog.verboselog(syslog.LOG_DEBUG, _msg) _msg = " config=%s, import-config=%s" % (config_path, self.import_config_path) self.wlog.verboselog(syslog.LOG_DEBUG, _msg) _msg = " date=%s" % options.date self.wlog.verboselog(syslog.LOG_DEBUG, _msg) _msg = " dry-run=%s, calc-missing=%s" % (self.dry_run, self.calc_missing) self.wlog.verboselog(syslog.LOG_DEBUG, _msg) _msg = " tranche=%s, interval=%s" % (self.tranche, self.interval) self.wlog.verboselog(syslog.LOG_DEBUG, _msg) _msg = " UV=%s, radiation=%s" % (self.UV_sensor, self.solar_sensor) self.wlog.verboselog(syslog.LOG_DEBUG, _msg) _msg = "Using database binding '%s', which is bound to database '%s'" % (self.db_binding_wx, self.dbm.database_name) self.wlog.printlog(syslog.LOG_INFO, _msg) _msg = "Destination table '%s' unit system is '%#04x' (%s)." % (self.dbm.table_name, self.archive_unit_sys, unit_nicknames[self.archive_unit_sys]) self.wlog.printlog(syslog.LOG_INFO, _msg) if self.calc_missing: print "Missing derived observations will be calculated." if not self.UV_sensor: print "All weewx UV fields will be set to None." if not self.solar_sensor: print "All weewx radiation fields will be set to None." if options.date: print "Observations timestamped after %s and up to and" % (timestamp_to_string(self.first_ts), ) print "including %s will be imported." % (timestamp_to_string(self.last_ts), ) if self.dry_run: print "This is a dry run, imported data will not be saved to archive."