def add_record(self, record): """Add a new record, then return the difference in value divided by the difference in time.""" # If the type does not appear in the incoming record, then we can't calculate the derivative if self.obs_type not in record: raise weewx.CannotCalculate(self.obs_type) derivative = None if record[self.obs_type] is not None: # We can't calculate anything if we don't have an old record if self.old_timestamp: # Check to make sure the incoming record is later than the retained record if record['dateTime'] < self.old_timestamp: raise weewx.ViolatedPrecondition( "Records presented out of order (%s vs %s)" % (record['dateTime'], self.old_timestamp)) # Calculate the time derivative only if there is a delta in time, # and the old record is not too old. if record['dateTime'] != self.old_timestamp \ and (record['dateTime'] - self.old_timestamp) <= self.stale_age: # All OK. derivative = (record[self.obs_type] - self.old_value) \ / (record['dateTime'] - self.old_timestamp) # Save the current values self.old_timestamp = record['dateTime'] self.old_value = record[self.obs_type] return derivative
def __init__(self, config_dict, config_path, cumulus_config_dict, import_config_path, options): # call our parents __init__ super(CumulusSource, self).__init__(config_dict, cumulus_config_dict, options) # save our import config path self.import_config_path = import_config_path # save our import config dict self.cumulus_config_dict = cumulus_config_dict # wind dir bounds self.wind_dir = [0, 360] # field delimiter used in monthly log files, default to comma self.delimiter = str(cumulus_config_dict.get('delimiter', ',')) # decimal separator used in monthly log files, default to decimal point self.decimal = cumulus_config_dict.get('decimal', '.') # date separator used in monthly log files, default to solidus separator = cumulus_config_dict.get('separator', '/') # we combine Cumulus date and time fields to give a fixed format # date-time string self.raw_datetime_format = separator.join(('%d', '%m', '%y %H:%M')) # Cumulus log files provide a number of cumulative rainfall fields. We # cannot use the daily rainfall as this may reset at some time of day # other than midnight (as required by WeeWX). So we use field 26, total # rainfall since midnight and treat it as a cumulative value. self.rain = 'cumulative' # initialise our import field-to-WeeWX archive field map self.map = None # Cumulus log files have a number of 'rain' fields that can be used to # derive the WeeWX rain field. Which one is available depends on the # Cumulus version that created the logs. The preferred field is field # 26(AA) - total rainfall since midnight but it is only available in # Cumulus v1.9.4 or later. If that field is not available then the # preferred field in field 09(J) - total rainfall today then field # 11(L) - total rainfall counter. Initialise the rain_source_confirmed # property now and we will deal with it later when we have some source # data. self.rain_source_confirmed = None # Units of measure for some obs (eg temperatures) cannot be derived from # the Cumulus monthly log files. These units must be specified by the # user in the import config file. Read these units and fill in the # missing unit data in the header map. Do some basic error checking and # validation, if one of the fields is missing or invalid then we need # to catch the error and raise it as we can't go on. # Temperature try: temp_u = cumulus_config_dict['Units'].get('temperature') except KeyError: _msg = "No units specified for Cumulus temperature " \ "fields in %s." % (self.import_config_path, ) raise weewx.UnitError(_msg) else: if temp_u in weewx.units.default_unit_format_dict: self._header_map['cur_out_temp']['units'] = temp_u self._header_map['curr_in_temp']['units'] = temp_u self._header_map['cur_dewpoint']['units'] = temp_u self._header_map['cur_heatindex']['units'] = temp_u self._header_map['cur_windchill']['units'] = temp_u self._header_map['cur_app_temp']['units'] = temp_u else: _msg = "Unknown units '%s' specified for Cumulus " \ "temperature fields in %s." % (temp_u, self.import_config_path) raise weewx.UnitError(_msg) # Pressure try: press_u = cumulus_config_dict['Units'].get('pressure') except KeyError: _msg = "No units specified for Cumulus pressure " \ "fields in %s." % (self.import_config_path, ) raise weewx.UnitError(_msg) else: if press_u in ['inHg', 'mbar', 'hPa']: self._header_map['cur_slp']['units'] = press_u else: _msg = "Unknown units '%s' specified for Cumulus " \ "pressure fields in %s." % (press_u, self.import_config_path) raise weewx.UnitError(_msg) # Rain try: rain_u = cumulus_config_dict['Units'].get('rain') except KeyError: _msg = "No units specified for Cumulus " \ "rain fields in %s." % (self.import_config_path, ) raise weewx.UnitError(_msg) else: if rain_u in rain_units_dict: self._header_map['midnight_rain']['units'] = rain_u self._header_map['cur_rain_rate']['units'] = rain_units_dict[ rain_u] else: _msg = "Unknown units '%s' specified for Cumulus " \ "rain fields in %s." % (rain_u, self.import_config_path) raise weewx.UnitError(_msg) # Speed try: speed_u = cumulus_config_dict['Units'].get('speed') except KeyError: _msg = "No units specified for Cumulus " \ "speed fields in %s." % (self.import_config_path, ) raise weewx.UnitError(_msg) else: if speed_u in weewx.units.default_unit_format_dict: self._header_map['avg_wind_speed']['units'] = speed_u self._header_map['gust_wind_speed']['units'] = speed_u else: _msg = "Unknown units '%s' specified for Cumulus " \ "speed fields in %s." % (speed_u, self.import_config_path) raise weewx.UnitError(_msg) # get our source file path try: self.source = cumulus_config_dict['directory'] except KeyError: _msg = "Cumulus monthly logs directory not specified in '%s'." % import_config_path raise weewx.ViolatedPrecondition(_msg) # get the source file encoding, default to utf-8-sig self.source_encoding = self.cumulus_config_dict.get( 'source_encoding', 'utf-8-sig') # property holding the current log file name being processed self.file_name = None # Now get a list on monthly log files sorted from oldest to newest month_log_list = glob.glob(self.source + '/?????log.txt') _temp = [(fn, fn[-9:-7], time.strptime(fn[-12:-9], '%b').tm_mon) for fn in month_log_list] self.log_list = [ a[0] for a in sorted(_temp, key=lambda el: (el[1], el[2])) ] if len(self.log_list) == 0: raise weeimport.WeeImportIOError( "No Cumulus monthly logs found in directory '%s'." % self.source) # tell the user/log what we intend to do _msg = "Cumulus monthly log files in the '%s' directory will be imported" % self.source print(_msg) log.info(_msg) _msg = "The following options will be used:" if self.verbose: print(_msg) log.debug(_msg) _msg = " config=%s, import-config=%s" % (config_path, self.import_config_path) if self.verbose: print(_msg) log.debug(_msg) if options.date: _msg = " date=%s" % options.date else: # we must have --from and --to _msg = " from=%s, to=%s" % (options.date_from, options.date_to) if self.verbose: print(_msg) log.debug(_msg) _msg = " dry-run=%s, calc_missing=%s, " \ "ignore_invalid_data=%s" % (self.dry_run, self.calc_missing, self.ignore_invalid_data) if self.verbose: print(_msg) log.debug(_msg) _msg = " tranche=%s, interval=%s" % (self.tranche, self.interval) if self.verbose: print(_msg) log.debug(_msg) _msg = " UV=%s, radiation=%s" % (self.UV_sensor, self.solar_sensor) if self.verbose: print(_msg) log.debug(_msg) _msg = "Using database binding '%s', which is bound " \ "to database '%s'" % (self.db_binding_wx, self.dbm.database_name) print(_msg) log.info(_msg) _msg = "Destination table '%s' unit system " \ "is '%#04x' (%s)." % (self.dbm.table_name, self.archive_unit_sys, unit_nicknames[self.archive_unit_sys]) print(_msg) log.info(_msg) if self.calc_missing: print("Missing derived observations will be calculated.") if not self.UV_sensor: print("All WeeWX UV fields will be set to None.") if not self.solar_sensor: print("All WeeWX radiation fields will be set to None.") if options.date or options.date_from: print("Observations timestamped after %s and " "up to and" % timestamp_to_string(self.first_ts)) print("including %s will be imported." % timestamp_to_string(self.last_ts)) if self.dry_run: print( "This is a dry run, imported data will not be saved to archive." )
def __init__(self, config_dict, config_path, wu_config_dict, import_config_path, options): # call our parents __init__ super(WUSource, self).__init__(config_dict, wu_config_dict, options) # save our import config path self.import_config_path = import_config_path # save our import config dict self.wu_config_dict = wu_config_dict # get our WU station ID try: self.station_id = wu_config_dict['station_id'] except KeyError: _msg = "Weather Underground station ID not specified in '%s'." % import_config_path raise weewx.ViolatedPrecondition(_msg) # get our WU API key try: self.api_key = wu_config_dict['api_key'] except KeyError: _msg = "Weather Underground API key not specified in '%s'." % import_config_path raise weewx.ViolatedPrecondition(_msg) # wind dir bounds _wind_direction = option_as_list( wu_config_dict.get('wind_direction', '0,360')) try: if float(_wind_direction[0]) <= float(_wind_direction[1]): self.wind_dir = [ float(_wind_direction[0]), float(_wind_direction[1]) ] else: self.wind_dir = [0, 360] except (IndexError, TypeError): self.wind_dir = [0, 360] # some properties we know because of the format of the returned WU data # WU returns a fixed format date-time string self.raw_datetime_format = '%Y-%m-%d %H:%M:%S' # WU only provides hourly rainfall and a daily cumulative rainfall. # We use the latter so force 'cumulative' for rain. self.rain = 'cumulative' # initialise our import field-to-WeeWX archive field map self.map = None # For a WU import we might have to import multiple days but we can only # get one day at a time from WU. So our start and end properties # (counters) are datetime objects and our increment is a timedelta. # Get datetime objects for any date or date range specified on the # command line, if there wasn't one then default to today. self.start = dt.fromtimestamp(startOfDay(self.first_ts)) self.end = dt.fromtimestamp(startOfDay(self.last_ts)) # set our increment self.increment = datetime.timedelta(days=1) # property holding the current period being processed self.period = None # tell the user/log what we intend to do _msg = "Observation history for Weather Underground station '%s' will be imported." % self.station_id print(_msg) log.info(_msg) _msg = "The following options will be used:" if self.verbose: print(_msg) log.debug(_msg) _msg = " config=%s, import-config=%s" % (config_path, self.import_config_path) if self.verbose: print(_msg) log.debug(_msg) if options.date: _msg = " station=%s, date=%s" % (self.station_id, options.date) else: # we must have --from and --to _msg = " station=%s, from=%s, to=%s" % ( self.station_id, options.date_from, options.date_to) if self.verbose: print(_msg) log.debug(_msg) _obf_api_key_msg = '='.join( [' apiKey', '*' * (len(self.api_key) - 4) + self.api_key[-4:]]) if self.verbose: print(_obf_api_key_msg) log.debug(_obf_api_key_msg) _msg = " dry-run=%s, calc_missing=%s, ignore_invalid_data=%s" % ( self.dry_run, self.calc_missing, self.ignore_invalid_data) if self.verbose: print(_msg) log.debug(_msg) _msg = " tranche=%s, interval=%s, wind_direction=%s" % ( self.tranche, self.interval, self.wind_dir) if self.verbose: print(_msg) log.debug(_msg) _msg = "Using database binding '%s', which is bound to database '%s'" % ( self.db_binding_wx, self.dbm.database_name) print(_msg) log.info(_msg) _msg = "Destination table '%s' unit system is '%#04x' (%s)." % ( self.dbm.table_name, self.archive_unit_sys, unit_nicknames[self.archive_unit_sys]) print(_msg) log.info(_msg) if self.calc_missing: print("Missing derived observations will be calculated.") if options.date or options.date_from: print("Observations timestamped after %s and up to and" % timestamp_to_string(self.first_ts)) print("including %s will be imported." % timestamp_to_string(self.last_ts)) if self.dry_run: print( "This is a dry run, imported data will not be saved to archive." )
def __init__(self, config_dict, config_path, csv_config_dict, import_config_path, options): # call our parents __init__ super(CSVSource, self).__init__(config_dict, csv_config_dict, options) # save our import config path self.import_config_path = import_config_path # save our import config dict self.csv_config_dict = csv_config_dict # get a few config settings from our CSV config dict # csv field delimiter self.delimiter = str(self.csv_config_dict.get('delimiter', ',')) # string format used to decode the imported field holding our dateTime self.raw_datetime_format = self.csv_config_dict.get( 'raw_datetime_format', '%Y-%m-%d %H:%M:%S') # is our rain discrete or cumulative self.rain = self.csv_config_dict.get('rain', 'cumulative') # determine valid range for imported wind direction _wind_direction = option_as_list( self.csv_config_dict.get('wind_direction', '0,360')) try: if float(_wind_direction[0]) <= float(_wind_direction[1]): self.wind_dir = [ float(_wind_direction[0]), float(_wind_direction[1]) ] else: self.wind_dir = [-360, 360] except (KeyError, ValueError): self.wind_dir = [-360, 360] # get our source file path try: self.source = csv_config_dict['file'] except KeyError: raise weewx.ViolatedPrecondition( "CSV source file not specified in '%s'." % import_config_path) # get the source file encoding, default to utf-8-sig self.source_encoding = self.csv_config_dict.get( 'source_encoding', 'utf-8-sig') # initialise our import field-to-WeeWX archive field map self.map = None # initialise some other properties we will need self.start = 1 self.end = 1 self.increment = 1 # tell the user/log what we intend to do _msg = "A CSV import from source file '%s' has been requested." % self.source print(_msg) log.info(_msg) _msg = "The following options will be used:" if self.verbose: print(_msg) log.debug(_msg) _msg = " config=%s, import-config=%s" % (config_path, self.import_config_path) if self.verbose: print(_msg) log.debug(_msg) if options.date: _msg = " source=%s, date=%s" % (self.source, options.date) else: # we must have --from and --to _msg = " source=%s, from=%s, to=%s" % ( self.source, options.date_from, options.date_to) if self.verbose: print(_msg) log.debug(_msg) _msg = " dry-run=%s, calc_missing=%s, ignore_invalid_data=%s" % ( self.dry_run, self.calc_missing, self.ignore_invalid_data) if self.verbose: print(_msg) log.debug(_msg) _msg = " tranche=%s, interval=%s, date/time_string_format=%s" % ( self.tranche, self.interval, self.raw_datetime_format) if self.verbose: print(_msg) log.debug(_msg) _msg = " delimiter='%s', rain=%s, wind_direction=%s" % ( self.delimiter, self.rain, self.wind_dir) if self.verbose: print(_msg) log.debug(_msg) _msg = " UV=%s, radiation=%s" % (self.UV_sensor, self.solar_sensor) if self.verbose: print(_msg) log.debug(_msg) _msg = "Using database binding '%s', which is bound to database '%s'" % ( self.db_binding_wx, self.dbm.database_name) print(_msg) log.info(_msg) _msg = "Destination table '%s' unit system is '%#04x' (%s)." % ( self.dbm.table_name, self.archive_unit_sys, unit_nicknames[self.archive_unit_sys]) print(_msg) log.info(_msg) if self.calc_missing: _msg = "Missing derived observations will be calculated." print(_msg) log.info(_msg) if not self.UV_sensor: _msg = "All WeeWX UV fields will be set to None." print(_msg) log.info(_msg) if not self.solar_sensor: _msg = "All WeeWX radiation fields will be set to None." print(_msg) log.info(_msg) if options.date or options.date_from: _msg = "Observations timestamped after %s and up to and" % timestamp_to_string( self.first_ts) print(_msg) log.info(_msg) _msg = "including %s will be imported." % timestamp_to_string( self.last_ts) print(_msg) log.info(_msg) if self.dry_run: _msg = "This is a dry run, imported data will not be saved to archive." print(_msg) log.info(_msg)
def __init__(self, config_dict, config_path, wd_config_dict, import_config_path, options): # call our parents __init__ super(WDSource, self).__init__(config_dict, wd_config_dict, options) # save the import config path self.import_config_path = import_config_path # save the import config dict self.wd_config_dict = wd_config_dict # our parent uses 'derive' as the default interval setting, for WD the # default should be 1 (minute) so redo the interval setting with our # default self.interval = wd_config_dict.get('interval', 1) # wind dir bounds self.wind_dir = [0, 360] # How the WeeWX field 'rain' is populated depends on the source rain # data. If the only data available is cumulative then the WeeWX rain # field is calculated as the difference between successive cumulative # values. WD provides a rain per interval field so that data can be # used to map directly to the WeeWX rain field. If rain is to be # calculated from a cumulative value then self.rain must be set to # 'cumulative', to map directly to the WeeWX rain field self.rain must # be set to None. self.rain = None # field delimiter used in text format monthly log files, default to # space self.txt_delimiter = str(wd_config_dict.get('txt_delimiter', ' ')) # field delimiter used in csv format monthly log files, default to # comma self.csv_delimiter = str(wd_config_dict.get('csv_delimiter', ',')) # decimal separator used in monthly log files, default to decimal point self.decimal = wd_config_dict.get('decimal', '.') # ignore extreme > 255.0 values for temperature and humidity fields self.ignore_extreme_temp_hum = weeutil.weeutil.tobool( wd_config_dict.get('ignore_extreme_temp_hum', True)) # initialise the import field-to-WeeWX archive field map self.map = None # property holding the current log file name being processed self.file_name = None # WD logs use either US or Metric units. The units used in each case # are: # Metric units: C, knots, hPa, mm # US units: F, mph, inHg, inch # # The user must specify the units to be used in the import config file. # This can be by either by specifying the log units as Metric or US # using the 'units' config option. Alternatively temperature, pressure, # rainfall and speed units can be specified individually under the # [Units] stanza. First check for a valid 'units' config option then # check for individual group units. Do some basic error checking and # validation, if one of the fields is missing or invalid then we need # to catch the error and raise it as we can't go on. log_unit_config = wd_config_dict.get('Units') if log_unit_config is not None: # get the units config option log_unit_sys = wd_config_dict['Units'].get('units') # accept any capitalization of USA as == US log_unit_sys = log_unit_sys if log_unit_sys.upper( ) != 'USA' else 'US' # does the units config option specify a valid log unit system if log_unit_sys is None or log_unit_sys.upper() not in [ 'METRIC', 'US' ]: # log unit system not specified look for individual entries # temperature temp_u = wd_config_dict['Units'].get('temperature') if temp_u is not None: if temp_u in weewx.units.default_unit_format_dict: self._header_map['temperature']['units'] = temp_u self._header_map['dewpoint']['units'] = temp_u self._header_map['heatindex']['units'] = temp_u self._header_map['soiltemp']['units'] = temp_u self._header_map['temp1']['units'] = temp_u self._header_map['temp2']['units'] = temp_u self._header_map['temp3']['units'] = temp_u self._header_map['temp4']['units'] = temp_u self._header_map['temp5']['units'] = temp_u self._header_map['temp6']['units'] = temp_u self._header_map['temp7']['units'] = temp_u else: _msg = "Unknown units '%s' specified for Weather Display " \ "temperature fields in %s." % (temp_u, self.import_config_path) raise weewx.UnitError(_msg) else: _msg = "No units specified for Weather Display temperature " \ "fields in %s." % (self.import_config_path,) raise weewx.UnitError(_msg) # pressure press_u = wd_config_dict['Units'].get('pressure') if press_u is not None: if press_u in ['inHg', 'hPa']: self._header_map['barometer']['units'] = press_u else: _msg = "Unknown units '%s' specified for Weather Display " \ "pressure fields in %s." % (press_u, self.import_config_path) raise weewx.UnitError(_msg) else: _msg = "No units specified for Weather Display pressure " \ "fields in %s." % (self.import_config_path,) raise weewx.UnitError(_msg) # rain rain_u = wd_config_dict['Units'].get('rain') if rain_u is not None: if rain_u in ['inch', 'mm']: self._header_map['rainlastmin']['units'] = rain_u self._header_map['dailyrain']['units'] = rain_u self._header_map['monthlyrain']['units'] = rain_u self._header_map['yearlyrain']['units'] = rain_u self._header_map['dailyet']['units'] = rain_u else: _msg = "Unknown units '%s' specified for Weather Display " \ "rain fields in %s." % (rain_u, self.import_config_path) raise weewx.UnitError(_msg) else: _msg = "No units specified for Weather Display rain fields " \ "in %s." % (self.import_config_path,) raise weewx.UnitError(_msg) # speed speed_u = wd_config_dict['Units'].get('speed') if speed_u is not None: if speed_u in ['inch', 'mm']: self._header_map['windspeed']['units'] = speed_u self._header_map['gustspeed']['units'] = speed_u else: _msg = "Unknown units '%s' specified for Weather Display " \ "speed fields in %s." % (speed_u, self.import_config_path) raise weewx.UnitError(_msg) else: _msg = "No units specified for Weather Display speed fields " \ "in %s." % (self.import_config_path,) raise weewx.UnitError(_msg) else: # log unit system specified _unit_sys = log_unit_sys.upper() # do we have a valid log unit system if _unit_sys in ['METRIC', 'US']: # valid log unit system so assign units as applicable self._header_map['temperature'][ 'units'] = self.wd_unit_sys['temperature'][_unit_sys] self._header_map['dewpoint']['units'] = self.wd_unit_sys[ 'temperature'][_unit_sys] self._header_map['heatindex']['units'] = self.wd_unit_sys[ 'temperature'][_unit_sys] self._header_map['barometer']['units'] = self.wd_unit_sys[ 'barometer'][_unit_sys] self._header_map['windspeed']['units'] = self.wd_unit_sys[ 'windspeed'][_unit_sys] self._header_map['gustspeed']['units'] = self.wd_unit_sys[ 'gustspeed'][_unit_sys] self._header_map['rainlastmin'][ 'units'] = self.wd_unit_sys['rainlastmin'][_unit_sys] self._header_map['soiltemp']['units'] = self.wd_unit_sys[ 'soiltemp'][_unit_sys] for _num in range(1, 8): _temp = 'temp%s' % _num self._header_map[_temp]['units'] = self.wd_unit_sys[ _temp][_unit_sys] else: # no valid Units config found, we can't go on so raise an error raise weewx.UnitError( "Invalid setting for 'units' config option.") else: # there is no Units config, we can't go on so raise an error raise weewx.UnitError("No Weather Display units config found.") # obtain a list of logs files to be processed _to_process = wd_config_dict.get('logs_to_process', list(self.logs.keys())) self.logs_to_process = weeutil.weeutil.option_as_list(_to_process) # can missing log files be ignored self.ignore_missing_log = weeutil.weeutil.to_bool( wd_config_dict.get('ignore_missing_log', True)) # get our source file path try: self.source = wd_config_dict['directory'] except KeyError: _msg = "Weather Display monthly logs directory not specified in '%s'." % import_config_path raise weewx.ViolatedPrecondition(_msg) # get the source file encoding, default to utf-8-sig self.source_encoding = self.wd_config_dict.get('source_encoding', 'utf-8-sig') # Now get a list on monthly log files sorted from oldest to newest. # This is complicated by the log file naming convention used by WD. # first the 1 digit months _lg_5_list = glob.glob(self.source + '/' + '[0-9]' * 5 + 'lg.txt') # and the 2 digit months _lg_6_list = glob.glob(self.source + '/' + '[0-9]' * 6 + 'lg.txt') # concatenate the two lists to get the complete list month_lg_list = _lg_5_list + _lg_6_list # create a list of log files in chronological order (month, year) _temp = [] # create a list of log files, adding year and month fields for sorting for p in month_lg_list: # obtain the file name fn = os.path.split(p)[1] # obtain the numeric part of the file name _digits = ''.join(c for c in fn if c.isdigit()) # append a list of format [path+file name, month, year] _temp.append([p, int(_digits[:-4]), int(_digits[-4:])]) # now sort the list keeping just the log file path and name self.log_list = [ a[0] for a in sorted(_temp, key=lambda el: (el[2], el[1])) ] # if there are no log files then there is nothing to be done if len(self.log_list) == 0: raise weeimport.WeeImportIOError( "No Weather Display monthly logs found in directory '%s'." % self.source) # Some log files have entries that belong in a different month. # Initialise a list to hold these extra records for processing during # the appropriate month self.extras = {} for l in self.logs_to_process: self.extras[l] = [] # tell the user/log what we intend to do _msg = "Weather Display monthly log files in the '%s' directory will be imported" % self.source print(_msg) log.info(_msg) _msg = "The following options will be used:" if self.verbose: print(_msg) log.debug(_msg) _msg = " config=%s, import-config=%s" % (config_path, self.import_config_path) if self.verbose: print(_msg) log.debug(_msg) if options.date: _msg = " date=%s" % options.date else: # we must have --from and --to _msg = " from=%s, to=%s" % (options.date_from, options.date_to) if self.verbose: print(_msg) log.debug(_msg) _msg = " dry-run=%s, calc_missing=%s, ignore_invalid_data=%s" % ( self.dry_run, self.calc_missing, self.ignore_invalid_data) if self.verbose: print(_msg) log.debug(_msg) if log_unit_sys is not None and log_unit_sys.upper() in [ 'METRIC', 'US' ]: # valid unit system specified _msg = " monthly logs are in %s units" % log_unit_sys.upper() if self.verbose: print(_msg) log.debug(_msg) else: # group units specified _msg = " monthly logs use the following units:" if self.verbose: print(_msg) log.debug(_msg) _msg = " temperature=%s pressure=%s" % (temp_u, press_u) if self.verbose: print(_msg) log.debug(_msg) _msg = " rain=%s speed=%s" % (rain_u, speed_u) if self.verbose: print(_msg) log.debug(_msg) _msg = " tranche=%s, interval=%s" % (self.tranche, self.interval) if self.verbose: print(_msg) log.debug(_msg) _msg = " UV=%s, radiation=%s ignore extreme temperature and humidity=%s" % ( self.UV_sensor, self.solar_sensor, self.ignore_extreme_temp_hum) if self.verbose: print(_msg) log.debug(_msg) _msg = "Using database binding '%s', which is bound to database '%s'" % ( self.db_binding_wx, self.dbm.database_name) print(_msg) log.info(_msg) _msg = "Destination table '%s' unit system is '%#04x' (%s)." % ( self.dbm.table_name, self.archive_unit_sys, unit_nicknames[self.archive_unit_sys]) print(_msg) log.info(_msg) if self.calc_missing: print("Missing derived observations will be calculated.") if not self.UV_sensor: print("All WeeWX UV fields will be set to None.") if not self.solar_sensor: print("All WeeWX radiation fields will be set to None.") if options.date or options.date_from: print("Observations timestamped after %s and up to and" % timestamp_to_string(self.first_ts)) print("including %s will be imported." % timestamp_to_string(self.last_ts)) if self.dry_run: print( "This is a dry run, imported data will not be saved to archive." )
def __init__(self, config_dict, config_path, cumulus_config_dict, import_config_path, options, log): # call our parents __init__ super(CumulusSource, self).__init__(config_dict, cumulus_config_dict, options, log) # save our import config path self.import_config_path = import_config_path # save our import config dict self.cumulus_config_dict = cumulus_config_dict # wind dir bounds self.wind_dir = [0, 360] # Decimal separator used in monthly log files, default to decimal point self.decimal = cumulus_config_dict.get('decimal', '.') # Field delimiter used in monthly log files, default to comma self.delimiter = cumulus_config_dict.get('delimiter', ',') # We combine Cumulus date and time fields to give a fixed format # date-time string self.raw_datetime_format = '%d/%m/%y %H:%M' # Cumulus log files provide a number of cumulative rainfall fields. We # cannot use the daily rainfall as this may reset at some time of day # other than midnight (as required by weewx). So we use field 26, total # rainfall since midnight and treat it as a cumulative value. self.rain = 'cumulative' # initialise our import field-to-weewx archive field map self.map = None # Units of measure for some obs (eg temperatures) cannot be derived from # the Cumulus monthly log files. These units must be specified by the # user in the import config file. Read these units and fill in the # missing unit data in the header map. Do some basic error checking and # validation, if one of the fields is missing or invalid then we need # to catch the error and raise it as we can't go on. # Temperature try: temp_u = cumulus_config_dict['Units'].get('temperature') except: _msg = "No units specified for Cumulus temperature fields in %s." % (self.import_config_path, ) raise weewx.UnitError(_msg) else: if temp_u in weewx.units.default_unit_format_dict: self._header_map['cur_out_temp']['units'] = temp_u self._header_map['curr_in_temp']['units'] = temp_u self._header_map['cur_dewpoint']['units'] = temp_u self._header_map['cur_heatindex']['units'] = temp_u self._header_map['cur_windchill']['units'] = temp_u self._header_map['cur_app_temp']['units'] = temp_u else: _msg = "Unknown units '%s' specified for Cumulus temperature fields in %s." % (temp_u, self.import_config_path) raise weewx.UnitError(_msg) # Pressure try: press_u = cumulus_config_dict['Units'].get('pressure') except: _msg = "No units specified for Cumulus pressure fields in %s." % (self.import_config_path, ) raise weewx.UnitError(_msg) else: if press_u in ['inHg', 'mbar', 'hPa']: self._header_map['cur_slp']['units'] = press_u else: _msg = "Unknown units '%s' specified for Cumulus pressure fields in %s." % (press_u, self.import_config_path) raise weewx.UnitError(_msg) # Rain try: rain_u = cumulus_config_dict['Units'].get('rain') except: _msg = "No units specified for Cumulus rain fields in %s." % (self.import_config_path, ) raise weewx.UnitError(_msg) else: if rain_u in rain_units_dict: self._header_map['midnight_rain']['units'] = rain_u self._header_map['cur_rain_rate']['units'] = rain_units_dict[rain_u] else: _msg = "Unknown units '%s' specified for Cumulus rain fields in %s." % (rain_u, self.import_config_path) raise weewx.UnitError(_msg) # Speed try: speed_u = cumulus_config_dict['Units'].get('speed') except: _msg = "No units specified for Cumulus speed fields in %s." % (self.import_config_path, ) raise weewx.UnitError(_msg) else: if speed_u in weewx.units.default_unit_format_dict: self._header_map['avg_wind_speed']['units'] = speed_u self._header_map['gust_wind_speed']['units'] = speed_u else: _msg = "Unknown units '%s' specified for Cumulus speed fields in %s." % (speed_u, self.import_config_path) raise weewx.UnitError(_msg) # get our source file path try: self.source = cumulus_config_dict['directory'] except KeyError: raise weewx.ViolatedPrecondition("Cumulus monthly logs directory not specified in '%s'." % import_config_path) # Now get a list on monthly log files sorted from oldest to newest month_log_list = glob.glob(self.source + '/?????log.txt') _temp = [(fn, fn[-9:-7], time.strptime(fn[-12:-9],'%b').tm_mon) for fn in month_log_list] self.log_list = [a[0] for a in sorted(_temp, key = lambda el : (el[1], el[2]))] if len(self.log_list) == 0: raise weeimport.WeeImportIOError( "No Cumulus monthly logs found in directory '%s'." % self.source) # tell the user/log what we intend to do _msg = "Cumulus monthly log files in the '%s' directory will be imported" % self.source self.wlog.printlog(syslog.LOG_INFO, _msg) _msg = "The following options will be used:" self.wlog.verboselog(syslog.LOG_DEBUG, _msg) _msg = " config=%s, import-config=%s" % (config_path, self.import_config_path) self.wlog.verboselog(syslog.LOG_DEBUG, _msg) _msg = " date=%s" % options.date self.wlog.verboselog(syslog.LOG_DEBUG, _msg) _msg = " dry-run=%s, calc-missing=%s" % (self.dry_run, self.calc_missing) self.wlog.verboselog(syslog.LOG_DEBUG, _msg) _msg = " tranche=%s, interval=%s" % (self.tranche, self.interval) self.wlog.verboselog(syslog.LOG_DEBUG, _msg) _msg = " UV=%s, radiation=%s" % (self.UV_sensor, self.solar_sensor) self.wlog.verboselog(syslog.LOG_DEBUG, _msg) _msg = "Using database binding '%s', which is bound to database '%s'" % (self.db_binding_wx, self.dbm.database_name) self.wlog.printlog(syslog.LOG_INFO, _msg) _msg = "Destination table '%s' unit system is '%#04x' (%s)." % (self.dbm.table_name, self.archive_unit_sys, unit_nicknames[self.archive_unit_sys]) self.wlog.printlog(syslog.LOG_INFO, _msg) if self.calc_missing: print "Missing derived observations will be calculated." if not self.UV_sensor: print "All weewx UV fields will be set to None." if not self.solar_sensor: print "All weewx radiation fields will be set to None." if options.date: print "Observations timestamped after %s and up to and" % (timestamp_to_string(self.first_ts), ) print "including %s will be imported." % (timestamp_to_string(self.last_ts), ) if self.dry_run: print "This is a dry run, imported data will not be saved to archive."
def __init__(self, queue, database, username=None, password=None, line_format='single-line', tags=None, unit_system=None, augment_record=True, inputs=dict(), obs_to_upload='all', append_units_label=True, server_url=_DEFAULT_SERVER_URL, skip_upload=False, manager_dict=None, post_interval=None, max_backlog=sys.maxint, stale=None, log_success=True, log_failure=True, timeout=60, max_tries=3, retry_wait=5): super(InfluxThread, self).__init__(queue, protocol_name='Influx', manager_dict=manager_dict, post_interval=post_interval, max_backlog=max_backlog, stale=stale, log_success=log_success, log_failure=log_failure, max_tries=max_tries, timeout=timeout, retry_wait=retry_wait) self.database = database self.username = username self.password = password self.tags = tags self.upload_all = True if obs_to_upload.lower() == 'all' else False self.append_units_label = append_units_label self.inputs = inputs self.server_url = server_url self.skip_upload = to_bool(skip_upload) self.unit_system = unit_system self.augment_record = augment_record self.templates = dict() self.line_format = line_format # ensure that the database exists qstr = urllib.urlencode({'q': 'CREATE DATABASE %s' % self.database}) url = '%s/query?%s' % (self.server_url, qstr) req = urllib2.Request(url) req.add_header("User-Agent", "weewx/%s" % weewx.__version__) if self.username is not None: b64s = base64.encodestring( '%s:%s' % (self.username, self.password)).replace('\n', '') req.add_header("Authorization", "Basic %s" % b64s) try: self.post_with_retries(req) except (weewx.restx.FailedPost, weewx.restx.AbortedPost), e: raise weewx.ViolatedPrecondition(e)
def __init__(self, config_dict, config_path, weathercat_config_dict, import_config_path, options): # call our parents __init__ super(WeatherCatSource, self).__init__(config_dict, weathercat_config_dict, options) # save our import config path self.import_config_path = import_config_path # save our import config dict self.weathercat_config_dict = weathercat_config_dict # wind dir bounds self.wind_dir = [0, 360] # decimal separator used in monthly log files, default to decimal point self.decimal = weathercat_config_dict.get('decimal', '.') # Cumulus log files provide a number of cumulative rainfall fields. We # cannot use the daily rainfall as this may reset at some time of day # other than midnight (as required by WeeWX). So we use field 26, total # rainfall since midnight and treat it as a cumulative value. self.rain = 'cumulative' # The WeatherCat .cat file structure is well defined so we can # construct our import field-to-WeeWX archive field map now. The user # can specify the units used in the WeatherCat .cat file so first # construct a default field map then go through and adjust the units # where necessary. # first initialise with a default field map self.map = self.default_header_map # now check the [[Units]] stanza in the import config file and adjust # any units as required if 'Units' in weathercat_config_dict and len(weathercat_config_dict['Units']) > 0: # we have [[Units]] settings so iterate over each for group, value in six.iteritems(weathercat_config_dict['Units']): # is this group (eg 'temperature', 'rain' etc one that we know # about if group in self.weathercat_unit_groups: # it is, so iterate over each import field that could be affected by # this unit setting for field in self.weathercat_unit_groups[group]: # set the units field accordingly self.map[field]['units'] = value # We have one special 'derived' unit setting, rainRate. The # rainRate units are derived from the rain units by simply # appending '_per_hour' if group == 'precipitation': self.map['rainRate']['units'] = ''.join([value, '_per_hour']) # property holding the current log file name being processed self.file_name = None # get our source file path try: self.source = weathercat_config_dict['directory'] except KeyError: raise weewx.ViolatedPrecondition( "WeatherCat directory not specified in '%s'." % import_config_path) # Get a list of monthly .cat files sorted from oldest to newest. # Remember the .cat files are in 'year' folders. # first get a list of all the 'year' folders including path _y_list = [os.path.join(self.source, d) for d in os.listdir(self.source) if os.path.isdir(os.path.join(self.source, d))] # initialise our list of .cat files f_list = [] # iterate over the 'year' directories for _dir in _y_list: # find any .cat files in the 'year' directory and add them to the # file list f_list += glob.glob(''.join([_dir, '/*[0-9]_WeatherCatData.cat'])) # now get an intermediate list that we can use to sort the .cat file # list from oldest to newest _temp = [(fn, os.path.basename(os.path.dirname(fn)), os.path.basename(fn).split('_')[0].zfill(2)) for fn in f_list] # now do the sorting self.cat_list = [a[0] for a in sorted(_temp, key=lambda el: (el[1], el[2]))] if len(self.cat_list) == 0: raise weeimport.WeeImportIOError( "No WeatherCat monthly .cat files found in directory '%s'." % self.source) # tell the user/log what we intend to do _msg = "WeatherCat monthly .cat files in the '%s' directory " \ "will be imported" % self.source print(_msg) log.info(_msg) _msg = "The following options will be used:" if self.verbose: print(_msg) log.debug(_msg) _msg = " config=%s, import-config=%s" % (config_path, self.import_config_path) if self.verbose: print(_msg) log.debug(_msg) if options.date: _msg = " date=%s" % options.date else: # we must have --from and --to _msg = " from=%s, to=%s" % (options.date_from, options.date_to) if self.verbose: print(_msg) log.debug(_msg) _msg = " dry-run=%s, calc-missing=%s" % (self.dry_run, self.calc_missing) if self.verbose: print(_msg) log.debug(_msg) _msg = " tranche=%s, interval=%s" % (self.tranche, self.interval) if self.verbose: print(_msg) log.debug(_msg) _msg = " UV=%s, radiation=%s" % (self.UV_sensor, self.solar_sensor) if self.verbose: print(_msg) log.debug(_msg) _msg = "Using database binding '%s', which is " \ "bound to database '%s'" % (self.db_binding_wx, self.dbm.database_name) print(_msg) log.info(_msg) _msg = "Destination table '%s' unit system " \ "is '%#04x' (%s)." % (self.dbm.table_name, self.archive_unit_sys, unit_nicknames[self.archive_unit_sys]) print(_msg) log.info(_msg) if self.calc_missing: print("Missing derived observations will be calculated.") if not self.UV_sensor: print("All WeeWX UV fields will be set to None.") if not self.solar_sensor: print("All WeeWX radiation fields will be set to None.") if options.date or options.date_from: print("Observations timestamped after %s and " "up to and" % (timestamp_to_string(self.first_ts),)) print("including %s will be imported." % (timestamp_to_string(self.last_ts),)) if self.dry_run: print("This is a dry run, imported data will not be saved to archive.")
def __init__(self, config_dict, config_path, wu_config_dict, import_config_path, options, log): # call our parents __init__ super(WUSource, self).__init__(config_dict, wu_config_dict, options, log) # save our import config path self.import_config_path = import_config_path # save our import config dict self.wu_config_dict = wu_config_dict # get our WU station ID try: self.station_id = wu_config_dict['station_id'] except KeyError: raise weewx.ViolatedPrecondition( "Weather Underground station ID not specified in '%s'." % import_config_path) # wind dir bounds _wind_direction = option_as_list( wu_config_dict.get('wind_direction', '0,360')) try: if float(_wind_direction[0]) <= float(_wind_direction[1]): self.wind_dir = [ float(_wind_direction[0]), float(_wind_direction[1]) ] else: self.wind_dir = [0, 360] except: self.wind_dir = [0, 360] # some properties we know because of the format of the returned WU data # WU returns a fixed format date-time string self.raw_datetime_format = '%Y-%m-%d %H:%M:%S' # WU only provides hourly rainfall and a daily cumulative rainfall. # We use the latter so force 'cumulative' for rain. self.rain = 'cumulative' # initialise our import field-to-weewx archive field map self.map = None # For a WU import we might have to import multiple days but we can only # get one day at a time from WU. So our start and end properties # (counters) are datetime objects and our increment is a timedelta. # Get datetime objects for any date or date range specified on the # command line, if there wasn't one then default to today. self.start = dt.fromtimestamp(startOfDay(self.first_ts)) self.end = dt.fromtimestamp(startOfDay(self.last_ts)) # set our increment self.increment = datetime.timedelta(days=1) # tell the user/log what we intend to do _msg = "Observation history for Weather Underground station '%s' will be imported." % self.station_id self.wlog.printlog(syslog.LOG_INFO, _msg) _msg = "The following options will be used:" self.wlog.verboselog(syslog.LOG_DEBUG, _msg) _msg = " config=%s, import-config=%s" % (config_path, self.import_config_path) self.wlog.verboselog(syslog.LOG_DEBUG, _msg) _msg = " station=%s, date=%s" % (self.station_id, options.date) self.wlog.verboselog(syslog.LOG_DEBUG, _msg) _msg = " dry-run=%s, calc-missing=%s" % (self.dry_run, self.calc_missing) self.wlog.verboselog(syslog.LOG_DEBUG, _msg) _msg = " tranche=%s, interval=%s, wind_direction=%s" % ( self.tranche, self.interval, self.wind_dir) self.wlog.verboselog(syslog.LOG_DEBUG, _msg) _msg = "Using database binding '%s', which is bound to database '%s'" % ( self.db_binding_wx, self.dbm.database_name) self.wlog.printlog(syslog.LOG_INFO, _msg) _msg = "Destination table '%s' unit system is '%#04x' (%s)." % ( self.dbm.table_name, self.archive_unit_sys, unit_nicknames[self.archive_unit_sys]) self.wlog.printlog(syslog.LOG_INFO, _msg) if self.calc_missing: print "Missing derived observations will be calculated." if options.date: print "Observations timestamped after %s and up to and" % ( timestamp_to_string(self.first_ts), ) print "including %s will be imported." % (timestamp_to_string( self.last_ts), ) if self.dry_run: print "This is a dry run, imported data will not be saved to archive."
def __init__(self, config_dict, config_path, csv_config_dict, import_config_path, options, log): # call our parents __init__ super(CSVSource, self).__init__(config_dict, csv_config_dict, options, log) # save our import config path self.import_config_path = import_config_path # save our import config dict self.csv_config_dict = csv_config_dict # get a few config settings from our CSV config dict # string format used to decode the imported field holding our dateTime self.raw_datetime_format = self.csv_config_dict.get( 'raw_datetime_format', '%Y-%m-%d %H:%M:%S') # is our rain discrete or cumulative self.rain = self.csv_config_dict.get('rain', 'cumulative') # determine valid range for imported wind direction _wind_direction = option_as_list( self.csv_config_dict.get('wind_direction', '0,360')) try: if float(_wind_direction[0]) <= float(_wind_direction[1]): self.wind_dir = [ float(_wind_direction[0]), float(_wind_direction[1]) ] else: self.wind_dir = [-360, 360] except: self.wind_dir = [-360, 360] # get our source file path try: self.source = csv_config_dict['file'] except KeyError: raise weewx.ViolatedPrecondition( "CSV source file not specified in '%s'." % import_config_path) # initialise our import field-to-weewx archive field map self.map = None # initialise some other properties we will need self.start = 1 self.end = 1 self.increment = 1 # tell the user/log what we intend to do _msg = "A CSV import from source file '%s' has been requested." % self.source self.wlog.printlog(syslog.LOG_INFO, _msg) _msg = "The following options will be used:" self.wlog.verboselog(syslog.LOG_DEBUG, _msg) _msg = " config=%s, import-config=%s" % (config_path, self.import_config_path) self.wlog.verboselog(syslog.LOG_DEBUG, _msg) _msg = " source=%s, date=%s" % (self.source, options.date) self.wlog.verboselog(syslog.LOG_DEBUG, _msg) _msg = " dry-run=%s, calc-missing=%s" % (self.dry_run, self.calc_missing) self.wlog.verboselog(syslog.LOG_DEBUG, _msg) _msg = " tranche=%s, interval=%s, date/time_string_format=%s" % ( self.tranche, self.interval, self.raw_datetime_format) self.wlog.verboselog(syslog.LOG_DEBUG, _msg) _msg = " rain=%s, wind_direction=%s" % (self.rain, self.wind_dir) self.wlog.verboselog(syslog.LOG_DEBUG, _msg) _msg = " UV=%s, radiation=%s" % (self.UV_sensor, self.solar_sensor) self.wlog.verboselog(syslog.LOG_DEBUG, _msg) _msg = "Using database binding '%s', which is bound to database '%s'" % ( self.db_binding_wx, self.dbm.database_name) self.wlog.printlog(syslog.LOG_INFO, _msg) _msg = "Destination table '%s' unit system is '%#04x' (%s)." % ( self.dbm.table_name, self.archive_unit_sys, unit_nicknames[self.archive_unit_sys]) self.wlog.printlog(syslog.LOG_INFO, _msg) if self.calc_missing: print "Missing derived observations will be calculated." if not self.UV_sensor: print "All weewx UV fields will be set to None." if not self.solar_sensor: print "All weewx radiation fields will be set to None." if options.date: print "Observations timestamped after %s and up to and" % ( timestamp_to_string(self.first_ts), ) print "including %s will be imported." % (timestamp_to_string( self.last_ts), ) if self.dry_run: print "This is a dry run, imported data will not be saved to archive."
def addRecord(self, record_obj, log_level=syslog.LOG_NOTICE): """Commit a single record or a collection of records to the archive. record_obj: Either a data record, or an iterable that can return data records. Each data record must look like a dictionary, where the keys are the SQL types and the values are the values to be stored in the database.""" # Determine if record_obj is just a single dictionary instance (in which # case it will have method 'keys'). If so, wrap it in something iterable # (a list): record_list = [record_obj] if hasattr(record_obj, 'keys') else record_obj with weedb.Transaction(self.connection) as cursor: for record in record_list: if record['dateTime'] is None: syslog.syslog( syslog.LOG_ERR, "Archive: archive record with null time encountered.") raise weewx.ViolatedPrecondition( "Archive record with null time encountered.") # Check to make sure the incoming record is in the same unit system as the # records already in the database: if self.std_unit_system: if record['usUnits'] != self.std_unit_system: raise ValueError("Unit system of incoming record (0x%x) "\ "differs from the archive database (0x%x)" % (record['usUnits'], self.std_unit_system)) else: # This is the first record. Remember the unit system to check # against subsequent records: self.std_unit_system = record['usUnits'] # Only data types that appear in the database schema can be inserted. # To find them, form the intersection between the set of all record # keys and the set of all sql keys record_key_set = set(record.keys()) insert_key_set = record_key_set.intersection(self.sqlkeys) # Convert to an ordered list: key_list = list(insert_key_set) # Get the values in the same order: value_list = [record[k] for k in key_list] # This will a string of sql types, separated by commas. Because # some of the weewx sql keys (notably 'interval') are reserved # words in MySQL, put them in backquotes. k_str = ','.join(["`%s`" % k for k in key_list]) # This will be a string with the correct number of placeholder question marks: q_str = ','.join('?' * len(key_list)) # Form the SQL insert statement: sql_insert_stmt = "INSERT INTO %s (%s) VALUES (%s)" % ( self.table, k_str, q_str) try: cursor.execute(sql_insert_stmt, value_list) syslog.syslog( log_level, "Archive: added %s record %s" % (self.table, weeutil.weeutil.timestamp_to_string( record['dateTime']))) except Exception, e: syslog.syslog( syslog.LOG_ERR, "Archive: unable to add archive record %s" % weeutil.weeutil.timestamp_to_string( record['dateTime'])) syslog.syslog(syslog.LOG_ERR, " **** Reason: %s" % e)