def __init__(self, engine, config_dict):
# Pass the initialization information on to my superclass:
super(BatteryAlarm, self).__init__(engine, config_dict)
# This will hold the time when the last alarm message went out:
self.last_msg_ts = 0
# This will hold the count of the number of times the VP2 has signaled
# a low battery alarm this archive period
self.alarm_count = 0
try:
# Dig the needed options out of the configuration dictionary.
# If a critical option is missing, an exception will be thrown and
# the alarm will not be set.
self.time_wait = int(config_dict['Alarm'].get('time_wait', 3600))
self.count_threshold = int(config_dict['Alarm'].get('count_threshold', 50))
self.smtp_host = config_dict['Alarm']['smtp_host']
self.smtp_user = config_dict['Alarm'].get('smtp_user')
self.smtp_password = config_dict['Alarm'].get('smtp_password')
self.SUBJECT = config_dict['Alarm'].get('subject', "Low battery alarm message from weewx")
self.FROM = config_dict['Alarm'].get('from', '*****@*****.**')
self.TO = option_as_list(config_dict['Alarm']['mailto'])
syslog.syslog(syslog.LOG_INFO, "lowBattery: LowBattery alarm turned on. Count threshold is %d" % self.count_threshold)
# If we got this far, it's ok to start intercepting events:
self.bind(weewx.NEW_LOOP_PACKET, self.newLoopPacket)
self.bind(weewx.NEW_ARCHIVE_RECORD, self.newArchiveRecord)
except Exception, e:
syslog.syslog(syslog.LOG_INFO, "lowBattery: No alarm set. %s" % e)
def __init__(self, engine, config_dict):
# Pass the initialization information on to my superclass:
super(BatteryAlarm, self).__init__(engine, config_dict)
# This will hold the time when the last alarm message went out:
self.last_msg_ts = 0
# This will hold the count of the number of times the VP2 has signaled
# a low battery alarm this archive period
self.alarm_count = 0
try:
# Dig the needed options out of the configuration dictionary.
# If a critical option is missing, an exception will be thrown and
# the alarm will not be set.
self.time_wait = int(config_dict['Alarm'].get('time_wait', 3600))
self.count_threshold = int(config_dict['Alarm'].get(
'count_threshold', 10))
self.smtp_host = config_dict['Alarm']['smtp_host']
self.smtp_user = config_dict['Alarm'].get('smtp_user')
self.smtp_password = config_dict['Alarm'].get('smtp_password')
self.SUBJECT = config_dict['Alarm'].get(
'subject', "Low battery alarm message from weewx")
self.FROM = config_dict['Alarm'].get('from', '*****@*****.**')
self.TO = option_as_list(config_dict['Alarm']['mailto'])
except KeyError as e:
log.info("No alarm set. Missing parameter: %s", e)
else:
# If we got this far, it's ok to start intercepting events:
self.bind(weewx.NEW_LOOP_PACKET, self.new_loop_packet)
self.bind(weewx.NEW_ARCHIVE_RECORD, self.new_archive_record)
log.info("LowBattery alarm enabled. Count threshold is %d",
self.count_threshold)
def __init__(self, engine, config_dict):
# Pass the initialization information on to my superclass:
super(MyAlarm, self).__init__(engine, config_dict)
# This will hold the time when the last alarm message went out:
self.last_msg_ts = 0
try:
# Dig the needed options out of the configuration dictionary.
# If a critical option is missing, an exception will be thrown and
# the alarm will not be set.
self.expression = config_dict['Alarm']['expression']
self.time_wait = int(config_dict['Alarm'].get('time_wait', 3600))
self.smtp_host = config_dict['Alarm']['smtp_host']
self.smtp_user = config_dict['Alarm'].get('smtp_user')
self.smtp_password = config_dict['Alarm'].get('smtp_password')
self.SUBJECT = config_dict['Alarm'].get('subject', "Alarm message from weewx")
self.FROM = config_dict['Alarm'].get('from', '*****@*****.**')
self.TO = option_as_list(config_dict['Alarm']['mailto'])
syslog.syslog(syslog.LOG_INFO, "alarm: Alarm set for expression: \"%s\"" % self.expression)
# If we got this far, it's ok to start intercepting events:
self.bind(weewx.NEW_ARCHIVE_RECORD, self.newArchiveRecord) # NOTE 1
except Exception, e:
syslog.syslog(syslog.LOG_INFO, "alarm: No alarm set. %s" % e)
def __init__(self, engine, config_dict):
# Pass the initialization information on to my superclass:
super(WindAlarm, self).__init__(engine, config_dict)
# This will hold the time when the last alarm message went out:
self.last_msg_ts = 0
# En esta variable se acumula tiempo en segundos
self.last_event_time = time.time()
# Esta variable indica si previamente la alarma ha sido disparada
self.in_event = False
try:
# Dig the needed options out of the configuration dictionary.
# If a critical option is missing, an exception will be raised and
# the alarm will not be set.
self.expression = config_dict['Alarm']['wind_expression']
self.time_event = int(config_dict['Alarm']['wind_time_event'])
self.time_wait = int(config_dict['Alarm']['wind_time_wait'])
self.smtp_host = config_dict['Alarm']['smtp_host']
self.smtp_user = config_dict['Alarm'].get('smtp_user')
self.smtp_password = config_dict['Alarm'].get('smtp_password')
self.SUBJECT = config_dict['Alarm'].get('wind_subject', "Alarm message from weewx")
self.FROM = config_dict['Alarm'].get('from', '*****@*****.**')
self.TO = option_as_list(config_dict['Alarm']['mailto'])
syslog.syslog(syslog.LOG_INFO, "alarm: Alarm set for wind_expression: '%s'" % self.expression)
# If we got this far, it's ok to start intercepting events:
self.bind(weewx.NEW_ARCHIVE_RECORD, self.newArchiveRecord) # NOTE 1
except KeyError, e:
syslog.syslog(syslog.LOG_INFO, "alarm: No alarm set. %s" % e)
def __init__(self, engine, config_dict):
# Pass the initialization information on to my superclass:
super(MyAlarm, self).__init__(engine, config_dict)
# This will hold the time when the last alarm message went out:
self.last_msg_ts = 0
try:
# Dig the needed options out of the configuration dictionary.
# If a critical option is missing, an exception will be raised and
# the alarm will not be set.
self.expression = config_dict['Alarm']['expression']
self.time_wait = int(config_dict['Alarm'].get('time_wait', 3600))
self.timeout = int(config_dict['Alarm'].get('timeout', 10))
self.smtp_host = config_dict['Alarm']['smtp_host']
self.smtp_user = config_dict['Alarm'].get('smtp_user')
self.smtp_password = config_dict['Alarm'].get('smtp_password')
self.SUBJECT = config_dict['Alarm'].get(
'subject', "Alarm message from weewx")
self.FROM = config_dict['Alarm'].get('from', '*****@*****.**')
self.TO = option_as_list(config_dict['Alarm']['mailto'])
syslog.syslog(
syslog.LOG_INFO,
"alarm: Alarm set for expression: '%s'" % self.expression)
# If we got this far, it's ok to start intercepting events:
self.bind(weewx.NEW_ARCHIVE_RECORD,
self.new_archive_record) # NOTE 1
except KeyError as e:
syslog.syslog(syslog.LOG_INFO,
"alarm: No alarm set. Missing parameter: %s" % e)
def __init__(self, config_dict, config_path, csv_config_dict,
import_config_path, options):
# call our parents __init__
super(CSVSource, self).__init__(config_dict, csv_config_dict, options)
# save our import config path
self.import_config_path = import_config_path
# save our import config dict
self.csv_config_dict = csv_config_dict
# get a few config settings from our CSV config dict
# csv field delimiter
self.delimiter = str(self.csv_config_dict.get('delimiter', ','))
# string format used to decode the imported field holding our dateTime
self.raw_datetime_format = self.csv_config_dict.get(
'raw_datetime_format', '%Y-%m-%d %H:%M:%S')
# is our rain discrete or cumulative
self.rain = self.csv_config_dict.get('rain', 'cumulative')
# determine valid range for imported wind direction
_wind_direction = option_as_list(
self.csv_config_dict.get('wind_direction', '0,360'))
try:
if float(_wind_direction[0]) <= float(_wind_direction[1]):
self.wind_dir = [
float(_wind_direction[0]),
float(_wind_direction[1])
]
else:
self.wind_dir = [-360, 360]
except (KeyError, ValueError):
self.wind_dir = [-360, 360]
# get our source file path
try:
self.source = csv_config_dict['file']
except KeyError:
raise weewx.ViolatedPrecondition(
"CSV source file not specified in '%s'." % import_config_path)
# get the source file encoding, default to utf-8-sig
self.source_encoding = self.csv_config_dict.get(
'source_encoding', 'utf-8-sig')
# initialise our import field-to-WeeWX archive field map
self.map = None
# initialise some other properties we will need
self.start = 1
self.end = 1
self.increment = 1
# tell the user/log what we intend to do
_msg = "A CSV import from source file '%s' has been requested." % self.source
print(_msg)
log.info(_msg)
_msg = "The following options will be used:"
if self.verbose:
print(_msg)
log.debug(_msg)
_msg = " config=%s, import-config=%s" % (config_path,
self.import_config_path)
if self.verbose:
print(_msg)
log.debug(_msg)
if options.date:
_msg = " source=%s, date=%s" % (self.source, options.date)
else:
# we must have --from and --to
_msg = " source=%s, from=%s, to=%s" % (
self.source, options.date_from, options.date_to)
if self.verbose:
print(_msg)
log.debug(_msg)
_msg = " dry-run=%s, calc_missing=%s, ignore_invalid_data=%s" % (
self.dry_run, self.calc_missing, self.ignore_invalid_data)
if self.verbose:
print(_msg)
log.debug(_msg)
_msg = " tranche=%s, interval=%s, date/time_string_format=%s" % (
self.tranche, self.interval, self.raw_datetime_format)
if self.verbose:
print(_msg)
log.debug(_msg)
_msg = " delimiter='%s', rain=%s, wind_direction=%s" % (
self.delimiter, self.rain, self.wind_dir)
if self.verbose:
print(_msg)
log.debug(_msg)
_msg = " UV=%s, radiation=%s" % (self.UV_sensor, self.solar_sensor)
if self.verbose:
print(_msg)
log.debug(_msg)
_msg = "Using database binding '%s', which is bound to database '%s'" % (
self.db_binding_wx, self.dbm.database_name)
print(_msg)
log.info(_msg)
_msg = "Destination table '%s' unit system is '%#04x' (%s)." % (
self.dbm.table_name, self.archive_unit_sys,
unit_nicknames[self.archive_unit_sys])
print(_msg)
log.info(_msg)
if self.calc_missing:
_msg = "Missing derived observations will be calculated."
print(_msg)
log.info(_msg)
if not self.UV_sensor:
_msg = "All WeeWX UV fields will be set to None."
print(_msg)
log.info(_msg)
if not self.solar_sensor:
_msg = "All WeeWX radiation fields will be set to None."
print(_msg)
log.info(_msg)
if options.date or options.date_from:
_msg = "Observations timestamped after %s and up to and" % timestamp_to_string(
self.first_ts)
print(_msg)
log.info(_msg)
_msg = "including %s will be imported." % timestamp_to_string(
self.last_ts)
print(_msg)
log.info(_msg)
if self.dry_run:
_msg = "This is a dry run, imported data will not be saved to archive."
print(_msg)
log.info(_msg)
def run(self):
"""Main entry point for calculating missing derived fields.
Calculate the missing derived fields for the timespan concerned, save
the calculated data to archive and recalculate the daily summaries.
"""
# record the current time
t1 = time.time()
# obtain a wxservices.WXCalculate object to calculate the missing fields
# first we need station altitude, latitude and longitude
stn_dict = self.config_dict['Station']
altitude_t = option_as_list(stn_dict.get('altitude', (None, None)))
try:
altitude_vt = weewx.units.ValueTuple(float(altitude_t[0]),
altitude_t[1],
"group_altitude")
except KeyError as e:
raise weewx.ViolatedPrecondition(
"Value 'altitude' needs a unit (%s)" % e)
latitude_f = float(stn_dict['latitude'])
longitude_f = float(stn_dict['longitude'])
# now we can create a WXCalculate object
wxcalculate = weewx.wxservices.WXCalculate(self.config_dict,
altitude_vt,
latitude_f,
longitude_f)
# initialise some counters so we know what we have processed
days_updated = 0
days_processed = 0
total_records_processed = 0
total_records_updated = 0
# obtain gregorian days for our start and stop timestamps
start_greg = weeutil.weeutil.toGregorianDay(self.start_ts)
stop_greg = weeutil.weeutil.toGregorianDay(self.stop_ts)
# start at the first day
day = start_greg
while day <= stop_greg:
# get the start and stop timestamps for this tranche
tr_start_ts = weeutil.weeutil.startOfGregorianDay(day)
tr_stop_ts = min(weeutil.weeutil.startOfGregorianDay(stop_greg + 1),
weeutil.weeutil.startOfGregorianDay(day + self.trans_days))
# start the transaction
with weedb.Transaction(self.dbm.connection) as _cursor:
# iterate over each day in the tranche we are to work in
for tranche_day in weeutil.weeutil.genDaySpans(tr_start_ts, tr_stop_ts):
# initialise a counter for records processed on this day
records_updated = 0
# iterate over each record in this day
for record in self.dbm.genBatchRecords(startstamp=tranche_day.start,
stopstamp=tranche_day.stop):
# but we are only concerned with records after the
# start and before or equal to the stop timestamps
if self.start_ts < record['dateTime'] <= self.stop_ts:
# first obtain a list of the fields that may be calculated
extras_list = []
for obs in wxcalculate.svc_dict['Calculations']:
directive = wxcalculate.svc_dict['Calculations'][obs]
if directive == 'software' \
or directive == 'prefer_hardware' and (
obs not in record or record[obs] is None):
extras_list.append(obs)
# calculate the missing derived fields for the record
wxcalculate.do_calculations(data_dict=record,
data_type='archive')
# Obtain a dict containing only those fields that
# WXCalculate calculated. We could do this as a
# dictionary comprehension but python2.6 does not
# support dictionary comprehensions.
extras_dict = {}
for k in extras_list:
if k in record.keys():
extras_dict[k] = record[k]
# update the archive with the calculated data
records_updated += self.update_record_fields(record['dateTime'],
extras_dict)
# update the total records processed
total_records_processed += 1
# Give the user some information on progress
if total_records_processed % 1000 == 0:
p_msg = "Processing record: %d; Last record: %s" % (total_records_processed,
timestamp_to_string(record['dateTime']))
self._progress(p_msg)
# update the total records updated
total_records_updated += records_updated
# if we updated any records on this day increment the count
# of days updated
days_updated += 1 if records_updated > 0 else 0
days_processed += 1
# advance to the next tranche
day += self.trans_days
# finished, so give the user some final information on progress, mainly
# so the total tallies with the log
p_msg = "Processing record: %d; Last record: %s" % (total_records_processed,
timestamp_to_string(tr_stop_ts))
self._progress(p_msg, overprint=False)
# now update the daily summaries, but only if this is not a dry run
if not self.dry_run:
print("Recalculating daily summaries...")
# first we need a start and stop date object
start_d = datetime.date.fromtimestamp(self.start_ts)
# Since each daily summary is identified by the midnight timestamp
# for that day we need to make sure we our stop timestamp is not on
# a midnight boundary or we will rebuild the following days sumamry
# as well. if it is on a midnight boundary just subtract 1 second
# and use that.
summary_stop_ts = self.stop_ts
if weeutil.weeutil.isMidnight(self.stop_ts):
summary_stop_ts -= 1
stop_d = datetime.date.fromtimestamp(summary_stop_ts)
# do the update
self.dbm.backfill_day_summary(start_d=start_d, stop_d=stop_d)
print(file=sys.stdout)
print("Finished recalculating daily summaries")
else:
# it's a dry run so say the rebuild was skipped
print("This is a dry run, recalculation of daily summaries was skipped")
tdiff = time.time() - t1
# we are done so log and inform the user
_day_processed_str = "day" if days_processed == 1 else "days"
_day_updated_str = "day" if days_updated == 1 else "days"
if not self.dry_run:
log.info("Processed %d %s consisting of %d records. "
"%d %s consisting of %d records were updated "
"in %0.2f seconds." % (days_processed,
_day_processed_str,
total_records_processed,
days_updated,
_day_updated_str,
total_records_updated,
tdiff))
else:
# this was a dry run
log.info("Processed %d %s consisting of %d records. "
"%d %s consisting of %d records would have been updated "
"in %0.2f seconds." % (days_processed,
_day_processed_str,
total_records_processed,
days_updated,
_day_updated_str,
total_records_updated,
tdiff))
def __init__(self, config_dict, skin_dict, gen_ts,
first_run, stn_info, record=None):
# Initialise my superclass
super(ImageStackedWindRoseGenerator, self).__init__(config_dict,
skin_dict,
gen_ts,
first_run,
stn_info,
record)
# Get a manager for our archive
_binding = self.config_dict['StdArchive'].get('data_binding',
'wx_binding')
self.archive = self.db_binder.get_manager(_binding)
self.log_success = to_bool(skin_dict.get('log_success', True))
self.log_failure = to_bool(skin_dict.get('log_failure', True))
# Set a few properties we will need
self.image_dict = self.skin_dict['ImageStackedWindRoseGenerator']
self.title_dict = self.skin_dict['Labels']['Generic']
self.converter = Converter.fromSkinDict(self.skin_dict)
# Set image attributes
self.image_width = int(self.image_dict['image_width'])
self.image_height = int(self.image_dict['image_height'])
self.image_back_box_color = int(self.image_dict['image_background_box_color'], 0)
self.image_back_circle_color = int(self.image_dict['image_background_circle_color'], 0)
self.image_back_range_ring_color = int(self.image_dict['image_background_range_ring_color'], 0)
self.image_back_image = self.image_dict['image_background_image']
# Set compass point abbreviations
_compass = option_as_list(self.skin_dict['Labels'].get('compass_points',
'N, S, E, W'))
self.north = _compass[0]
self.south = _compass[1]
self.east = _compass[2]
self.west = _compass[3]
# Set windrose attributes
self.plot_border = int(self.image_dict['windrose_plot_border'])
self.legend_bar_width = int(self.image_dict['windrose_legend_bar_width'])
self.font_path = self.image_dict['windrose_font_path']
self.plot_font_size = int(self.image_dict['windrose_plot_font_size'])
self.plot_font_color = int(self.image_dict['windrose_plot_font_color'], 0)
self.legend_font_size = int(self.image_dict['windrose_legend_font_size'])
self.legend_font_color = int(self.image_dict['windrose_legend_font_color'], 0)
self.label_font_size = int(self.image_dict['windrose_label_font_size'])
self.label_font_color = int(self.image_dict['windrose_label_font_color'], 0)
# Look for petal colours, if not defined then set some defaults
_colors = option_as_list(self.image_dict.get('windrose_plot_petal_colors',
DEFAULT_PETAL_COLORS))
_colors = DEFAULT_PETAL_COLORS if len(_colors) < 7 else _colors
self.petal_colors = []
for _color in _colors:
try:
# Can it be converted to a number?
self.petal_colors.append(int(_color, 0))
except ValueError: # Cannot convert to a number, assume it is
# a colour word so append it as is
self.petal_colors.append(_color)
# Get petal width, if not defined then set default to 16 (degrees)
try:
self.petal_width = int(self.image_dict['windrose_plot_petal_width'])
except KeyError:
self.petal_width = 16
# Boundaries for speed range bands, these mark the colour boundaries
# on the stacked bar in the legend. 7 elements only (ie 0, 10% of max,
# 20% of max...100% of max)
self.speedFactor = [0.0, 0.1, 0.2, 0.3, 0.5, 0.7, 1.0]
self.period = None
self.p_gen_ts = None
self.label = None
self.t_stamp = None
self.t_stamp_loc = None
self.obName = None
self.dirName = None
self.units = None
self.max_ring_value = None
self.label_dir = None
self.draw = None
self.plotFont = None
self.legendFont = None
self.labelFont = None
self.roseMaxDiameter = None
self.originX = None
self.originY = None
self.image = None
def __init__(self, config_dict, config_path, csv_config_dict,
import_config_path, options, log):
# call our parents __init__
super(CSVSource, self).__init__(config_dict, csv_config_dict, options,
log)
# save our import config path
self.import_config_path = import_config_path
# save our import config dict
self.csv_config_dict = csv_config_dict
# get a few config settings from our CSV config dict
# string format used to decode the imported field holding our dateTime
self.raw_datetime_format = self.csv_config_dict.get(
'raw_datetime_format', '%Y-%m-%d %H:%M:%S')
# is our rain discrete or cumulative
self.rain = self.csv_config_dict.get('rain', 'cumulative')
# determine valid range for imported wind direction
_wind_direction = option_as_list(
self.csv_config_dict.get('wind_direction', '0,360'))
try:
if float(_wind_direction[0]) <= float(_wind_direction[1]):
self.wind_dir = [
float(_wind_direction[0]),
float(_wind_direction[1])
]
else:
self.wind_dir = [-360, 360]
except:
self.wind_dir = [-360, 360]
# get our source file path
try:
self.source = csv_config_dict['file']
except KeyError:
raise weewx.ViolatedPrecondition(
"CSV source file not specified in '%s'." % import_config_path)
# initialise our import field-to-weewx archive field map
self.map = None
# initialise some other properties we will need
self.start = 1
self.end = 1
self.increment = 1
# tell the user/log what we intend to do
_msg = "A CSV import from source file '%s' has been requested." % self.source
self.wlog.printlog(syslog.LOG_INFO, _msg)
_msg = "The following options will be used:"
self.wlog.verboselog(syslog.LOG_DEBUG, _msg)
_msg = " config=%s, import-config=%s" % (config_path,
self.import_config_path)
self.wlog.verboselog(syslog.LOG_DEBUG, _msg)
_msg = " source=%s, date=%s" % (self.source, options.date)
self.wlog.verboselog(syslog.LOG_DEBUG, _msg)
_msg = " dry-run=%s, calc-missing=%s" % (self.dry_run,
self.calc_missing)
self.wlog.verboselog(syslog.LOG_DEBUG, _msg)
_msg = " tranche=%s, interval=%s, date/time_string_format=%s" % (
self.tranche, self.interval, self.raw_datetime_format)
self.wlog.verboselog(syslog.LOG_DEBUG, _msg)
_msg = " rain=%s, wind_direction=%s" % (self.rain, self.wind_dir)
self.wlog.verboselog(syslog.LOG_DEBUG, _msg)
_msg = " UV=%s, radiation=%s" % (self.UV_sensor, self.solar_sensor)
self.wlog.verboselog(syslog.LOG_DEBUG, _msg)
_msg = "Using database binding '%s', which is bound to database '%s'" % (
self.db_binding_wx, self.dbm.database_name)
self.wlog.printlog(syslog.LOG_INFO, _msg)
_msg = "Destination table '%s' unit system is '%#04x' (%s)." % (
self.dbm.table_name, self.archive_unit_sys,
unit_nicknames[self.archive_unit_sys])
self.wlog.printlog(syslog.LOG_INFO, _msg)
if self.calc_missing:
print "Missing derived observations will be calculated."
if not self.UV_sensor:
print "All weewx UV fields will be set to None."
if not self.solar_sensor:
print "All weewx radiation fields will be set to None."
if options.date:
print "Observations timestamped after %s and up to and" % (
timestamp_to_string(self.first_ts), )
print "including %s will be imported." % (timestamp_to_string(
self.last_ts), )
if self.dry_run:
print "This is a dry run, imported data will not be saved to archive."
def __init__(self, config_dict, config_path, wu_config_dict, import_config_path, options, log):
# call our parents __init__
super(WUSource, self).__init__(config_dict,
wu_config_dict,
options,
log)
# save our import config path
self.import_config_path = import_config_path
# save our import config dict
self.wu_config_dict = wu_config_dict
# get our WU station ID
try:
self.station_id = wu_config_dict['station_id']
except KeyError:
raise weewx.ViolatedPrecondition("Weather Underground station ID not specified in '%s'." % import_config_path)
# wind dir bounds
_wind_direction = option_as_list(wu_config_dict.get('wind_direction',
'0,360'))
try:
if float(_wind_direction[0]) <= float(_wind_direction[1]):
self.wind_dir = [float(_wind_direction[0]),
float(_wind_direction[1])]
else:
self.wind_dir = [0, 360]
except:
self.wind_dir = [0, 360]
# some properties we know because of the format of the returned WU data
# WU returns a fixed format date-time string
self.raw_datetime_format = '%Y-%m-%d %H:%M:%S'
# WU only provides hourly rainfall and a daily cumulative rainfall.
# We use the latter so force 'cumulative' for rain.
self.rain = 'cumulative'
# initialise our import field-to-weewx archive field map
self.map = None
# For a WU import we might have to import multiple days but we can only
# get one day at a time from WU. So our start and end properties
# (counters) are datetime objects and our increment is a timedelta.
# Get datetime objects for any date or date range specified on the
# command line, if there wasn't one then default to today.
self.start = dt.fromtimestamp(startOfDay(self.first_ts))
self.end = dt.fromtimestamp(startOfDay(self.last_ts))
# set our increment
self.increment = datetime.timedelta(days=1)
# tell the user/log what we intend to do
_msg = "Observation history for Weather Underground station '%s' will be imported." % self.station_id
self.wlog.printlog(syslog.LOG_INFO, _msg)
_msg = "The following options will be used:"
self.wlog.verboselog(syslog.LOG_DEBUG, _msg)
_msg = " config=%s, import-config=%s" % (config_path,
self.import_config_path)
self.wlog.verboselog(syslog.LOG_DEBUG, _msg)
if options.date:
_msg = " station=%s, date=%s" % (self.station_id, options.date)
else:
# we must have --from and --to
_msg = " station=%s, from=%s, to=%s" % (self.station_id,
options.date_from,
options.date_to)
self.wlog.verboselog(syslog.LOG_DEBUG, _msg)
_msg = " dry-run=%s, calc-missing=%s" % (self.dry_run,
self.calc_missing)
self.wlog.verboselog(syslog.LOG_DEBUG, _msg)
_msg = " tranche=%s, interval=%s, wind_direction=%s" % (self.tranche,
self.interval,
self.wind_dir)
self.wlog.verboselog(syslog.LOG_DEBUG, _msg)
_msg = "Using database binding '%s', which is bound to database '%s'" % (self.db_binding_wx,
self.dbm.database_name)
self.wlog.printlog(syslog.LOG_INFO, _msg)
_msg = "Destination table '%s' unit system is '%#04x' (%s)." % (self.dbm.table_name,
self.archive_unit_sys,
unit_nicknames[self.archive_unit_sys])
self.wlog.printlog(syslog.LOG_INFO, _msg)
if self.calc_missing:
print "Missing derived observations will be calculated."
if options.date or options.date_from:
print "Observations timestamped after %s and up to and" % (timestamp_to_string(self.first_ts), )
print "including %s will be imported." % (timestamp_to_string(self.last_ts), )
if self.dry_run:
print "This is a dry run, imported data will not be saved to archive."
def test_option_as_list(self):
self.assertEqual(option_as_list("abc"), ['abc'])
self.assertEqual(option_as_list(['a', 'b']), ['a', 'b'])
self.assertEqual(option_as_list(None), None)
self.assertEqual(option_as_list(''), [''])
def __init__(self,
queue,
alert_dict,
server_url=OpsGenieAlerts.alert_url,
post_interval=3600,
max_backlog=0,
stale=None,
log_success=True,
log_failure=True,
timeout=60,
max_tries=1,
retry_wait=5):
"""Initialize an instance of OpsGenieAlertsThread.
Required parameters:
queue: An instance of Queue.Queue where the records will appear.
alert: Name of the alert for tracking.
config: Config dictionary for the alert.
Optional parameters:
server_url: The OpsGenie alert URL.
Default is 'https://api.opsgenie.com/v1/json/alert'
post_interval: How long to wait between alert notifications.
Default is 3600 seconds (1 hour).
max_backlog: How many records are allowed to accumulate in the queue
before the queue is trimmed.
Default is zero (no backlog at all).
stale: How old a record can be and still considered useful.
Default is None (never becomes too old).
log_success: If True, log a successful post in the system log.
Default is True.
log_failure: If True, log an unsuccessful post in the system log.
Default is True.
timeout: How long to wait for the server to respond before giving up.
Default is 10 seconds.
max_tries: How many times to try the post before giving up.
Default is 3
retry_wait: How long to wait between retries when failures.
Default is 5 seconds.
"""
super(OpsGenieAlertsThread,
self).__init__(queue,
protocol_name='OpsGenieAlert',
post_interval=post_interval,
max_backlog=max_backlog,
stale=stale,
log_success=log_success,
log_failure=log_failure,
timeout=timeout,
max_tries=max_tries,
retry_wait=retry_wait)
self.server_url = server_url
self.apiKey = alert_dict['apiKey']
self.Alias = alert_dict['Alias']
self.Message = alert_dict['Message']
self.Recipients = option_as_list(alert_dict.get('Recipients'))
self.Entity = alert_dict.get('Entity')
self.Source = alert_dict.get('Source')
self.Tags = alert_dict.get('Tags')
self.Actions = alert_dict.get('Actions')
self.Description = alert_dict.get('Description')
self.Details = option_as_list(alert_dict.get('Details'))
def __init__(self, config_dict, config_path, wu_config_dict,
import_config_path, options, log):
# call our parents __init__
super(WUSource, self).__init__(config_dict, wu_config_dict, options,
log)
# save our import config path
self.import_config_path = import_config_path
# save our import config dict
self.wu_config_dict = wu_config_dict
# get our WU station ID
try:
self.station_id = wu_config_dict['station_id']
except KeyError:
raise weewx.ViolatedPrecondition(
"Weather Underground station ID not specified in '%s'." %
import_config_path)
# wind dir bounds
_wind_direction = option_as_list(
wu_config_dict.get('wind_direction', '0,360'))
try:
if float(_wind_direction[0]) <= float(_wind_direction[1]):
self.wind_dir = [
float(_wind_direction[0]),
float(_wind_direction[1])
]
else:
self.wind_dir = [0, 360]
except:
self.wind_dir = [0, 360]
# some properties we know because of the format of the returned WU data
# WU returns a fixed format date-time string
self.raw_datetime_format = '%Y-%m-%d %H:%M:%S'
# WU only provides hourly rainfall and a daily cumulative rainfall.
# We use the latter so force 'cumulative' for rain.
self.rain = 'cumulative'
# initialise our import field-to-weewx archive field map
self.map = None
# For a WU import we might have to import multiple days but we can only
# get one day at a time from WU. So our start and end properties
# (counters) are datetime objects and our increment is a timedelta.
# Get datetime objects for any date or date range specified on the
# command line, if there wasn't one then default to today.
self.start = dt.fromtimestamp(startOfDay(self.first_ts))
self.end = dt.fromtimestamp(startOfDay(self.last_ts))
# set our increment
self.increment = datetime.timedelta(days=1)
# tell the user/log what we intend to do
_msg = "Observation history for Weather Underground station '%s' will be imported." % self.station_id
self.wlog.printlog(syslog.LOG_INFO, _msg)
_msg = "The following options will be used:"
self.wlog.verboselog(syslog.LOG_DEBUG, _msg)
_msg = " config=%s, import-config=%s" % (config_path,
self.import_config_path)
self.wlog.verboselog(syslog.LOG_DEBUG, _msg)
_msg = " station=%s, date=%s" % (self.station_id, options.date)
self.wlog.verboselog(syslog.LOG_DEBUG, _msg)
_msg = " dry-run=%s, calc-missing=%s" % (self.dry_run,
self.calc_missing)
self.wlog.verboselog(syslog.LOG_DEBUG, _msg)
_msg = " tranche=%s, interval=%s, wind_direction=%s" % (
self.tranche, self.interval, self.wind_dir)
self.wlog.verboselog(syslog.LOG_DEBUG, _msg)
_msg = "Using database binding '%s', which is bound to database '%s'" % (
self.db_binding_wx, self.dbm.database_name)
self.wlog.printlog(syslog.LOG_INFO, _msg)
_msg = "Destination table '%s' unit system is '%#04x' (%s)." % (
self.dbm.table_name, self.archive_unit_sys,
unit_nicknames[self.archive_unit_sys])
self.wlog.printlog(syslog.LOG_INFO, _msg)
if self.calc_missing:
print "Missing derived observations will be calculated."
if options.date:
print "Observations timestamped after %s and up to and" % (
timestamp_to_string(self.first_ts), )
print "including %s will be imported." % (timestamp_to_string(
self.last_ts), )
if self.dry_run:
print "This is a dry run, imported data will not be saved to archive."
def run(self):
"""Main entry point for calculating missing derived fields.
Calculate the missing derived fields for the timespan concerned, save
the calculated data to archive and recalculate the daily summaries.
"""
# record the current time
t1 = time.time()
# obtain a wxservices.WXCalculate object to calculate the missing
# fields, first we need to get a DBBinder object ...
db_binder = weewx.manager.DBBinder(self.config_dict)
# ... then a database manager ...
db_manager = db_binder.get_manager(
data_binding=self.config_dict['StdWXCalculate']['data_binding'])
# ... then station altitude, latitude and longitude
stn_dict = self.config_dict['Station']
altitude_t = option_as_list(stn_dict.get('altitude', (None, None)))
try:
altitude_vt = weewx.units.ValueTuple(float(altitude_t[0]),
altitude_t[1],
"group_altitude")
except KeyError as e:
raise weewx.ViolatedPrecondition(
"Value 'altitude' needs a unit (%s)" % e)
latitude_f = float(stn_dict['latitude'])
longitude_f = float(stn_dict['longitude'])
# now we can create a WXCalculate object
wxcalculate = weewx.wxservices.WXCalculate(self.config_dict,
altitude_vt, latitude_f,
longitude_f, db_manager)
# initialise some counters so we know what we have processed
days_updated = 0
days_processed = 0
total_records_processed = 0
total_records_updated = 0
# obtain gregorian days for our start and stop timestamps
start_greg = weeutil.weeutil.toGregorianDay(self.start_ts)
stop_greg = weeutil.weeutil.toGregorianDay(self.stop_ts)
# start at the first day
day = start_greg
while day <= stop_greg:
# get the start and stop timestamps for this tranche
tr_start_ts = weeutil.weeutil.startOfGregorianDay(day)
tr_stop_ts = min(
weeutil.weeutil.startOfGregorianDay(stop_greg + 1),
weeutil.weeutil.startOfGregorianDay(day + self.trans_days))
# start the transaction
with weedb.Transaction(self.dbm.connection) as _cursor:
# iterate over each day in the tranche we are to work in
for tranche_day in weeutil.weeutil.genDaySpans(
tr_start_ts, tr_stop_ts):
# initialise a counter for records processed on this day
records_updated = 0
# iterate over each record in this day
for record in self.dbm.genBatchRecords(
startstamp=tranche_day.start,
stopstamp=tranche_day.stop):
# but we are only concerned with records after the
# start and before or equal to the stop timestamps
if self.start_ts < record['dateTime'] <= self.stop_ts:
# calculate the missing derived fields for the record
wxcalculate.do_calculations(data_dict=record,
data_type='archive')
# obtain a dict containing only those fields that
# WXCalculate calculated
extras_dict = {
k: record[k]
for k in record.keys()
if k in wxcalculate.calculations.keys()
}
# update the archive with the calculated data
records_updated += self.update_record_fields(
record['dateTime'], extras_dict)
# update the total records updated
total_records_updated += records_updated
total_records_processed += 1
# Give the user some information on progress
if total_records_processed % 1000 == 0:
p_msg = "Processing record: %d; Last date: %s" % (
total_records_processed,
timestamp_to_string(record['dateTime']))
self._progress(p_msg)
# if we updated any records on this day increment the count
# of days updated
days_updated += 1 if records_updated > 0 else 0
days_processed += 1
# advance to the next tranche
day += self.trans_days
# finished, so give the user some final information on progress, mainly
# so the total tallies with the log
p_msg = "Processing record: %d; (%s)" % (
total_records_processed, timestamp_to_string(tr_stop_ts))
self._progress(p_msg, overprint=False)
# now update the daily summaries
print("Recalculating daily summaries...")
# first we need a start and stop date object
start_d = datetime.date.fromtimestamp(self.start_ts)
stop_d = datetime.date.fromtimestamp(self.stop_ts)
# do the update
self.dbm.backfill_day_summary(start_d=start_d, stop_d=stop_d)
print(file=sys.stdout)
print("Finished recalculating daily summaries")
tdiff = time.time() - t1
# we are done so log and inform the user
log.info(
"calcmissing: Processed %d days consisting of %d records. "
"%d days consisting of %d records were updated in %0.2f seconds." %
(days_processed, total_records_processed, days_updated,
total_records_updated, tdiff))
if self.dry_run:
log.info("calcmissing: "
"This was a dry run. %s was not applied." % self.name)
def __init__(self, config_dict, config_path, csv_config_dict, import_config_path, options, log):
# call our parents __init__
super(CSVSource, self).__init__(config_dict,
csv_config_dict,
options,
log)
# save our import config path
self.import_config_path = import_config_path
# save our import config dict
self.csv_config_dict = csv_config_dict
# get a few config settings from our CSV config dict
# string format used to decode the imported field holding our dateTime
self.raw_datetime_format = self.csv_config_dict.get('raw_datetime_format',
'%Y-%m-%d %H:%M:%S')
# is our rain discrete or cumulative
self.rain = self.csv_config_dict.get('rain', 'cumulative')
# determine valid range for imported wind direction
_wind_direction = option_as_list(self.csv_config_dict.get('wind_direction',
'0,360'))
try:
if float(_wind_direction[0]) <= float(_wind_direction[1]):
self.wind_dir = [float(_wind_direction[0]),
float(_wind_direction[1])]
else:
self.wind_dir = [-360, 360]
except:
self.wind_dir = [-360, 360]
# get our source file path
try:
self.source = csv_config_dict['file']
except KeyError:
raise weewx.ViolatedPrecondition("CSV source file not specified in '%s'." % import_config_path)
# initialise our import field-to-weewx archive field map
self.map = None
# initialise some other properties we will need
self.start = 1
self.end = 1
self.increment = 1
# tell the user/log what we intend to do
_msg = "A CSV import from source file '%s' has been requested." % self.source
self.wlog.printlog(syslog.LOG_INFO, _msg)
_msg = "The following options will be used:"
self.wlog.verboselog(syslog.LOG_DEBUG, _msg)
_msg = " config=%s, import-config=%s" % (config_path,
self.import_config_path)
self.wlog.verboselog(syslog.LOG_DEBUG, _msg)
if options.date:
_msg = " source=%s, date=%s" % (self.source, options.date)
else:
# we must have --from and --to
_msg = " source=%s, from=%s, to=%s" % (self.source,
options.date_from,
options.date_to)
self.wlog.verboselog(syslog.LOG_DEBUG, _msg)
_msg = " dry-run=%s, calc-missing=%s" % (self.dry_run,
self.calc_missing)
self.wlog.verboselog(syslog.LOG_DEBUG, _msg)
_msg = " tranche=%s, interval=%s, date/time_string_format=%s" % (self.tranche,
self.interval,
self.raw_datetime_format)
self.wlog.verboselog(syslog.LOG_DEBUG, _msg)
_msg = " rain=%s, wind_direction=%s" % (self.rain, self.wind_dir)
self.wlog.verboselog(syslog.LOG_DEBUG, _msg)
_msg = " UV=%s, radiation=%s" % (self.UV_sensor, self.solar_sensor)
self.wlog.verboselog(syslog.LOG_DEBUG, _msg)
_msg = "Using database binding '%s', which is bound to database '%s'" % (self.db_binding_wx,
self.dbm.database_name)
self.wlog.printlog(syslog.LOG_INFO, _msg)
_msg = "Destination table '%s' unit system is '%#04x' (%s)." % (self.dbm.table_name,
self.archive_unit_sys,
unit_nicknames[self.archive_unit_sys])
self.wlog.printlog(syslog.LOG_INFO, _msg)
if self.calc_missing:
print "Missing derived observations will be calculated."
if not self.UV_sensor:
print "All weewx UV fields will be set to None."
if not self.solar_sensor:
print "All weewx radiation fields will be set to None."
if options.date or options.date_from:
print "Observations timestamped after %s and up to and" % (timestamp_to_string(self.first_ts), )
print "including %s will be imported." % (timestamp_to_string(self.last_ts), )
if self.dry_run:
print "This is a dry run, imported data will not be saved to archive."
def __init__(self, config_dict, config_path, wu_config_dict,
import_config_path, options):
# call our parents __init__
super(WUSource, self).__init__(config_dict, wu_config_dict, options)
# save our import config path
self.import_config_path = import_config_path
# save our import config dict
self.wu_config_dict = wu_config_dict
# get our WU station ID
try:
self.station_id = wu_config_dict['station_id']
except KeyError:
_msg = "Weather Underground station ID not specified in '%s'." % import_config_path
raise weewx.ViolatedPrecondition(_msg)
# get our WU API key
try:
self.api_key = wu_config_dict['api_key']
except KeyError:
_msg = "Weather Underground API key not specified in '%s'." % import_config_path
raise weewx.ViolatedPrecondition(_msg)
# wind dir bounds
_wind_direction = option_as_list(
wu_config_dict.get('wind_direction', '0,360'))
try:
if float(_wind_direction[0]) <= float(_wind_direction[1]):
self.wind_dir = [
float(_wind_direction[0]),
float(_wind_direction[1])
]
else:
self.wind_dir = [0, 360]
except (IndexError, TypeError):
self.wind_dir = [0, 360]
# some properties we know because of the format of the returned WU data
# WU returns a fixed format date-time string
self.raw_datetime_format = '%Y-%m-%d %H:%M:%S'
# WU only provides hourly rainfall and a daily cumulative rainfall.
# We use the latter so force 'cumulative' for rain.
self.rain = 'cumulative'
# initialise our import field-to-WeeWX archive field map
self.map = None
# For a WU import we might have to import multiple days but we can only
# get one day at a time from WU. So our start and end properties
# (counters) are datetime objects and our increment is a timedelta.
# Get datetime objects for any date or date range specified on the
# command line, if there wasn't one then default to today.
self.start = dt.fromtimestamp(startOfDay(self.first_ts))
self.end = dt.fromtimestamp(startOfDay(self.last_ts))
# set our increment
self.increment = datetime.timedelta(days=1)
# property holding the current period being processed
self.period = None
# tell the user/log what we intend to do
_msg = "Observation history for Weather Underground station '%s' will be imported." % self.station_id
print(_msg)
log.info(_msg)
_msg = "The following options will be used:"
if self.verbose:
print(_msg)
log.debug(_msg)
_msg = " config=%s, import-config=%s" % (config_path,
self.import_config_path)
if self.verbose:
print(_msg)
log.debug(_msg)
if options.date:
_msg = " station=%s, date=%s" % (self.station_id, options.date)
else:
# we must have --from and --to
_msg = " station=%s, from=%s, to=%s" % (
self.station_id, options.date_from, options.date_to)
if self.verbose:
print(_msg)
log.debug(_msg)
_obf_api_key_msg = '='.join(
[' apiKey', '*' * (len(self.api_key) - 4) + self.api_key[-4:]])
if self.verbose:
print(_obf_api_key_msg)
log.debug(_obf_api_key_msg)
_msg = " dry-run=%s, calc_missing=%s, ignore_invalid_data=%s" % (
self.dry_run, self.calc_missing, self.ignore_invalid_data)
if self.verbose:
print(_msg)
log.debug(_msg)
_msg = " tranche=%s, interval=%s, wind_direction=%s" % (
self.tranche, self.interval, self.wind_dir)
if self.verbose:
print(_msg)
log.debug(_msg)
_msg = "Using database binding '%s', which is bound to database '%s'" % (
self.db_binding_wx, self.dbm.database_name)
print(_msg)
log.info(_msg)
_msg = "Destination table '%s' unit system is '%#04x' (%s)." % (
self.dbm.table_name, self.archive_unit_sys,
unit_nicknames[self.archive_unit_sys])
print(_msg)
log.info(_msg)
if self.calc_missing:
print("Missing derived observations will be calculated.")
if options.date or options.date_from:
print("Observations timestamped after %s and up to and" %
timestamp_to_string(self.first_ts))
print("including %s will be imported." %
timestamp_to_string(self.last_ts))
if self.dry_run:
print(
"This is a dry run, imported data will not be saved to archive."
)
def test_option_as_list(self):
self.assertEqual(option_as_list("abc"), ['abc'])
self.assertEqual(option_as_list(['a', 'b']), ['a', 'b'])
self.assertEqual(option_as_list(None), None)
self.assertEqual(option_as_list(''), [''])
def __init__(self, config_dict, skin_dict, gen_ts,
first_run, stn_info, record=None):
# Initialise my superclass
super(ImageStackedWindRoseGenerator, self).__init__(config_dict,
skin_dict,
gen_ts,
first_run,
stn_info,
record)
# Get a manager for our archive
_binding = self.config_dict['StdArchive'].get('data_binding',
'wx_binding')
self.archive = self.db_binder.get_manager(_binding)
# Set a few properties we will need
self.image_dict = self.skin_dict['ImageStackedWindRoseGenerator']
self.title_dict = self.skin_dict['Labels']['Generic']
self.converter = Converter.fromSkinDict(self.skin_dict)
# Set image attributes
self.image_width = int(self.image_dict['image_width'])
self.image_height = int(self.image_dict['image_height'])
self.image_back_box_color = int(self.image_dict['image_background_box_color'], 0)
self.image_back_circle_color = int(self.image_dict['image_background_circle_color'], 0)
self.image_back_range_ring_color = int(self.image_dict['image_background_range_ring_color'], 0)
self.image_back_image = self.image_dict['image_background_image']
# Set compass point abbreviations
_compass = option_as_list(self.skin_dict['Labels'].get('compass_points',
'N, S, E, W'))
self.north = _compass[0]
self.south = _compass[1]
self.east = _compass[2]
self.west = _compass[3]
# Set windrose attributes
self.plot_border = int(self.image_dict['windrose_plot_border'])
self.legend_bar_width = int(self.image_dict['windrose_legend_bar_width'])
self.font_path = self.image_dict['windrose_font_path']
self.plot_font_size = int(self.image_dict['windrose_plot_font_size'])
self.plot_font_color = int(self.image_dict['windrose_plot_font_color'], 0)
self.legend_font_size = int(self.image_dict['windrose_legend_font_size'])
self.legend_font_color = int(self.image_dict['windrose_legend_font_color'], 0)
self.label_font_size = int(self.image_dict['windrose_label_font_size'])
self.label_font_color = int(self.image_dict['windrose_label_font_color'], 0)
# Look for petal colours, if not defined then set some defaults
_colors = option_as_list(self.image_dict.get('windrose_plot_petal_colors',
DEFAULT_PETAL_COLORS))
_colors = DEFAULT_PETAL_COLORS if len(_colors) < 7 else _colors
self.petal_colors=[]
for _color in _colors:
try:
# Can it be converted to a number?
self.petal_colors.append(int(_color, 0))
except ValueError: # Cannot convert to a number, assume it is
# a colour word so append it as is
self.petal_colors.append(_color)
# Get petal width, if not defined then set default to 16 (degrees)
try:
self.petal_width = int(self.image_dict['windrose_plot_petal_width'])
except KeyError:
self.petal_width = 16
# Boundaries for speed range bands, these mark the colour boundaries
# on the stacked bar in the legend. 7 elements only (ie 0, 10% of max,
# 20% of max...100% of max)
self.speedFactor = [0.0, 0.1, 0.2, 0.3, 0.5, 0.7, 1.0]