def time_query(client): """Time how long it takes to create a vector with daily max temperatures.""" print "start time=", timestamp_to_string(start_ts) print "stop time=", timestamp_to_string(stop_ts) print "(Approximately %.1f months of data)" % ((stop_ts - start_ts)/(30*24*3600)) print "\nRunning a query for each day" t0 = time.time() vec = [] for span in weeutil.weeutil.genDaySpans(start_ts, stop_ts): query = 'SELECT MAX(outTemp) FROM "wxpacket" WHERE time > %d AND time <= %d' % (span[0] * giga, span[1] * giga) result_set = client.query(query, database='example') for x in result_set.get_points(): vec.append(x["max"]) t1 = time.time() print "Elapsed query time=", t1-t0 print vec print "\nRunning a single query, with group by day" t0 = time.time() vec = [] query = 'SELECT MAX(outTemp) FROM "wxpacket" WHERE time > %d AND time <= %d GROUP BY time(1d)' % (start_ts * giga, stop_ts * giga) result_set = client.query(query, database="example") for x in result_set.get_points(): vec.append(x["max"]) t1 = time.time() print "Elapsed query time=", t1-t0 print vec
def time_query(config_dict): """A low-level approach. Use a SQL query""" manager_dict = weewx.manager.get_manager_dict_from_config(config_dict, 'wx_binding') # Just use a regular ol' manager to avoid using the daily summary optimization: manager_dict['manager'] = 'weewx.manager.Manager' manager = weewx.manager.open_manager(manager_dict,initialize=False) print "start time=", timestamp_to_string(start_ts) print "stop time=", timestamp_to_string(stop_ts) print "Approximately %.1f months of data" % ((stop_ts - start_ts)/(30*24*3600)) t0 = time.time() vec = [] for span in weeutil.weeutil.genDaySpans(start_ts, stop_ts): query = 'SELECT dateTime, outTemp FROM archive WHERE dateTime > %d AND dateTime <= %d '\ 'AND outTemp = (SELECT MAX(outTemp) FROM archive WHERE dateTime > %d AND dateTime <= %d)' % (span + span) tup = manager.getSql(query) vec.append(tup) t1 = time.time() print "Elapsed query time=", t1-t0 print vec
def get_aggregate(obs_type, timespan, aggregate_type, db_manager, **option_dict): """Returns an aggregation of pm2_5_aqi over a timespan by using the main archive table. obs_type timespan: An instance of weeutil.Timespan with the time period over which aggregation is to be done. aggregate_type: The type of aggregation to be done. For this function, must be 'avg', 'sum', 'count', 'first', 'last', 'min', or 'max'. Anything else will cause weewx.UnknownAggregation to be raised. db_manager: An instance of weewx.manager.Manager or subclass. option_dict: Not used in this version. returns: A ValueTuple containing the result. """ if obs_type not in [ 'pm2_5_aqi', 'pm2_5_aqi_color' ]: raise weewx.UnknownType(obs_type) log.debug('get_aggregate(%s, %s, %s, aggregate:%s)' % ( obs_type, timestamp_to_string(timespan.start), timestamp_to_string(timespan.stop), aggregate_type)) aggregate_type = aggregate_type.lower() # Raise exception if we don't know about this type of aggregation if aggregate_type not in list(AQI.agg_sql_dict.keys()): raise weewx.UnknownAggregation(aggregate_type) # Form the interpolation dictionary interpolation_dict = { 'start': timespan.start, 'stop': timespan.stop, 'table_name': db_manager.table_name } select_stmt = AQI.agg_sql_dict[aggregate_type] % interpolation_dict row = db_manager.getSql(select_stmt) if row: value, std_unit_system = row else: value = None std_unit_system = None if value is not None: if obs_type == 'pm2_5_aqi': value = AQI.compute_pm2_5_aqi(value) if obs_type == 'pm2_5_aqi_color': value = AQI.compute_pm2_5_aqi_color(AQI.compute_pm2_5_aqi(value)) t, g = weewx.units.getStandardUnitType(std_unit_system, obs_type, aggregate_type) # Form the ValueTuple and return it: log.debug('get_aggregate(%s, %s, %s, aggregate:%s, select_stmt: %s, returning %s)' % ( obs_type, timestamp_to_string(timespan.start), timestamp_to_string(timespan.stop), aggregate_type, select_stmt, value)) return weewx.units.ValueTuple(value, t, g)
def skip_this_post(self, time_ts): """Determine whether a post is to be skipped or not. Use one or more checks to determine whether a post is to be skipped or not. In this case the post is skipped if the record is: - Too old (based on the 'stale' property). This check is kept to honor the existing 'stale' property, PVOutput has a max age for which status data can be posted, we store this value as self.update_period. - Outside the maximum age that PVOutput will accept (varies by donation status) - Posted too soon after our last post. """ # don't post if this record is too old (stale) if self.stale is not None: _how_old = time.time() - time_ts if _how_old > self.stale: syslog.syslog( syslog.LOG_DEBUG, "pvoutput: %s: record %s is stale (%d > %d)." % (self.protocol_name, timestamp_to_string(time_ts), _how_old, self.stale)) return True # don't post if this record is older than that accepted by PVOutput if self.update_period is not None: now_dt = datetime.datetime.fromtimestamp(time.time()) _earliest_dt = now_dt - datetime.timedelta(days=self.update_period) _earliest_ts = time.mktime(_earliest_dt.timetuple()) if time_ts < _earliest_ts: how_old = (now_dt - datetime.datetime.fromtimestamp(time_ts)).days syslog.syslog( syslog.LOG_DEBUG, "pvoutput: %s: record %s is older than PVOuptut imposed limits (%d > %d)." % (self.protocol_name, timestamp_to_string(time_ts), _how_old, self.self.update_period)) return True # if we have a minimum interval between posts then don't post if that # interval has not passed if self.post_interval is not None: _how_long = time_ts - self.lastpost if _how_long < self.post_interval: syslog.syslog( syslog.LOG_DEBUG, "pvoutput: %s: wait interval (%d < %d) has not passed for record %s" % (self.protocol_name, _how_long, self.post_interval, timestamp_to_string(time_ts))) return True self.lastpost = time_ts return False
def get_series(obs_type, timespan, db_manager, aggregate_type=None, aggregate_interval=None): """Get a series, possibly with aggregation. """ if obs_type not in [ 'pm2_5_aqi', 'pm2_5_aqi_color' ]: raise weewx.UnknownType(obs_type) log.debug('get_series(%s, %s, %s, aggregate:%s, aggregate_interval:%s)' % ( obs_type, timestamp_to_string(timespan.start), timestamp_to_string( timespan.stop), aggregate_type, aggregate_interval)) # Prepare the lists that will hold the final results. start_vec = list() stop_vec = list() data_vec = list() # Is aggregation requested? if aggregate_type: # Yes. Just use the regular series function. return weewx.xtypes.ArchiveTable.get_series(obs_type, timespan, db_manager, aggregate_type, aggregate_interval) else: # No aggregation. sql_str = 'SELECT dateTime, usUnits, `interval`, pm2_5 FROM %s ' \ 'WHERE dateTime >= ? AND dateTime <= ? AND pm2_5 IS NOT NULL' \ % db_manager.table_name std_unit_system = None for record in db_manager.genSql(sql_str, timespan): ts, unit_system, interval, pm2_5 = record if std_unit_system: if std_unit_system != unit_system: raise weewx.UnsupportedFeature( "Unit type cannot change within a time interval.") else: std_unit_system = unit_system if obs_type == 'pm2_5_aqi': value = AQI.compute_pm2_5_aqi(pm2_5) if obs_type == 'pm2_5_aqi_color': value = AQI.compute_pm2_5_aqi_color(AQI.compute_pm2_5_aqi(pm2_5)) log.debug('get_series(%s): %s - %s - %s' % (obs_type, timestamp_to_string(ts - interval * 60), timestamp_to_string(ts), value)) start_vec.append(ts - interval * 60) stop_vec.append(ts) data_vec.append(value) unit, unit_group = weewx.units.getStandardUnitType(std_unit_system, obs_type, aggregate_type) return (ValueTuple(start_vec, 'unix_epoch', 'group_time'), ValueTuple(stop_vec, 'unix_epoch', 'group_time'), ValueTuple(data_vec, unit, unit_group))
def set_time(self, ts): local_time = time.localtime(ts) set_time_cmd = "ST%2.2d%2.2d%2.2d" % ( local_time.tm_hour, local_time.tm_min, local_time.tm_sec) logdbg("set station time to %s (%s)" % (timestamp_to_string(ts), set_time_cmd)) self.send_AT_cmd(set_time_cmd) set_date_cmd = "SD%2.2d%2.2d%2.2d" % ( local_time.tm_year % 100, local_time.tm_mon, local_time.tm_mday) logdbg("set station date to %s (%s)" % (timestamp_to_string(ts), set_date_cmd)) self.send_AT_cmd(set_date_cmd)
def time_query(collection): """Time how long it takes to create a vector with daily max temperatures.""" print "start time=", timestamp_to_string(start_ts) print "stop time=", timestamp_to_string(stop_ts) print "(Approximately %.1f months of data)" % ((stop_ts - start_ts) / (30 * 24 * 3600)) # # This would give the max temperature in each UTC day (unfortunately # # we want the max temperature in each local day) # rs = collection.aggregate([{ "$project": {"day": { "$dayOfYear": "$dateTime" }, # "dateTime": 1, # "outTemp": 1 # }}, # { "$sort": { "day": 1, "outTemp":-1 } }, # { "$group": { # "_id" : "$day", # "max_temperature": { "$first": "$outTemp" }, # "timestamp": { "$first": "$dateTime" } # }}, # { "$sort": { "_id":1 } } # ]) epoch = datetime.datetime.utcfromtimestamp(0) vec = [] t0 = time.time() for span in weeutil.weeutil.genDaySpans(start_ts, stop_ts): rs = collection.aggregate([{"$match" : {"dateTime" : {"$gt" : datetime.datetime.utcfromtimestamp(span[0]), "$lte" : datetime.datetime.utcfromtimestamp(span[1])}, "outTemp" : {"$ne" : None} }}, {"$project" : {"dateTime" : 1, "outTemp" : 1}}, {"$sort" : {"outTemp" : -1 } }, {"$limit" : 1} ]) for x in rs: # Convert from a (timezone naive) datetime object to unix epoch time: delta = x['dateTime'] - epoch vec.append((delta.total_seconds(), x['outTemp'])) t1 = time.time() print "Elapsed query time=", (t1 - t0) print vec with open("mongo.out", "w") as fd: for x in vec: fd.write("%s %.2f\n" % (weeutil.weeutil.timestamp_to_string(x[0]), x[1]))
def get_time(self): try: sta_time = int(self.send_AT_cmd("RT")) sta_date = int(self.send_AT_cmd("RD")) # break sta_time into hh:mm:ss hh = sta_time // 10000 mm = sta_time // 100 - (hh * 100) ss = sta_time - (mm * 100) - (hh * 10000) # break sta_date into YY:MM:DD YY = sta_date // 10000 MM = sta_date // 100 - (YY * 100) DD = sta_date - (MM * 100) - (YY * 10000) # two-digit year hack - this station defaults to a epoch of 1987 # when power is lost. If the year is greater than 86, we assume # it's the 20th century, otherwise it's the 21st. # Note - someone (not me!) will have to revisit this code in 2086. if YY > 86: year = 1900 + YY else: year = 2000 + YY # We keep the station clock in GMT, which eliminates the # DST silliness. ts = time.mktime((year, MM, DD, hh, mm, ss, 0, 0, -1)) logdbg("station date: %s, time: %s, (%s)" % (sta_date, sta_time, timestamp_to_string(ts))) return ts except (serial.serialutil.SerialException, weewx.WeeWxIOError) as e: logerr("get_time failed: %s" % e) return int(time.time())
def _progress(ndays, last_time): """Utility function to show our progress while processing the fix.""" _msg = "Updating 'windSpeed' daily summary: %d; " \ "Timestamp: %s\r" % (ndays, timestamp_to_string(last_time, format_str="%Y-%m-%d")) print(_msg, end='', file=sys.stdout) sys.stdout.flush()
def uploadFiles(self): start_ts = time.time() t_str = timestamp_to_string(start_ts) syslog.syslog(syslog.LOG_INFO, "S3upload: start upload at %s" % t_str) # Build command cmd = ["/usr/local/bin/s3cmd"] cmd.extend(["sync"]) cmd.extend(["--no-mime-magic"]) cmd.extend(["--access_key=%s" % self.access_key]) cmd.extend(["--secret_key=%s" % self.secret_token]) cmd.extend([self.local_root]) cmd.extend(["s3://%s" % self.bucket_name]) syslog.syslog(syslog.LOG_DEBUG, "S3upload command: %s" % cmd) try: S3upload_cmd = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) stdout = S3upload_cmd.communicate()[0] stroutput = stdout.strip() except OSError, e: if e.errno == errno.ENOENT: syslog.syslog( syslog.LOG_ERR, "S3upload: s3cmd does not appear to be installed on this system. (errno %d, \"%s\")" % (e.errno, e.strerror)) raise
def _progress(ndays, last_time): """Utility function to show our progress while processing the fix.""" print("Weighting daily summary: %d; Timestamp: %s\r" % (ndays, timestamp_to_string(last_time, format_str="%Y-%m-%d")), end='', file=sys.stdout) sys.stdout.flush()
def genStartupRecords(self, since_ts): loginf("reading records since %s" % timestamp_to_string(since_ts)) hbuf = None last_ts = None cnt = 0 while True: try: buf = self.station.read() if buf is not None: if buf[0] == 0xD2: hbuf = buf buf = None elif buf[0] == 0x7F and hbuf is not None: # FIXME: need better indicator of second half history buf = hbuf + buf hbuf = None if buf is not None and buf[0] == 0xD2: self.last_record = Station.get_record_index(buf) ts = Station._extract_ts(buf[4:9]) if ts is not None and ts > since_ts: keep = True if last_ts is not None else False pkt = Station.decode(buf) packet = self.convert_historical(pkt, ts, last_ts) last_ts = ts if keep: logdbg("historical record: %s" % packet) cnt += 1 yield packet if buf is not None and buf[0] == 0x57: idx = Station.get_latest_index(buf) msg = "count=%s last_index=%s latest_index=%s" % (cnt, self.last_record, idx) if self.last_record + 1 >= idx: loginf("catchup complete: %s" % msg) break loginf("catchup in progress: %s" % msg) if buf is not None and buf[0] == 0x41 and buf[3] == 0x65: nxtrec = Station.get_next_index(self.last_record) logdbg("request records starting with %s" % nxtrec) cmd = [0xCD, 0x18, 0x30, 0x62, _hi(nxtrec), _lo(nxtrec)] sent = self.station.write(cmd) if time.time() - self.last_a6 > self.heartbeat: logdbg("request station status: %s (%02x)" % (self.last_record, _lo(self.last_record))) cmd = [0xA6, 0x91, 0xCA, 0x45, 0x52, _lo(self.last_record)] sent = self.station.write(cmd) self.last_a6 = time.time() if self.last_7x == 0: # FIXME: what does 72/73 do? cmd = [0x73, 0xE5, 0x0A, 0x26, 0x88, 0x8B] sent = self.station.write(cmd) self.last_7x = time.time() if time.time() - self.last_65 > self.history_retry: logdbg("initiate record request: %s (%02x)" % (self.last_record, _lo(self.last_record))) cmd = [0x65, 0x19, 0xE5, 0x04, 0x52, _lo(self.last_record)] sent = self.station.write(cmd) self.last_65 = time.time() except usb.USBError, e: if not e.args[0].find("No data available"): raise weewx.WeeWxIOError(e) except (WrongLength, BadChecksum), e: loginf(e)
def get_scalar(obs_type, record, db_manager=None): log.debug('get_scalar(%s)' % obs_type) if obs_type not in [ 'pm2_5_aqi', 'pm2_5_aqi_color' ]: raise weewx.UnknownType(obs_type) log.debug('get_scalar(%s)' % obs_type) if record is None: log.debug('get_scalar called where record is None.') raise weewx.CannotCalculate(obs_type) if 'pm2_5' not in record: # Should not see this as pm2_5 is part of the extended schema that is required for this plugin. # Returning CannotCalculate causes exception in ImageGenerator, return UnknownType instead. # ERROR weewx.reportengine: Caught unrecoverable exception in generator 'weewx.imagegenerator.ImageGenerator' log.info('get_scalar called where record does not contain pm2_5. This is unexpected.') raise weewx.UnknownType(obs_type) if record['pm2_5'] is None: # Returning CannotCalculate causes exception in ImageGenerator, return UnknownType instead. # ERROR weewx.reportengine: Caught unrecoverable exception in generator 'weewx.imagegenerator.ImageGenerator' # Any archive catchup records will have None for pm2_5. log.debug('get_scalar called where record[pm2_5] is None: %s. Probably a catchup record.' % timestamp_to_string(record['dateTime'])) raise weewx.UnknownType(obs_type) try: pm2_5 = record['pm2_5'] if obs_type == 'pm2_5_aqi': value = AQI.compute_pm2_5_aqi(pm2_5) if obs_type == 'pm2_5_aqi_color': value = AQI.compute_pm2_5_aqi_color(AQI.compute_pm2_5_aqi(pm2_5)) t, g = weewx.units.getStandardUnitType(record['usUnits'], obs_type) # Form the ValueTuple and return it: return weewx.units.ValueTuple(value, t, g) except KeyError: # Don't have everything we need. Raise an exception. raise weewx.CannotCalculate(obs_type)
def run_manager(manager): print "start time=", timestamp_to_string(start_ts) print "stop time=", timestamp_to_string(stop_ts) print "Approximately %.1f months of data" % ((stop_ts - start_ts)/(30*24*3600)) t0 = time.time() vec = [] for span in weeutil.weeutil.genDaySpans(start_ts, stop_ts): maxTemp = manager.getAggregate(span, 'outTemp', 'max')[0] ts = manager.getAggregate(span, 'outTemp', 'maxtime')[0] vec.append((ts, maxTemp)) t1 = time.time() print "Elapsed query time=", t1-t0 return vec
def soundTheAlarm(self, timestamp, battery_status, alarm_count): """This function is called when the alarm has been triggered.""" # Get the time and convert to a string: t_str = timestamp_to_string(timestamp) # Log it in the system log: syslog.syslog( syslog.LOG_INFO, "lowBattery: Low battery alarm (0x%04x) sounded at %s." % (battery_status, t_str)) # Form the message text: msg_text = """The low battery alarm (0x%04x) has been seen %d times since the last archive period.\n\n"""\ """Alarm sounded at %s\n\n""" % (battery_status, alarm_count, t_str) # Convert to MIME: msg = MIMEText(msg_text) # Fill in MIME headers: msg['Subject'] = self.SUBJECT msg['From'] = self.FROM msg['To'] = ','.join(self.TO) # Create an instance of class SMTP for the given SMTP host: s = smtplib.SMTP(self.smtp_host) try: # Some servers (eg, gmail) require encrypted transport. # Be prepared to catch an exception if the server # doesn't support it. s.ehlo() s.starttls() s.ehlo() syslog.syslog(syslog.LOG_DEBUG, "lowBattery: using encrypted transport") except smtplib.SMTPException: syslog.syslog(syslog.LOG_DEBUG, "lowBattery: using unencrypted transport") try: # If a username has been given, assume that login is required for this host: if self.smtp_user: s.login(self.smtp_user, self.smtp_password) syslog.syslog( syslog.LOG_DEBUG, "lowBattery: logged in with user name %s" % (self.smtp_user, )) # Send the email: s.sendmail(msg['From'], self.TO, msg.as_string()) # Log out of the server: s.quit() except Exception, e: syslog.syslog( syslog.LOG_ERR, "lowBattery: SMTP mailer refused message with error %s" % (e, )) raise
def main(): print "start time=", timestamp_to_string(start_ts) print "stop time= ", timestamp_to_string(stop_ts) print "(Approximately %.1f days of data)" % ((stop_ts - start_ts)/(24*3600.0)) print "***** SQLITE *****" create_table(sqlite_db_dict) connect = weedb.connect(sqlite_db_dict) time_query(connect, 'outTemp') time_query(connect, 'barometer') connect.close() print "***** MySQL *****" create_table(mysql_db_dict) connect = weedb.connect(mysql_db_dict) time_query(connect, 'outTemp') time_query(connect, 'barometer') connect.close()
def _progress(record, ts): """Utility function to show our progress while processing the fix. Override in derived class to provide a different progress display. To do nothing override with a pass statement. """ print >>sys.stdout, "Fixing database record: %d; Timestamp: %s\r" % \ (record, timestamp_to_string(ts)), sys.stdout.flush()
def _progress(record, ts): """Utility function to show our progress while processing the fix. Override in derived class to provide a different progress display. To do nothing override with a pass statement. """ _msg = "Fixing database record: %d; Timestamp: %s\r" % (record, timestamp_to_string(ts)) print(_msg, end='', file=sys.stdout) sys.stdout.flush()
def do_alarm(self, record): """Send an email out""" # Get the time and convert to a string: t_str = timestamp_to_string(record['dateTime']) # Log the alarm log.info('Alarm expression "%s" evaluated True at %s' % (self.expression, t_str)) # Form the message text: msg_text = 'Alarm expression "%s" evaluated True at %s\nRecord:\n%s' \ % (self.expression, t_str, str(record)) # Convert to MIME: msg = MIMEText(msg_text) # Fill in MIME headers: msg['Subject'] = self.SUBJECT msg['From'] = self.FROM msg['To'] = ','.join(self.TO) try: # First try end-to-end encryption s = smtplib.SMTP_SSL(self.smtp_host, timeout=self.timeout) log.debug("Using SMTP_SSL") except (AttributeError, socket.timeout, socket.error) as e: log.debug("Unable to use SMTP_SSL connection. Reason: %s", e) # If that doesn't work, try creating an insecure host, then upgrading s = smtplib.SMTP(self.smtp_host, timeout=self.timeout) try: # Be prepared to catch an exception if the server # does not support encrypted transport. s.ehlo() s.starttls() s.ehlo() log.debug("Using SMTP encrypted transport") except smtplib.SMTPException as e: log.debug("Using SMTP unencrypted transport. Reason: %s", e) try: # If a username has been given, assume that login is required for this host: if self.smtp_user: s.login(self.smtp_user, self.smtp_password) log.debug("Logged in with user name %s", self.smtp_user) # Send the email: s.sendmail(msg['From'], self.TO, msg.as_string()) # Log out of the server: s.quit() except Exception as e: log.error("SMTP mailer refused message with error %s", e) raise # Log sending the email: log.info("Email sent to: %s", self.TO)
def soundTheAlarm(self, rec, status=True): """This function is called when the given expression evaluates True or False.""" # Get the time and convert to a string: t_str = timestamp_to_string(rec['dateTime']) # Log it in the system log: syslog.syslog( syslog.LOG_INFO, "alarm: Alarm expression '{}' evaluated {} at {}" % (self.expression, status, t_str)) # Form the message text: msg_text = "Alarm expression '{}' evaluated {} at {}\nRecord:\n{}".format( self.expression, status, t_str, str(rec)) # Convert to MIME: msg = MIMEText(msg_text) # Fill in MIME headers: msg['Subject'] = self.SUBJECT msg['From'] = self.FROM msg['To'] = ','.join(self.TO) # Create an instance of class SMTP for the given SMTP host: s = smtplib.SMTP(self.smtp_host) try: # Some servers (eg, gmail) require encrypted transport. # Be prepared to catch an exception if the server # doesn't support it. s.ehlo() s.starttls() s.ehlo() syslog.syslog(syslog.LOG_DEBUG, " **** using encrypted transport") except smtplib.SMTPException: syslog.syslog(syslog.LOG_DEBUG, " **** using unencrypted transport") try: # If a username has been given, assume that login is required for this host: if self.smtp_user: s.login(self.smtp_user, self.smtp_password) syslog.syslog( syslog.LOG_DEBUG, " **** logged in with user name %s" % (self.smtp_user, )) # Send the email: s.sendmail(msg['From'], self.TO, msg.as_string()) # Log out of the server: s.quit() except Exception, e: syslog.syslog( syslog.LOG_ERR, "alarm: SMTP mailer refused message with error %s" % (e, )) raise
def _time_sync(self, ts): logdbg("time sync to %s (%s)" % (ts, timestamp_to_string(ts))) t = time.localtime(ts) cmd = [WH23xxStation.TIME_SYNC, t.tm_year - 2000, t.tm_mon, t.tm_mday, t.tm_hour, t.tm_min, t.tm_sec, 0] chksum = _calc_checksum(cmd) buf = [0x02, 0x09] buf.extend(cmd) buf.append(chksum) self._write("time_sync", buf)
def soundTheAlarm(self, timestamp, battery_status, alarm_count): """This function is called when the low battery alarm has been sounded.""" # Get the time and convert to a string: t_str = timestamp_to_string(timestamp) # Log it in the system log: syslog.syslog( syslog.LOG_INFO, "lowBattery: Low battery alarm (0x%04x) sounded at %s." % (battery_status, t_str) ) # Form the message text: msg_text = ( """The low battery alarm (0x%04x) has been seen %d times since the last archive period.\n\n""" """Alarm sounded at %s\n\n""" % (battery_status, alarm_count, t_str) ) # Convert to MIME: msg = MIMEText(msg_text) # Fill in MIME headers: msg["Subject"] = self.SUBJECT msg["From"] = self.FROM msg["To"] = ",".join(self.TO) # Create an instance of class SMTP for the given SMTP host: s = smtplib.SMTP(self.smtp_host) try: # Some servers (eg, gmail) require encrypted transport. # Be prepared to catch an exception if the server # doesn't support it. s.ehlo() s.starttls() s.ehlo() syslog.syslog(syslog.LOG_DEBUG, " **** using encrypted transport") except smtplib.SMTPException: syslog.syslog(syslog.LOG_DEBUG, " **** using unencrypted transport") try: # If a username has been given, assume that login is required for this host: if self.smtp_user: s.login(self.smtp_user, self.smtp_password) syslog.syslog(syslog.LOG_DEBUG, " **** logged in with user name %s" % (self.smtp_user,)) # Send the email: s.sendmail(msg["From"], self.TO, msg.as_string()) # Log out of the server: s.quit() except Exception, e: syslog.syslog(syslog.LOG_ERR, "lowBattery: SMTP mailer refused message with error %s" % (e,)) raise
def set_time(self, ts): # go to modem mode so we do not get logger chatter self.set_modem_mode() # set time should work on all models tt = time.localtime(ts) cmd = b">A%04d%04d" % (tt.tm_yday - 1, tt.tm_min + tt.tm_hour * 60) log.debug("Set station time to %s (%s)", timestamp_to_string(ts), cmd) self.serial_port.write(b"%s\r" % cmd) # year works only for models 2004 and later if self.can_set_year: cmd = b">U%s" % tt.tm_year log.debug("Set station year to %s (%s)", tt.tm_year, cmd) self.serial_port.write(b"%s\r" % cmd)
def set_time(self, ts): # go to modem mode so we do not get logger chatter self.set_modem_mode() # set time should work on all models tt = time.localtime(ts) cmd = ">A%04d%04d" % ( tt.tm_yday - 1, tt.tm_min + tt.tm_hour * 60) logdbg("set station time to %s (%s)" % (timestamp_to_string(ts), cmd)) self.serial_port.write("%s\r" % cmd) # year works only for models 2004 and later if self.can_set_year: cmd = ">U%s" % tt.tm_year logdbg("set station year to %s (%s)" % (tt.tm_year, cmd)) self.serial_port.write("%s\r" % cmd)
def get_time(self): try: self.set_logger_mode() buf = self.get_readings_with_retry() data = Station.parse_readings(buf) d = data['day_of_year'] # seems to start at 0 m = data['minute_of_day'] # 0 is midnight before start of day tt = time.localtime() y = tt.tm_year s = tt.tm_sec ts = time.mktime((y,1,1,0,0,s,0,0,-1)) + d * 86400 + m * 60 logdbg("station time: day:%s min:%s (%s)" % (d, m, timestamp_to_string(ts))) return ts except (serial.serialutil.SerialException, weewx.WeeWxIOError), e: logerr("get_time failed: %s" % e)
def get_time(self): try: self.set_logger_mode() buf = self.get_readings_with_retry() data = Station.parse_readings(buf) d = data['day_of_year'] # seems to start at 0 m = data['minute_of_day'] # 0 is midnight before start of day tt = time.localtime() y = tt.tm_year s = tt.tm_sec ts = time.mktime((y, 1, 1, 0, 0, s, 0, 0, -1)) + d * 86400 + m * 60 log.debug("Station time: day:%s min:%s (%s)", d, m, timestamp_to_string(ts)) return ts except (serial.serialutil.SerialException, weewx.WeeWxIOError) as e: log.error("get_time failed: %s", e) return int(time.time())
def addRecord(self, record, add_hilo=True, weight=1): """Add a record to my running statistics. The record must have keys 'dateTime' and 'usUnits'.""" # Check to see if the record is within my observation timespan if not self.timespan.includesArchiveTime(record['dateTime']): raise OutOfSpan( "Attempt to add out-of-interval record (%s) to timespan (%s)" % (timestamp_to_string(record['dateTime']), self.timespan)) for obs_type in record: # Get the proper function ... func = get_add_function(obs_type) # ... then call it. func(self, record, obs_type, add_hilo, weight)
def new_loop_packet(self, event): log.debug('new_loop_packet(%s)' % event) with self.cfg.lock: log.debug('new_loop_packet: self.cfg.concentrations: %s' % self.cfg.concentrations) if self.cfg.concentrations is not None and \ self.cfg.concentrations.timestamp is not None and \ self.cfg.concentrations.timestamp + \ self.cfg.archive_interval >= time.time(): log.debug( 'Time of reading being inserted: %s' % timestamp_to_string(self.cfg.concentrations.timestamp)) # Insert pm1_0, pm2_5, pm10_0, aqi and aqic into loop packet. if self.cfg.concentrations.pm1_0 is not None: event.packet['pm1_0'] = self.cfg.concentrations.pm1_0 log.debug('Inserted packet[pm1_0]: %f into packet.' % event.packet['pm1_0']) if self.cfg.concentrations.pm2_5_cf_1_b is not None: b_reading = self.cfg.concentrations.pm2_5_cf_1_b else: b_reading = self.cfg.concentrations.pm2_5_cf_1 # Dup A sensor reading if (self.cfg.concentrations.pm2_5_cf_1 is not None and b_reading is not None and self.cfg.concentrations.current_humidity is not None and self.cfg.concentrations.current_temp_f): event.packet[ 'pm2_5'] = AQI.compute_pm2_5_us_epa_correction( self.cfg.concentrations.pm2_5_cf_1, b_reading, self.cfg.concentrations.current_humidity, self.cfg.concentrations.current_temp_f) log.debug('Inserted packet[pm2_5]: %f into packet.' % event.packet['pm2_5']) if self.cfg.concentrations.pm10_0 is not None: event.packet['pm10_0'] = self.cfg.concentrations.pm10_0 log.debug('Inserted packet[pm10_0]: %f into packet.' % event.packet['pm10_0']) if 'pm2_5' in event.packet: event.packet['pm2_5_aqi'] = AQI.compute_pm2_5_aqi( event.packet['pm2_5']) if 'pm2_5_aqi' in event.packet: event.packet[ 'pm2_5_aqi_color'] = AQI.compute_pm2_5_aqi_color( event.packet['pm2_5_aqi']) else: log.error('Found no fresh concentrations to insert.')
def soundTheAlarm(self, rec): """This function is called when the given expression evaluates True.""" # Get the time and convert to a string: t_str = timestamp_to_string(rec['dateTime']) # Log it in the system log: syslog.syslog(syslog.LOG_INFO, "alarm: Alarm expression \"%s\" evaluated True at %s" % (self.expression, t_str)) # Form the message text: msg_text = "Alarm expression \"%s\" evaluated True at %s\nRecord:\n%s" % (self.expression, t_str, str(rec)) # Convert to MIME: msg = MIMEText(msg_text) # Fill in MIME headers: msg['Subject'] = self.SUBJECT msg['From'] = self.FROM msg['To'] = ','.join(self.TO) # Create an instance of class SMTP for the given SMTP host: s = smtplib.SMTP(self.smtp_host) try: # Some servers (eg, gmail) require encrypted transport. # Be prepared to catch an exception if the server # doesn't support it. s.ehlo() s.starttls() s.ehlo() syslog.syslog(syslog.LOG_DEBUG, " **** using encrypted transport") except smtplib.SMTPException: syslog.syslog(syslog.LOG_DEBUG, " **** using unencrypted transport") try: # If a username has been given, assume that login is required for this host: if self.smtp_user: s.login(self.smtp_user, self.smtp_password) syslog.syslog(syslog.LOG_DEBUG, " **** logged in with user name %s" % (self.smtp_user,)) # Send the email: s.sendmail(msg['From'], self.TO, msg.as_string()) # Log out of the server: s.quit() except Exception, e: syslog.syslog(syslog.LOG_ERR, "alarm: SMTP mailer refused message with error %s" % (e,)) raise
def uploadFiles(self): start_ts = time.time() t_str = timestamp_to_string(start_ts) syslog.syslog(syslog.LOG_INFO, "FirebaseUpload: start upload at %s" % t_str) cmd = ["/usr/bin/node"] cmd.extend(["/srv/weewx/deploy.js"]) syslog.syslog(syslog.LOG_DEBUG, "FirebaseUpload command: %s" % cmd) try: FirebaseUpload_cmd = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) stdout = FirebaseUpload_cmd.communicate()[0] stroutput = stdout.strip() except OSError, e: if e.errno == errno.ENOENT: syslog.syslog( syslog.LOG_ERR, "FirebaseUpload: can't deploy. (errno %d, \"%s\")" % (e.errno, e.strerror)) raise
def uploadFiles(self): start_ts = time.time() t_str = timestamp_to_string(start_ts) syslog.syslog(syslog.LOG_DEBUG, "s3uploadgenerator: start upload at %s" % t_str) # Build command cmd = ["/usr/bin/s3cmd"] cmd.extend(["sync"]) cmd.extend(["--config=/home/weewx/.s3cfg"]) cmd.extend([self.local_root]) cmd.extend(["s3://%s" % self.bucket_name]) syslog.syslog(syslog.LOG_DEBUG, "s3uploadgenerator: command: %s" % cmd) try: S3upload_cmd = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) stdout = S3upload_cmd.communicate()[0] stroutput = stdout.strip() except OSError, e: if e.errno == errno.ENOENT: syslog.syslog(syslog.LOG_ERR, "s3uploadgenerator: s3cmd does not appear to be installed on this system. (errno %d, \"%s\")" % (e.errno, e.strerror)) raise
def soundTheAlarm(self, rec, alarm_count): """This function is called when the low battery alarm has been sounded.""" # Get the time and convert to a string: t_str = timestamp_to_string(rec['dateTime']) # Form the message text: msg_text = """The low battery alarm (0x%04x) has been seen %d times since the last archive period.\n\n"""\ """Alarm sounded at %s\n\n"""\ """LOOP record:\n%s""" % (rec['txBatteryStatus'], alarm_count, t_str, str(rec)) # Convert to MIME: msg = MIMEText(msg_text) # Fill in MIME headers: msg['Subject'] = "Low battery alarm message from weewx" msg['From'] = "weewx" msg['To'] = self.TO # Create an instance of class SMTP for the given SMTP host: s = smtplib.SMTP(self.smtp_host) try: # Some servers (eg, gmail) require encrypted transport. # Be prepared to catch an exception if the server # doesn't support it. s.ehlo() s.starttls() s.ehlo() except smtplib.SMTPException: pass # If a username has been given, assume that login is required for this host: if self.smtp_user: s.login(self.smtp_user, self.smtp_password) # Send the email: s.sendmail(msg['From'], [self.TO], msg.as_string()) # Log out of the server: s.quit() # Log it in the system log: syslog.syslog(syslog.LOG_INFO, "lowBattery: Low battery alarm (0x%04x) sounded." % rec['txBatteryStatus']) syslog.syslog(syslog.LOG_INFO, " *** email sent to: %s" % self.TO)
def soundTheAlarm(self, rec): """This function is called when the given expression evaluates True.""" # Get the time and convert to a string: t_str = timestamp_to_string(rec['dateTime']) # Form the message text: msg_text = "Alarm expression \"%s\" evaluated True at %s\nRecord:\n%s" % (self.expression, t_str, str(rec)) # Convert to MIME: msg = MIMEText(msg_text) # Fill in MIME headers: msg['Subject'] = "Alarm message from weewx" msg['From'] = "weewx" msg['To'] = self.TO # Create an instance of class SMTP for the given SMTP host: s = smtplib.SMTP(self.smtp_host) try: # Some servers (eg, gmail) require encrypted transport. # Be prepared to catch an exception if the server # doesn't support it. s.ehlo() s.starttls() s.ehlo() except smtplib.SMTPException: pass # If a username has been given, assume that login is required for this host: if self.smtp_user: s.login(self.smtp_user, self.smtp_password) # Send the email: s.sendmail(msg['From'], [self.TO], msg.as_string()) # Log out of the server: s.quit() # Log it in the system log: syslog.syslog(syslog.LOG_INFO, "alarm: Alarm sounded for expression: \"%s\"" % self.expression) syslog.syslog(syslog.LOG_INFO, " *** email sent to: %s" % self.TO)
def run(self): """Main entry point for calculating missing derived fields. Calculate the missing derived fields for the timespan concerned, save the calculated data to archive and recalculate the daily summaries. """ # record the current time t1 = time.time() # Instantiate a dummy engine, to be used to calculate derived variables. This will # cause all the xtype services to get loaded. engine = weewx.engine.DummyEngine(self.config_dict) # While the above instantiated an instance of StdWXCalculate, we have no way of # retrieving it. So, instantiate another one, then use that to calculate derived types. wxcalculate = weewx.wxservices.StdWXCalculate(engine, self.config_dict) # initialise some counters so we know what we have processed days_updated = 0 days_processed = 0 total_records_processed = 0 total_records_updated = 0 # obtain gregorian days for our start and stop timestamps start_greg = weeutil.weeutil.toGregorianDay(self.start_ts) stop_greg = weeutil.weeutil.toGregorianDay(self.stop_ts) # start at the first day day = start_greg while day <= stop_greg: # get the start and stop timestamps for this tranche tr_start_ts = weeutil.weeutil.startOfGregorianDay(day) tr_stop_ts = min(weeutil.weeutil.startOfGregorianDay(stop_greg + 1), weeutil.weeutil.startOfGregorianDay(day + self.trans_days)) # start the transaction with weedb.Transaction(self.dbm.connection) as _cursor: # iterate over each day in the tranche we are to work in for tranche_day in weeutil.weeutil.genDaySpans(tr_start_ts, tr_stop_ts): # initialise a counter for records processed on this day records_updated = 0 # iterate over each record in this day for record in self.dbm.genBatchRecords(startstamp=tranche_day.start, stopstamp=tranche_day.stop): # but we are only concerned with records after the # start and before or equal to the stop timestamps if self.start_ts < record['dateTime'] <= self.stop_ts: # first obtain a list of the fields that may be calculated extras_list = [] for obs in wxcalculate.calc_dict: directive = wxcalculate.calc_dict[obs] if directive == 'software' \ or directive == 'prefer_hardware' \ and (obs not in record or record[obs] is None): extras_list.append(obs) # calculate the missing derived fields for the record wxcalculate.do_calculations(record) # Obtain a new record dictionary that contains only those items # that wxcalculate calculated. Use dictionary comprehension. extras_dict = {k:v for (k,v) in record.items() if k in extras_list} # update the archive with the calculated data records_updated += self.update_record_fields(record['dateTime'], extras_dict) # update the total records processed total_records_processed += 1 # Give the user some information on progress if total_records_processed % 1000 == 0: p_msg = "Processing record: %d; Last record: %s" % (total_records_processed, timestamp_to_string(record['dateTime'])) self._progress(p_msg) # update the total records updated total_records_updated += records_updated # if we updated any records on this day increment the count # of days updated days_updated += 1 if records_updated > 0 else 0 days_processed += 1 # advance to the next tranche day += self.trans_days # finished, so give the user some final information on progress, mainly # so the total tallies with the log p_msg = "Processing record: %d; Last record: %s" % (total_records_processed, timestamp_to_string(tr_stop_ts)) self._progress(p_msg, overprint=False) # now update the daily summaries, but only if this is not a dry run if not self.dry_run: print("Recalculating daily summaries...") # first we need a start and stop date object start_d = datetime.date.fromtimestamp(self.start_ts) # Since each daily summary is identified by the midnight timestamp # for that day we need to make sure we our stop timestamp is not on # a midnight boundary or we will rebuild the following days sumamry # as well. if it is on a midnight boundary just subtract 1 second # and use that. summary_stop_ts = self.stop_ts if weeutil.weeutil.isMidnight(self.stop_ts): summary_stop_ts -= 1 stop_d = datetime.date.fromtimestamp(summary_stop_ts) # do the update self.dbm.backfill_day_summary(start_d=start_d, stop_d=stop_d) print(file=sys.stdout) print("Finished recalculating daily summaries") else: # it's a dry run so say the rebuild was skipped print("This is a dry run, recalculation of daily summaries was skipped") tdiff = time.time() - t1 # we are done so log and inform the user _day_processed_str = "day" if days_processed == 1 else "days" _day_updated_str = "day" if days_updated == 1 else "days" if not self.dry_run: log.info("Processed %d %s consisting of %d records. " "%d %s consisting of %d records were updated " "in %0.2f seconds." % (days_processed, _day_processed_str, total_records_processed, days_updated, _day_updated_str, total_records_updated, tdiff)) else: # this was a dry run log.info("Processed %d %s consisting of %d records. " "%d %s consisting of %d records would have been updated " "in %0.2f seconds." % (days_processed, _day_processed_str, total_records_processed, days_updated, _day_updated_str, total_records_updated, tdiff))
def do_fix(self, np_ts): """Apply the interval weighting fix to the daily summaries.""" # do we need to weight? Only weight if next day to weight ts is None or # there are records in the archive from that day if np_ts is None or self.dbm.last_timestamp > np_ts: t1 = time.time() syslog.syslog(syslog.LOG_INFO, "intervalweighting: Applying %s..." % self.name) _days = 0 # Get the earliest daily summary ts and the obs that it came from first_ts, obs = self.first_summary() # Get the start and stop ts for our first transaction days _tr_start_ts = np_ts if np_ts is not None else first_ts _tr_stop_dt = datetime.datetime.fromtimestamp(_tr_start_ts) + datetime.timedelta(days=self.trans_days) _tr_stop_ts = time.mktime(_tr_stop_dt.timetuple()) _tr_stop_ts = min(startOfDay(self.dbm.last_timestamp), _tr_stop_ts) last_start = None while True: with weedb.Transaction(self.dbm.connection) as _cursor: for _day_span in self.genSummaryDaySpans(_tr_start_ts, _tr_stop_ts, obs): # Get the weight to be applied for the day _weight = self.get_interval(_day_span) * 60 # Get the current day stats in an accumulator _day_accum = self.dbm._get_day_summary(_day_span.start) # Set the unit system for the accumulator _day_accum.unit_system = self.dbm.std_unit_system # Weight the necessary accumulator stats, use a # try..except in case something goes wrong last_key = None try: for _day_key in self.dbm.daykeys: last_key = _day_key _day_accum[_day_key].wsum *= _weight _day_accum[_day_key].sumtime *= _weight # Do we have a vecstats accumulator? if hasattr(_day_accum[_day_key], 'wsquaresum'): # Yes, so update the weighted vector stats _day_accum[_day_key].wsquaresum *= _weight _day_accum[_day_key].xsum *= _weight _day_accum[_day_key].ysum *= _weight _day_accum[_day_key].dirsumtime *= _weight except Exception, e: # log the exception and re-raise it syslog.syslog(syslog.LOG_INFO, "intervalweighting: Interval weighting of '%s' daily summary " "for %s failed: %s" % (last_key, timestamp_to_string(_day_span.start, format="%Y-%m-%d"), e)) raise # Update the daily summary with the weighted accumulator if not self.dry_run: self.dbm._set_day_summary(_day_accum, None, _cursor) _days += 1 # Save the ts of the weighted daily summary as the # 'lastWeightPatch' value in the archive_day__metadata # table if not self.dry_run: self.dbm._write_metadata('lastWeightPatch', _day_span.start, _cursor) # Give the user some information on progress if _days % 50 == 0: self._progress(_days, _day_span.start) last_start = _day_span.start # Setup our next tranche # Have we reached the end, if so break to finish if _tr_stop_ts >= startOfDay(self.dbm.last_timestamp): break # More to process so set our start and stop for the next # transaction _tr_start_dt = datetime.datetime.fromtimestamp(_tr_stop_ts) + datetime.timedelta(days=1) _tr_start_ts = time.mktime(_tr_start_dt.timetuple()) _tr_stop_dt = datetime.datetime.fromtimestamp(_tr_start_ts) + datetime.timedelta(days=self.trans_days) _tr_stop_ts = time.mktime(_tr_stop_dt.timetuple()) _tr_stop_ts = min(self.dbm.last_timestamp, _tr_stop_ts) # We have finished. Get rid of the no longer needed lastWeightPatch with weedb.Transaction(self.dbm.connection) as _cursor: _cursor.execute("DELETE FROM %s_day__metadata WHERE name=?" % self.dbm.table_name, ('lastWeightPatch',)) # Give the user some final information on progress, # mainly so the total tallies with the log self._progress(_days, last_start) print >>sys.stdout tdiff = time.time() - t1 # We are done so log and inform the user syslog.syslog(syslog.LOG_INFO, "intervalweighting: calculated weighting for %s days in %0.2f seconds." % (_days, tdiff)) if self.dry_run: syslog.syslog(syslog.LOG_INFO, "intervalweighting: This was a dry run. %s was not applied." % self.name)
def soundTheAlarm(self, timestamp, battery_flags, alarm_count): """This function is called when the alarm has been triggered.""" # Get the time and convert to a string: t_str = timestamp_to_string(timestamp) # Log it in the system log: syslog.syslog(syslog.LOG_INFO, "lowBattery: Low battery status sounded at %s: %s" % (t_str, battery_flags)) # Form the message text: indicator_strings = [] for bat in battery_flags: indicator_strings.append("%s: %04x" % (bat, battery_flags[bat])) msg_text = """ The low battery indicator has been seen %d times since the last archive period. Alarm sounded at %s Low battery indicators: %s """ % (alarm_count, t_str, '\n'.join(indicator_strings)) # Convert to MIME: msg = MIMEText(msg_text) # Fill in MIME headers: msg['Subject'] = self.SUBJECT msg['From'] = self.FROM msg['To'] = ','.join(self.TO) # Create an instance of class SMTP for the given SMTP host: s = smtplib.SMTP(self.smtp_host) try: # Some servers (eg, gmail) require encrypted transport. # Be prepared to catch an exception if the server # doesn't support it. s.ehlo() s.starttls() s.ehlo() syslog.syslog(syslog.LOG_DEBUG, "lowBattery: using encrypted transport") except smtplib.SMTPException: syslog.syslog(syslog.LOG_DEBUG, "lowBattery: using unencrypted transport") try: # If a username has been given, assume that login is required # for this host: if self.smtp_user: s.login(self.smtp_user, self.smtp_password) syslog.syslog(syslog.LOG_DEBUG, "lowBattery: logged in as %s" % self.smtp_user) # Send the email: s.sendmail(msg['From'], self.TO, msg.as_string()) # Log out of the server: s.quit() except Exception, e: syslog.syslog(syslog.LOG_ERR, "lowBattery: send email failed: %s" % (e,)) raise
def __init__(self, config_dict, config_path, csv_config_dict, import_config_path, options, log): # call our parents __init__ super(CSVSource, self).__init__(config_dict, csv_config_dict, options, log) # save our import config path self.import_config_path = import_config_path # save our import config dict self.csv_config_dict = csv_config_dict # get a few config settings from our CSV config dict # string format used to decode the imported field holding our dateTime self.raw_datetime_format = self.csv_config_dict.get('raw_datetime_format', '%Y-%m-%d %H:%M:%S') # is our rain discrete or cumulative self.rain = self.csv_config_dict.get('rain', 'cumulative') # determine valid range for imported wind direction _wind_direction = option_as_list(self.csv_config_dict.get('wind_direction', '0,360')) try: if float(_wind_direction[0]) <= float(_wind_direction[1]): self.wind_dir = [float(_wind_direction[0]), float(_wind_direction[1])] else: self.wind_dir = [-360, 360] except: self.wind_dir = [-360, 360] # get our source file path try: self.source = csv_config_dict['file'] except KeyError: raise weewx.ViolatedPrecondition("CSV source file not specified in '%s'." % import_config_path) # initialise our import field-to-weewx archive field map self.map = None # initialise some other properties we will need self.start = 1 self.end = 1 self.increment = 1 # tell the user/log what we intend to do _msg = "A CSV import from source file '%s' has been requested." % self.source self.wlog.printlog(syslog.LOG_INFO, _msg) _msg = "The following options will be used:" self.wlog.verboselog(syslog.LOG_DEBUG, _msg) _msg = " config=%s, import-config=%s" % (config_path, self.import_config_path) self.wlog.verboselog(syslog.LOG_DEBUG, _msg) if options.date: _msg = " source=%s, date=%s" % (self.source, options.date) else: # we must have --from and --to _msg = " source=%s, from=%s, to=%s" % (self.source, options.date_from, options.date_to) self.wlog.verboselog(syslog.LOG_DEBUG, _msg) _msg = " dry-run=%s, calc-missing=%s" % (self.dry_run, self.calc_missing) self.wlog.verboselog(syslog.LOG_DEBUG, _msg) _msg = " tranche=%s, interval=%s, date/time_string_format=%s" % (self.tranche, self.interval, self.raw_datetime_format) self.wlog.verboselog(syslog.LOG_DEBUG, _msg) _msg = " rain=%s, wind_direction=%s" % (self.rain, self.wind_dir) self.wlog.verboselog(syslog.LOG_DEBUG, _msg) _msg = " UV=%s, radiation=%s" % (self.UV_sensor, self.solar_sensor) self.wlog.verboselog(syslog.LOG_DEBUG, _msg) _msg = "Using database binding '%s', which is bound to database '%s'" % (self.db_binding_wx, self.dbm.database_name) self.wlog.printlog(syslog.LOG_INFO, _msg) _msg = "Destination table '%s' unit system is '%#04x' (%s)." % (self.dbm.table_name, self.archive_unit_sys, unit_nicknames[self.archive_unit_sys]) self.wlog.printlog(syslog.LOG_INFO, _msg) if self.calc_missing: print "Missing derived observations will be calculated." if not self.UV_sensor: print "All weewx UV fields will be set to None." if not self.solar_sensor: print "All weewx radiation fields will be set to None." if options.date or options.date_from: print "Observations timestamped after %s and up to and" % (timestamp_to_string(self.first_ts), ) print "including %s will be imported." % (timestamp_to_string(self.last_ts), ) if self.dry_run: print "This is a dry run, imported data will not be saved to archive."
def _progress(ndays, last_time): """Utility function to show our progress while processing the fix.""" print >>sys.stdout, "Weighting daily summary: %d; Timestamp: %s\r" % \ (ndays, timestamp_to_string(last_time, format_str="%Y-%m-%d")), sys.stdout.flush()
def fill_in_packet(cfg: Configuration, packet: Dict): with cfg.lock: log.debug('new_loop_packet: cfg.concentrations: %s' % cfg.concentrations) if cfg.concentrations is not None and \ cfg.concentrations.timestamp is not None and \ cfg.concentrations.timestamp + \ cfg.archive_interval >= time.time(): log.debug('Time of reading being inserted: %s' % timestamp_to_string(cfg.concentrations.timestamp)) # Insert pm1_0, pm2_5, pm10_0, aqi and aqic into loop packet. if cfg.concentrations.pm_1_last is not None: packet['pm1_0'] = cfg.concentrations.pm_1_last log.debug('Inserted packet[pm1_0]: %f into packet.' % cfg.concentrations.pm_1_last) if (cfg.concentrations.pm_2p5_last is not None and cfg.concentrations.hum is not None and cfg.concentrations.temp is not None): packet['pm2_5'] = AQI.compute_pm_2p5_us_epa_correction( cfg.concentrations.pm_2p5_last, cfg.concentrations.hum, cfg.concentrations.temp) log.debug('Inserted packet[pm2_5]: %f into packet.' % cfg.concentrations.pm_2p5_last) # Put aqi and color in the packet. packet['pm2_5_aqi'] = AQI.compute_pm2_5_aqi(packet['pm2_5']) packet['pm2_5_aqi_color'] = AQI.compute_pm2_5_aqi_color(packet['pm2_5_aqi']) if cfg.concentrations.pm_10_last is not None: packet['pm10_0'] = cfg.concentrations.pm_10_last log.debug('Inserted packet[pm10_0]: %f into packet.' % cfg.concentrations.pm_10_last) # Also insert one minute averages as these averages are more useful for showing in realtime. # If 1m averages are not available, use last instead. if cfg.concentrations.pm_1 is not None: packet['pm1_0_1m'] = cfg.concentrations.pm_1 elif cfg.concentrations.pm_1_last is not None: packet['pm1_0_1m'] = cfg.concentrations.pm_1_last if cfg.concentrations.pm_2p5 is not None: packet['pm2_5_1m'] = AQI.compute_pm_2p5_us_epa_correction( cfg.concentrations.pm_2p5, cfg.concentrations.hum, cfg.concentrations.temp) elif (cfg.concentrations.pm_2p5_last is not None and cfg.concentrations.hum is not None and cfg.concentrations.temp is not None): packet['pm2_5_1m'] = AQI.compute_pm_2p5_us_epa_correction( cfg.concentrations.pm_2p5_last, cfg.concentrations.hum, cfg.concentrations.temp) if cfg.concentrations.pm_10 is not None: packet['pm10_0_1m'] = cfg.concentrations.pm_10 elif cfg.concentrations.pm_10_last is not None: packet['pm10_0_1m'] = cfg.concentrations.pm_10_last # Add 1m aqi and color if 'pm2_5_1m' in packet: packet['pm2_5_1m_aqi'] = AQI.compute_pm2_5_aqi(packet['pm2_5_1m']) packet['pm2_5_1m_aqi_color'] = AQI.compute_pm2_5_aqi_color(packet['pm2_5_1m_aqi']) # And insert nowcast for pm 2.5 and 10 as some might want to report that. # If nowcast not available, don't substitute. if (cfg.concentrations.pm_2p5_nowcast is not None and cfg.concentrations.hum is not None and cfg.concentrations.temp is not None): packet['pm2_5_nowcast'] = AQI.compute_pm_2p5_us_epa_correction( cfg.concentrations.pm_2p5_nowcast, cfg.concentrations.hum, cfg.concentrations.temp) packet['pm2_5_nowcast_aqi'] = AQI.compute_pm2_5_aqi(packet['pm2_5_nowcast']) packet['pm2_5_nowcast_aqi_color'] = AQI.compute_pm2_5_aqi_color(packet['pm2_5_nowcast_aqi']) if cfg.concentrations.pm_10_nowcast is not None: packet['pm10_0_nowcast'] = cfg.concentrations.pm_10_nowcast else: log.error('Found no concentrations to insert.')
def run(self): """Main entry point for calculating missing derived fields. Calculate the missing derived fields for the timespan concerned, save the calculated data to archive and recalculate the daily summaries. """ # record the current time t1 = time.time() # obtain a wxservices.WXCalculate object to calculate the missing fields # first we need station altitude, latitude and longitude stn_dict = self.config_dict['Station'] altitude_t = option_as_list(stn_dict.get('altitude', (None, None))) try: altitude_vt = weewx.units.ValueTuple(float(altitude_t[0]), altitude_t[1], "group_altitude") except KeyError as e: raise weewx.ViolatedPrecondition( "Value 'altitude' needs a unit (%s)" % e) latitude_f = float(stn_dict['latitude']) longitude_f = float(stn_dict['longitude']) # now we can create a WXCalculate object wxcalculate = weewx.wxservices.WXCalculate(self.config_dict, altitude_vt, latitude_f, longitude_f) # initialise some counters so we know what we have processed days_updated = 0 days_processed = 0 total_records_processed = 0 total_records_updated = 0 # obtain gregorian days for our start and stop timestamps start_greg = weeutil.weeutil.toGregorianDay(self.start_ts) stop_greg = weeutil.weeutil.toGregorianDay(self.stop_ts) # start at the first day day = start_greg while day <= stop_greg: # get the start and stop timestamps for this tranche tr_start_ts = weeutil.weeutil.startOfGregorianDay(day) tr_stop_ts = min(weeutil.weeutil.startOfGregorianDay(stop_greg + 1), weeutil.weeutil.startOfGregorianDay(day + self.trans_days)) # start the transaction with weedb.Transaction(self.dbm.connection) as _cursor: # iterate over each day in the tranche we are to work in for tranche_day in weeutil.weeutil.genDaySpans(tr_start_ts, tr_stop_ts): # initialise a counter for records processed on this day records_updated = 0 # iterate over each record in this day for record in self.dbm.genBatchRecords(startstamp=tranche_day.start, stopstamp=tranche_day.stop): # but we are only concerned with records after the # start and before or equal to the stop timestamps if self.start_ts < record['dateTime'] <= self.stop_ts: # first obtain a list of the fields that may be calculated extras_list = [] for obs in wxcalculate.svc_dict['Calculations']: directive = wxcalculate.svc_dict['Calculations'][obs] if directive == 'software' \ or directive == 'prefer_hardware' and ( obs not in record or record[obs] is None): extras_list.append(obs) # calculate the missing derived fields for the record wxcalculate.do_calculations(data_dict=record, data_type='archive') # Obtain a dict containing only those fields that # WXCalculate calculated. We could do this as a # dictionary comprehension but python2.6 does not # support dictionary comprehensions. extras_dict = {} for k in extras_list: if k in record.keys(): extras_dict[k] = record[k] # update the archive with the calculated data records_updated += self.update_record_fields(record['dateTime'], extras_dict) # update the total records processed total_records_processed += 1 # Give the user some information on progress if total_records_processed % 1000 == 0: p_msg = "Processing record: %d; Last record: %s" % (total_records_processed, timestamp_to_string(record['dateTime'])) self._progress(p_msg) # update the total records updated total_records_updated += records_updated # if we updated any records on this day increment the count # of days updated days_updated += 1 if records_updated > 0 else 0 days_processed += 1 # advance to the next tranche day += self.trans_days # finished, so give the user some final information on progress, mainly # so the total tallies with the log p_msg = "Processing record: %d; Last record: %s" % (total_records_processed, timestamp_to_string(tr_stop_ts)) self._progress(p_msg, overprint=False) # now update the daily summaries, but only if this is not a dry run if not self.dry_run: print("Recalculating daily summaries...") # first we need a start and stop date object start_d = datetime.date.fromtimestamp(self.start_ts) # Since each daily summary is identified by the midnight timestamp # for that day we need to make sure we our stop timestamp is not on # a midnight boundary or we will rebuild the following days sumamry # as well. if it is on a midnight boundary just subtract 1 second # and use that. summary_stop_ts = self.stop_ts if weeutil.weeutil.isMidnight(self.stop_ts): summary_stop_ts -= 1 stop_d = datetime.date.fromtimestamp(summary_stop_ts) # do the update self.dbm.backfill_day_summary(start_d=start_d, stop_d=stop_d) print(file=sys.stdout) print("Finished recalculating daily summaries") else: # it's a dry run so say the rebuild was skipped print("This is a dry run, recalculation of daily summaries was skipped") tdiff = time.time() - t1 # we are done so log and inform the user _day_processed_str = "day" if days_processed == 1 else "days" _day_updated_str = "day" if days_updated == 1 else "days" if not self.dry_run: log.info("Processed %d %s consisting of %d records. " "%d %s consisting of %d records were updated " "in %0.2f seconds." % (days_processed, _day_processed_str, total_records_processed, days_updated, _day_updated_str, total_records_updated, tdiff)) else: # this was a dry run log.info("Processed %d %s consisting of %d records. " "%d %s consisting of %d records would have been updated " "in %0.2f seconds." % (days_processed, _day_processed_str, total_records_processed, days_updated, _day_updated_str, total_records_updated, tdiff))
S3upload_message = "uploaded %d files (%s bytes) in %%0.2f seconds" % (int(file_cnt), byte_cnt) else: S3upload_message = "executed in %0.2f seconds" except: S3upload_message = "executed in %0.2f seconds" else: # suspect we have an s3cmd error so display a message syslog.syslog(syslog.LOG_INFO, "s3uploadgenerator: s3cmd reported errors") for line in iter(stroutput.splitlines()): syslog.syslog(syslog.LOG_INFO, "s3uploadgenerator: s3cmd error: %s" % line) S3upload_message = "executed in %0.2f seconds" stop_ts = time.time() syslog.syslog(syslog.LOG_INFO, "s3uploadgenerator: results: " + S3upload_message % (stop_ts - start_ts)) t_str = timestamp_to_string(stop_ts) syslog.syslog(syslog.LOG_DEBUG, "s3uploadgenerator: end upload at %s" % t_str) if __name__ == '__main__': """This section is used for testing the code. """ # Note that this fails! import sys import configobj from optparse import OptionParser usage_string ="""Usage: S3upload.py config_path Arguments:
def __init__(self, config_dict, config_path, wd_config_dict, import_config_path, options): # call our parents __init__ super(WDSource, self).__init__(config_dict, wd_config_dict, options) # save the import config path self.import_config_path = import_config_path # save the import config dict self.wd_config_dict = wd_config_dict # our parent uses 'derive' as the default interval setting, for WD the # default should be 1 (minute) so redo the interval setting with our # default self.interval = wd_config_dict.get('interval', 1) # wind dir bounds self.wind_dir = [0, 360] # How the WeeWX field 'rain' is populated depends on the source rain # data. If the only data available is cumulative then the WeeWX rain # field is calculated as the difference between successive cumulative # values. WD provides a rain per interval field so that data can be # used to map directly to the WeeWX rain field. If rain is to be # calculated from a cumulative value then self.rain must be set to # 'cumulative', to map directly to the WeeWX rain field self.rain must # be set to None. self.rain = None # field delimiter used in text format monthly log files, default to # space self.txt_delimiter = str(wd_config_dict.get('txt_delimiter', ' ')) # field delimiter used in csv format monthly log files, default to # comma self.csv_delimiter = str(wd_config_dict.get('csv_delimiter', ',')) # decimal separator used in monthly log files, default to decimal point self.decimal = wd_config_dict.get('decimal', '.') # ignore extreme > 255.0 values for temperature and humidity fields self.ignore_extreme_temp_hum = weeutil.weeutil.tobool( wd_config_dict.get('ignore_extreme_temp_hum', True)) # initialise the import field-to-WeeWX archive field map self.map = None # property holding the current log file name being processed self.file_name = None # WD logs use either US or Metric units. The units used in each case # are: # Metric units: C, knots, hPa, mm # US units: F, mph, inHg, inch # # The user must specify the units to be used in the import config file. # This can be by either by specifying the log units as Metric or US # using the 'units' config option. Alternatively temperature, pressure, # rainfall and speed units can be specified individually under the # [Units] stanza. First check for a valid 'units' config option then # check for individual group units. Do some basic error checking and # validation, if one of the fields is missing or invalid then we need # to catch the error and raise it as we can't go on. log_unit_config = wd_config_dict.get('Units') if log_unit_config is not None: # get the units config option log_unit_sys = wd_config_dict['Units'].get('units') # accept any capitalization of USA as == US log_unit_sys = log_unit_sys if log_unit_sys.upper( ) != 'USA' else 'US' # does the units config option specify a valid log unit system if log_unit_sys is None or log_unit_sys.upper() not in [ 'METRIC', 'US' ]: # log unit system not specified look for individual entries # temperature temp_u = wd_config_dict['Units'].get('temperature') if temp_u is not None: if temp_u in weewx.units.default_unit_format_dict: self._header_map['temperature']['units'] = temp_u self._header_map['dewpoint']['units'] = temp_u self._header_map['heatindex']['units'] = temp_u self._header_map['soiltemp']['units'] = temp_u self._header_map['temp1']['units'] = temp_u self._header_map['temp2']['units'] = temp_u self._header_map['temp3']['units'] = temp_u self._header_map['temp4']['units'] = temp_u self._header_map['temp5']['units'] = temp_u self._header_map['temp6']['units'] = temp_u self._header_map['temp7']['units'] = temp_u else: _msg = "Unknown units '%s' specified for Weather Display " \ "temperature fields in %s." % (temp_u, self.import_config_path) raise weewx.UnitError(_msg) else: _msg = "No units specified for Weather Display temperature " \ "fields in %s." % (self.import_config_path,) raise weewx.UnitError(_msg) # pressure press_u = wd_config_dict['Units'].get('pressure') if press_u is not None: if press_u in ['inHg', 'hPa']: self._header_map['barometer']['units'] = press_u else: _msg = "Unknown units '%s' specified for Weather Display " \ "pressure fields in %s." % (press_u, self.import_config_path) raise weewx.UnitError(_msg) else: _msg = "No units specified for Weather Display pressure " \ "fields in %s." % (self.import_config_path,) raise weewx.UnitError(_msg) # rain rain_u = wd_config_dict['Units'].get('rain') if rain_u is not None: if rain_u in ['inch', 'mm']: self._header_map['rainlastmin']['units'] = rain_u self._header_map['dailyrain']['units'] = rain_u self._header_map['monthlyrain']['units'] = rain_u self._header_map['yearlyrain']['units'] = rain_u self._header_map['dailyet']['units'] = rain_u else: _msg = "Unknown units '%s' specified for Weather Display " \ "rain fields in %s." % (rain_u, self.import_config_path) raise weewx.UnitError(_msg) else: _msg = "No units specified for Weather Display rain fields " \ "in %s." % (self.import_config_path,) raise weewx.UnitError(_msg) # speed speed_u = wd_config_dict['Units'].get('speed') if speed_u is not None: if speed_u in ['inch', 'mm']: self._header_map['windspeed']['units'] = speed_u self._header_map['gustspeed']['units'] = speed_u else: _msg = "Unknown units '%s' specified for Weather Display " \ "speed fields in %s." % (speed_u, self.import_config_path) raise weewx.UnitError(_msg) else: _msg = "No units specified for Weather Display speed fields " \ "in %s." % (self.import_config_path,) raise weewx.UnitError(_msg) else: # log unit system specified _unit_sys = log_unit_sys.upper() # do we have a valid log unit system if _unit_sys in ['METRIC', 'US']: # valid log unit system so assign units as applicable self._header_map['temperature'][ 'units'] = self.wd_unit_sys['temperature'][_unit_sys] self._header_map['dewpoint']['units'] = self.wd_unit_sys[ 'temperature'][_unit_sys] self._header_map['heatindex']['units'] = self.wd_unit_sys[ 'temperature'][_unit_sys] self._header_map['barometer']['units'] = self.wd_unit_sys[ 'barometer'][_unit_sys] self._header_map['windspeed']['units'] = self.wd_unit_sys[ 'windspeed'][_unit_sys] self._header_map['gustspeed']['units'] = self.wd_unit_sys[ 'gustspeed'][_unit_sys] self._header_map['rainlastmin'][ 'units'] = self.wd_unit_sys['rainlastmin'][_unit_sys] self._header_map['soiltemp']['units'] = self.wd_unit_sys[ 'soiltemp'][_unit_sys] for _num in range(1, 8): _temp = 'temp%s' % _num self._header_map[_temp]['units'] = self.wd_unit_sys[ _temp][_unit_sys] else: # no valid Units config found, we can't go on so raise an error raise weewx.UnitError( "Invalid setting for 'units' config option.") else: # there is no Units config, we can't go on so raise an error raise weewx.UnitError("No Weather Display units config found.") # obtain a list of logs files to be processed _to_process = wd_config_dict.get('logs_to_process', list(self.logs.keys())) self.logs_to_process = weeutil.weeutil.option_as_list(_to_process) # can missing log files be ignored self.ignore_missing_log = weeutil.weeutil.to_bool( wd_config_dict.get('ignore_missing_log', True)) # get our source file path try: self.source = wd_config_dict['directory'] except KeyError: _msg = "Weather Display monthly logs directory not specified in '%s'." % import_config_path raise weewx.ViolatedPrecondition(_msg) # get the source file encoding, default to utf-8-sig self.source_encoding = self.wd_config_dict.get('source_encoding', 'utf-8-sig') # Now get a list on monthly log files sorted from oldest to newest. # This is complicated by the log file naming convention used by WD. # first the 1 digit months _lg_5_list = glob.glob(self.source + '/' + '[0-9]' * 5 + 'lg.txt') # and the 2 digit months _lg_6_list = glob.glob(self.source + '/' + '[0-9]' * 6 + 'lg.txt') # concatenate the two lists to get the complete list month_lg_list = _lg_5_list + _lg_6_list # create a list of log files in chronological order (month, year) _temp = [] # create a list of log files, adding year and month fields for sorting for p in month_lg_list: # obtain the file name fn = os.path.split(p)[1] # obtain the numeric part of the file name _digits = ''.join(c for c in fn if c.isdigit()) # append a list of format [path+file name, month, year] _temp.append([p, int(_digits[:-4]), int(_digits[-4:])]) # now sort the list keeping just the log file path and name self.log_list = [ a[0] for a in sorted(_temp, key=lambda el: (el[2], el[1])) ] # if there are no log files then there is nothing to be done if len(self.log_list) == 0: raise weeimport.WeeImportIOError( "No Weather Display monthly logs found in directory '%s'." % self.source) # Some log files have entries that belong in a different month. # Initialise a list to hold these extra records for processing during # the appropriate month self.extras = {} for l in self.logs_to_process: self.extras[l] = [] # tell the user/log what we intend to do _msg = "Weather Display monthly log files in the '%s' directory will be imported" % self.source print(_msg) log.info(_msg) _msg = "The following options will be used:" if self.verbose: print(_msg) log.debug(_msg) _msg = " config=%s, import-config=%s" % (config_path, self.import_config_path) if self.verbose: print(_msg) log.debug(_msg) if options.date: _msg = " date=%s" % options.date else: # we must have --from and --to _msg = " from=%s, to=%s" % (options.date_from, options.date_to) if self.verbose: print(_msg) log.debug(_msg) _msg = " dry-run=%s, calc_missing=%s, ignore_invalid_data=%s" % ( self.dry_run, self.calc_missing, self.ignore_invalid_data) if self.verbose: print(_msg) log.debug(_msg) if log_unit_sys is not None and log_unit_sys.upper() in [ 'METRIC', 'US' ]: # valid unit system specified _msg = " monthly logs are in %s units" % log_unit_sys.upper() if self.verbose: print(_msg) log.debug(_msg) else: # group units specified _msg = " monthly logs use the following units:" if self.verbose: print(_msg) log.debug(_msg) _msg = " temperature=%s pressure=%s" % (temp_u, press_u) if self.verbose: print(_msg) log.debug(_msg) _msg = " rain=%s speed=%s" % (rain_u, speed_u) if self.verbose: print(_msg) log.debug(_msg) _msg = " tranche=%s, interval=%s" % (self.tranche, self.interval) if self.verbose: print(_msg) log.debug(_msg) _msg = " UV=%s, radiation=%s ignore extreme temperature and humidity=%s" % ( self.UV_sensor, self.solar_sensor, self.ignore_extreme_temp_hum) if self.verbose: print(_msg) log.debug(_msg) _msg = "Using database binding '%s', which is bound to database '%s'" % ( self.db_binding_wx, self.dbm.database_name) print(_msg) log.info(_msg) _msg = "Destination table '%s' unit system is '%#04x' (%s)." % ( self.dbm.table_name, self.archive_unit_sys, unit_nicknames[self.archive_unit_sys]) print(_msg) log.info(_msg) if self.calc_missing: print("Missing derived observations will be calculated.") if not self.UV_sensor: print("All WeeWX UV fields will be set to None.") if not self.solar_sensor: print("All WeeWX radiation fields will be set to None.") if options.date or options.date_from: print("Observations timestamped after %s and up to and" % timestamp_to_string(self.first_ts)) print("including %s will be imported." % timestamp_to_string(self.last_ts)) if self.dry_run: print( "This is a dry run, imported data will not be saved to archive." )
def do_fix(self, np_ts): """Apply the interval weighting fix to the daily summaries.""" # do we need to weight? Only weight if next day to weight ts is None or # there are records in the archive from that day if np_ts is None or self.dbm.last_timestamp > np_ts: t1 = time.time() log.info("intervalweighting: Applying %s..." % self.name) _days = 0 # Get the earliest daily summary ts and the obs that it came from first_ts, obs = self.first_summary() # Get the start and stop ts for our first transaction days _tr_start_ts = np_ts if np_ts is not None else first_ts _tr_stop_dt = datetime.datetime.fromtimestamp(_tr_start_ts) \ + datetime.timedelta(days=self.trans_days) _tr_stop_ts = time.mktime(_tr_stop_dt.timetuple()) _tr_stop_ts = min(startOfDay(self.dbm.last_timestamp), _tr_stop_ts) last_start = None while True: with weedb.Transaction(self.dbm.connection) as _cursor: for _day_span in self.genSummaryDaySpans(_tr_start_ts, _tr_stop_ts, obs): # Get the weight to be applied for the day _weight = self.get_interval(_day_span) * 60 # Get the current day stats in an accumulator _day_accum = self.dbm._get_day_summary(_day_span.start) # Set the unit system for the accumulator _day_accum.unit_system = self.dbm.std_unit_system # Weight the necessary accumulator stats, use a # try..except in case something goes wrong last_key = None try: for _day_key in self.dbm.daykeys: last_key = _day_key _day_accum[_day_key].wsum *= _weight _day_accum[_day_key].sumtime *= _weight # Do we have a vecstats accumulator? if hasattr(_day_accum[_day_key], 'wsquaresum'): # Yes, so update the weighted vector stats _day_accum[_day_key].wsquaresum *= _weight _day_accum[_day_key].xsum *= _weight _day_accum[_day_key].ysum *= _weight _day_accum[_day_key].dirsumtime *= _weight except Exception as e: # log the exception and re-raise it log.info("intervalweighting: Interval weighting of '%s' daily summary " "for %s failed: %s" % (last_key, timestamp_to_string(_day_span.start, format_str="%Y-%m-%d"), e)) raise # Update the daily summary with the weighted accumulator if not self.dry_run: self.dbm._set_day_summary(_day_accum, None, _cursor) _days += 1 # Save the ts of the weighted daily summary as the # 'lastWeightPatch' value in the archive_day__metadata # table if not self.dry_run: self.dbm._write_metadata('lastWeightPatch', _day_span.start, _cursor) # Give the user some information on progress if _days % 50 == 0: self._progress(_days, _day_span.start) last_start = _day_span.start # Setup our next tranche # Have we reached the end, if so break to finish if _tr_stop_ts >= startOfDay(self.dbm.last_timestamp): break # More to process so set our start and stop for the next # transaction _tr_start_dt = datetime.datetime.fromtimestamp(_tr_stop_ts) \ + datetime.timedelta(days=1) _tr_start_ts = time.mktime(_tr_start_dt.timetuple()) _tr_stop_dt = datetime.datetime.fromtimestamp(_tr_start_ts) \ + datetime.timedelta(days=self.trans_days) _tr_stop_ts = time.mktime(_tr_stop_dt.timetuple()) _tr_stop_ts = min(self.dbm.last_timestamp, _tr_stop_ts) # We have finished. Get rid of the no longer needed lastWeightPatch with weedb.Transaction(self.dbm.connection) as _cursor: _cursor.execute("DELETE FROM %s_day__metadata WHERE name=?" % self.dbm.table_name, ('lastWeightPatch',)) # Give the user some final information on progress, # mainly so the total tallies with the log self._progress(_days, last_start) print(file=sys.stdout) tdiff = time.time() - t1 # We are done so log and inform the user log.info("intervalweighting: Calculated weighting " "for %s days in %0.2f seconds." % (_days, tdiff)) if self.dry_run: log.info("intervalweighting: " "This was a dry run. %s was not applied." % self.name) else: # we didn't need to weight so inform the user log.info("intervalweighting: %s has already been applied." % self.name)
def sound_the_alarm(self, timestamp, battery_flags, alarm_count): """This function is called when the alarm has been triggered.""" # Get the time and convert to a string: t_str = timestamp_to_string(timestamp) # Log it in the system log: log.info("Low battery status sounded at %s: %s" % (t_str, battery_flags)) # Form the message text: indicator_strings = [] for bat in battery_flags: indicator_strings.append("%s: %04x" % (bat, int(battery_flags[bat]))) msg_text = """ The low battery indicator has been seen %d times since the last archive period. Alarm sounded at %s Low battery indicators: %s """ % (alarm_count, t_str, '\n'.join(indicator_strings)) # Convert to MIME: msg = MIMEText(msg_text) # Fill in MIME headers: msg['Subject'] = self.SUBJECT msg['From'] = self.FROM msg['To'] = ','.join(self.TO) try: # First try end-to-end encryption s = smtplib.SMTP_SSL(self.smtp_host) log.debug("Using SMTP_SSL") except AttributeError: # If that doesn't work, try creating an insecure host, then upgrading s = smtplib.SMTP(self.smtp_host) try: # Be prepared to catch an exception if the server # does not support encrypted transport. s.ehlo() s.starttls() s.ehlo() log.debug("Using SMTP encrypted transport") except smtplib.SMTPException: log.debug("Using SMTP unencrypted transport") try: # If a username has been given, assume that login is required # for this host: if self.smtp_user: s.login(self.smtp_user, self.smtp_password) log.debug("Logged in as %s", self.smtp_user) # Send the email: s.sendmail(msg['From'], self.TO, msg.as_string()) # Log out of the server: s.quit() except Exception as e: log.error("Send email failed: %s", e) raise # Log sending the email: log.info("Email sent to: %s", self.TO)
def generate(self, section, gen_ts): """Generate one or more reports for the indicated section. Each section in a period is a report. A report has one or more templates. section: A ConfigObj dictionary, holding the templates to be generated. Any subsections in the dictionary will be recursively processed as well. gen_ts: The report will be current to this time. """ ngen = 0 # Go through each subsection (if any) of this section, # generating from any templates they may contain for subsection in section.sections: # Sections 'SummaryByMonth' and 'SummaryByYear' imply summarize_by # certain time spans if not section[subsection].has_key('summarize_by'): if subsection == 'SummaryByDay': section[subsection]['summarize_by'] = 'SummaryByDay' elif subsection == 'SummaryByMonth': section[subsection]['summarize_by'] = 'SummaryByMonth' elif subsection == 'SummaryByYear': section[subsection]['summarize_by'] = 'SummaryByYear' # Call recursively, to generate any templates in this subsection ngen += self.generate(section[subsection], gen_ts) # We have finished recursively processing any subsections in this # section. Time to do the section itself. If there is no option # 'template', then there isn't anything to do. Return. if not section.has_key('template'): return ngen # Change directory to the skin subdirectory. We use absolute paths # for cheetah, so the directory change is not necessary for generating # files. However, changing to the skin directory provides a known # location so that calls to os.getcwd() in any templates will return # a predictable result. os.chdir(os.path.join(self.config_dict['WEEWX_ROOT'], self.skin_dict['SKIN_ROOT'], self.skin_dict['skin'])) report_dict = weeutil.weeutil.accumulateLeaves(section) (template, dest_dir, encoding, default_binding) = self._prepGen(report_dict) # Get start and stop times default_archive = self.db_binder.get_manager(default_binding) start_ts = default_archive.firstGoodStamp() if not start_ts: loginf('Skipping template %s: cannot find start time' % section['template']) return ngen if gen_ts: record = default_archive.getRecord(gen_ts, max_delta=to_int(report_dict.get('max_delta'))) if record: stop_ts = record['dateTime'] else: loginf('Skipping template %s: generate time %s not in database' % (section['template'], timestamp_to_string(gen_ts)) ) return ngen else: stop_ts = default_archive.lastGoodStamp() # Get an appropriate generator function summarize_by = report_dict['summarize_by'] if summarize_by in CheetahGenerator.generator_dict: _spangen = CheetahGenerator.generator_dict[summarize_by] else: # Just a single timespan to generate. Use a lambda expression. _spangen = lambda start_ts, stop_ts : [weeutil.weeutil.TimeSpan(start_ts, stop_ts)] # Use the generator function for timespan in _spangen(start_ts, stop_ts): start_tt = time.localtime(timespan.start) stop_tt = time.localtime(timespan.stop) if summarize_by in CheetahGenerator.format_dict: # This is a "SummaryBy" type generation. If it hasn't been done already, save the # date as a string, to be used inside the document date_str = time.strftime(CheetahGenerator.format_dict[summarize_by], start_tt) if date_str not in self.outputted_dict[summarize_by]: self.outputted_dict[summarize_by].append(date_str) # For these "SummaryBy" generations, the file name comes from the start of the timespan: _filename = self._getFileName(template, start_tt) else: # This is a "ToDate" generation. File name comes # from the stop (i.e., present) time: _filename = self._getFileName(template, stop_tt) # Get the absolute path for the target of this template _fullname = os.path.join(dest_dir, _filename) # Skip summary files outside the timespan if report_dict['summarize_by'] in CheetahGenerator.generator_dict \ and os.path.exists(_fullname) \ and not timespan.includesArchiveTime(stop_ts): continue # skip files that are fresh, but only if staleness is defined stale = to_int(report_dict.get('stale_age')) if stale is not None: t_now = time.time() try: last_mod = os.path.getmtime(_fullname) if t_now - last_mod < stale: logdbg("Skip '%s': last_mod=%s age=%s stale=%s" % (_filename, last_mod, t_now - last_mod, stale)) continue except os.error: pass searchList = self._getSearchList(encoding, timespan, default_binding) tmpname = _fullname + '.tmp' try: compiled_template = Cheetah.Template.Template( file=template, searchList=searchList, filter=encoding, filtersLib=weewx.cheetahgenerator) with open(tmpname, mode='w') as _file: print >> _file, compiled_template os.rename(tmpname, _fullname) except Exception, e: # We would like to get better feedback when there are cheetah # compiler failures, but there seem to be no hooks for this. # For example, if we could get make cheetah emit the source # on which the compiler is working, one could compare that with # the template to figure out exactly where the problem is. # In Cheetah.Compile.ModuleCompiler the source is manipulated # a bit then handed off to parserClass. Unfortunately there # are no hooks to intercept the source and spit it out. So # the best we can do is indicate the template that was being # processed when the failure ocurred. logerr("Generate failed with exception '%s'" % type(e)) logerr("**** Ignoring template %s" % template) logerr("**** Reason: %s" % e) weeutil.weeutil.log_traceback("**** ") else: ngen += 1 finally:
def __init__(self, config_dict, config_path, cumulus_config_dict, import_config_path, options, log): # call our parents __init__ super(CumulusSource, self).__init__(config_dict, cumulus_config_dict, options, log) # save our import config path self.import_config_path = import_config_path # save our import config dict self.cumulus_config_dict = cumulus_config_dict # wind dir bounds self.wind_dir = [0, 360] # Decimal separator used in monthly log files, default to decimal point self.decimal = cumulus_config_dict.get('decimal', '.') # Field delimiter used in monthly log files, default to comma self.delimiter = cumulus_config_dict.get('delimiter', ',') # We combine Cumulus date and time fields to give a fixed format # date-time string self.raw_datetime_format = '%d/%m/%y %H:%M' # Cumulus log files provide a number of cumulative rainfall fields. We # cannot use the daily rainfall as this may reset at some time of day # other than midnight (as required by weewx). So we use field 26, total # rainfall since midnight and treat it as a cumulative value. self.rain = 'cumulative' # initialise our import field-to-weewx archive field map self.map = None # Units of measure for some obs (eg temperatures) cannot be derived from # the Cumulus monthly log files. These units must be specified by the # user in the import config file. Read these units and fill in the # missing unit data in the header map. Do some basic error checking and # validation, if one of the fields is missing or invalid then we need # to catch the error and raise it as we can't go on. # Temperature try: temp_u = cumulus_config_dict['Units'].get('temperature') except: _msg = "No units specified for Cumulus temperature fields in %s." % (self.import_config_path, ) raise weewx.UnitError(_msg) else: if temp_u in weewx.units.default_unit_format_dict: self._header_map['cur_out_temp']['units'] = temp_u self._header_map['curr_in_temp']['units'] = temp_u self._header_map['cur_dewpoint']['units'] = temp_u self._header_map['cur_heatindex']['units'] = temp_u self._header_map['cur_windchill']['units'] = temp_u self._header_map['cur_app_temp']['units'] = temp_u else: _msg = "Unknown units '%s' specified for Cumulus temperature fields in %s." % (temp_u, self.import_config_path) raise weewx.UnitError(_msg) # Pressure try: press_u = cumulus_config_dict['Units'].get('pressure') except: _msg = "No units specified for Cumulus pressure fields in %s." % (self.import_config_path, ) raise weewx.UnitError(_msg) else: if press_u in ['inHg', 'mbar', 'hPa']: self._header_map['cur_slp']['units'] = press_u else: _msg = "Unknown units '%s' specified for Cumulus pressure fields in %s." % (press_u, self.import_config_path) raise weewx.UnitError(_msg) # Rain try: rain_u = cumulus_config_dict['Units'].get('rain') except: _msg = "No units specified for Cumulus rain fields in %s." % (self.import_config_path, ) raise weewx.UnitError(_msg) else: if rain_u in rain_units_dict: self._header_map['midnight_rain']['units'] = rain_u self._header_map['cur_rain_rate']['units'] = rain_units_dict[rain_u] else: _msg = "Unknown units '%s' specified for Cumulus rain fields in %s." % (rain_u, self.import_config_path) raise weewx.UnitError(_msg) # Speed try: speed_u = cumulus_config_dict['Units'].get('speed') except: _msg = "No units specified for Cumulus speed fields in %s." % (self.import_config_path, ) raise weewx.UnitError(_msg) else: if speed_u in weewx.units.default_unit_format_dict: self._header_map['avg_wind_speed']['units'] = speed_u self._header_map['gust_wind_speed']['units'] = speed_u else: _msg = "Unknown units '%s' specified for Cumulus speed fields in %s." % (speed_u, self.import_config_path) raise weewx.UnitError(_msg) # get our source file path try: self.source = cumulus_config_dict['directory'] except KeyError: raise weewx.ViolatedPrecondition("Cumulus monthly logs directory not specified in '%s'." % import_config_path) # Now get a list on monthly log files sorted from oldest to newest month_log_list = glob.glob(self.source + '/?????log.txt') _temp = [(fn, fn[-9:-7], time.strptime(fn[-12:-9],'%b').tm_mon) for fn in month_log_list] self.log_list = [a[0] for a in sorted(_temp, key = lambda el : (el[1], el[2]))] if len(self.log_list) == 0: raise weeimport.WeeImportIOError( "No Cumulus monthly logs found in directory '%s'." % self.source) # tell the user/log what we intend to do _msg = "Cumulus monthly log files in the '%s' directory will be imported" % self.source self.wlog.printlog(syslog.LOG_INFO, _msg) _msg = "The following options will be used:" self.wlog.verboselog(syslog.LOG_DEBUG, _msg) _msg = " config=%s, import-config=%s" % (config_path, self.import_config_path) self.wlog.verboselog(syslog.LOG_DEBUG, _msg) if options.date: _msg = " date=%s" % options.date else: # we must have --from and --to _msg = " from=%s, to=%s" % (options.date_from, options.date_to) self.wlog.verboselog(syslog.LOG_DEBUG, _msg) _msg = " dry-run=%s, calc-missing=%s" % (self.dry_run, self.calc_missing) self.wlog.verboselog(syslog.LOG_DEBUG, _msg) _msg = " tranche=%s, interval=%s" % (self.tranche, self.interval) self.wlog.verboselog(syslog.LOG_DEBUG, _msg) _msg = " UV=%s, radiation=%s" % (self.UV_sensor, self.solar_sensor) self.wlog.verboselog(syslog.LOG_DEBUG, _msg) _msg = "Using database binding '%s', which is bound to database '%s'" % (self.db_binding_wx, self.dbm.database_name) self.wlog.printlog(syslog.LOG_INFO, _msg) _msg = "Destination table '%s' unit system is '%#04x' (%s)." % (self.dbm.table_name, self.archive_unit_sys, unit_nicknames[self.archive_unit_sys]) self.wlog.printlog(syslog.LOG_INFO, _msg) if self.calc_missing: print "Missing derived observations will be calculated." if not self.UV_sensor: print "All weewx UV fields will be set to None." if not self.solar_sensor: print "All weewx radiation fields will be set to None." if options.date or options.date_from: print "Observations timestamped after %s and up to and" % (timestamp_to_string(self.first_ts), ) print "including %s will be imported." % (timestamp_to_string(self.last_ts), ) if self.dry_run: print "This is a dry run, imported data will not be saved to archive."
def generate(self, section, gen_ts): """Generate one or more reports for the indicated section. Each section in a period is a report. A report has one or more templates. section: A ConfigObj dictionary, holding the templates to be generated. Any subsections in the dictionary will be recursively processed as well. gen_ts: The report will be current to this time. """ ngen = 0 # Go through each subsection (if any) of this section, # generating from any templates they may contain for subsection in section.sections: # Sections 'SummaryByMonth' and 'SummaryByYear' imply summarize_by certain time spans if not section[subsection].has_key('summarize_by'): if subsection == 'SummaryByMonth': section[subsection]['summarize_by'] = 'SummaryByMonth' elif subsection == 'SummaryByYear': section[subsection]['summarize_by'] = 'SummaryByYear' # Call myself recursively, to generate any templates in this subsection ngen += self.generate(section[subsection], gen_ts) # We have finished recursively processing any subsections in this # section. Time to do the section itself. If there is no option # 'template', then there isn't anything to do. Return. if not section.has_key('template'): return ngen # Change directory to the skin subdirectory. We use absolute paths # for cheetah, so the directory change is not necessary for generating # files. However, changing to the skin directory provides a known # location so that calls to os.getcwd() in any templates will return # a predictable result. os.chdir( os.path.join(self.config_dict['WEEWX_ROOT'], self.skin_dict['SKIN_ROOT'], self.skin_dict['skin'])) report_dict = weeutil.weeutil.accumulateLeaves(section) (template, dest_dir, encoding, default_binding) = self._prepGen(report_dict) # Get start and stop times default_archive = self.db_binder.get_manager(default_binding) start_ts = default_archive.firstGoodStamp() if not start_ts: loginf('Skipping template %s: cannot find start time' % section['template']) return ngen if gen_ts: record = default_archive.getRecord( gen_ts, max_delta=to_int(report_dict.get('max_delta'))) if record: stop_ts = record['dateTime'] else: loginf( 'Skipping template %s; generate time %s not in database' % (section['template'], timestamp_to_string(gen_ts))) return ngen else: stop_ts = default_archive.lastGoodStamp() # Get an appropriate generator function summarize_by = report_dict['summarize_by'] if summarize_by in CheetahGenerator.generator_dict: _spangen = CheetahGenerator.generator_dict[summarize_by] else: # Just a single timespan to generate. Use a lambda expression. _spangen = lambda start_ts, stop_ts: [ weeutil.weeutil.TimeSpan(start_ts, stop_ts) ] # Use the generator function for timespan in _spangen(start_ts, stop_ts): # Save YYYY-MM so they can be used within the document if summarize_by in CheetahGenerator.generator_dict: timespan_start_tt = time.localtime(timespan.start) _yr_str = "%4d" % timespan_start_tt[0] if summarize_by == 'SummaryByMonth': _mo_str = "%02d" % timespan_start_tt[1] if _mo_str not in self.outputted_dict[summarize_by]: self.outputted_dict[summarize_by].append( "%s-%s" % (_yr_str, _mo_str)) if summarize_by == 'SummaryByYear' and _yr_str not in self.outputted_dict[ summarize_by]: self.outputted_dict[summarize_by].append(_yr_str) # figure out the filename for this template _filename = self._getFileName(template, timespan) _fullname = os.path.join(dest_dir, _filename) # Skip summary files outside the timespan if report_dict['summarize_by'] in CheetahGenerator.generator_dict \ and os.path.exists(_fullname) \ and not timespan.includesArchiveTime(stop_ts): continue # skip files that are fresh, but only if staleness is defined stale = to_int(report_dict.get('stale_age')) if stale is not None: t_now = time.time() try: last_mod = os.path.getmtime(_fullname) if t_now - last_mod < stale: logdbg("Skip '%s': last_mod=%s age=%s stale=%s" % (_filename, last_mod, t_now - last_mod, stale)) continue except os.error: pass searchList = self._getSearchList(encoding, timespan, default_binding) tmpname = _fullname + '.tmp' try: text = Cheetah.Template.Template( file=template, searchList=searchList, filter=encoding, filtersLib=weewx.cheetahgenerator) with open(tmpname, mode='w') as _file: print >> _file, text os.rename(tmpname, _fullname) except Exception, e: logerr("Generate failed with exception '%s'" % type(e)) logerr("**** Ignoring template %s" % template) logerr("**** Reason: %s" % e) weeutil.weeutil.log_traceback("**** ") else: ngen += 1 finally:
def generate(self, section, gen_ts): """Generate one or more reports for the indicated section. Each section in a period is a report. A report has one or more templates. section: A ConfigObj dictionary, holding the templates to be generated. Any subsections in the dictionary will be recursively processed as well. gen_ts: The report will be current to this time. """ ngen = 0 # Go through each subsection (if any) of this section, # generating from any templates they may contain for subsection in section.sections: # Sections 'SummaryByMonth' and 'SummaryByYear' imply summarize_by # certain time spans if not section[subsection].has_key('summarize_by'): if subsection == 'SummaryByDay': section[subsection]['summarize_by'] = 'SummaryByDay' elif subsection == 'SummaryByMonth': section[subsection]['summarize_by'] = 'SummaryByMonth' elif subsection == 'SummaryByYear': section[subsection]['summarize_by'] = 'SummaryByYear' # Call recursively, to generate any templates in this subsection ngen += self.generate(section[subsection], gen_ts) # We have finished recursively processing any subsections in this # section. Time to do the section itself. If there is no option # 'template', then there isn't anything to do. Return. if not section.has_key('template'): return ngen # Change directory to the skin subdirectory. We use absolute paths # for cheetah, so the directory change is not necessary for generating # files. However, changing to the skin directory provides a known # location so that calls to os.getcwd() in any templates will return # a predictable result. os.chdir( os.path.join(self.config_dict['WEEWX_ROOT'], self.skin_dict['SKIN_ROOT'], self.skin_dict['skin'])) report_dict = weeutil.weeutil.accumulateLeaves(section) (template, dest_dir, encoding, default_binding) = self._prepGen(report_dict) # Get start and stop times default_archive = self.db_binder.get_manager(default_binding) start_ts = default_archive.firstGoodStamp() if not start_ts: loginf('Skipping template %s: cannot find start time' % section['template']) return ngen if gen_ts: record = default_archive.getRecord( gen_ts, max_delta=to_int(report_dict.get('max_delta'))) if record: stop_ts = record['dateTime'] else: loginf( 'Skipping template %s: generate time %s not in database' % (section['template'], timestamp_to_string(gen_ts))) return ngen else: stop_ts = default_archive.lastGoodStamp() # Get an appropriate generator function summarize_by = report_dict['summarize_by'] if summarize_by in CheetahGenerator.generator_dict: _spangen = CheetahGenerator.generator_dict[summarize_by] else: # Just a single timespan to generate. Use a lambda expression. _spangen = lambda start_ts, stop_ts: [ weeutil.weeutil.TimeSpan(start_ts, stop_ts) ] # Use the generator function for timespan in _spangen(start_ts, stop_ts): start_tt = time.localtime(timespan.start) stop_tt = time.localtime(timespan.stop) if summarize_by in CheetahGenerator.format_dict: # This is a "SummaryBy" type generation. If it hasn't been done already, save the # date as a string, to be used inside the document date_str = time.strftime( CheetahGenerator.format_dict[summarize_by], start_tt) if date_str not in self.outputted_dict[summarize_by]: self.outputted_dict[summarize_by].append(date_str) # For these "SummaryBy" generations, the file name comes from the start of the timespan: _filename = self._getFileName(template, start_tt) else: # This is a "ToDate" generation. File name comes # from the stop (i.e., present) time: _filename = self._getFileName(template, stop_tt) # Get the absolute path for the target of this template _fullname = os.path.join(dest_dir, _filename) # Skip summary files outside the timespan if report_dict['summarize_by'] in CheetahGenerator.generator_dict \ and os.path.exists(_fullname) \ and not timespan.includesArchiveTime(stop_ts): continue # skip files that are fresh, but only if staleness is defined stale = to_int(report_dict.get('stale_age')) if stale is not None: t_now = time.time() try: last_mod = os.path.getmtime(_fullname) if t_now - last_mod < stale: logdbg("Skip '%s': last_mod=%s age=%s stale=%s" % (_filename, last_mod, t_now - last_mod, stale)) continue except os.error: pass searchList = self._getSearchList(encoding, timespan, default_binding) tmpname = _fullname + '.tmp' try: text = Cheetah.Template.Template( file=template, searchList=searchList, filter=encoding, filtersLib=weewx.cheetahgenerator) with open(tmpname, mode='w') as _file: print >> _file, text os.rename(tmpname, _fullname) except Exception, e: # We would like to get better feedback when there are cheetah # compiler failures, but there seem to be no hooks for this. # For example, if we could get make cheetah emit the source # on which the compiler is working, one could compare that with # the template to figure out exactly where the problem is. # In Cheetah.Compile.ModuleCompiler the source is manipulated # a bit then handed off to parserClass. Unfortunately there # are no hooks to intercept the source and spit it out. So # the best we can do is indicate the template that was being # processed when the failure ocurred. logerr("Generate failed with exception '%s'" % type(e)) logerr("**** Ignoring template %s" % template) logerr("**** Reason: %s" % e) weeutil.weeutil.log_traceback("**** ") else: ngen += 1 finally:
def generate(self, section, gen_ts): """Generate one or more reports for the indicated section. Each section in a period is a report. A report has one or more templates. section: A ConfigObj dictionary, holding the templates to be generated. Any subsections in the dictionary will be recursively processed as well. gen_ts: The report will be current to this time. """ ngen = 0 # Go through each subsection (if any) of this section, # generating from any templates they may contain for subsection in section.sections: # Sections 'SummaryByMonth' and 'SummaryByYear' imply summarize_by certain time spans if not section[subsection].has_key('summarize_by'): if subsection == 'SummaryByMonth': section[subsection]['summarize_by'] = 'SummaryByMonth' elif subsection == 'SummaryByYear': section[subsection]['summarize_by'] = 'SummaryByYear' # Call myself recursively, to generate any templates in this subsection ngen += self.generate(section[subsection], gen_ts) # We have finished recursively processing any subsections in this # section. Time to do the section itself. If there is no option # 'template', then there isn't anything to do. Return. if not section.has_key('template'): return ngen # Change directory to the skin subdirectory. We use absolute paths # for cheetah, so the directory change is not necessary for generating # files. However, changing to the skin directory provides a known # location so that calls to os.getcwd() in any templates will return # a predictable result. os.chdir(os.path.join(self.config_dict['WEEWX_ROOT'], self.skin_dict['SKIN_ROOT'], self.skin_dict['skin'])) report_dict = weeutil.weeutil.accumulateLeaves(section) (template, dest_dir, encoding, default_binding) = self._prepGen(report_dict) # Get start and stop times default_archive = self.db_binder.get_manager(default_binding) start_ts = default_archive.firstGoodStamp() if not start_ts: loginf('Skipping template %s: cannot find start time' % section['template']) return ngen if gen_ts: record = default_archive.getRecord(gen_ts, max_delta=to_int(report_dict.get('max_delta'))) if record: stop_ts = record['dateTime'] else: loginf('Skipping template %s; generate time %s not in database' % (section['template'], timestamp_to_string(gen_ts)) ) return ngen else: stop_ts = default_archive.lastGoodStamp() # Get an appropriate generator function summarize_by = report_dict['summarize_by'] if summarize_by in CheetahGenerator.generator_dict: _spangen = CheetahGenerator.generator_dict[summarize_by] else: # Just a single timespan to generate. Use a lambda expression. _spangen = lambda start_ts, stop_ts : [weeutil.weeutil.TimeSpan(start_ts, stop_ts)] # Use the generator function for timespan in _spangen(start_ts, stop_ts): # Save YYYY-MM so they can be used within the document if summarize_by in CheetahGenerator.generator_dict: timespan_start_tt = time.localtime(timespan.start) _yr_str = "%4d" % timespan_start_tt[0] if summarize_by == 'SummaryByMonth': _mo_str = "%02d" % timespan_start_tt[1] if _mo_str not in self.outputted_dict[summarize_by]: self.outputted_dict[summarize_by].append("%s-%s" % (_yr_str, _mo_str)) if summarize_by == 'SummaryByYear' and _yr_str not in self.outputted_dict[summarize_by]: self.outputted_dict[summarize_by].append(_yr_str) # figure out the filename for this template _filename = self._getFileName(template, timespan) _fullname = os.path.join(dest_dir, _filename) # Skip summary files outside the timespan if report_dict['summarize_by'] in CheetahGenerator.generator_dict \ and os.path.exists(_fullname) \ and not timespan.includesArchiveTime(stop_ts): continue # skip files that are fresh, but only if staleness is defined stale = to_int(report_dict.get('stale_age')) if stale is not None: t_now = time.time() try: last_mod = os.path.getmtime(_fullname) if t_now - last_mod < stale: logdbg("Skip '%s': last_mod=%s age=%s stale=%s" % (_filename, last_mod, t_now - last_mod, stale)) continue except os.error: pass searchList = self._getSearchList(encoding, timespan, default_binding) tmpname = _fullname + '.tmp' try: text = Cheetah.Template.Template(file=template, searchList=searchList, filter=encoding, filtersLib=weewx.cheetahgenerator) with open(tmpname, mode='w') as _file: print >> _file, text os.rename(tmpname, _fullname) except Exception, e: logerr("Generate failed with exception '%s'" % type(e)) logerr("**** Ignoring template %s" % template) logerr("**** Reason: %s" % e) weeutil.weeutil.log_traceback("**** ") else: ngen += 1 finally:
def __init__(self, config_dict, config_path, csv_config_dict, import_config_path, options): # call our parents __init__ super(CSVSource, self).__init__(config_dict, csv_config_dict, options) # save our import config path self.import_config_path = import_config_path # save our import config dict self.csv_config_dict = csv_config_dict # get a few config settings from our CSV config dict # csv field delimiter self.delimiter = str(self.csv_config_dict.get('delimiter', ',')) # string format used to decode the imported field holding our dateTime self.raw_datetime_format = self.csv_config_dict.get( 'raw_datetime_format', '%Y-%m-%d %H:%M:%S') # is our rain discrete or cumulative self.rain = self.csv_config_dict.get('rain', 'cumulative') # determine valid range for imported wind direction _wind_direction = option_as_list( self.csv_config_dict.get('wind_direction', '0,360')) try: if float(_wind_direction[0]) <= float(_wind_direction[1]): self.wind_dir = [ float(_wind_direction[0]), float(_wind_direction[1]) ] else: self.wind_dir = [-360, 360] except (KeyError, ValueError): self.wind_dir = [-360, 360] # get our source file path try: self.source = csv_config_dict['file'] except KeyError: raise weewx.ViolatedPrecondition( "CSV source file not specified in '%s'." % import_config_path) # get the source file encoding, default to utf-8-sig self.source_encoding = self.csv_config_dict.get( 'source_encoding', 'utf-8-sig') # initialise our import field-to-WeeWX archive field map self.map = None # initialise some other properties we will need self.start = 1 self.end = 1 self.increment = 1 # tell the user/log what we intend to do _msg = "A CSV import from source file '%s' has been requested." % self.source print(_msg) log.info(_msg) _msg = "The following options will be used:" if self.verbose: print(_msg) log.debug(_msg) _msg = " config=%s, import-config=%s" % (config_path, self.import_config_path) if self.verbose: print(_msg) log.debug(_msg) if options.date: _msg = " source=%s, date=%s" % (self.source, options.date) else: # we must have --from and --to _msg = " source=%s, from=%s, to=%s" % ( self.source, options.date_from, options.date_to) if self.verbose: print(_msg) log.debug(_msg) _msg = " dry-run=%s, calc_missing=%s, ignore_invalid_data=%s" % ( self.dry_run, self.calc_missing, self.ignore_invalid_data) if self.verbose: print(_msg) log.debug(_msg) _msg = " tranche=%s, interval=%s, date/time_string_format=%s" % ( self.tranche, self.interval, self.raw_datetime_format) if self.verbose: print(_msg) log.debug(_msg) _msg = " delimiter='%s', rain=%s, wind_direction=%s" % ( self.delimiter, self.rain, self.wind_dir) if self.verbose: print(_msg) log.debug(_msg) _msg = " UV=%s, radiation=%s" % (self.UV_sensor, self.solar_sensor) if self.verbose: print(_msg) log.debug(_msg) _msg = "Using database binding '%s', which is bound to database '%s'" % ( self.db_binding_wx, self.dbm.database_name) print(_msg) log.info(_msg) _msg = "Destination table '%s' unit system is '%#04x' (%s)." % ( self.dbm.table_name, self.archive_unit_sys, unit_nicknames[self.archive_unit_sys]) print(_msg) log.info(_msg) if self.calc_missing: _msg = "Missing derived observations will be calculated." print(_msg) log.info(_msg) if not self.UV_sensor: _msg = "All WeeWX UV fields will be set to None." print(_msg) log.info(_msg) if not self.solar_sensor: _msg = "All WeeWX radiation fields will be set to None." print(_msg) log.info(_msg) if options.date or options.date_from: _msg = "Observations timestamped after %s and up to and" % timestamp_to_string( self.first_ts) print(_msg) log.info(_msg) _msg = "including %s will be imported." % timestamp_to_string( self.last_ts) print(_msg) log.info(_msg) if self.dry_run: _msg = "This is a dry run, imported data will not be saved to archive." print(_msg) log.info(_msg)
def __init__(self, config_dict, config_path, wu_config_dict, import_config_path, options): # call our parents __init__ super(WUSource, self).__init__(config_dict, wu_config_dict, options) # save our import config path self.import_config_path = import_config_path # save our import config dict self.wu_config_dict = wu_config_dict # get our WU station ID try: self.station_id = wu_config_dict['station_id'] except KeyError: _msg = "Weather Underground station ID not specified in '%s'." % import_config_path raise weewx.ViolatedPrecondition(_msg) # get our WU API key try: self.api_key = wu_config_dict['api_key'] except KeyError: _msg = "Weather Underground API key not specified in '%s'." % import_config_path raise weewx.ViolatedPrecondition(_msg) # wind dir bounds _wind_direction = option_as_list( wu_config_dict.get('wind_direction', '0,360')) try: if float(_wind_direction[0]) <= float(_wind_direction[1]): self.wind_dir = [ float(_wind_direction[0]), float(_wind_direction[1]) ] else: self.wind_dir = [0, 360] except (IndexError, TypeError): self.wind_dir = [0, 360] # some properties we know because of the format of the returned WU data # WU returns a fixed format date-time string self.raw_datetime_format = '%Y-%m-%d %H:%M:%S' # WU only provides hourly rainfall and a daily cumulative rainfall. # We use the latter so force 'cumulative' for rain. self.rain = 'cumulative' # initialise our import field-to-WeeWX archive field map self.map = None # For a WU import we might have to import multiple days but we can only # get one day at a time from WU. So our start and end properties # (counters) are datetime objects and our increment is a timedelta. # Get datetime objects for any date or date range specified on the # command line, if there wasn't one then default to today. self.start = dt.fromtimestamp(startOfDay(self.first_ts)) self.end = dt.fromtimestamp(startOfDay(self.last_ts)) # set our increment self.increment = datetime.timedelta(days=1) # property holding the current period being processed self.period = None # tell the user/log what we intend to do _msg = "Observation history for Weather Underground station '%s' will be imported." % self.station_id print(_msg) log.info(_msg) _msg = "The following options will be used:" if self.verbose: print(_msg) log.debug(_msg) _msg = " config=%s, import-config=%s" % (config_path, self.import_config_path) if self.verbose: print(_msg) log.debug(_msg) if options.date: _msg = " station=%s, date=%s" % (self.station_id, options.date) else: # we must have --from and --to _msg = " station=%s, from=%s, to=%s" % ( self.station_id, options.date_from, options.date_to) if self.verbose: print(_msg) log.debug(_msg) _obf_api_key_msg = '='.join( [' apiKey', '*' * (len(self.api_key) - 4) + self.api_key[-4:]]) if self.verbose: print(_obf_api_key_msg) log.debug(_obf_api_key_msg) _msg = " dry-run=%s, calc_missing=%s, ignore_invalid_data=%s" % ( self.dry_run, self.calc_missing, self.ignore_invalid_data) if self.verbose: print(_msg) log.debug(_msg) _msg = " tranche=%s, interval=%s, wind_direction=%s" % ( self.tranche, self.interval, self.wind_dir) if self.verbose: print(_msg) log.debug(_msg) _msg = "Using database binding '%s', which is bound to database '%s'" % ( self.db_binding_wx, self.dbm.database_name) print(_msg) log.info(_msg) _msg = "Destination table '%s' unit system is '%#04x' (%s)." % ( self.dbm.table_name, self.archive_unit_sys, unit_nicknames[self.archive_unit_sys]) print(_msg) log.info(_msg) if self.calc_missing: print("Missing derived observations will be calculated.") if options.date or options.date_from: print("Observations timestamped after %s and up to and" % timestamp_to_string(self.first_ts)) print("including %s will be imported." % timestamp_to_string(self.last_ts)) if self.dry_run: print( "This is a dry run, imported data will not be saved to archive." )
def __init__(self, config_dict, config_path, cumulus_config_dict, import_config_path, options): # call our parents __init__ super(CumulusSource, self).__init__(config_dict, cumulus_config_dict, options) # save our import config path self.import_config_path = import_config_path # save our import config dict self.cumulus_config_dict = cumulus_config_dict # wind dir bounds self.wind_dir = [0, 360] # field delimiter used in monthly log files, default to comma self.delimiter = str(cumulus_config_dict.get('delimiter', ',')) # decimal separator used in monthly log files, default to decimal point self.decimal = cumulus_config_dict.get('decimal', '.') # date separator used in monthly log files, default to solidus separator = cumulus_config_dict.get('separator', '/') # we combine Cumulus date and time fields to give a fixed format # date-time string self.raw_datetime_format = separator.join(('%d', '%m', '%y %H:%M')) # Cumulus log files provide a number of cumulative rainfall fields. We # cannot use the daily rainfall as this may reset at some time of day # other than midnight (as required by WeeWX). So we use field 26, total # rainfall since midnight and treat it as a cumulative value. self.rain = 'cumulative' # initialise our import field-to-WeeWX archive field map self.map = None # Cumulus log files have a number of 'rain' fields that can be used to # derive the WeeWX rain field. Which one is available depends on the # Cumulus version that created the logs. The preferred field is field # 26(AA) - total rainfall since midnight but it is only available in # Cumulus v1.9.4 or later. If that field is not available then the # preferred field in field 09(J) - total rainfall today then field # 11(L) - total rainfall counter. Initialise the rain_source_confirmed # property now and we will deal with it later when we have some source # data. self.rain_source_confirmed = None # Units of measure for some obs (eg temperatures) cannot be derived from # the Cumulus monthly log files. These units must be specified by the # user in the import config file. Read these units and fill in the # missing unit data in the header map. Do some basic error checking and # validation, if one of the fields is missing or invalid then we need # to catch the error and raise it as we can't go on. # Temperature try: temp_u = cumulus_config_dict['Units'].get('temperature') except KeyError: _msg = "No units specified for Cumulus temperature " \ "fields in %s." % (self.import_config_path, ) raise weewx.UnitError(_msg) else: if temp_u in weewx.units.default_unit_format_dict: self._header_map['cur_out_temp']['units'] = temp_u self._header_map['curr_in_temp']['units'] = temp_u self._header_map['cur_dewpoint']['units'] = temp_u self._header_map['cur_heatindex']['units'] = temp_u self._header_map['cur_windchill']['units'] = temp_u self._header_map['cur_app_temp']['units'] = temp_u else: _msg = "Unknown units '%s' specified for Cumulus " \ "temperature fields in %s." % (temp_u, self.import_config_path) raise weewx.UnitError(_msg) # Pressure try: press_u = cumulus_config_dict['Units'].get('pressure') except KeyError: _msg = "No units specified for Cumulus pressure " \ "fields in %s." % (self.import_config_path, ) raise weewx.UnitError(_msg) else: if press_u in ['inHg', 'mbar', 'hPa']: self._header_map['cur_slp']['units'] = press_u else: _msg = "Unknown units '%s' specified for Cumulus " \ "pressure fields in %s." % (press_u, self.import_config_path) raise weewx.UnitError(_msg) # Rain try: rain_u = cumulus_config_dict['Units'].get('rain') except KeyError: _msg = "No units specified for Cumulus " \ "rain fields in %s." % (self.import_config_path, ) raise weewx.UnitError(_msg) else: if rain_u in rain_units_dict: self._header_map['midnight_rain']['units'] = rain_u self._header_map['cur_rain_rate']['units'] = rain_units_dict[ rain_u] else: _msg = "Unknown units '%s' specified for Cumulus " \ "rain fields in %s." % (rain_u, self.import_config_path) raise weewx.UnitError(_msg) # Speed try: speed_u = cumulus_config_dict['Units'].get('speed') except KeyError: _msg = "No units specified for Cumulus " \ "speed fields in %s." % (self.import_config_path, ) raise weewx.UnitError(_msg) else: if speed_u in weewx.units.default_unit_format_dict: self._header_map['avg_wind_speed']['units'] = speed_u self._header_map['gust_wind_speed']['units'] = speed_u else: _msg = "Unknown units '%s' specified for Cumulus " \ "speed fields in %s." % (speed_u, self.import_config_path) raise weewx.UnitError(_msg) # get our source file path try: self.source = cumulus_config_dict['directory'] except KeyError: _msg = "Cumulus monthly logs directory not specified in '%s'." % import_config_path raise weewx.ViolatedPrecondition(_msg) # get the source file encoding, default to utf-8-sig self.source_encoding = self.cumulus_config_dict.get( 'source_encoding', 'utf-8-sig') # property holding the current log file name being processed self.file_name = None # Now get a list on monthly log files sorted from oldest to newest month_log_list = glob.glob(self.source + '/?????log.txt') _temp = [(fn, fn[-9:-7], time.strptime(fn[-12:-9], '%b').tm_mon) for fn in month_log_list] self.log_list = [ a[0] for a in sorted(_temp, key=lambda el: (el[1], el[2])) ] if len(self.log_list) == 0: raise weeimport.WeeImportIOError( "No Cumulus monthly logs found in directory '%s'." % self.source) # tell the user/log what we intend to do _msg = "Cumulus monthly log files in the '%s' directory will be imported" % self.source print(_msg) log.info(_msg) _msg = "The following options will be used:" if self.verbose: print(_msg) log.debug(_msg) _msg = " config=%s, import-config=%s" % (config_path, self.import_config_path) if self.verbose: print(_msg) log.debug(_msg) if options.date: _msg = " date=%s" % options.date else: # we must have --from and --to _msg = " from=%s, to=%s" % (options.date_from, options.date_to) if self.verbose: print(_msg) log.debug(_msg) _msg = " dry-run=%s, calc_missing=%s, " \ "ignore_invalid_data=%s" % (self.dry_run, self.calc_missing, self.ignore_invalid_data) if self.verbose: print(_msg) log.debug(_msg) _msg = " tranche=%s, interval=%s" % (self.tranche, self.interval) if self.verbose: print(_msg) log.debug(_msg) _msg = " UV=%s, radiation=%s" % (self.UV_sensor, self.solar_sensor) if self.verbose: print(_msg) log.debug(_msg) _msg = "Using database binding '%s', which is bound " \ "to database '%s'" % (self.db_binding_wx, self.dbm.database_name) print(_msg) log.info(_msg) _msg = "Destination table '%s' unit system " \ "is '%#04x' (%s)." % (self.dbm.table_name, self.archive_unit_sys, unit_nicknames[self.archive_unit_sys]) print(_msg) log.info(_msg) if self.calc_missing: print("Missing derived observations will be calculated.") if not self.UV_sensor: print("All WeeWX UV fields will be set to None.") if not self.solar_sensor: print("All WeeWX radiation fields will be set to None.") if options.date or options.date_from: print("Observations timestamped after %s and " "up to and" % timestamp_to_string(self.first_ts)) print("including %s will be imported." % timestamp_to_string(self.last_ts)) if self.dry_run: print( "This is a dry run, imported data will not be saved to archive." )
def __init__(self, config_dict, config_path, wu_config_dict, import_config_path, options, log): # call our parents __init__ super(WUSource, self).__init__(config_dict, wu_config_dict, options, log) # save our import config path self.import_config_path = import_config_path # save our import config dict self.wu_config_dict = wu_config_dict # get our WU station ID try: self.station_id = wu_config_dict['station_id'] except KeyError: raise weewx.ViolatedPrecondition("Weather Underground station ID not specified in '%s'." % import_config_path) # wind dir bounds _wind_direction = option_as_list(wu_config_dict.get('wind_direction', '0,360')) try: if float(_wind_direction[0]) <= float(_wind_direction[1]): self.wind_dir = [float(_wind_direction[0]), float(_wind_direction[1])] else: self.wind_dir = [0, 360] except: self.wind_dir = [0, 360] # some properties we know because of the format of the returned WU data # WU returns a fixed format date-time string self.raw_datetime_format = '%Y-%m-%d %H:%M:%S' # WU only provides hourly rainfall and a daily cumulative rainfall. # We use the latter so force 'cumulative' for rain. self.rain = 'cumulative' # initialise our import field-to-weewx archive field map self.map = None # For a WU import we might have to import multiple days but we can only # get one day at a time from WU. So our start and end properties # (counters) are datetime objects and our increment is a timedelta. # Get datetime objects for any date or date range specified on the # command line, if there wasn't one then default to today. self.start = dt.fromtimestamp(startOfDay(self.first_ts)) self.end = dt.fromtimestamp(startOfDay(self.last_ts)) # set our increment self.increment = datetime.timedelta(days=1) # tell the user/log what we intend to do _msg = "Observation history for Weather Underground station '%s' will be imported." % self.station_id self.wlog.printlog(syslog.LOG_INFO, _msg) _msg = "The following options will be used:" self.wlog.verboselog(syslog.LOG_DEBUG, _msg) _msg = " config=%s, import-config=%s" % (config_path, self.import_config_path) self.wlog.verboselog(syslog.LOG_DEBUG, _msg) if options.date: _msg = " station=%s, date=%s" % (self.station_id, options.date) else: # we must have --from and --to _msg = " station=%s, from=%s, to=%s" % (self.station_id, options.date_from, options.date_to) self.wlog.verboselog(syslog.LOG_DEBUG, _msg) _msg = " dry-run=%s, calc-missing=%s" % (self.dry_run, self.calc_missing) self.wlog.verboselog(syslog.LOG_DEBUG, _msg) _msg = " tranche=%s, interval=%s, wind_direction=%s" % (self.tranche, self.interval, self.wind_dir) self.wlog.verboselog(syslog.LOG_DEBUG, _msg) _msg = "Using database binding '%s', which is bound to database '%s'" % (self.db_binding_wx, self.dbm.database_name) self.wlog.printlog(syslog.LOG_INFO, _msg) _msg = "Destination table '%s' unit system is '%#04x' (%s)." % (self.dbm.table_name, self.archive_unit_sys, unit_nicknames[self.archive_unit_sys]) self.wlog.printlog(syslog.LOG_INFO, _msg) if self.calc_missing: print "Missing derived observations will be calculated." if options.date or options.date_from: print "Observations timestamped after %s and up to and" % (timestamp_to_string(self.first_ts), ) print "including %s will be imported." % (timestamp_to_string(self.last_ts), ) if self.dry_run: print "This is a dry run, imported data will not be saved to archive."