Example #1
0
def time_query(client):
    """Time how long it takes to create a vector with daily max temperatures."""
    
    print "start time=", timestamp_to_string(start_ts)
    print "stop time=",  timestamp_to_string(stop_ts)
    print "(Approximately %.1f months of data)" % ((stop_ts - start_ts)/(30*24*3600))
    
    print "\nRunning a query for each day"
    t0 = time.time()
    vec = []
    
    for span in weeutil.weeutil.genDaySpans(start_ts, stop_ts):
        query = 'SELECT MAX(outTemp) FROM "wxpacket" WHERE time > %d AND time <= %d' % (span[0] * giga, span[1] * giga)
        result_set = client.query(query, database='example')
        for x in result_set.get_points():
            vec.append(x["max"])

    t1 = time.time()
    print "Elapsed query time=", t1-t0
        
    print vec
    
    print "\nRunning a single query, with group by day"
    t0 = time.time()
    vec = []
    
    query = 'SELECT MAX(outTemp) FROM "wxpacket" WHERE time > %d AND time <= %d GROUP BY time(1d)' % (start_ts * giga, stop_ts * giga)
    result_set = client.query(query, database="example")
    for x in result_set.get_points():
        vec.append(x["max"])
    
    t1 = time.time()
    print "Elapsed query time=", t1-t0
    print vec
Example #2
0
def time_query(config_dict):
    """A low-level approach. Use a SQL query"""
    
    manager_dict = weewx.manager.get_manager_dict_from_config(config_dict, 'wx_binding')
    # Just use a regular ol' manager to avoid using the daily summary optimization:
    manager_dict['manager'] = 'weewx.manager.Manager'

    manager = weewx.manager.open_manager(manager_dict,initialize=False)
    
    print "start time=", timestamp_to_string(start_ts)
    print "stop time=",  timestamp_to_string(stop_ts)
    print "Approximately %.1f months of data" % ((stop_ts - start_ts)/(30*24*3600))
    
    t0 = time.time()
    vec = []
    
    for span in weeutil.weeutil.genDaySpans(start_ts, stop_ts):
        query = 'SELECT dateTime, outTemp FROM archive WHERE dateTime > %d AND dateTime <= %d '\
                    'AND outTemp = (SELECT MAX(outTemp) FROM archive WHERE dateTime > %d AND dateTime <= %d)' % (span + span)
        tup = manager.getSql(query)
        vec.append(tup)
    t1 = time.time()
    print "Elapsed query time=", t1-t0
        
    print vec
Example #3
0
    def get_aggregate(obs_type, timespan, aggregate_type, db_manager, **option_dict):
        """Returns an aggregation of pm2_5_aqi over a timespan by using the main archive
        table.

        obs_type

        timespan: An instance of weeutil.Timespan with the time period over which aggregation is to
        be done.

        aggregate_type: The type of aggregation to be done. For this function, must be 'avg',
        'sum', 'count', 'first', 'last', 'min', or 'max'. Anything else will cause
        weewx.UnknownAggregation to be raised.

        db_manager: An instance of weewx.manager.Manager or subclass.

        option_dict: Not used in this version.

        returns: A ValueTuple containing the result.
        """
        if obs_type not in [ 'pm2_5_aqi', 'pm2_5_aqi_color' ]:
            raise weewx.UnknownType(obs_type)

        log.debug('get_aggregate(%s, %s, %s, aggregate:%s)' % (
            obs_type, timestamp_to_string(timespan.start),
            timestamp_to_string(timespan.stop), aggregate_type))

        aggregate_type = aggregate_type.lower()

        # Raise exception if we don't know about this type of aggregation
        if aggregate_type not in list(AQI.agg_sql_dict.keys()):
            raise weewx.UnknownAggregation(aggregate_type)

        # Form the interpolation dictionary
        interpolation_dict = {
            'start': timespan.start,
            'stop': timespan.stop,
            'table_name': db_manager.table_name
        }

        select_stmt = AQI.agg_sql_dict[aggregate_type] % interpolation_dict
        row = db_manager.getSql(select_stmt)
        if row:
            value, std_unit_system = row
        else:
            value = None
            std_unit_system = None

        if value is not None:
            if obs_type == 'pm2_5_aqi':
                value = AQI.compute_pm2_5_aqi(value)
            if obs_type == 'pm2_5_aqi_color':
                value = AQI.compute_pm2_5_aqi_color(AQI.compute_pm2_5_aqi(value))
        t, g = weewx.units.getStandardUnitType(std_unit_system, obs_type, aggregate_type)
        # Form the ValueTuple and return it:
        log.debug('get_aggregate(%s, %s, %s, aggregate:%s, select_stmt: %s, returning %s)' % (
            obs_type, timestamp_to_string(timespan.start), timestamp_to_string(timespan.stop),
            aggregate_type, select_stmt, value))
        return weewx.units.ValueTuple(value, t, g)
Example #4
0
    def skip_this_post(self, time_ts):
        """Determine whether a post is to be skipped or not.

        Use one or more checks to determine whether a post is to be skipped or
        not. In this case the post is skipped if the record is:
        -   Too old (based on the 'stale' property). This check is kept to
            honor the existing 'stale' property, PVOutput has a max age for
            which status data can be posted, we store this value as
            self.update_period.
        -   Outside the maximum age that PVOutput will accept (varies by
            donation status)
        -   Posted too soon after our last post.
        """

        # don't post if this record is too old (stale)
        if self.stale is not None:
            _how_old = time.time() - time_ts
            if _how_old > self.stale:
                syslog.syslog(
                    syslog.LOG_DEBUG,
                    "pvoutput: %s: record %s is stale (%d > %d)." %
                    (self.protocol_name, timestamp_to_string(time_ts),
                     _how_old, self.stale))
                return True

        # don't post if this record is older than that accepted by PVOutput
        if self.update_period is not None:
            now_dt = datetime.datetime.fromtimestamp(time.time())
            _earliest_dt = now_dt - datetime.timedelta(days=self.update_period)
            _earliest_ts = time.mktime(_earliest_dt.timetuple())
            if time_ts < _earliest_ts:
                how_old = (now_dt -
                           datetime.datetime.fromtimestamp(time_ts)).days
                syslog.syslog(
                    syslog.LOG_DEBUG,
                    "pvoutput: %s: record %s is older than PVOuptut imposed limits (%d > %d)."
                    % (self.protocol_name, timestamp_to_string(time_ts),
                       _how_old, self.self.update_period))
                return True

        # if we have a minimum interval between posts then don't post if that
        # interval has not passed
        if self.post_interval is not None:
            _how_long = time_ts - self.lastpost
            if _how_long < self.post_interval:
                syslog.syslog(
                    syslog.LOG_DEBUG,
                    "pvoutput: %s: wait interval (%d < %d) has not passed for record %s"
                    % (self.protocol_name, _how_long, self.post_interval,
                       timestamp_to_string(time_ts)))
                return True

        self.lastpost = time_ts
        return False
Example #5
0
    def get_series(obs_type, timespan, db_manager, aggregate_type=None, aggregate_interval=None):
        """Get a series, possibly with aggregation.
        """

        if obs_type not in [ 'pm2_5_aqi', 'pm2_5_aqi_color' ]:
            raise weewx.UnknownType(obs_type)

        log.debug('get_series(%s, %s, %s, aggregate:%s, aggregate_interval:%s)' % (
            obs_type, timestamp_to_string(timespan.start), timestamp_to_string(
            timespan.stop), aggregate_type, aggregate_interval))

        #  Prepare the lists that will hold the final results.
        start_vec = list()
        stop_vec = list()
        data_vec = list()

        # Is aggregation requested?
        if aggregate_type:
            # Yes. Just use the regular series function.
            return weewx.xtypes.ArchiveTable.get_series(obs_type, timespan, db_manager, aggregate_type,
                                           aggregate_interval)
        else:
            # No aggregation.
            sql_str = 'SELECT dateTime, usUnits, `interval`, pm2_5 FROM %s ' \
                      'WHERE dateTime >= ? AND dateTime <= ? AND pm2_5 IS NOT NULL' \
                      % db_manager.table_name
            std_unit_system = None

            for record in db_manager.genSql(sql_str, timespan):
                ts, unit_system, interval, pm2_5 = record
                if std_unit_system:
                    if std_unit_system != unit_system:
                        raise weewx.UnsupportedFeature(
                            "Unit type cannot change within a time interval.")
                else:
                    std_unit_system = unit_system

                if obs_type == 'pm2_5_aqi':
                    value = AQI.compute_pm2_5_aqi(pm2_5)
                if obs_type == 'pm2_5_aqi_color':
                    value = AQI.compute_pm2_5_aqi_color(AQI.compute_pm2_5_aqi(pm2_5))
                log.debug('get_series(%s): %s - %s - %s' % (obs_type,
                    timestamp_to_string(ts - interval * 60),
                    timestamp_to_string(ts), value))
                start_vec.append(ts - interval * 60)
                stop_vec.append(ts)
                data_vec.append(value)

            unit, unit_group = weewx.units.getStandardUnitType(std_unit_system, obs_type,
                                                               aggregate_type)

        return (ValueTuple(start_vec, 'unix_epoch', 'group_time'),
                ValueTuple(stop_vec, 'unix_epoch', 'group_time'),
                ValueTuple(data_vec, unit, unit_group))
Example #6
0
    def set_time(self, ts):
        local_time = time.localtime(ts)
        set_time_cmd = "ST%2.2d%2.2d%2.2d" % (
            local_time.tm_hour, local_time.tm_min, local_time.tm_sec)
        logdbg("set station time to %s (%s)" %
               (timestamp_to_string(ts), set_time_cmd))
        self.send_AT_cmd(set_time_cmd)

        set_date_cmd = "SD%2.2d%2.2d%2.2d" % (
            local_time.tm_year % 100, local_time.tm_mon, local_time.tm_mday)
        logdbg("set station date to %s (%s)" %
               (timestamp_to_string(ts), set_date_cmd))
        self.send_AT_cmd(set_date_cmd)
Example #7
0
def time_query(collection):
    """Time how long it takes to create a vector with daily max temperatures."""
    
    print "start time=", timestamp_to_string(start_ts)
    print "stop time=", timestamp_to_string(stop_ts)
    print "(Approximately %.1f months of data)" % ((stop_ts - start_ts) / (30 * 24 * 3600))
    
#     # This would give the max temperature in each UTC day (unfortunately
#     # we want the max temperature in each local day)
#     rs = collection.aggregate([{ "$project": {"day": { "$dayOfYear": "$dateTime" },
#                                               "dateTime": 1,
#                                               "outTemp": 1
#                                               }},
#                                { "$sort": { "day": 1, "outTemp":-1 } },
#                                { "$group": {
#                                             "_id" : "$day",
#                                             "max_temperature": { "$first": "$outTemp" },
#                                             "timestamp": { "$first": "$dateTime" }
#                                             }},
#                                { "$sort": { "_id":1 } }
#                                ])


    epoch = datetime.datetime.utcfromtimestamp(0)

    vec = []
    t0 = time.time()
    for span in weeutil.weeutil.genDaySpans(start_ts, stop_ts):
        rs = collection.aggregate([{"$match"   : {"dateTime" : {"$gt"  : datetime.datetime.utcfromtimestamp(span[0]), 
                                                                "$lte" : datetime.datetime.utcfromtimestamp(span[1])},
                                                  "outTemp"  : {"$ne" : None}
                                                  }}, 
                                   {"$project" : {"dateTime" : 1, "outTemp" : 1}},
                                   {"$sort" : {"outTemp" : -1 } },
                                   {"$limit" : 1}
                                   ])
        for x in rs:
            # Convert from a (timezone naive) datetime object to unix epoch time: 
            delta = x['dateTime'] - epoch
            vec.append((delta.total_seconds(), x['outTemp']))

    t1 = time.time()
    print "Elapsed query time=", (t1 - t0)
    print vec
 
    with open("mongo.out", "w") as fd:
        for x in vec:
            fd.write("%s %.2f\n" % (weeutil.weeutil.timestamp_to_string(x[0]), x[1]))
Example #8
0
    def get_time(self):
        try:
            sta_time = int(self.send_AT_cmd("RT"))
            sta_date = int(self.send_AT_cmd("RD"))
            # break sta_time into hh:mm:ss
            hh = sta_time // 10000
            mm = sta_time // 100 - (hh * 100)
            ss = sta_time - (mm * 100) - (hh * 10000)

            # break sta_date into YY:MM:DD
            YY = sta_date // 10000
            MM = sta_date // 100 - (YY * 100)
            DD = sta_date - (MM * 100) - (YY * 10000)

            # two-digit year hack - this station defaults to a epoch of 1987
            # when power is lost. If the year is greater than 86, we assume
            # it's the 20th century, otherwise it's the 21st.
            # Note - someone (not me!) will have to revisit this code in 2086.
            if YY > 86:
                year = 1900 + YY
            else:
                year = 2000 + YY

            # We keep the station clock in GMT, which eliminates the
            # DST silliness.
            ts = time.mktime((year, MM, DD, hh, mm, ss, 0, 0, -1))
            logdbg("station date: %s, time: %s, (%s)" %
                   (sta_date, sta_time, timestamp_to_string(ts)))
            return ts

        except (serial.serialutil.SerialException, weewx.WeeWxIOError) as e:
            logerr("get_time failed: %s" % e)
        return int(time.time())
Example #9
0
    def _progress(ndays, last_time):
        """Utility function to show our progress while processing the fix."""

        _msg = "Updating 'windSpeed' daily summary: %d; " \
               "Timestamp: %s\r" % (ndays, timestamp_to_string(last_time, format_str="%Y-%m-%d"))
        print(_msg, end='', file=sys.stdout)
        sys.stdout.flush()
Example #10
0
    def uploadFiles(self):
        start_ts = time.time()
        t_str = timestamp_to_string(start_ts)
        syslog.syslog(syslog.LOG_INFO, "S3upload: start upload at %s" % t_str)

        # Build command
        cmd = ["/usr/local/bin/s3cmd"]
        cmd.extend(["sync"])
        cmd.extend(["--no-mime-magic"])
        cmd.extend(["--access_key=%s" % self.access_key])
        cmd.extend(["--secret_key=%s" % self.secret_token])
        cmd.extend([self.local_root])
        cmd.extend(["s3://%s" % self.bucket_name])

        syslog.syslog(syslog.LOG_DEBUG, "S3upload command: %s" % cmd)
        try:
            S3upload_cmd = subprocess.Popen(cmd,
                                            stdout=subprocess.PIPE,
                                            stderr=subprocess.STDOUT)

            stdout = S3upload_cmd.communicate()[0]
            stroutput = stdout.strip()
        except OSError, e:
            if e.errno == errno.ENOENT:
                syslog.syslog(
                    syslog.LOG_ERR,
                    "S3upload: s3cmd does not appear to be installed on this system. (errno %d, \"%s\")"
                    % (e.errno, e.strerror))
            raise
Example #11
0
    def _progress(ndays, last_time):
        """Utility function to show our progress while processing the fix."""

        print("Weighting daily summary: %d; Timestamp: %s\r"
              % (ndays, timestamp_to_string(last_time, format_str="%Y-%m-%d")),
              end='', file=sys.stdout)
        sys.stdout.flush()
Example #12
0
 def genStartupRecords(self, since_ts):
     loginf("reading records since %s" % timestamp_to_string(since_ts))
     hbuf = None
     last_ts = None
     cnt = 0
     while True:
         try:
             buf = self.station.read()
             if buf is not None:
                 if buf[0] == 0xD2:
                     hbuf = buf
                     buf = None
                 elif buf[0] == 0x7F and hbuf is not None:
                     # FIXME: need better indicator of second half history
                     buf = hbuf + buf
                     hbuf = None
             if buf is not None and buf[0] == 0xD2:
                 self.last_record = Station.get_record_index(buf)
                 ts = Station._extract_ts(buf[4:9])
                 if ts is not None and ts > since_ts:
                     keep = True if last_ts is not None else False
                     pkt = Station.decode(buf)
                     packet = self.convert_historical(pkt, ts, last_ts)
                     last_ts = ts
                     if keep:
                         logdbg("historical record: %s" % packet)
                         cnt += 1
                         yield packet
             if buf is not None and buf[0] == 0x57:
                 idx = Station.get_latest_index(buf)
                 msg = "count=%s last_index=%s latest_index=%s" % (cnt, self.last_record, idx)
                 if self.last_record + 1 >= idx:
                     loginf("catchup complete: %s" % msg)
                     break
                 loginf("catchup in progress: %s" % msg)
             if buf is not None and buf[0] == 0x41 and buf[3] == 0x65:
                 nxtrec = Station.get_next_index(self.last_record)
                 logdbg("request records starting with %s" % nxtrec)
                 cmd = [0xCD, 0x18, 0x30, 0x62, _hi(nxtrec), _lo(nxtrec)]
                 sent = self.station.write(cmd)
             if time.time() - self.last_a6 > self.heartbeat:
                 logdbg("request station status: %s (%02x)" % (self.last_record, _lo(self.last_record)))
                 cmd = [0xA6, 0x91, 0xCA, 0x45, 0x52, _lo(self.last_record)]
                 sent = self.station.write(cmd)
                 self.last_a6 = time.time()
             if self.last_7x == 0:
                 # FIXME: what does 72/73 do?
                 cmd = [0x73, 0xE5, 0x0A, 0x26, 0x88, 0x8B]
                 sent = self.station.write(cmd)
                 self.last_7x = time.time()
             if time.time() - self.last_65 > self.history_retry:
                 logdbg("initiate record request: %s (%02x)" % (self.last_record, _lo(self.last_record)))
                 cmd = [0x65, 0x19, 0xE5, 0x04, 0x52, _lo(self.last_record)]
                 sent = self.station.write(cmd)
                 self.last_65 = time.time()
         except usb.USBError, e:
             if not e.args[0].find("No data available"):
                 raise weewx.WeeWxIOError(e)
         except (WrongLength, BadChecksum), e:
             loginf(e)
Example #13
0
 def get_scalar(obs_type, record, db_manager=None):
     log.debug('get_scalar(%s)' % obs_type)
     if obs_type not in [ 'pm2_5_aqi', 'pm2_5_aqi_color' ]:
         raise weewx.UnknownType(obs_type)
     log.debug('get_scalar(%s)' % obs_type)
     if record is None:
         log.debug('get_scalar called where record is None.')
         raise weewx.CannotCalculate(obs_type)
     if 'pm2_5' not in record:
         # Should not see this as pm2_5 is part of the extended schema that is required for this plugin.
         # Returning CannotCalculate causes exception in ImageGenerator, return UnknownType instead.
         # ERROR weewx.reportengine: Caught unrecoverable exception in generator 'weewx.imagegenerator.ImageGenerator'
         log.info('get_scalar called where record does not contain pm2_5.  This is unexpected.')
         raise weewx.UnknownType(obs_type)
     if record['pm2_5'] is None:
         # Returning CannotCalculate causes exception in ImageGenerator, return UnknownType instead.
         # ERROR weewx.reportengine: Caught unrecoverable exception in generator 'weewx.imagegenerator.ImageGenerator'
         # Any archive catchup records will have None for pm2_5.
         log.debug('get_scalar called where record[pm2_5] is None: %s.  Probably a catchup record.' %
             timestamp_to_string(record['dateTime']))
         raise weewx.UnknownType(obs_type)
     try:
         pm2_5 = record['pm2_5']
         if obs_type == 'pm2_5_aqi':
             value = AQI.compute_pm2_5_aqi(pm2_5)
         if obs_type == 'pm2_5_aqi_color':
             value = AQI.compute_pm2_5_aqi_color(AQI.compute_pm2_5_aqi(pm2_5))
         t, g = weewx.units.getStandardUnitType(record['usUnits'], obs_type)
         # Form the ValueTuple and return it:
         return weewx.units.ValueTuple(value, t, g)
     except KeyError:
         # Don't have everything we need. Raise an exception.
         raise weewx.CannotCalculate(obs_type)
Example #14
0
def run_manager(manager):    
    print "start time=", timestamp_to_string(start_ts)
    print "stop time=",  timestamp_to_string(stop_ts)
    print "Approximately %.1f months of data" % ((stop_ts - start_ts)/(30*24*3600))
    
    t0 = time.time()
    vec = []
    
    for span in weeutil.weeutil.genDaySpans(start_ts, stop_ts):
        maxTemp = manager.getAggregate(span, 'outTemp', 'max')[0]
        ts = manager.getAggregate(span, 'outTemp', 'maxtime')[0]
        vec.append((ts, maxTemp))
        
    t1 = time.time()
    print "Elapsed query time=", t1-t0
        
    return vec
Example #15
0
    def soundTheAlarm(self, timestamp, battery_status, alarm_count):
        """This function is called when the alarm has been triggered."""

        # Get the time and convert to a string:
        t_str = timestamp_to_string(timestamp)

        # Log it in the system log:
        syslog.syslog(
            syslog.LOG_INFO,
            "lowBattery: Low battery alarm (0x%04x) sounded at %s." %
            (battery_status, t_str))

        # Form the message text:
        msg_text = """The low battery alarm (0x%04x) has been seen %d times since the last archive period.\n\n"""\
                   """Alarm sounded at %s\n\n""" % (battery_status, alarm_count, t_str)
        # Convert to MIME:
        msg = MIMEText(msg_text)

        # Fill in MIME headers:
        msg['Subject'] = self.SUBJECT
        msg['From'] = self.FROM
        msg['To'] = ','.join(self.TO)

        # Create an instance of class SMTP for the given SMTP host:
        s = smtplib.SMTP(self.smtp_host)
        try:
            # Some servers (eg, gmail) require encrypted transport.
            # Be prepared to catch an exception if the server
            # doesn't support it.
            s.ehlo()
            s.starttls()
            s.ehlo()
            syslog.syslog(syslog.LOG_DEBUG,
                          "lowBattery: using encrypted transport")
        except smtplib.SMTPException:
            syslog.syslog(syslog.LOG_DEBUG,
                          "lowBattery: using unencrypted transport")

        try:
            # If a username has been given, assume that login is required for this host:
            if self.smtp_user:
                s.login(self.smtp_user, self.smtp_password)
                syslog.syslog(
                    syslog.LOG_DEBUG,
                    "lowBattery: logged in with user name %s" %
                    (self.smtp_user, ))

            # Send the email:
            s.sendmail(msg['From'], self.TO, msg.as_string())
            # Log out of the server:
            s.quit()
        except Exception, e:
            syslog.syslog(
                syslog.LOG_ERR,
                "lowBattery: SMTP mailer refused message with error %s" %
                (e, ))
            raise
Example #16
0
def main():

    print "start time=", timestamp_to_string(start_ts)
    print "stop time= ",  timestamp_to_string(stop_ts)
    print "(Approximately %.1f days of data)" % ((stop_ts - start_ts)/(24*3600.0))
     
    print "***** SQLITE *****"
    create_table(sqlite_db_dict)
    connect = weedb.connect(sqlite_db_dict)
    time_query(connect, 'outTemp')
    time_query(connect, 'barometer')
    connect.close()
    
    print "***** MySQL *****"
    create_table(mysql_db_dict)
    connect = weedb.connect(mysql_db_dict)
    time_query(connect, 'outTemp')
    time_query(connect, 'barometer')
    connect.close()
Example #17
0
    def _progress(record, ts):
        """Utility function to show our progress while processing the fix.

            Override in derived class to provide a different progress display.
            To do nothing override with a pass statement.
        """

        print >>sys.stdout, "Fixing database record: %d; Timestamp: %s\r" % \
            (record, timestamp_to_string(ts)),
        sys.stdout.flush()
Example #18
0
    def _progress(record, ts):
        """Utility function to show our progress while processing the fix.

            Override in derived class to provide a different progress display.
            To do nothing override with a pass statement.
        """

        _msg = "Fixing database record: %d; Timestamp: %s\r" % (record, timestamp_to_string(ts))
        print(_msg, end='', file=sys.stdout)
        sys.stdout.flush()
Example #19
0
    def do_alarm(self, record):
        """Send an email out"""

        # Get the time and convert to a string:
        t_str = timestamp_to_string(record['dateTime'])

        # Log the alarm
        log.info('Alarm expression "%s" evaluated True at %s' %
                 (self.expression, t_str))

        # Form the message text:
        msg_text = 'Alarm expression "%s" evaluated True at %s\nRecord:\n%s' \
                   % (self.expression, t_str, str(record))
        # Convert to MIME:
        msg = MIMEText(msg_text)

        # Fill in MIME headers:
        msg['Subject'] = self.SUBJECT
        msg['From'] = self.FROM
        msg['To'] = ','.join(self.TO)

        try:
            # First try end-to-end encryption
            s = smtplib.SMTP_SSL(self.smtp_host, timeout=self.timeout)
            log.debug("Using SMTP_SSL")
        except (AttributeError, socket.timeout, socket.error) as e:
            log.debug("Unable to use SMTP_SSL connection. Reason: %s", e)
            # If that doesn't work, try creating an insecure host, then upgrading
            s = smtplib.SMTP(self.smtp_host, timeout=self.timeout)
            try:
                # Be prepared to catch an exception if the server
                # does not support encrypted transport.
                s.ehlo()
                s.starttls()
                s.ehlo()
                log.debug("Using SMTP encrypted transport")
            except smtplib.SMTPException as e:
                log.debug("Using SMTP unencrypted transport. Reason: %s", e)

        try:
            # If a username has been given, assume that login is required for this host:
            if self.smtp_user:
                s.login(self.smtp_user, self.smtp_password)
                log.debug("Logged in with user name %s", self.smtp_user)

            # Send the email:
            s.sendmail(msg['From'], self.TO, msg.as_string())
            # Log out of the server:
            s.quit()
        except Exception as e:
            log.error("SMTP mailer refused message with error %s", e)
            raise

        # Log sending the email:
        log.info("Email sent to: %s", self.TO)
Example #20
0
    def soundTheAlarm(self, rec, status=True):
        """This function is called when the given expression evaluates True or False."""

        # Get the time and convert to a string:
        t_str = timestamp_to_string(rec['dateTime'])

        # Log it in the system log:
        syslog.syslog(
            syslog.LOG_INFO,
            "alarm: Alarm expression '{}' evaluated {} at {}" %
            (self.expression, status, t_str))

        # Form the message text:
        msg_text = "Alarm expression '{}' evaluated {} at {}\nRecord:\n{}".format(
            self.expression, status, t_str, str(rec))
        # Convert to MIME:
        msg = MIMEText(msg_text)

        # Fill in MIME headers:
        msg['Subject'] = self.SUBJECT
        msg['From'] = self.FROM
        msg['To'] = ','.join(self.TO)

        # Create an instance of class SMTP for the given SMTP host:
        s = smtplib.SMTP(self.smtp_host)
        try:
            # Some servers (eg, gmail) require encrypted transport.
            # Be prepared to catch an exception if the server
            # doesn't support it.
            s.ehlo()
            s.starttls()
            s.ehlo()
            syslog.syslog(syslog.LOG_DEBUG, "  **** using encrypted transport")
        except smtplib.SMTPException:
            syslog.syslog(syslog.LOG_DEBUG,
                          "  **** using unencrypted transport")

        try:
            # If a username has been given, assume that login is required for this host:
            if self.smtp_user:
                s.login(self.smtp_user, self.smtp_password)
                syslog.syslog(
                    syslog.LOG_DEBUG,
                    "  **** logged in with user name %s" % (self.smtp_user, ))

            # Send the email:
            s.sendmail(msg['From'], self.TO, msg.as_string())
            # Log out of the server:
            s.quit()
        except Exception, e:
            syslog.syslog(
                syslog.LOG_ERR,
                "alarm: SMTP mailer refused message with error %s" % (e, ))
            raise
Example #21
0
 def _time_sync(self, ts):
     logdbg("time sync to %s (%s)" % (ts, timestamp_to_string(ts)))
     t = time.localtime(ts)
     cmd = [WH23xxStation.TIME_SYNC,
            t.tm_year - 2000, t.tm_mon, t.tm_mday,
            t.tm_hour, t.tm_min, t.tm_sec, 0]
     chksum = _calc_checksum(cmd)
     buf = [0x02, 0x09]
     buf.extend(cmd)
     buf.append(chksum)
     self._write("time_sync", buf)
Example #22
0
    def soundTheAlarm(self, timestamp, battery_status, alarm_count):
        """This function is called when the low battery alarm has been sounded."""

        # Get the time and convert to a string:
        t_str = timestamp_to_string(timestamp)

        # Log it in the system log:
        syslog.syslog(
            syslog.LOG_INFO, "lowBattery: Low battery alarm (0x%04x) sounded at %s." % (battery_status, t_str)
        )

        # Form the message text:
        msg_text = (
            """The low battery alarm (0x%04x) has been seen %d times since the last archive period.\n\n"""
            """Alarm sounded at %s\n\n""" % (battery_status, alarm_count, t_str)
        )
        # Convert to MIME:
        msg = MIMEText(msg_text)

        # Fill in MIME headers:
        msg["Subject"] = self.SUBJECT
        msg["From"] = self.FROM
        msg["To"] = ",".join(self.TO)

        # Create an instance of class SMTP for the given SMTP host:
        s = smtplib.SMTP(self.smtp_host)
        try:
            # Some servers (eg, gmail) require encrypted transport.
            # Be prepared to catch an exception if the server
            # doesn't support it.
            s.ehlo()
            s.starttls()
            s.ehlo()
            syslog.syslog(syslog.LOG_DEBUG, "  **** using encrypted transport")
        except smtplib.SMTPException:
            syslog.syslog(syslog.LOG_DEBUG, "  **** using unencrypted transport")

        try:
            # If a username has been given, assume that login is required for this host:
            if self.smtp_user:
                s.login(self.smtp_user, self.smtp_password)
                syslog.syslog(syslog.LOG_DEBUG, "  **** logged in with user name %s" % (self.smtp_user,))

            # Send the email:
            s.sendmail(msg["From"], self.TO, msg.as_string())
            # Log out of the server:
            s.quit()
        except Exception, e:
            syslog.syslog(syslog.LOG_ERR, "lowBattery: SMTP mailer refused message with error %s" % (e,))
            raise
Example #23
0
    def set_time(self, ts):
        # go to modem mode so we do not get logger chatter
        self.set_modem_mode()

        # set time should work on all models
        tt = time.localtime(ts)
        cmd = b">A%04d%04d" % (tt.tm_yday - 1, tt.tm_min + tt.tm_hour * 60)
        log.debug("Set station time to %s (%s)", timestamp_to_string(ts), cmd)
        self.serial_port.write(b"%s\r" % cmd)

        # year works only for models 2004 and later
        if self.can_set_year:
            cmd = b">U%s" % tt.tm_year
            log.debug("Set station year to %s (%s)", tt.tm_year, cmd)
            self.serial_port.write(b"%s\r" % cmd)
Example #24
0
    def set_time(self, ts):
        # go to modem mode so we do not get logger chatter
        self.set_modem_mode()

        # set time should work on all models
        tt = time.localtime(ts)
        cmd = ">A%04d%04d" % (
            tt.tm_yday - 1, tt.tm_min + tt.tm_hour * 60)
        logdbg("set station time to %s (%s)" % (timestamp_to_string(ts), cmd))
        self.serial_port.write("%s\r" % cmd)

        # year works only for models 2004 and later
        if self.can_set_year:
            cmd = ">U%s" % tt.tm_year
            logdbg("set station year to %s (%s)" % (tt.tm_year, cmd))
            self.serial_port.write("%s\r" % cmd)
Example #25
0
 def get_time(self):
     try:
         self.set_logger_mode()
         buf = self.get_readings_with_retry()
         data = Station.parse_readings(buf)
         d = data['day_of_year']  # seems to start at 0
         m = data['minute_of_day']  # 0 is midnight before start of day
         tt = time.localtime()
         y = tt.tm_year
         s = tt.tm_sec
         ts = time.mktime((y,1,1,0,0,s,0,0,-1)) + d * 86400 + m * 60
         logdbg("station time: day:%s min:%s (%s)" %
                (d, m, timestamp_to_string(ts)))
         return ts
     except (serial.serialutil.SerialException, weewx.WeeWxIOError), e:
         logerr("get_time failed: %s" % e)
Example #26
0
 def get_time(self):
     try:
         self.set_logger_mode()
         buf = self.get_readings_with_retry()
         data = Station.parse_readings(buf)
         d = data['day_of_year']  # seems to start at 0
         m = data['minute_of_day']  # 0 is midnight before start of day
         tt = time.localtime()
         y = tt.tm_year
         s = tt.tm_sec
         ts = time.mktime((y, 1, 1, 0, 0, s, 0, 0, -1)) + d * 86400 + m * 60
         log.debug("Station time: day:%s min:%s (%s)", d, m, timestamp_to_string(ts))
         return ts
     except (serial.serialutil.SerialException, weewx.WeeWxIOError) as e:
         log.error("get_time failed: %s", e)
     return int(time.time())
Example #27
0
    def addRecord(self, record, add_hilo=True, weight=1):
        """Add a record to my running statistics. 
        
        The record must have keys 'dateTime' and 'usUnits'."""

        # Check to see if the record is within my observation timespan
        if not self.timespan.includesArchiveTime(record['dateTime']):
            raise OutOfSpan(
                "Attempt to add out-of-interval record (%s) to timespan (%s)" %
                (timestamp_to_string(record['dateTime']), self.timespan))

        for obs_type in record:
            # Get the proper function ...
            func = get_add_function(obs_type)
            # ... then call it.
            func(self, record, obs_type, add_hilo, weight)
Example #28
0
 def new_loop_packet(self, event):
     log.debug('new_loop_packet(%s)' % event)
     with self.cfg.lock:
         log.debug('new_loop_packet: self.cfg.concentrations: %s' %
                   self.cfg.concentrations)
         if self.cfg.concentrations is not None and \
                 self.cfg.concentrations.timestamp is not None and \
                 self.cfg.concentrations.timestamp + \
                 self.cfg.archive_interval >= time.time():
             log.debug(
                 'Time of reading being inserted: %s' %
                 timestamp_to_string(self.cfg.concentrations.timestamp))
             # Insert pm1_0, pm2_5, pm10_0, aqi and aqic into loop packet.
             if self.cfg.concentrations.pm1_0 is not None:
                 event.packet['pm1_0'] = self.cfg.concentrations.pm1_0
                 log.debug('Inserted packet[pm1_0]: %f into packet.' %
                           event.packet['pm1_0'])
             if self.cfg.concentrations.pm2_5_cf_1_b is not None:
                 b_reading = self.cfg.concentrations.pm2_5_cf_1_b
             else:
                 b_reading = self.cfg.concentrations.pm2_5_cf_1  # Dup A sensor reading
             if (self.cfg.concentrations.pm2_5_cf_1 is not None
                     and b_reading is not None and
                     self.cfg.concentrations.current_humidity is not None
                     and self.cfg.concentrations.current_temp_f):
                 event.packet[
                     'pm2_5'] = AQI.compute_pm2_5_us_epa_correction(
                         self.cfg.concentrations.pm2_5_cf_1, b_reading,
                         self.cfg.concentrations.current_humidity,
                         self.cfg.concentrations.current_temp_f)
                 log.debug('Inserted packet[pm2_5]: %f into packet.' %
                           event.packet['pm2_5'])
             if self.cfg.concentrations.pm10_0 is not None:
                 event.packet['pm10_0'] = self.cfg.concentrations.pm10_0
                 log.debug('Inserted packet[pm10_0]: %f into packet.' %
                           event.packet['pm10_0'])
             if 'pm2_5' in event.packet:
                 event.packet['pm2_5_aqi'] = AQI.compute_pm2_5_aqi(
                     event.packet['pm2_5'])
             if 'pm2_5_aqi' in event.packet:
                 event.packet[
                     'pm2_5_aqi_color'] = AQI.compute_pm2_5_aqi_color(
                         event.packet['pm2_5_aqi'])
         else:
             log.error('Found no fresh concentrations to insert.')
Example #29
0
    def soundTheAlarm(self, rec):
        """This function is called when the given expression evaluates True."""
        
        # Get the time and convert to a string:
        t_str = timestamp_to_string(rec['dateTime'])

        # Log it in the system log:
        syslog.syslog(syslog.LOG_INFO, "alarm: Alarm expression \"%s\" evaluated True at %s" % (self.expression, t_str))

        # Form the message text:
        msg_text = "Alarm expression \"%s\" evaluated True at %s\nRecord:\n%s" % (self.expression, t_str, str(rec))
        # Convert to MIME:
        msg = MIMEText(msg_text)
        
        # Fill in MIME headers:
        msg['Subject'] = self.SUBJECT
        msg['From']    = self.FROM
        msg['To']      = ','.join(self.TO)
        
        # Create an instance of class SMTP for the given SMTP host:
        s = smtplib.SMTP(self.smtp_host)
        try:
            # Some servers (eg, gmail) require encrypted transport.
            # Be prepared to catch an exception if the server
            # doesn't support it.
            s.ehlo()
            s.starttls()
            s.ehlo()
            syslog.syslog(syslog.LOG_DEBUG, "  **** using encrypted transport")
        except smtplib.SMTPException:
            syslog.syslog(syslog.LOG_DEBUG, "  **** using unencrypted transport")

        try:
            # If a username has been given, assume that login is required for this host:
            if self.smtp_user:
                s.login(self.smtp_user, self.smtp_password)
                syslog.syslog(syslog.LOG_DEBUG, "  **** logged in with user name %s" % (self.smtp_user,))
            
            # Send the email:
            s.sendmail(msg['From'], self.TO,  msg.as_string())
            # Log out of the server:
            s.quit()
        except Exception, e:
            syslog.syslog(syslog.LOG_ERR, "alarm: SMTP mailer refused message with error %s" % (e,))
            raise
Example #30
0
 def uploadFiles(self):
     start_ts = time.time()
     t_str = timestamp_to_string(start_ts)
     syslog.syslog(syslog.LOG_INFO,
                   "FirebaseUpload: start upload at %s" % t_str)
     cmd = ["/usr/bin/node"]
     cmd.extend(["/srv/weewx/deploy.js"])
     syslog.syslog(syslog.LOG_DEBUG, "FirebaseUpload command: %s" % cmd)
     try:
         FirebaseUpload_cmd = subprocess.Popen(cmd,
                                               stdout=subprocess.PIPE,
                                               stderr=subprocess.STDOUT)
         stdout = FirebaseUpload_cmd.communicate()[0]
         stroutput = stdout.strip()
     except OSError, e:
         if e.errno == errno.ENOENT:
             syslog.syslog(
                 syslog.LOG_ERR,
                 "FirebaseUpload: can't deploy. (errno %d, \"%s\")" %
                 (e.errno, e.strerror))
         raise
Example #31
0
    def uploadFiles(self):
        start_ts = time.time()
        t_str = timestamp_to_string(start_ts)
        syslog.syslog(syslog.LOG_DEBUG, "s3uploadgenerator: start upload at %s" % t_str)

        # Build command
        cmd = ["/usr/bin/s3cmd"]
        cmd.extend(["sync"])
        cmd.extend(["--config=/home/weewx/.s3cfg"])
        cmd.extend([self.local_root])
        cmd.extend(["s3://%s" % self.bucket_name])

        syslog.syslog(syslog.LOG_DEBUG, "s3uploadgenerator: command: %s" % cmd)
        try:
            S3upload_cmd = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
            stdout = S3upload_cmd.communicate()[0]
            stroutput = stdout.strip()
        except OSError, e:
            if e.errno == errno.ENOENT:
                syslog.syslog(syslog.LOG_ERR, "s3uploadgenerator: s3cmd does not appear to be installed on this system. (errno %d, \"%s\")" % (e.errno, e.strerror))
            raise
Example #32
0
 def soundTheAlarm(self, rec, alarm_count):
     """This function is called when the low battery alarm has been sounded."""
     
     # Get the time and convert to a string:
     t_str = timestamp_to_string(rec['dateTime'])
     # Form the message text:
     msg_text = """The low battery alarm (0x%04x) has been seen %d times since the last archive period.\n\n"""\
                """Alarm sounded at %s\n\n"""\
                """LOOP record:\n%s""" % (rec['txBatteryStatus'], alarm_count, t_str, str(rec))
     # Convert to MIME:
     msg = MIMEText(msg_text)
     
     # Fill in MIME headers:
     msg['Subject'] = "Low battery alarm message from weewx"
     msg['From']    = "weewx"
     msg['To']      = self.TO
     
     # Create an instance of class SMTP for the given SMTP host:
     s = smtplib.SMTP(self.smtp_host)
     try:
         # Some servers (eg, gmail) require encrypted transport.
         # Be prepared to catch an exception if the server
         # doesn't support it.
         s.ehlo()
         s.starttls()
         s.ehlo()
     except smtplib.SMTPException:
         pass
     # If a username has been given, assume that login is required for this host:
     if self.smtp_user:
         s.login(self.smtp_user, self.smtp_password)
     # Send the email:
     s.sendmail(msg['From'], [self.TO],  msg.as_string())
     # Log out of the server:
     s.quit()
     # Log it in the system log:
     syslog.syslog(syslog.LOG_INFO, "lowBattery: Low battery alarm (0x%04x) sounded." % rec['txBatteryStatus'])
     syslog.syslog(syslog.LOG_INFO, "       ***  email sent to: %s" % self.TO)
Example #33
0
 def soundTheAlarm(self, rec):
     """This function is called when the given expression evaluates True."""
     
     # Get the time and convert to a string:
     t_str = timestamp_to_string(rec['dateTime'])
     # Form the message text:
     msg_text = "Alarm expression \"%s\" evaluated True at %s\nRecord:\n%s" % (self.expression, t_str, str(rec))
     # Convert to MIME:
     msg = MIMEText(msg_text)
     
     # Fill in MIME headers:
     msg['Subject'] = "Alarm message from weewx"
     msg['From']    = "weewx"
     msg['To']      = self.TO
     
     # Create an instance of class SMTP for the given SMTP host:
     s = smtplib.SMTP(self.smtp_host)
     try:
         # Some servers (eg, gmail) require encrypted transport.
         # Be prepared to catch an exception if the server
         # doesn't support it.
         s.ehlo()
         s.starttls()
         s.ehlo()
     except smtplib.SMTPException:
         pass
     # If a username has been given, assume that login is required for this host:
     if self.smtp_user:
         s.login(self.smtp_user, self.smtp_password)
     # Send the email:
     s.sendmail(msg['From'], [self.TO],  msg.as_string())
     # Log out of the server:
     s.quit()
     # Log it in the system log:
     syslog.syslog(syslog.LOG_INFO, "alarm: Alarm sounded for expression: \"%s\"" % self.expression)
     syslog.syslog(syslog.LOG_INFO, "       *** email sent to: %s" % self.TO)
Example #34
0
    def run(self):
        """Main entry point for calculating missing derived fields.

        Calculate the missing derived fields for the timespan concerned, save
        the calculated data to archive and recalculate the daily summaries.
        """

        # record the current time
        t1 = time.time()

        # Instantiate a dummy engine, to be used to calculate derived variables. This will
        # cause all the xtype services to get loaded.
        engine = weewx.engine.DummyEngine(self.config_dict)
        # While the above instantiated an instance of StdWXCalculate, we have no way of
        # retrieving it. So, instantiate another one, then use that to calculate derived types.
        wxcalculate = weewx.wxservices.StdWXCalculate(engine, self.config_dict)

        # initialise some counters so we know what we have processed
        days_updated = 0
        days_processed = 0
        total_records_processed = 0
        total_records_updated = 0

        # obtain gregorian days for our start and stop timestamps
        start_greg = weeutil.weeutil.toGregorianDay(self.start_ts)
        stop_greg = weeutil.weeutil.toGregorianDay(self.stop_ts)
        # start at the first day
        day = start_greg
        while day <= stop_greg:
            # get the start and stop timestamps for this tranche
            tr_start_ts = weeutil.weeutil.startOfGregorianDay(day)
            tr_stop_ts = min(weeutil.weeutil.startOfGregorianDay(stop_greg + 1),
                             weeutil.weeutil.startOfGregorianDay(day + self.trans_days))
            # start the transaction
            with weedb.Transaction(self.dbm.connection) as _cursor:
                # iterate over each day in the tranche we are to work in
                for tranche_day in weeutil.weeutil.genDaySpans(tr_start_ts, tr_stop_ts):
                    # initialise a counter for records processed on this day
                    records_updated = 0
                    # iterate over each record in this day
                    for record in self.dbm.genBatchRecords(startstamp=tranche_day.start,
                                                           stopstamp=tranche_day.stop):
                        # but we are only concerned with records after the
                        # start and before or equal to the stop timestamps
                        if self.start_ts < record['dateTime'] <= self.stop_ts:
                            # first obtain a list of the fields that may be calculated
                            extras_list = []
                            for obs in wxcalculate.calc_dict:
                                directive = wxcalculate.calc_dict[obs]
                                if directive == 'software' \
                                        or directive == 'prefer_hardware' \
                                        and (obs not in record or record[obs] is None):
                                    extras_list.append(obs)

                            # calculate the missing derived fields for the record
                            wxcalculate.do_calculations(record)

                            # Obtain a new record dictionary that contains only those items
                            # that wxcalculate calculated. Use dictionary comprehension.
                            extras_dict = {k:v for (k,v) in record.items() if k in extras_list}

                            # update the archive with the calculated data
                            records_updated += self.update_record_fields(record['dateTime'],
                                                                         extras_dict)
                            # update the total records processed
                            total_records_processed += 1
                        # Give the user some information on progress
                        if total_records_processed % 1000 == 0:
                            p_msg = "Processing record: %d; Last record: %s" % (total_records_processed,
                                                                                timestamp_to_string(record['dateTime']))
                            self._progress(p_msg)
                    # update the total records updated
                    total_records_updated += records_updated
                    # if we updated any records on this day increment the count
                    # of days updated
                    days_updated += 1 if records_updated > 0 else 0
                    days_processed += 1
            # advance to the next tranche
            day += self.trans_days
        # finished, so give the user some final information on progress, mainly
        # so the total tallies with the log
        p_msg = "Processing record: %d; Last record: %s" % (total_records_processed,
                                                            timestamp_to_string(tr_stop_ts))
        self._progress(p_msg, overprint=False)
        # now update the daily summaries, but only if this is not a dry run
        if not self.dry_run:
            print("Recalculating daily summaries...")
            # first we need a start and stop date object
            start_d = datetime.date.fromtimestamp(self.start_ts)
            # Since each daily summary is identified by the midnight timestamp
            # for that day we need to make sure we our stop timestamp is not on
            # a midnight boundary or we will rebuild the following days sumamry
            # as well. if it is on a midnight boundary just subtract 1 second
            # and use that.
            summary_stop_ts = self.stop_ts
            if weeutil.weeutil.isMidnight(self.stop_ts):
                summary_stop_ts -= 1
            stop_d = datetime.date.fromtimestamp(summary_stop_ts)
            # do the update
            self.dbm.backfill_day_summary(start_d=start_d, stop_d=stop_d)
            print(file=sys.stdout)
            print("Finished recalculating daily summaries")
        else:
            # it's a dry run so say the rebuild was skipped
            print("This is a dry run, recalculation of daily summaries was skipped")
        tdiff = time.time() - t1
        # we are done so log and inform the user
        _day_processed_str = "day" if days_processed == 1 else "days"
        _day_updated_str = "day" if days_updated == 1 else "days"
        if not self.dry_run:
            log.info("Processed %d %s consisting of %d records. "
                     "%d %s consisting of %d records were updated "
                     "in %0.2f seconds." % (days_processed,
                                            _day_processed_str,
                                            total_records_processed,
                                            days_updated,
                                            _day_updated_str,
                                            total_records_updated,
                                            tdiff))
        else:
            # this was a dry run
            log.info("Processed %d %s consisting of %d records. "
                     "%d %s consisting of %d records would have been updated "
                     "in %0.2f seconds." % (days_processed,
                                            _day_processed_str,
                                            total_records_processed,
                                            days_updated,
                                            _day_updated_str,
                                            total_records_updated,
                                            tdiff))
Example #35
0
    def do_fix(self, np_ts):
        """Apply the interval weighting fix to the daily summaries."""

        # do we need to weight? Only weight if next day to weight ts is None or
        # there are records in the archive from that day
        if np_ts is None or self.dbm.last_timestamp > np_ts:
            t1 = time.time()
            syslog.syslog(syslog.LOG_INFO, "intervalweighting: Applying %s..." % self.name)
            _days = 0
            # Get the earliest daily summary ts and the obs that it came from
            first_ts, obs = self.first_summary()
            # Get the start and stop ts for our first transaction days
            _tr_start_ts = np_ts if np_ts is not None else first_ts
            _tr_stop_dt = datetime.datetime.fromtimestamp(_tr_start_ts) + datetime.timedelta(days=self.trans_days)
            _tr_stop_ts = time.mktime(_tr_stop_dt.timetuple())
            _tr_stop_ts = min(startOfDay(self.dbm.last_timestamp), _tr_stop_ts)
            last_start = None
            while True:
                with weedb.Transaction(self.dbm.connection) as _cursor:
                    for _day_span in self.genSummaryDaySpans(_tr_start_ts, _tr_stop_ts, obs):
                        # Get the weight to be applied for the day
                        _weight = self.get_interval(_day_span) * 60
                        # Get the current day stats in an accumulator
                        _day_accum = self.dbm._get_day_summary(_day_span.start)
                        # Set the unit system for the accumulator
                        _day_accum.unit_system = self.dbm.std_unit_system
                        # Weight the necessary accumulator stats, use a
                        # try..except in case something goes wrong
                        last_key = None
                        try:
                            for _day_key in self.dbm.daykeys:
                                last_key = _day_key
                                _day_accum[_day_key].wsum *= _weight
                                _day_accum[_day_key].sumtime *= _weight
                                # Do we have a vecstats accumulator?
                                if hasattr(_day_accum[_day_key], 'wsquaresum'):
                                    # Yes, so update the weighted vector stats
                                    _day_accum[_day_key].wsquaresum *= _weight
                                    _day_accum[_day_key].xsum *= _weight
                                    _day_accum[_day_key].ysum *= _weight
                                    _day_accum[_day_key].dirsumtime *= _weight
                        except Exception, e:
                            # log the exception and re-raise it
                            syslog.syslog(syslog.LOG_INFO,
                                          "intervalweighting: Interval weighting of '%s' daily summary "
                                          "for %s failed: %s" %
                                          (last_key, timestamp_to_string(_day_span.start, format="%Y-%m-%d"), e))
                            raise
                        # Update the daily summary with the weighted accumulator
                        if not self.dry_run:
                            self.dbm._set_day_summary(_day_accum, None, _cursor)
                        _days += 1
                        # Save the ts of the weighted daily summary as the
                        # 'lastWeightPatch' value in the archive_day__metadata
                        # table
                        if not self.dry_run:
                            self.dbm._write_metadata('lastWeightPatch',
                                                     _day_span.start,
                                                     _cursor)
                        # Give the user some information on progress
                        if _days % 50 == 0:
                            self._progress(_days, _day_span.start)
                        last_start = _day_span.start

                    # Setup our next tranche
                    # Have we reached the end, if so break to finish
                    if _tr_stop_ts >= startOfDay(self.dbm.last_timestamp):
                        break
                    # More to process so set our start and stop for the next
                    # transaction
                    _tr_start_dt = datetime.datetime.fromtimestamp(_tr_stop_ts) + datetime.timedelta(days=1)
                    _tr_start_ts = time.mktime(_tr_start_dt.timetuple())
                    _tr_stop_dt = datetime.datetime.fromtimestamp(_tr_start_ts) + datetime.timedelta(days=self.trans_days)
                    _tr_stop_ts = time.mktime(_tr_stop_dt.timetuple())
                    _tr_stop_ts = min(self.dbm.last_timestamp, _tr_stop_ts)

            # We have finished. Get rid of the no longer needed lastWeightPatch
            with weedb.Transaction(self.dbm.connection) as _cursor:
                _cursor.execute("DELETE FROM %s_day__metadata WHERE name=?" % self.dbm.table_name, ('lastWeightPatch',))

            # Give the user some final information on progress,
            # mainly so the total tallies with the log
            self._progress(_days, last_start)
            print >>sys.stdout
            tdiff = time.time() - t1
            # We are done so log and inform the user
            syslog.syslog(syslog.LOG_INFO,
                          "intervalweighting: calculated weighting for %s days in %0.2f seconds." % (_days, tdiff))
            if self.dry_run:
                syslog.syslog(syslog.LOG_INFO, "intervalweighting: This was a dry run. %s was not applied." % self.name)
Example #36
0
    def soundTheAlarm(self, timestamp, battery_flags, alarm_count):
        """This function is called when the alarm has been triggered."""
        
        # Get the time and convert to a string:
        t_str = timestamp_to_string(timestamp)

        # Log it in the system log:
        syslog.syslog(syslog.LOG_INFO, "lowBattery: Low battery status sounded at %s: %s" % (t_str, battery_flags))

        # Form the message text:
        indicator_strings = []
        for bat in battery_flags:
            indicator_strings.append("%s: %04x" % (bat, battery_flags[bat]))
        msg_text = """
The low battery indicator has been seen %d times since the last archive period.

Alarm sounded at %s

Low battery indicators:
%s

""" % (alarm_count, t_str, '\n'.join(indicator_strings))
        # Convert to MIME:
        msg = MIMEText(msg_text)
        
        # Fill in MIME headers:
        msg['Subject'] = self.SUBJECT
        msg['From']    = self.FROM
        msg['To']      = ','.join(self.TO)
        
        # Create an instance of class SMTP for the given SMTP host:
        s = smtplib.SMTP(self.smtp_host)
        try:
            # Some servers (eg, gmail) require encrypted transport.
            # Be prepared to catch an exception if the server
            # doesn't support it.
            s.ehlo()
            s.starttls()
            s.ehlo()
            syslog.syslog(syslog.LOG_DEBUG,
                          "lowBattery: using encrypted transport")
        except smtplib.SMTPException:
            syslog.syslog(syslog.LOG_DEBUG,
                          "lowBattery: using unencrypted transport")

        try:
            # If a username has been given, assume that login is required
            # for this host:
            if self.smtp_user:
                s.login(self.smtp_user, self.smtp_password)
                syslog.syslog(syslog.LOG_DEBUG,
                              "lowBattery: logged in as %s" % self.smtp_user)

            # Send the email:
            s.sendmail(msg['From'], self.TO,  msg.as_string())
            # Log out of the server:
            s.quit()
        except Exception, e:
            syslog.syslog(syslog.LOG_ERR,
                          "lowBattery: send email failed: %s" % (e,))
            raise
Example #37
0
    def __init__(self, config_dict, config_path, csv_config_dict, import_config_path, options, log):

        # call our parents __init__
        super(CSVSource, self).__init__(config_dict,
                                        csv_config_dict,
                                        options,
                                        log)

        # save our import config path
        self.import_config_path = import_config_path
        # save our import config dict
        self.csv_config_dict = csv_config_dict

        # get a few config settings from our CSV config dict
        # string format used to decode the imported field holding our dateTime
        self.raw_datetime_format = self.csv_config_dict.get('raw_datetime_format',
                                                            '%Y-%m-%d %H:%M:%S')
        # is our rain discrete or cumulative
        self.rain = self.csv_config_dict.get('rain', 'cumulative')
        # determine valid range for imported wind direction
        _wind_direction = option_as_list(self.csv_config_dict.get('wind_direction',
                                                                  '0,360'))
        try:
            if float(_wind_direction[0]) <= float(_wind_direction[1]):
                self.wind_dir = [float(_wind_direction[0]),
                                 float(_wind_direction[1])]
            else:
                self.wind_dir = [-360, 360]
        except:
            self.wind_dir = [-360, 360]
        # get our source file path
        try:
            self.source = csv_config_dict['file']
        except KeyError:
            raise weewx.ViolatedPrecondition("CSV source file not specified in '%s'." % import_config_path)
        # initialise our import field-to-weewx archive field map
        self.map = None
        # initialise some other properties we will need
        self.start = 1
        self.end = 1
        self.increment = 1

        # tell the user/log what we intend to do
        _msg = "A CSV import from source file '%s' has been requested." % self.source
        self.wlog.printlog(syslog.LOG_INFO, _msg)
        _msg = "The following options will be used:"
        self.wlog.verboselog(syslog.LOG_DEBUG, _msg)
        _msg = "     config=%s, import-config=%s" % (config_path,
                                                     self.import_config_path)
        self.wlog.verboselog(syslog.LOG_DEBUG, _msg)
        if options.date:
            _msg = "     source=%s, date=%s" % (self.source, options.date)
        else:
            # we must have --from and --to
            _msg = "     source=%s, from=%s, to=%s" % (self.source,
                                                       options.date_from,
                                                       options.date_to)
        self.wlog.verboselog(syslog.LOG_DEBUG, _msg)
        _msg = "     dry-run=%s, calc-missing=%s" % (self.dry_run,
                                                     self.calc_missing)
        self.wlog.verboselog(syslog.LOG_DEBUG, _msg)
        _msg = "     tranche=%s, interval=%s, date/time_string_format=%s" % (self.tranche,
                                                                             self.interval,
                                                                             self.raw_datetime_format)
        self.wlog.verboselog(syslog.LOG_DEBUG, _msg)
        _msg = "     rain=%s, wind_direction=%s" % (self.rain, self.wind_dir)
        self.wlog.verboselog(syslog.LOG_DEBUG, _msg)
        _msg = "     UV=%s, radiation=%s" % (self.UV_sensor, self.solar_sensor)
        self.wlog.verboselog(syslog.LOG_DEBUG, _msg)
        _msg = "Using database binding '%s', which is bound to database '%s'" % (self.db_binding_wx,
                                                                                 self.dbm.database_name)
        self.wlog.printlog(syslog.LOG_INFO, _msg)
        _msg = "Destination table '%s' unit system is '%#04x' (%s)." % (self.dbm.table_name,
                                                                        self.archive_unit_sys,
                                                                        unit_nicknames[self.archive_unit_sys])
        self.wlog.printlog(syslog.LOG_INFO, _msg)
        if self.calc_missing:
            print "Missing derived observations will be calculated."
        if not self.UV_sensor:
            print "All weewx UV fields will be set to None."
        if not self.solar_sensor:
            print "All weewx radiation fields will be set to None."
        if options.date or options.date_from:
            print "Observations timestamped after %s and up to and" % (timestamp_to_string(self.first_ts), )
            print "including %s will be imported." % (timestamp_to_string(self.last_ts), )
        if self.dry_run:
            print "This is a dry run, imported data will not be saved to archive."
Example #38
0
    def _progress(ndays, last_time):
        """Utility function to show our progress while processing the fix."""

        print >>sys.stdout, "Weighting daily summary: %d; Timestamp: %s\r" % \
            (ndays, timestamp_to_string(last_time, format_str="%Y-%m-%d")),
        sys.stdout.flush()
Example #39
0
    def fill_in_packet(cfg: Configuration, packet: Dict):
        with cfg.lock:
            log.debug('new_loop_packet: cfg.concentrations: %s' % cfg.concentrations)
            if cfg.concentrations is not None and \
                    cfg.concentrations.timestamp is not None and \
                    cfg.concentrations.timestamp + \
                    cfg.archive_interval >= time.time():
                log.debug('Time of reading being inserted: %s' % timestamp_to_string(cfg.concentrations.timestamp))
                # Insert pm1_0, pm2_5, pm10_0, aqi and aqic into loop packet.
                if cfg.concentrations.pm_1_last is not None:
                    packet['pm1_0'] = cfg.concentrations.pm_1_last
                    log.debug('Inserted packet[pm1_0]: %f into packet.' % cfg.concentrations.pm_1_last)
                if (cfg.concentrations.pm_2p5_last is not None
                        and cfg.concentrations.hum is not None
                        and cfg.concentrations.temp is not None):
                    packet['pm2_5'] = AQI.compute_pm_2p5_us_epa_correction(
                            cfg.concentrations.pm_2p5_last, cfg.concentrations.hum, cfg.concentrations.temp)
                    log.debug('Inserted packet[pm2_5]: %f into packet.' % cfg.concentrations.pm_2p5_last)
                    # Put aqi and color in the packet.
                    packet['pm2_5_aqi'] = AQI.compute_pm2_5_aqi(packet['pm2_5'])
                    packet['pm2_5_aqi_color'] = AQI.compute_pm2_5_aqi_color(packet['pm2_5_aqi'])
                if cfg.concentrations.pm_10_last is not None:
                    packet['pm10_0'] = cfg.concentrations.pm_10_last
                    log.debug('Inserted packet[pm10_0]: %f into packet.' % cfg.concentrations.pm_10_last)

                # Also insert one minute averages as these averages are more useful for showing in realtime.
                # If 1m averages are not available, use last instead.
                if cfg.concentrations.pm_1 is not None:
                    packet['pm1_0_1m']       = cfg.concentrations.pm_1
                elif cfg.concentrations.pm_1_last is not None:
                    packet['pm1_0_1m']       = cfg.concentrations.pm_1_last
                if cfg.concentrations.pm_2p5 is not None:
                    packet['pm2_5_1m']       = AQI.compute_pm_2p5_us_epa_correction(
                            cfg.concentrations.pm_2p5, cfg.concentrations.hum, cfg.concentrations.temp)
                elif (cfg.concentrations.pm_2p5_last is not None
                        and cfg.concentrations.hum is not None
                        and cfg.concentrations.temp is not None):
                    packet['pm2_5_1m']       = AQI.compute_pm_2p5_us_epa_correction(
                            cfg.concentrations.pm_2p5_last, cfg.concentrations.hum, cfg.concentrations.temp)
                if cfg.concentrations.pm_10 is not None:
                    packet['pm10_0_1m']      = cfg.concentrations.pm_10
                elif cfg.concentrations.pm_10_last is not None:
                    packet['pm10_0_1m']      = cfg.concentrations.pm_10_last

                # Add 1m aqi and color
                if 'pm2_5_1m' in packet:
                    packet['pm2_5_1m_aqi'] = AQI.compute_pm2_5_aqi(packet['pm2_5_1m'])
                    packet['pm2_5_1m_aqi_color'] = AQI.compute_pm2_5_aqi_color(packet['pm2_5_1m_aqi'])

                # And insert nowcast for pm 2.5 and 10 as some might want to report that.
                # If nowcast not available, don't substitute.
                if (cfg.concentrations.pm_2p5_nowcast is not None
                        and cfg.concentrations.hum is not None
                        and cfg.concentrations.temp is not None):
                    packet['pm2_5_nowcast']  = AQI.compute_pm_2p5_us_epa_correction(
                            cfg.concentrations.pm_2p5_nowcast, cfg.concentrations.hum, cfg.concentrations.temp)
                    packet['pm2_5_nowcast_aqi'] = AQI.compute_pm2_5_aqi(packet['pm2_5_nowcast'])
                    packet['pm2_5_nowcast_aqi_color'] = AQI.compute_pm2_5_aqi_color(packet['pm2_5_nowcast_aqi'])
                if cfg.concentrations.pm_10_nowcast is not None:
                    packet['pm10_0_nowcast'] = cfg.concentrations.pm_10_nowcast
            else:
                log.error('Found no concentrations to insert.')
Example #40
0
    def run(self):
        """Main entry point for calculating missing derived fields.

        Calculate the missing derived fields for the timespan concerned, save
        the calculated data to archive and recalculate the daily summaries.
        """

        # record the current time
        t1 = time.time()
        # obtain a wxservices.WXCalculate object to calculate the missing fields
        # first we need station altitude, latitude and longitude
        stn_dict = self.config_dict['Station']
        altitude_t = option_as_list(stn_dict.get('altitude', (None, None)))
        try:
            altitude_vt = weewx.units.ValueTuple(float(altitude_t[0]),
                                                 altitude_t[1],
                                                 "group_altitude")
        except KeyError as e:
            raise weewx.ViolatedPrecondition(
                "Value 'altitude' needs a unit (%s)" % e)
        latitude_f = float(stn_dict['latitude'])
        longitude_f = float(stn_dict['longitude'])

        # now we can create a WXCalculate object
        wxcalculate = weewx.wxservices.WXCalculate(self.config_dict,
                                                   altitude_vt,
                                                   latitude_f,
                                                   longitude_f)

        # initialise some counters so we know what we have processed
        days_updated = 0
        days_processed = 0
        total_records_processed = 0
        total_records_updated = 0

        # obtain gregorian days for our start and stop timestamps
        start_greg = weeutil.weeutil.toGregorianDay(self.start_ts)
        stop_greg = weeutil.weeutil.toGregorianDay(self.stop_ts)
        # start at the first day
        day = start_greg
        while day <= stop_greg:
            # get the start and stop timestamps for this tranche
            tr_start_ts = weeutil.weeutil.startOfGregorianDay(day)
            tr_stop_ts = min(weeutil.weeutil.startOfGregorianDay(stop_greg + 1),
                             weeutil.weeutil.startOfGregorianDay(day + self.trans_days))
            # start the transaction
            with weedb.Transaction(self.dbm.connection) as _cursor:
                # iterate over each day in the tranche we are to work in
                for tranche_day in weeutil.weeutil.genDaySpans(tr_start_ts, tr_stop_ts):
                    # initialise a counter for records processed on this day
                    records_updated = 0
                    # iterate over each record in this day
                    for record in self.dbm.genBatchRecords(startstamp=tranche_day.start,
                                                           stopstamp=tranche_day.stop):
                        # but we are only concerned with records after the
                        # start and before or equal to the stop timestamps
                        if self.start_ts < record['dateTime'] <= self.stop_ts:
                            # first obtain a list of the fields that may be calculated
                            extras_list = []
                            for obs in wxcalculate.svc_dict['Calculations']:
                                directive = wxcalculate.svc_dict['Calculations'][obs]
                                if directive == 'software' \
                                        or directive == 'prefer_hardware' and (
                                        obs not in record or record[obs] is None):
                                    extras_list.append(obs)

                            # calculate the missing derived fields for the record
                            wxcalculate.do_calculations(data_dict=record,
                                                        data_type='archive')
                            # Obtain a dict containing only those fields that
                            # WXCalculate calculated. We could do this as a
                            # dictionary comprehension but python2.6 does not
                            # support dictionary comprehensions.
                            extras_dict = {}
                            for k in extras_list:
                                if k in record.keys():
                                    extras_dict[k] = record[k]
                            # update the archive with the calculated data
                            records_updated += self.update_record_fields(record['dateTime'],
                                                                         extras_dict)
                            # update the total records processed
                            total_records_processed += 1
                        # Give the user some information on progress
                        if total_records_processed % 1000 == 0:
                            p_msg = "Processing record: %d; Last record: %s" % (total_records_processed,
                                                                                timestamp_to_string(record['dateTime']))
                            self._progress(p_msg)
                    # update the total records updated
                    total_records_updated += records_updated
                    # if we updated any records on this day increment the count
                    # of days updated
                    days_updated += 1 if records_updated > 0 else 0
                    days_processed += 1
            # advance to the next tranche
            day += self.trans_days
        # finished, so give the user some final information on progress, mainly
        # so the total tallies with the log
        p_msg = "Processing record: %d; Last record: %s" % (total_records_processed,
                                                            timestamp_to_string(tr_stop_ts))
        self._progress(p_msg, overprint=False)
        # now update the daily summaries, but only if this is not a dry run
        if not self.dry_run:
            print("Recalculating daily summaries...")
            # first we need a start and stop date object
            start_d = datetime.date.fromtimestamp(self.start_ts)
            # Since each daily summary is identified by the midnight timestamp
            # for that day we need to make sure we our stop timestamp is not on
            # a midnight boundary or we will rebuild the following days sumamry
            # as well. if it is on a midnight boundary just subtract 1 second
            # and use that.
            summary_stop_ts = self.stop_ts
            if weeutil.weeutil.isMidnight(self.stop_ts):
                summary_stop_ts -= 1
            stop_d = datetime.date.fromtimestamp(summary_stop_ts)
            # do the update
            self.dbm.backfill_day_summary(start_d=start_d, stop_d=stop_d)
            print(file=sys.stdout)
            print("Finished recalculating daily summaries")
        else:
            # it's a dry run so say the rebuild was skipped
            print("This is a dry run, recalculation of daily summaries was skipped")
        tdiff = time.time() - t1
        # we are done so log and inform the user
        _day_processed_str = "day" if days_processed == 1 else "days"
        _day_updated_str = "day" if days_updated == 1 else "days"
        if not self.dry_run:
            log.info("Processed %d %s consisting of %d records. "
                     "%d %s consisting of %d records were updated "
                     "in %0.2f seconds." % (days_processed,
                                            _day_processed_str,
                                            total_records_processed,
                                            days_updated,
                                            _day_updated_str,
                                            total_records_updated,
                                            tdiff))
        else:
            # this was a dry run
            log.info("Processed %d %s consisting of %d records. "
                     "%d %s consisting of %d records would have been updated "
                     "in %0.2f seconds." % (days_processed,
                                            _day_processed_str,
                                            total_records_processed,
                                            days_updated,
                                            _day_updated_str,
                                            total_records_updated,
                                            tdiff))
Example #41
0
                    S3upload_message = "uploaded %d files (%s bytes) in %%0.2f seconds" % (int(file_cnt), byte_cnt)
                else:
                    S3upload_message = "executed in %0.2f seconds"
            except:
                S3upload_message = "executed in %0.2f seconds"
        else:
            # suspect we have an s3cmd error so display a message
            syslog.syslog(syslog.LOG_INFO, "s3uploadgenerator: s3cmd reported errors")
            for line in iter(stroutput.splitlines()):
                syslog.syslog(syslog.LOG_INFO, "s3uploadgenerator: s3cmd error: %s" % line)
            S3upload_message = "executed in %0.2f seconds"
        
        stop_ts = time.time()
        syslog.syslog(syslog.LOG_INFO, "s3uploadgenerator: results: "  + S3upload_message % (stop_ts - start_ts))

        t_str = timestamp_to_string(stop_ts)
        syslog.syslog(syslog.LOG_DEBUG, "s3uploadgenerator: end upload at %s" % t_str)

if __name__ == '__main__':
    """This section is used for testing the code. """
    # Note that this fails!
    import sys
    import configobj
    from optparse import OptionParser


    usage_string ="""Usage: 
    
    S3upload.py config_path 
    
    Arguments:
Example #42
0
    def __init__(self, config_dict, config_path, wd_config_dict,
                 import_config_path, options):

        # call our parents __init__
        super(WDSource, self).__init__(config_dict, wd_config_dict, options)

        # save the import config path
        self.import_config_path = import_config_path
        # save the import config dict
        self.wd_config_dict = wd_config_dict

        # our parent uses 'derive' as the default interval setting, for WD the
        # default should be 1 (minute) so redo the interval setting with our
        # default
        self.interval = wd_config_dict.get('interval', 1)

        # wind dir bounds
        self.wind_dir = [0, 360]

        # How the WeeWX field 'rain' is populated depends on the source rain
        # data. If the only data available is cumulative then the WeeWX rain
        # field is calculated as the difference between successive cumulative
        # values. WD provides a rain per interval field so that data can be
        # used to map directly to the WeeWX rain field. If rain is to be
        # calculated from a cumulative value then self.rain must be set to
        # 'cumulative', to map directly to the WeeWX rain field self.rain must
        # be set to None.
        self.rain = None

        # field delimiter used in text format monthly log files, default to
        # space
        self.txt_delimiter = str(wd_config_dict.get('txt_delimiter', ' '))
        # field delimiter used in csv format monthly log files, default to
        # comma
        self.csv_delimiter = str(wd_config_dict.get('csv_delimiter', ','))

        # decimal separator used in monthly log files, default to decimal point
        self.decimal = wd_config_dict.get('decimal', '.')

        # ignore extreme > 255.0 values for temperature and humidity fields
        self.ignore_extreme_temp_hum = weeutil.weeutil.tobool(
            wd_config_dict.get('ignore_extreme_temp_hum', True))

        # initialise the import field-to-WeeWX archive field map
        self.map = None

        # property holding the current log file name being processed
        self.file_name = None

        # WD logs use either US or Metric units. The units used in each case
        # are:
        # Metric units: C, knots, hPa, mm
        # US units: F, mph, inHg, inch
        #
        # The user must specify the units to be used in the import config file.
        # This can be by either by specifying the log units as Metric or US
        # using the 'units' config option. Alternatively temperature, pressure,
        # rainfall and speed units can be specified individually under the
        # [Units] stanza. First check for a valid 'units' config option then
        # check for individual group units. Do some basic error checking and
        # validation, if one of the fields is missing or invalid then we need
        # to catch the error and raise it as we can't go on.
        log_unit_config = wd_config_dict.get('Units')
        if log_unit_config is not None:
            # get the units config option
            log_unit_sys = wd_config_dict['Units'].get('units')
            # accept any capitalization of USA as == US
            log_unit_sys = log_unit_sys if log_unit_sys.upper(
            ) != 'USA' else 'US'
            # does the units config option specify a valid log unit system
            if log_unit_sys is None or log_unit_sys.upper() not in [
                    'METRIC', 'US'
            ]:
                # log unit system not specified look for individual entries
                # temperature
                temp_u = wd_config_dict['Units'].get('temperature')
                if temp_u is not None:
                    if temp_u in weewx.units.default_unit_format_dict:
                        self._header_map['temperature']['units'] = temp_u
                        self._header_map['dewpoint']['units'] = temp_u
                        self._header_map['heatindex']['units'] = temp_u
                        self._header_map['soiltemp']['units'] = temp_u
                        self._header_map['temp1']['units'] = temp_u
                        self._header_map['temp2']['units'] = temp_u
                        self._header_map['temp3']['units'] = temp_u
                        self._header_map['temp4']['units'] = temp_u
                        self._header_map['temp5']['units'] = temp_u
                        self._header_map['temp6']['units'] = temp_u
                        self._header_map['temp7']['units'] = temp_u
                    else:
                        _msg = "Unknown units '%s' specified for Weather Display " \
                               "temperature fields in %s." % (temp_u, self.import_config_path)
                        raise weewx.UnitError(_msg)
                else:
                    _msg = "No units specified for Weather Display temperature " \
                           "fields in %s." % (self.import_config_path,)
                    raise weewx.UnitError(_msg)

                # pressure
                press_u = wd_config_dict['Units'].get('pressure')
                if press_u is not None:
                    if press_u in ['inHg', 'hPa']:
                        self._header_map['barometer']['units'] = press_u
                    else:
                        _msg = "Unknown units '%s' specified for Weather Display " \
                               "pressure fields in %s." % (press_u, self.import_config_path)
                        raise weewx.UnitError(_msg)
                else:
                    _msg = "No units specified for Weather Display pressure " \
                           "fields in %s." % (self.import_config_path,)
                    raise weewx.UnitError(_msg)

                # rain
                rain_u = wd_config_dict['Units'].get('rain')
                if rain_u is not None:
                    if rain_u in ['inch', 'mm']:
                        self._header_map['rainlastmin']['units'] = rain_u
                        self._header_map['dailyrain']['units'] = rain_u
                        self._header_map['monthlyrain']['units'] = rain_u
                        self._header_map['yearlyrain']['units'] = rain_u
                        self._header_map['dailyet']['units'] = rain_u
                    else:
                        _msg = "Unknown units '%s' specified for Weather Display " \
                               "rain fields in %s." % (rain_u, self.import_config_path)
                        raise weewx.UnitError(_msg)
                else:
                    _msg = "No units specified for Weather Display rain fields " \
                           "in %s." % (self.import_config_path,)
                    raise weewx.UnitError(_msg)

                # speed
                speed_u = wd_config_dict['Units'].get('speed')
                if speed_u is not None:
                    if speed_u in ['inch', 'mm']:
                        self._header_map['windspeed']['units'] = speed_u
                        self._header_map['gustspeed']['units'] = speed_u
                    else:
                        _msg = "Unknown units '%s' specified for Weather Display " \
                               "speed fields in %s." % (speed_u, self.import_config_path)
                        raise weewx.UnitError(_msg)
                else:
                    _msg = "No units specified for Weather Display speed fields " \
                           "in %s." % (self.import_config_path,)
                    raise weewx.UnitError(_msg)

            else:
                # log unit system specified
                _unit_sys = log_unit_sys.upper()
                # do we have a valid log unit system
                if _unit_sys in ['METRIC', 'US']:
                    # valid log unit system so assign units as applicable
                    self._header_map['temperature'][
                        'units'] = self.wd_unit_sys['temperature'][_unit_sys]
                    self._header_map['dewpoint']['units'] = self.wd_unit_sys[
                        'temperature'][_unit_sys]
                    self._header_map['heatindex']['units'] = self.wd_unit_sys[
                        'temperature'][_unit_sys]
                    self._header_map['barometer']['units'] = self.wd_unit_sys[
                        'barometer'][_unit_sys]
                    self._header_map['windspeed']['units'] = self.wd_unit_sys[
                        'windspeed'][_unit_sys]
                    self._header_map['gustspeed']['units'] = self.wd_unit_sys[
                        'gustspeed'][_unit_sys]
                    self._header_map['rainlastmin'][
                        'units'] = self.wd_unit_sys['rainlastmin'][_unit_sys]
                    self._header_map['soiltemp']['units'] = self.wd_unit_sys[
                        'soiltemp'][_unit_sys]
                    for _num in range(1, 8):
                        _temp = 'temp%s' % _num
                        self._header_map[_temp]['units'] = self.wd_unit_sys[
                            _temp][_unit_sys]
                else:
                    # no valid Units config found, we can't go on so raise an error
                    raise weewx.UnitError(
                        "Invalid setting for 'units' config option.")
        else:
            # there is no Units config, we can't go on so raise an error
            raise weewx.UnitError("No Weather Display units config found.")

        # obtain a list of logs files to be processed
        _to_process = wd_config_dict.get('logs_to_process',
                                         list(self.logs.keys()))
        self.logs_to_process = weeutil.weeutil.option_as_list(_to_process)

        # can missing log files be ignored
        self.ignore_missing_log = weeutil.weeutil.to_bool(
            wd_config_dict.get('ignore_missing_log', True))

        # get our source file path
        try:
            self.source = wd_config_dict['directory']
        except KeyError:
            _msg = "Weather Display monthly logs directory not specified in '%s'." % import_config_path
            raise weewx.ViolatedPrecondition(_msg)

        # get the source file encoding, default to utf-8-sig
        self.source_encoding = self.wd_config_dict.get('source_encoding',
                                                       'utf-8-sig')

        # Now get a list on monthly log files sorted from oldest to newest.
        # This is complicated by the log file naming convention used by WD.
        # first the 1 digit months
        _lg_5_list = glob.glob(self.source + '/' + '[0-9]' * 5 + 'lg.txt')
        # and the 2 digit months
        _lg_6_list = glob.glob(self.source + '/' + '[0-9]' * 6 + 'lg.txt')
        # concatenate the two lists to get the complete list
        month_lg_list = _lg_5_list + _lg_6_list
        # create a list of log files in chronological order (month, year)
        _temp = []
        # create a list of log files, adding year and month fields for sorting
        for p in month_lg_list:
            # obtain the file name
            fn = os.path.split(p)[1]
            # obtain the numeric part of the file name
            _digits = ''.join(c for c in fn if c.isdigit())
            # append a list of format [path+file name, month, year]
            _temp.append([p, int(_digits[:-4]), int(_digits[-4:])])
        # now sort the list keeping just the log file path and name
        self.log_list = [
            a[0] for a in sorted(_temp, key=lambda el: (el[2], el[1]))
        ]
        # if there are no log files then there is nothing to be done
        if len(self.log_list) == 0:
            raise weeimport.WeeImportIOError(
                "No Weather Display monthly logs found in directory '%s'." %
                self.source)
        # Some log files have entries that belong in a different month.
        # Initialise a list to hold these extra records for processing during
        # the appropriate month
        self.extras = {}
        for l in self.logs_to_process:
            self.extras[l] = []

        # tell the user/log what we intend to do
        _msg = "Weather Display monthly log files in the '%s' directory will be imported" % self.source
        print(_msg)
        log.info(_msg)
        _msg = "The following options will be used:"
        if self.verbose:
            print(_msg)
        log.debug(_msg)
        _msg = "     config=%s, import-config=%s" % (config_path,
                                                     self.import_config_path)
        if self.verbose:
            print(_msg)
        log.debug(_msg)
        if options.date:
            _msg = "     date=%s" % options.date
        else:
            # we must have --from and --to
            _msg = "     from=%s, to=%s" % (options.date_from, options.date_to)
        if self.verbose:
            print(_msg)
        log.debug(_msg)
        _msg = "     dry-run=%s, calc_missing=%s, ignore_invalid_data=%s" % (
            self.dry_run, self.calc_missing, self.ignore_invalid_data)
        if self.verbose:
            print(_msg)
        log.debug(_msg)
        if log_unit_sys is not None and log_unit_sys.upper() in [
                'METRIC', 'US'
        ]:
            # valid unit system specified
            _msg = "     monthly logs are in %s units" % log_unit_sys.upper()
            if self.verbose:
                print(_msg)
            log.debug(_msg)
        else:
            # group units specified
            _msg = "     monthly logs use the following units:"
            if self.verbose:
                print(_msg)
            log.debug(_msg)
            _msg = "       temperature=%s pressure=%s" % (temp_u, press_u)
            if self.verbose:
                print(_msg)
            log.debug(_msg)
            _msg = "       rain=%s speed=%s" % (rain_u, speed_u)
            if self.verbose:
                print(_msg)
            log.debug(_msg)
        _msg = "     tranche=%s, interval=%s" % (self.tranche, self.interval)
        if self.verbose:
            print(_msg)
        log.debug(_msg)
        _msg = "     UV=%s, radiation=%s ignore extreme temperature and humidity=%s" % (
            self.UV_sensor, self.solar_sensor, self.ignore_extreme_temp_hum)
        if self.verbose:
            print(_msg)
        log.debug(_msg)
        _msg = "Using database binding '%s', which is bound to database '%s'" % (
            self.db_binding_wx, self.dbm.database_name)
        print(_msg)
        log.info(_msg)
        _msg = "Destination table '%s' unit system is '%#04x' (%s)." % (
            self.dbm.table_name, self.archive_unit_sys,
            unit_nicknames[self.archive_unit_sys])
        print(_msg)
        log.info(_msg)
        if self.calc_missing:
            print("Missing derived observations will be calculated.")
        if not self.UV_sensor:
            print("All WeeWX UV fields will be set to None.")
        if not self.solar_sensor:
            print("All WeeWX radiation fields will be set to None.")
        if options.date or options.date_from:
            print("Observations timestamped after %s and up to and" %
                  timestamp_to_string(self.first_ts))
            print("including %s will be imported." %
                  timestamp_to_string(self.last_ts))
        if self.dry_run:
            print(
                "This is a dry run, imported data will not be saved to archive."
            )
Example #43
0
    def do_fix(self, np_ts):
        """Apply the interval weighting fix to the daily summaries."""

        # do we need to weight? Only weight if next day to weight ts is None or
        # there are records in the archive from that day
        if np_ts is None or self.dbm.last_timestamp > np_ts:
            t1 = time.time()
            log.info("intervalweighting: Applying %s..." % self.name)
            _days = 0
            # Get the earliest daily summary ts and the obs that it came from
            first_ts, obs = self.first_summary()
            # Get the start and stop ts for our first transaction days
            _tr_start_ts = np_ts if np_ts is not None else first_ts
            _tr_stop_dt = datetime.datetime.fromtimestamp(_tr_start_ts) \
                + datetime.timedelta(days=self.trans_days)
            _tr_stop_ts = time.mktime(_tr_stop_dt.timetuple())
            _tr_stop_ts = min(startOfDay(self.dbm.last_timestamp), _tr_stop_ts)
            last_start = None
            while True:
                with weedb.Transaction(self.dbm.connection) as _cursor:
                    for _day_span in self.genSummaryDaySpans(_tr_start_ts, _tr_stop_ts, obs):
                        # Get the weight to be applied for the day
                        _weight = self.get_interval(_day_span) * 60
                        # Get the current day stats in an accumulator
                        _day_accum = self.dbm._get_day_summary(_day_span.start)
                        # Set the unit system for the accumulator
                        _day_accum.unit_system = self.dbm.std_unit_system
                        # Weight the necessary accumulator stats, use a
                        # try..except in case something goes wrong
                        last_key = None
                        try:
                            for _day_key in self.dbm.daykeys:
                                last_key = _day_key
                                _day_accum[_day_key].wsum *= _weight
                                _day_accum[_day_key].sumtime *= _weight
                                # Do we have a vecstats accumulator?
                                if hasattr(_day_accum[_day_key], 'wsquaresum'):
                                    # Yes, so update the weighted vector stats
                                    _day_accum[_day_key].wsquaresum *= _weight
                                    _day_accum[_day_key].xsum *= _weight
                                    _day_accum[_day_key].ysum *= _weight
                                    _day_accum[_day_key].dirsumtime *= _weight
                        except Exception as e:
                            # log the exception and re-raise it
                            log.info("intervalweighting: Interval weighting of '%s' daily summary "
                                     "for %s failed: %s"
                                     % (last_key, timestamp_to_string(_day_span.start,
                                                                      format_str="%Y-%m-%d"), e))
                            raise
                        # Update the daily summary with the weighted accumulator
                        if not self.dry_run:
                            self.dbm._set_day_summary(_day_accum, None, _cursor)
                        _days += 1
                        # Save the ts of the weighted daily summary as the
                        # 'lastWeightPatch' value in the archive_day__metadata
                        # table
                        if not self.dry_run:
                            self.dbm._write_metadata('lastWeightPatch',
                                                     _day_span.start,
                                                     _cursor)
                        # Give the user some information on progress
                        if _days % 50 == 0:
                            self._progress(_days, _day_span.start)
                        last_start = _day_span.start

                    # Setup our next tranche
                    # Have we reached the end, if so break to finish
                    if _tr_stop_ts >= startOfDay(self.dbm.last_timestamp):
                        break
                    # More to process so set our start and stop for the next
                    # transaction
                    _tr_start_dt = datetime.datetime.fromtimestamp(_tr_stop_ts) \
                        + datetime.timedelta(days=1)
                    _tr_start_ts = time.mktime(_tr_start_dt.timetuple())
                    _tr_stop_dt = datetime.datetime.fromtimestamp(_tr_start_ts) \
                        + datetime.timedelta(days=self.trans_days)
                    _tr_stop_ts = time.mktime(_tr_stop_dt.timetuple())
                    _tr_stop_ts = min(self.dbm.last_timestamp, _tr_stop_ts)

            # We have finished. Get rid of the no longer needed lastWeightPatch
            with weedb.Transaction(self.dbm.connection) as _cursor:
                _cursor.execute("DELETE FROM %s_day__metadata WHERE name=?"
                                % self.dbm.table_name, ('lastWeightPatch',))

            # Give the user some final information on progress,
            # mainly so the total tallies with the log
            self._progress(_days, last_start)
            print(file=sys.stdout)
            tdiff = time.time() - t1
            # We are done so log and inform the user
            log.info("intervalweighting: Calculated weighting "
                     "for %s days in %0.2f seconds." % (_days, tdiff))
            if self.dry_run:
                log.info("intervalweighting: "
                         "This was a dry run. %s was not applied." % self.name)
        else:
            # we didn't need to weight so inform the user
            log.info("intervalweighting: %s has already been applied." % self.name)
Example #44
0
    def sound_the_alarm(self, timestamp, battery_flags, alarm_count):
        """This function is called when the alarm has been triggered."""

        # Get the time and convert to a string:
        t_str = timestamp_to_string(timestamp)

        # Log it in the system log:
        log.info("Low battery status sounded at %s: %s" %
                 (t_str, battery_flags))

        # Form the message text:
        indicator_strings = []
        for bat in battery_flags:
            indicator_strings.append("%s: %04x" %
                                     (bat, int(battery_flags[bat])))
        msg_text = """
The low battery indicator has been seen %d times since the last archive period.

Alarm sounded at %s

Low battery indicators:
%s

""" % (alarm_count, t_str, '\n'.join(indicator_strings))
        # Convert to MIME:
        msg = MIMEText(msg_text)

        # Fill in MIME headers:
        msg['Subject'] = self.SUBJECT
        msg['From'] = self.FROM
        msg['To'] = ','.join(self.TO)

        try:
            # First try end-to-end encryption
            s = smtplib.SMTP_SSL(self.smtp_host)
            log.debug("Using SMTP_SSL")
        except AttributeError:
            # If that doesn't work, try creating an insecure host, then upgrading
            s = smtplib.SMTP(self.smtp_host)
            try:
                # Be prepared to catch an exception if the server
                # does not support encrypted transport.
                s.ehlo()
                s.starttls()
                s.ehlo()
                log.debug("Using SMTP encrypted transport")
            except smtplib.SMTPException:
                log.debug("Using SMTP unencrypted transport")

        try:
            # If a username has been given, assume that login is required
            # for this host:
            if self.smtp_user:
                s.login(self.smtp_user, self.smtp_password)
                log.debug("Logged in as %s", self.smtp_user)

            # Send the email:
            s.sendmail(msg['From'], self.TO, msg.as_string())
            # Log out of the server:
            s.quit()
        except Exception as e:
            log.error("Send email failed: %s", e)
            raise

        # Log sending the email:
        log.info("Email sent to: %s", self.TO)
Example #45
0
    def generate(self, section, gen_ts):
        """Generate one or more reports for the indicated section.  Each
        section in a period is a report.  A report has one or more templates.

        section: A ConfigObj dictionary, holding the templates to be
        generated.  Any subsections in the dictionary will be recursively
        processed as well.
        
        gen_ts: The report will be current to this time.
        """
        
        ngen = 0
        # Go through each subsection (if any) of this section,
        # generating from any templates they may contain
        for subsection in section.sections:
            # Sections 'SummaryByMonth' and 'SummaryByYear' imply summarize_by
            # certain time spans
            if not section[subsection].has_key('summarize_by'):
                if subsection == 'SummaryByDay':
                    section[subsection]['summarize_by'] = 'SummaryByDay'
                elif subsection == 'SummaryByMonth':
                    section[subsection]['summarize_by'] = 'SummaryByMonth'
                elif subsection == 'SummaryByYear':
                    section[subsection]['summarize_by'] = 'SummaryByYear'
            # Call recursively, to generate any templates in this subsection
            ngen += self.generate(section[subsection], gen_ts)

        # We have finished recursively processing any subsections in this
        # section. Time to do the section itself. If there is no option
        # 'template', then there isn't anything to do. Return.
        if not section.has_key('template'):
            return ngen
        
        # Change directory to the skin subdirectory.  We use absolute paths
        # for cheetah, so the directory change is not necessary for generating
        # files.  However, changing to the skin directory provides a known
        # location so that calls to os.getcwd() in any templates will return
        # a predictable result.
        os.chdir(os.path.join(self.config_dict['WEEWX_ROOT'],
                              self.skin_dict['SKIN_ROOT'],
                              self.skin_dict['skin']))

        report_dict = weeutil.weeutil.accumulateLeaves(section)
        
        (template, dest_dir, encoding, default_binding) = self._prepGen(report_dict)

        # Get start and stop times        
        default_archive = self.db_binder.get_manager(default_binding)
        start_ts = default_archive.firstGoodStamp()
        if not start_ts:
            loginf('Skipping template %s: cannot find start time' % section['template'])
            return ngen

        if gen_ts:
            record = default_archive.getRecord(gen_ts,
                                               max_delta=to_int(report_dict.get('max_delta')))
            if record:
                stop_ts = record['dateTime']
            else:
                loginf('Skipping template %s: generate time %s not in database' % (section['template'], timestamp_to_string(gen_ts)) )
                return ngen
        else:
            stop_ts = default_archive.lastGoodStamp()
        
        # Get an appropriate generator function
        summarize_by = report_dict['summarize_by']
        if summarize_by in CheetahGenerator.generator_dict:
            _spangen = CheetahGenerator.generator_dict[summarize_by]
        else:
            # Just a single timespan to generate. Use a lambda expression.
            _spangen = lambda start_ts, stop_ts : [weeutil.weeutil.TimeSpan(start_ts, stop_ts)]

        # Use the generator function
        for timespan in _spangen(start_ts, stop_ts):
            start_tt = time.localtime(timespan.start)
            stop_tt  = time.localtime(timespan.stop)

            if summarize_by in CheetahGenerator.format_dict:
                # This is a "SummaryBy" type generation. If it hasn't been done already, save the
                # date as a string, to be used inside the document
                date_str = time.strftime(CheetahGenerator.format_dict[summarize_by], start_tt)
                if date_str not in self.outputted_dict[summarize_by]:
                    self.outputted_dict[summarize_by].append(date_str)
                # For these "SummaryBy" generations, the file name comes from the start of the timespan:
                _filename = self._getFileName(template, start_tt)
            else:
                # This is a "ToDate" generation. File name comes 
                # from the stop (i.e., present) time:
                _filename = self._getFileName(template, stop_tt)

            # Get the absolute path for the target of this template
            _fullname = os.path.join(dest_dir, _filename)

            # Skip summary files outside the timespan
            if report_dict['summarize_by'] in CheetahGenerator.generator_dict \
                    and os.path.exists(_fullname) \
                    and not timespan.includesArchiveTime(stop_ts):
                continue

            # skip files that are fresh, but only if staleness is defined
            stale = to_int(report_dict.get('stale_age'))
            if stale is not None:
                t_now = time.time()
                try:
                    last_mod = os.path.getmtime(_fullname)
                    if t_now - last_mod < stale:
                        logdbg("Skip '%s': last_mod=%s age=%s stale=%s" %
                               (_filename, last_mod, t_now - last_mod, stale))
                        continue
                except os.error:
                    pass

            searchList = self._getSearchList(encoding, timespan,
                                             default_binding)
            tmpname = _fullname + '.tmp'
            
            try:
                compiled_template = Cheetah.Template.Template(
                    file=template,
                    searchList=searchList,
                    filter=encoding,
                    filtersLib=weewx.cheetahgenerator)
                with open(tmpname, mode='w') as _file:
                    print >> _file, compiled_template
                os.rename(tmpname, _fullname)
            except Exception, e:
                # We would like to get better feedback when there are cheetah
                # compiler failures, but there seem to be no hooks for this.
                # For example, if we could get make cheetah emit the source
                # on which the compiler is working, one could compare that with
                # the template to figure out exactly where the problem is.
                # In Cheetah.Compile.ModuleCompiler the source is manipulated
                # a bit then handed off to parserClass.  Unfortunately there
                # are no hooks to intercept the source and spit it out.  So
                # the best we can do is indicate the template that was being
                # processed when the failure ocurred.
                logerr("Generate failed with exception '%s'" % type(e))
                logerr("**** Ignoring template %s" % template)
                logerr("**** Reason: %s" % e)
                weeutil.weeutil.log_traceback("****  ")
            else:
                ngen += 1
            finally:
Example #46
0
    def __init__(self, config_dict, config_path, cumulus_config_dict, import_config_path, options, log):

        # call our parents __init__
        super(CumulusSource, self).__init__(config_dict,
                                            cumulus_config_dict,
                                            options,
                                            log)

        # save our import config path
        self.import_config_path = import_config_path
        # save our import config dict
        self.cumulus_config_dict = cumulus_config_dict

        # wind dir bounds
        self.wind_dir = [0, 360]

        # Decimal separator used in monthly log files, default to decimal point
        self.decimal = cumulus_config_dict.get('decimal', '.')
        # Field delimiter used in monthly log files, default to comma
        self.delimiter = cumulus_config_dict.get('delimiter', ',')

        # We combine Cumulus date and time fields to give a fixed format
        # date-time string
        self.raw_datetime_format = '%d/%m/%y %H:%M'
        # Cumulus log files provide a number of cumulative rainfall fields. We
        # cannot use the daily rainfall as this may reset at some time of day
        # other than midnight (as required by weewx). So we use field 26, total
        # rainfall since midnight and treat it as a cumulative value.
        self.rain = 'cumulative'

        # initialise our import field-to-weewx archive field map
        self.map = None

        # Units of measure for some obs (eg temperatures) cannot be derived from
        # the Cumulus monthly log files. These units must be specified by the
        # user in the import config file. Read these units and fill in the
        # missing unit data in the header map. Do some basic error checking and
        # validation, if one of the fields is missing or invalid then we need
        # to catch the error and raise it as we can't go on.
        # Temperature
        try:
            temp_u = cumulus_config_dict['Units'].get('temperature')
        except:
            _msg = "No units specified for Cumulus temperature fields in %s." % (self.import_config_path, )
            raise weewx.UnitError(_msg)
        else:
            if temp_u in weewx.units.default_unit_format_dict:
                self._header_map['cur_out_temp']['units'] = temp_u
                self._header_map['curr_in_temp']['units'] = temp_u
                self._header_map['cur_dewpoint']['units'] = temp_u
                self._header_map['cur_heatindex']['units'] = temp_u
                self._header_map['cur_windchill']['units'] = temp_u
                self._header_map['cur_app_temp']['units'] = temp_u
            else:
                _msg = "Unknown units '%s' specified for Cumulus temperature fields in %s." % (temp_u,
                                                                                               self.import_config_path)
                raise weewx.UnitError(_msg)
        # Pressure
        try:
            press_u = cumulus_config_dict['Units'].get('pressure')
        except:
            _msg = "No units specified for Cumulus pressure fields in %s." % (self.import_config_path, )
            raise weewx.UnitError(_msg)
        else:
            if press_u in ['inHg', 'mbar', 'hPa']:
                self._header_map['cur_slp']['units'] = press_u
            else:
                _msg = "Unknown units '%s' specified for Cumulus pressure fields in %s." % (press_u,
                                                                                            self.import_config_path)
                raise weewx.UnitError(_msg)
        # Rain
        try:
            rain_u = cumulus_config_dict['Units'].get('rain')
        except:
            _msg = "No units specified for Cumulus rain fields in %s." % (self.import_config_path, )
            raise weewx.UnitError(_msg)
        else:
            if rain_u in rain_units_dict:
                self._header_map['midnight_rain']['units'] = rain_u
                self._header_map['cur_rain_rate']['units'] = rain_units_dict[rain_u]

            else:
                _msg = "Unknown units '%s' specified for Cumulus rain fields in %s." % (rain_u,
                                                                                        self.import_config_path)
                raise weewx.UnitError(_msg)
        # Speed
        try:
            speed_u = cumulus_config_dict['Units'].get('speed')
        except:
            _msg = "No units specified for Cumulus speed fields in %s." % (self.import_config_path, )
            raise weewx.UnitError(_msg)
        else:
            if speed_u in weewx.units.default_unit_format_dict:
                self._header_map['avg_wind_speed']['units'] = speed_u
                self._header_map['gust_wind_speed']['units'] = speed_u
            else:
                _msg = "Unknown units '%s' specified for Cumulus speed fields in %s." % (speed_u,
                                                                                         self.import_config_path)
                raise weewx.UnitError(_msg)

        # get our source file path
        try:
            self.source = cumulus_config_dict['directory']
        except KeyError:
            raise weewx.ViolatedPrecondition("Cumulus monthly logs directory not specified in '%s'." % import_config_path)

        # Now get a list on monthly log files sorted from oldest to newest
        month_log_list = glob.glob(self.source + '/?????log.txt')
        _temp = [(fn, fn[-9:-7], time.strptime(fn[-12:-9],'%b').tm_mon) for fn in month_log_list]
        self.log_list = [a[0] for a in sorted(_temp,
                                              key = lambda el : (el[1], el[2]))]
        if len(self.log_list) == 0:
            raise weeimport.WeeImportIOError(
                "No Cumulus monthly logs found in directory '%s'." % self.source)

        # tell the user/log what we intend to do
        _msg = "Cumulus monthly log files in the '%s' directory will be imported" % self.source
        self.wlog.printlog(syslog.LOG_INFO, _msg)
        _msg = "The following options will be used:"
        self.wlog.verboselog(syslog.LOG_DEBUG, _msg)
        _msg = "     config=%s, import-config=%s" % (config_path,
                                                     self.import_config_path)
        self.wlog.verboselog(syslog.LOG_DEBUG, _msg)
        if options.date:
            _msg = "     date=%s" % options.date
        else:
            # we must have --from and --to
            _msg = "     from=%s, to=%s" % (options.date_from, options.date_to)
        self.wlog.verboselog(syslog.LOG_DEBUG, _msg)
        _msg = "     dry-run=%s, calc-missing=%s" % (self.dry_run,
                                                     self.calc_missing)
        self.wlog.verboselog(syslog.LOG_DEBUG, _msg)
        _msg = "     tranche=%s, interval=%s" % (self.tranche,
                                                 self.interval)
        self.wlog.verboselog(syslog.LOG_DEBUG, _msg)
        _msg = "     UV=%s, radiation=%s" % (self.UV_sensor, self.solar_sensor)
        self.wlog.verboselog(syslog.LOG_DEBUG, _msg)
        _msg = "Using database binding '%s', which is bound to database '%s'" % (self.db_binding_wx,
                                                                                 self.dbm.database_name)
        self.wlog.printlog(syslog.LOG_INFO, _msg)
        _msg = "Destination table '%s' unit system is '%#04x' (%s)." % (self.dbm.table_name,
                                                                        self.archive_unit_sys,
                                                                        unit_nicknames[self.archive_unit_sys])
        self.wlog.printlog(syslog.LOG_INFO, _msg)
        if self.calc_missing:
            print "Missing derived observations will be calculated."
        if not self.UV_sensor:
            print "All weewx UV fields will be set to None."
        if not self.solar_sensor:
            print "All weewx radiation fields will be set to None."
        if options.date or options.date_from:
            print "Observations timestamped after %s and up to and" % (timestamp_to_string(self.first_ts), )
            print "including %s will be imported." % (timestamp_to_string(self.last_ts), )
        if self.dry_run:
            print "This is a dry run, imported data will not be saved to archive."
Example #47
0
    def generate(self, section, gen_ts):
        """Generate one or more reports for the indicated section.  Each section
        in a period is a report.  A report has one or more templates.

        section: A ConfigObj dictionary, holding the templates to be generated. Any
        subsections in the dictionary will be recursively processed as well.
        
        gen_ts: The report will be current to this time.
        """

        ngen = 0
        # Go through each subsection (if any) of this section,
        # generating from any templates they may contain
        for subsection in section.sections:
            # Sections 'SummaryByMonth' and 'SummaryByYear' imply summarize_by certain time spans
            if not section[subsection].has_key('summarize_by'):
                if subsection == 'SummaryByMonth':
                    section[subsection]['summarize_by'] = 'SummaryByMonth'
                elif subsection == 'SummaryByYear':
                    section[subsection]['summarize_by'] = 'SummaryByYear'
            # Call myself recursively, to generate any templates in this subsection
            ngen += self.generate(section[subsection], gen_ts)

        # We have finished recursively processing any subsections in this
        # section. Time to do the section itself. If there is no option
        # 'template', then there isn't anything to do. Return.
        if not section.has_key('template'):
            return ngen

        # Change directory to the skin subdirectory.  We use absolute paths
        # for cheetah, so the directory change is not necessary for generating
        # files.  However, changing to the skin directory provides a known
        # location so that calls to os.getcwd() in any templates will return
        # a predictable result.
        os.chdir(
            os.path.join(self.config_dict['WEEWX_ROOT'],
                         self.skin_dict['SKIN_ROOT'], self.skin_dict['skin']))

        report_dict = weeutil.weeutil.accumulateLeaves(section)

        (template, dest_dir, encoding,
         default_binding) = self._prepGen(report_dict)

        # Get start and stop times
        default_archive = self.db_binder.get_manager(default_binding)
        start_ts = default_archive.firstGoodStamp()
        if not start_ts:
            loginf('Skipping template %s: cannot find start time' %
                   section['template'])
            return ngen

        if gen_ts:
            record = default_archive.getRecord(
                gen_ts, max_delta=to_int(report_dict.get('max_delta')))
            if record:
                stop_ts = record['dateTime']
            else:
                loginf(
                    'Skipping template %s; generate time %s not in database' %
                    (section['template'], timestamp_to_string(gen_ts)))
                return ngen
        else:
            stop_ts = default_archive.lastGoodStamp()

        # Get an appropriate generator function
        summarize_by = report_dict['summarize_by']
        if summarize_by in CheetahGenerator.generator_dict:
            _spangen = CheetahGenerator.generator_dict[summarize_by]
        else:
            # Just a single timespan to generate. Use a lambda expression.
            _spangen = lambda start_ts, stop_ts: [
                weeutil.weeutil.TimeSpan(start_ts, stop_ts)
            ]

        # Use the generator function
        for timespan in _spangen(start_ts, stop_ts):

            # Save YYYY-MM so they can be used within the document
            if summarize_by in CheetahGenerator.generator_dict:
                timespan_start_tt = time.localtime(timespan.start)
                _yr_str = "%4d" % timespan_start_tt[0]
                if summarize_by == 'SummaryByMonth':
                    _mo_str = "%02d" % timespan_start_tt[1]
                    if _mo_str not in self.outputted_dict[summarize_by]:
                        self.outputted_dict[summarize_by].append(
                            "%s-%s" % (_yr_str, _mo_str))
                if summarize_by == 'SummaryByYear' and _yr_str not in self.outputted_dict[
                        summarize_by]:
                    self.outputted_dict[summarize_by].append(_yr_str)

            # figure out the filename for this template
            _filename = self._getFileName(template, timespan)
            _fullname = os.path.join(dest_dir, _filename)

            # Skip summary files outside the timespan
            if report_dict['summarize_by'] in CheetahGenerator.generator_dict \
                    and os.path.exists(_fullname) \
                    and not timespan.includesArchiveTime(stop_ts):
                continue

            # skip files that are fresh, but only if staleness is defined
            stale = to_int(report_dict.get('stale_age'))
            if stale is not None:
                t_now = time.time()
                try:
                    last_mod = os.path.getmtime(_fullname)
                    if t_now - last_mod < stale:
                        logdbg("Skip '%s': last_mod=%s age=%s stale=%s" %
                               (_filename, last_mod, t_now - last_mod, stale))
                        continue
                except os.error:
                    pass

            searchList = self._getSearchList(encoding, timespan,
                                             default_binding)
            tmpname = _fullname + '.tmp'

            try:
                text = Cheetah.Template.Template(
                    file=template,
                    searchList=searchList,
                    filter=encoding,
                    filtersLib=weewx.cheetahgenerator)

                with open(tmpname, mode='w') as _file:
                    print >> _file, text
                os.rename(tmpname, _fullname)
            except Exception, e:
                logerr("Generate failed with exception '%s'" % type(e))
                logerr("**** Ignoring template %s" % template)
                logerr("**** Reason: %s" % e)
                weeutil.weeutil.log_traceback("****  ")
            else:
                ngen += 1
            finally:
Example #48
0
    def generate(self, section, gen_ts):
        """Generate one or more reports for the indicated section.  Each
        section in a period is a report.  A report has one or more templates.

        section: A ConfigObj dictionary, holding the templates to be
        generated.  Any subsections in the dictionary will be recursively
        processed as well.
        
        gen_ts: The report will be current to this time.
        """

        ngen = 0
        # Go through each subsection (if any) of this section,
        # generating from any templates they may contain
        for subsection in section.sections:
            # Sections 'SummaryByMonth' and 'SummaryByYear' imply summarize_by
            # certain time spans
            if not section[subsection].has_key('summarize_by'):
                if subsection == 'SummaryByDay':
                    section[subsection]['summarize_by'] = 'SummaryByDay'
                elif subsection == 'SummaryByMonth':
                    section[subsection]['summarize_by'] = 'SummaryByMonth'
                elif subsection == 'SummaryByYear':
                    section[subsection]['summarize_by'] = 'SummaryByYear'
            # Call recursively, to generate any templates in this subsection
            ngen += self.generate(section[subsection], gen_ts)

        # We have finished recursively processing any subsections in this
        # section. Time to do the section itself. If there is no option
        # 'template', then there isn't anything to do. Return.
        if not section.has_key('template'):
            return ngen

        # Change directory to the skin subdirectory.  We use absolute paths
        # for cheetah, so the directory change is not necessary for generating
        # files.  However, changing to the skin directory provides a known
        # location so that calls to os.getcwd() in any templates will return
        # a predictable result.
        os.chdir(
            os.path.join(self.config_dict['WEEWX_ROOT'],
                         self.skin_dict['SKIN_ROOT'], self.skin_dict['skin']))

        report_dict = weeutil.weeutil.accumulateLeaves(section)

        (template, dest_dir, encoding,
         default_binding) = self._prepGen(report_dict)

        # Get start and stop times
        default_archive = self.db_binder.get_manager(default_binding)
        start_ts = default_archive.firstGoodStamp()
        if not start_ts:
            loginf('Skipping template %s: cannot find start time' %
                   section['template'])
            return ngen

        if gen_ts:
            record = default_archive.getRecord(
                gen_ts, max_delta=to_int(report_dict.get('max_delta')))
            if record:
                stop_ts = record['dateTime']
            else:
                loginf(
                    'Skipping template %s: generate time %s not in database' %
                    (section['template'], timestamp_to_string(gen_ts)))
                return ngen
        else:
            stop_ts = default_archive.lastGoodStamp()

        # Get an appropriate generator function
        summarize_by = report_dict['summarize_by']
        if summarize_by in CheetahGenerator.generator_dict:
            _spangen = CheetahGenerator.generator_dict[summarize_by]
        else:
            # Just a single timespan to generate. Use a lambda expression.
            _spangen = lambda start_ts, stop_ts: [
                weeutil.weeutil.TimeSpan(start_ts, stop_ts)
            ]

        # Use the generator function
        for timespan in _spangen(start_ts, stop_ts):
            start_tt = time.localtime(timespan.start)
            stop_tt = time.localtime(timespan.stop)

            if summarize_by in CheetahGenerator.format_dict:
                # This is a "SummaryBy" type generation. If it hasn't been done already, save the
                # date as a string, to be used inside the document
                date_str = time.strftime(
                    CheetahGenerator.format_dict[summarize_by], start_tt)
                if date_str not in self.outputted_dict[summarize_by]:
                    self.outputted_dict[summarize_by].append(date_str)
                # For these "SummaryBy" generations, the file name comes from the start of the timespan:
                _filename = self._getFileName(template, start_tt)
            else:
                # This is a "ToDate" generation. File name comes
                # from the stop (i.e., present) time:
                _filename = self._getFileName(template, stop_tt)

            # Get the absolute path for the target of this template
            _fullname = os.path.join(dest_dir, _filename)

            # Skip summary files outside the timespan
            if report_dict['summarize_by'] in CheetahGenerator.generator_dict \
                    and os.path.exists(_fullname) \
                    and not timespan.includesArchiveTime(stop_ts):
                continue

            # skip files that are fresh, but only if staleness is defined
            stale = to_int(report_dict.get('stale_age'))
            if stale is not None:
                t_now = time.time()
                try:
                    last_mod = os.path.getmtime(_fullname)
                    if t_now - last_mod < stale:
                        logdbg("Skip '%s': last_mod=%s age=%s stale=%s" %
                               (_filename, last_mod, t_now - last_mod, stale))
                        continue
                except os.error:
                    pass

            searchList = self._getSearchList(encoding, timespan,
                                             default_binding)
            tmpname = _fullname + '.tmp'

            try:
                text = Cheetah.Template.Template(
                    file=template,
                    searchList=searchList,
                    filter=encoding,
                    filtersLib=weewx.cheetahgenerator)
                with open(tmpname, mode='w') as _file:
                    print >> _file, text
                os.rename(tmpname, _fullname)
            except Exception, e:
                # We would like to get better feedback when there are cheetah
                # compiler failures, but there seem to be no hooks for this.
                # For example, if we could get make cheetah emit the source
                # on which the compiler is working, one could compare that with
                # the template to figure out exactly where the problem is.
                # In Cheetah.Compile.ModuleCompiler the source is manipulated
                # a bit then handed off to parserClass.  Unfortunately there
                # are no hooks to intercept the source and spit it out.  So
                # the best we can do is indicate the template that was being
                # processed when the failure ocurred.
                logerr("Generate failed with exception '%s'" % type(e))
                logerr("**** Ignoring template %s" % template)
                logerr("**** Reason: %s" % e)
                weeutil.weeutil.log_traceback("****  ")
            else:
                ngen += 1
            finally:
Example #49
0
    def generate(self, section, gen_ts):
        """Generate one or more reports for the indicated section.  Each section
        in a period is a report.  A report has one or more templates.

        section: A ConfigObj dictionary, holding the templates to be generated. Any
        subsections in the dictionary will be recursively processed as well.
        
        gen_ts: The report will be current to this time.
        """
        
        ngen = 0
        # Go through each subsection (if any) of this section,
        # generating from any templates they may contain
        for subsection in section.sections:
            # Sections 'SummaryByMonth' and 'SummaryByYear' imply summarize_by certain time spans
            if not section[subsection].has_key('summarize_by'):
                if subsection == 'SummaryByMonth':
                    section[subsection]['summarize_by'] = 'SummaryByMonth'
                elif subsection == 'SummaryByYear':
                    section[subsection]['summarize_by'] = 'SummaryByYear'
            # Call myself recursively, to generate any templates in this subsection
            ngen += self.generate(section[subsection], gen_ts)

        # We have finished recursively processing any subsections in this
        # section. Time to do the section itself. If there is no option
        # 'template', then there isn't anything to do. Return.
        if not section.has_key('template'):
            return ngen
        
        # Change directory to the skin subdirectory.  We use absolute paths
        # for cheetah, so the directory change is not necessary for generating
        # files.  However, changing to the skin directory provides a known
        # location so that calls to os.getcwd() in any templates will return
        # a predictable result.
        os.chdir(os.path.join(self.config_dict['WEEWX_ROOT'],
                              self.skin_dict['SKIN_ROOT'],
                              self.skin_dict['skin']))

        report_dict = weeutil.weeutil.accumulateLeaves(section)
        
        (template, dest_dir, encoding, default_binding) = self._prepGen(report_dict)

        # Get start and stop times        
        default_archive = self.db_binder.get_manager(default_binding)
        start_ts = default_archive.firstGoodStamp()
        if not start_ts:
            loginf('Skipping template %s: cannot find start time' % section['template'])
            return ngen

        if gen_ts:
            record = default_archive.getRecord(gen_ts, max_delta=to_int(report_dict.get('max_delta')))
            if record:
                stop_ts = record['dateTime']
            else:
                loginf('Skipping template %s; generate time %s not in database' % (section['template'], timestamp_to_string(gen_ts)) )
                return ngen
        else:
            stop_ts = default_archive.lastGoodStamp()
        
        # Get an appropriate generator function
        summarize_by = report_dict['summarize_by']
        if summarize_by in CheetahGenerator.generator_dict:
            _spangen = CheetahGenerator.generator_dict[summarize_by]
        else:
            # Just a single timespan to generate. Use a lambda expression.
            _spangen = lambda start_ts, stop_ts : [weeutil.weeutil.TimeSpan(start_ts, stop_ts)]

        # Use the generator function
        for timespan in _spangen(start_ts, stop_ts):

            # Save YYYY-MM so they can be used within the document
            if summarize_by in CheetahGenerator.generator_dict:
                timespan_start_tt = time.localtime(timespan.start)
                _yr_str = "%4d" % timespan_start_tt[0]
                if summarize_by == 'SummaryByMonth':
                    _mo_str = "%02d" % timespan_start_tt[1]
                    if _mo_str not in self.outputted_dict[summarize_by]:
                        self.outputted_dict[summarize_by].append("%s-%s" % (_yr_str, _mo_str))
                if summarize_by == 'SummaryByYear' and _yr_str not in self.outputted_dict[summarize_by]:
                    self.outputted_dict[summarize_by].append(_yr_str)

            # figure out the filename for this template
            _filename = self._getFileName(template, timespan)
            _fullname = os.path.join(dest_dir, _filename)

            # Skip summary files outside the timespan
            if report_dict['summarize_by'] in CheetahGenerator.generator_dict \
                    and os.path.exists(_fullname) \
                    and not timespan.includesArchiveTime(stop_ts):
                continue

            # skip files that are fresh, but only if staleness is defined
            stale = to_int(report_dict.get('stale_age'))
            if stale is not None:
                t_now = time.time()
                try:
                    last_mod = os.path.getmtime(_fullname)
                    if t_now - last_mod < stale:
                        logdbg("Skip '%s': last_mod=%s age=%s stale=%s" %
                               (_filename, last_mod, t_now - last_mod, stale))
                        continue
                except os.error:
                    pass

            searchList = self._getSearchList(encoding, timespan,
                                             default_binding)
            tmpname = _fullname + '.tmp'
            
            try:
                text = Cheetah.Template.Template(file=template,
                                                 searchList=searchList,
                                                 filter=encoding,
                                                 filtersLib=weewx.cheetahgenerator)
                
                with open(tmpname, mode='w') as _file:
                    print >> _file, text
                os.rename(tmpname, _fullname)
            except Exception, e:
                logerr("Generate failed with exception '%s'" % type(e))
                logerr("**** Ignoring template %s" % template)
                logerr("**** Reason: %s" % e)
                weeutil.weeutil.log_traceback("****  ")
            else:
                ngen += 1
            finally:
Example #50
0
    def __init__(self, config_dict, config_path, csv_config_dict,
                 import_config_path, options):

        # call our parents __init__
        super(CSVSource, self).__init__(config_dict, csv_config_dict, options)

        # save our import config path
        self.import_config_path = import_config_path
        # save our import config dict
        self.csv_config_dict = csv_config_dict

        # get a few config settings from our CSV config dict
        # csv field delimiter
        self.delimiter = str(self.csv_config_dict.get('delimiter', ','))
        # string format used to decode the imported field holding our dateTime
        self.raw_datetime_format = self.csv_config_dict.get(
            'raw_datetime_format', '%Y-%m-%d %H:%M:%S')
        # is our rain discrete or cumulative
        self.rain = self.csv_config_dict.get('rain', 'cumulative')
        # determine valid range for imported wind direction
        _wind_direction = option_as_list(
            self.csv_config_dict.get('wind_direction', '0,360'))
        try:
            if float(_wind_direction[0]) <= float(_wind_direction[1]):
                self.wind_dir = [
                    float(_wind_direction[0]),
                    float(_wind_direction[1])
                ]
            else:
                self.wind_dir = [-360, 360]
        except (KeyError, ValueError):
            self.wind_dir = [-360, 360]
        # get our source file path
        try:
            self.source = csv_config_dict['file']
        except KeyError:
            raise weewx.ViolatedPrecondition(
                "CSV source file not specified in '%s'." % import_config_path)
        # get the source file encoding, default to utf-8-sig
        self.source_encoding = self.csv_config_dict.get(
            'source_encoding', 'utf-8-sig')
        # initialise our import field-to-WeeWX archive field map
        self.map = None
        # initialise some other properties we will need
        self.start = 1
        self.end = 1
        self.increment = 1

        # tell the user/log what we intend to do
        _msg = "A CSV import from source file '%s' has been requested." % self.source
        print(_msg)
        log.info(_msg)
        _msg = "The following options will be used:"
        if self.verbose:
            print(_msg)
        log.debug(_msg)
        _msg = "     config=%s, import-config=%s" % (config_path,
                                                     self.import_config_path)
        if self.verbose:
            print(_msg)
        log.debug(_msg)
        if options.date:
            _msg = "     source=%s, date=%s" % (self.source, options.date)
        else:
            # we must have --from and --to
            _msg = "     source=%s, from=%s, to=%s" % (
                self.source, options.date_from, options.date_to)
        if self.verbose:
            print(_msg)
        log.debug(_msg)
        _msg = "     dry-run=%s, calc_missing=%s, ignore_invalid_data=%s" % (
            self.dry_run, self.calc_missing, self.ignore_invalid_data)
        if self.verbose:
            print(_msg)
        log.debug(_msg)
        _msg = "     tranche=%s, interval=%s, date/time_string_format=%s" % (
            self.tranche, self.interval, self.raw_datetime_format)
        if self.verbose:
            print(_msg)
        log.debug(_msg)
        _msg = "     delimiter='%s', rain=%s, wind_direction=%s" % (
            self.delimiter, self.rain, self.wind_dir)
        if self.verbose:
            print(_msg)
        log.debug(_msg)
        _msg = "     UV=%s, radiation=%s" % (self.UV_sensor, self.solar_sensor)
        if self.verbose:
            print(_msg)
        log.debug(_msg)
        _msg = "Using database binding '%s', which is bound to database '%s'" % (
            self.db_binding_wx, self.dbm.database_name)
        print(_msg)
        log.info(_msg)
        _msg = "Destination table '%s' unit system is '%#04x' (%s)." % (
            self.dbm.table_name, self.archive_unit_sys,
            unit_nicknames[self.archive_unit_sys])
        print(_msg)
        log.info(_msg)
        if self.calc_missing:
            _msg = "Missing derived observations will be calculated."
            print(_msg)
            log.info(_msg)

        if not self.UV_sensor:
            _msg = "All WeeWX UV fields will be set to None."
            print(_msg)
            log.info(_msg)
        if not self.solar_sensor:
            _msg = "All WeeWX radiation fields will be set to None."
            print(_msg)
            log.info(_msg)
        if options.date or options.date_from:
            _msg = "Observations timestamped after %s and up to and" % timestamp_to_string(
                self.first_ts)
            print(_msg)
            log.info(_msg)
            _msg = "including %s will be imported." % timestamp_to_string(
                self.last_ts)
            print(_msg)
            log.info(_msg)
        if self.dry_run:
            _msg = "This is a dry run, imported data will not be saved to archive."
            print(_msg)
            log.info(_msg)
Example #51
0
    def __init__(self, config_dict, config_path, wu_config_dict,
                 import_config_path, options):

        # call our parents __init__
        super(WUSource, self).__init__(config_dict, wu_config_dict, options)

        # save our import config path
        self.import_config_path = import_config_path
        # save our import config dict
        self.wu_config_dict = wu_config_dict

        # get our WU station ID
        try:
            self.station_id = wu_config_dict['station_id']
        except KeyError:
            _msg = "Weather Underground station ID not specified in '%s'." % import_config_path
            raise weewx.ViolatedPrecondition(_msg)

        # get our WU API key
        try:
            self.api_key = wu_config_dict['api_key']
        except KeyError:
            _msg = "Weather Underground API key not specified in '%s'." % import_config_path
            raise weewx.ViolatedPrecondition(_msg)

        # wind dir bounds
        _wind_direction = option_as_list(
            wu_config_dict.get('wind_direction', '0,360'))
        try:
            if float(_wind_direction[0]) <= float(_wind_direction[1]):
                self.wind_dir = [
                    float(_wind_direction[0]),
                    float(_wind_direction[1])
                ]
            else:
                self.wind_dir = [0, 360]
        except (IndexError, TypeError):
            self.wind_dir = [0, 360]

        # some properties we know because of the format of the returned WU data
        # WU returns a fixed format date-time string
        self.raw_datetime_format = '%Y-%m-%d %H:%M:%S'
        # WU only provides hourly rainfall and a daily cumulative rainfall.
        # We use the latter so force 'cumulative' for rain.
        self.rain = 'cumulative'

        # initialise our import field-to-WeeWX archive field map
        self.map = None
        # For a WU import we might have to import multiple days but we can only
        # get one day at a time from WU. So our start and end properties
        # (counters) are datetime objects and our increment is a timedelta.
        # Get datetime objects for any date or date range specified on the
        # command line, if there wasn't one then default to today.
        self.start = dt.fromtimestamp(startOfDay(self.first_ts))
        self.end = dt.fromtimestamp(startOfDay(self.last_ts))
        # set our increment
        self.increment = datetime.timedelta(days=1)

        # property holding the current period being processed
        self.period = None

        # tell the user/log what we intend to do
        _msg = "Observation history for Weather Underground station '%s' will be imported." % self.station_id
        print(_msg)
        log.info(_msg)
        _msg = "The following options will be used:"
        if self.verbose:
            print(_msg)
        log.debug(_msg)
        _msg = "     config=%s, import-config=%s" % (config_path,
                                                     self.import_config_path)
        if self.verbose:
            print(_msg)
        log.debug(_msg)
        if options.date:
            _msg = "     station=%s, date=%s" % (self.station_id, options.date)
        else:
            # we must have --from and --to
            _msg = "     station=%s, from=%s, to=%s" % (
                self.station_id, options.date_from, options.date_to)
        if self.verbose:
            print(_msg)
        log.debug(_msg)
        _obf_api_key_msg = '='.join(
            ['     apiKey', '*' * (len(self.api_key) - 4) + self.api_key[-4:]])
        if self.verbose:
            print(_obf_api_key_msg)
        log.debug(_obf_api_key_msg)
        _msg = "     dry-run=%s, calc_missing=%s, ignore_invalid_data=%s" % (
            self.dry_run, self.calc_missing, self.ignore_invalid_data)
        if self.verbose:
            print(_msg)
        log.debug(_msg)
        _msg = "     tranche=%s, interval=%s, wind_direction=%s" % (
            self.tranche, self.interval, self.wind_dir)
        if self.verbose:
            print(_msg)
        log.debug(_msg)
        _msg = "Using database binding '%s', which is bound to database '%s'" % (
            self.db_binding_wx, self.dbm.database_name)
        print(_msg)
        log.info(_msg)
        _msg = "Destination table '%s' unit system is '%#04x' (%s)." % (
            self.dbm.table_name, self.archive_unit_sys,
            unit_nicknames[self.archive_unit_sys])
        print(_msg)
        log.info(_msg)
        if self.calc_missing:
            print("Missing derived observations will be calculated.")
        if options.date or options.date_from:
            print("Observations timestamped after %s and up to and" %
                  timestamp_to_string(self.first_ts))
            print("including %s will be imported." %
                  timestamp_to_string(self.last_ts))
        if self.dry_run:
            print(
                "This is a dry run, imported data will not be saved to archive."
            )
Example #52
0
    def __init__(self, config_dict, config_path, cumulus_config_dict,
                 import_config_path, options):

        # call our parents __init__
        super(CumulusSource, self).__init__(config_dict, cumulus_config_dict,
                                            options)

        # save our import config path
        self.import_config_path = import_config_path
        # save our import config dict
        self.cumulus_config_dict = cumulus_config_dict

        # wind dir bounds
        self.wind_dir = [0, 360]

        # field delimiter used in monthly log files, default to comma
        self.delimiter = str(cumulus_config_dict.get('delimiter', ','))
        # decimal separator used in monthly log files, default to decimal point
        self.decimal = cumulus_config_dict.get('decimal', '.')

        # date separator used in monthly log files, default to solidus
        separator = cumulus_config_dict.get('separator', '/')
        # we combine Cumulus date and time fields to give a fixed format
        # date-time string
        self.raw_datetime_format = separator.join(('%d', '%m', '%y %H:%M'))

        # Cumulus log files provide a number of cumulative rainfall fields. We
        # cannot use the daily rainfall as this may reset at some time of day
        # other than midnight (as required by WeeWX). So we use field 26, total
        # rainfall since midnight and treat it as a cumulative value.
        self.rain = 'cumulative'

        # initialise our import field-to-WeeWX archive field map
        self.map = None

        # Cumulus log files have a number of 'rain' fields that can be used to
        # derive the WeeWX rain field. Which one is available depends on the
        # Cumulus version that created the logs. The preferred field is field
        # 26(AA) - total rainfall since midnight but it is only available in
        # Cumulus v1.9.4 or later. If that field is not available then the
        # preferred field in field 09(J) - total rainfall today then field
        # 11(L) - total rainfall counter. Initialise the rain_source_confirmed
        # property now and we will deal with it later when we have some source
        # data.
        self.rain_source_confirmed = None

        # Units of measure for some obs (eg temperatures) cannot be derived from
        # the Cumulus monthly log files. These units must be specified by the
        # user in the import config file. Read these units and fill in the
        # missing unit data in the header map. Do some basic error checking and
        # validation, if one of the fields is missing or invalid then we need
        # to catch the error and raise it as we can't go on.
        # Temperature
        try:
            temp_u = cumulus_config_dict['Units'].get('temperature')
        except KeyError:
            _msg = "No units specified for Cumulus temperature " \
                   "fields in %s." % (self.import_config_path, )
            raise weewx.UnitError(_msg)
        else:
            if temp_u in weewx.units.default_unit_format_dict:
                self._header_map['cur_out_temp']['units'] = temp_u
                self._header_map['curr_in_temp']['units'] = temp_u
                self._header_map['cur_dewpoint']['units'] = temp_u
                self._header_map['cur_heatindex']['units'] = temp_u
                self._header_map['cur_windchill']['units'] = temp_u
                self._header_map['cur_app_temp']['units'] = temp_u
            else:
                _msg = "Unknown units '%s' specified for Cumulus " \
                       "temperature fields in %s." % (temp_u,
                                                      self.import_config_path)
                raise weewx.UnitError(_msg)
        # Pressure
        try:
            press_u = cumulus_config_dict['Units'].get('pressure')
        except KeyError:
            _msg = "No units specified for Cumulus pressure " \
                   "fields in %s." % (self.import_config_path, )
            raise weewx.UnitError(_msg)
        else:
            if press_u in ['inHg', 'mbar', 'hPa']:
                self._header_map['cur_slp']['units'] = press_u
            else:
                _msg = "Unknown units '%s' specified for Cumulus " \
                       "pressure fields in %s." % (press_u,
                                                   self.import_config_path)
                raise weewx.UnitError(_msg)
        # Rain
        try:
            rain_u = cumulus_config_dict['Units'].get('rain')
        except KeyError:
            _msg = "No units specified for Cumulus " \
                   "rain fields in %s." % (self.import_config_path, )
            raise weewx.UnitError(_msg)
        else:
            if rain_u in rain_units_dict:
                self._header_map['midnight_rain']['units'] = rain_u
                self._header_map['cur_rain_rate']['units'] = rain_units_dict[
                    rain_u]

            else:
                _msg = "Unknown units '%s' specified for Cumulus " \
                       "rain fields in %s." % (rain_u,
                                               self.import_config_path)
                raise weewx.UnitError(_msg)
        # Speed
        try:
            speed_u = cumulus_config_dict['Units'].get('speed')
        except KeyError:
            _msg = "No units specified for Cumulus " \
                   "speed fields in %s." % (self.import_config_path, )
            raise weewx.UnitError(_msg)
        else:
            if speed_u in weewx.units.default_unit_format_dict:
                self._header_map['avg_wind_speed']['units'] = speed_u
                self._header_map['gust_wind_speed']['units'] = speed_u
            else:
                _msg = "Unknown units '%s' specified for Cumulus " \
                       "speed fields in %s." % (speed_u,
                                                self.import_config_path)
                raise weewx.UnitError(_msg)

        # get our source file path
        try:
            self.source = cumulus_config_dict['directory']
        except KeyError:
            _msg = "Cumulus monthly logs directory not specified in '%s'." % import_config_path
            raise weewx.ViolatedPrecondition(_msg)

        # get the source file encoding, default to utf-8-sig
        self.source_encoding = self.cumulus_config_dict.get(
            'source_encoding', 'utf-8-sig')

        # property holding the current log file name being processed
        self.file_name = None

        # Now get a list on monthly log files sorted from oldest to newest
        month_log_list = glob.glob(self.source + '/?????log.txt')
        _temp = [(fn, fn[-9:-7], time.strptime(fn[-12:-9], '%b').tm_mon)
                 for fn in month_log_list]
        self.log_list = [
            a[0] for a in sorted(_temp, key=lambda el: (el[1], el[2]))
        ]
        if len(self.log_list) == 0:
            raise weeimport.WeeImportIOError(
                "No Cumulus monthly logs found in directory '%s'." %
                self.source)

        # tell the user/log what we intend to do
        _msg = "Cumulus monthly log files in the '%s' directory will be imported" % self.source
        print(_msg)
        log.info(_msg)
        _msg = "The following options will be used:"
        if self.verbose:
            print(_msg)
        log.debug(_msg)
        _msg = "     config=%s, import-config=%s" % (config_path,
                                                     self.import_config_path)
        if self.verbose:
            print(_msg)
        log.debug(_msg)
        if options.date:
            _msg = "     date=%s" % options.date
        else:
            # we must have --from and --to
            _msg = "     from=%s, to=%s" % (options.date_from, options.date_to)
        if self.verbose:
            print(_msg)
        log.debug(_msg)
        _msg = "     dry-run=%s, calc_missing=%s, " \
               "ignore_invalid_data=%s" % (self.dry_run,
                                           self.calc_missing,
                                           self.ignore_invalid_data)
        if self.verbose:
            print(_msg)
        log.debug(_msg)
        _msg = "     tranche=%s, interval=%s" % (self.tranche, self.interval)
        if self.verbose:
            print(_msg)
        log.debug(_msg)
        _msg = "     UV=%s, radiation=%s" % (self.UV_sensor, self.solar_sensor)
        if self.verbose:
            print(_msg)
        log.debug(_msg)
        _msg = "Using database binding '%s', which is bound " \
               "to database '%s'" % (self.db_binding_wx,
                                     self.dbm.database_name)
        print(_msg)
        log.info(_msg)
        _msg = "Destination table '%s' unit system " \
               "is '%#04x' (%s)." % (self.dbm.table_name,
                                     self.archive_unit_sys,
                                     unit_nicknames[self.archive_unit_sys])
        print(_msg)
        log.info(_msg)
        if self.calc_missing:
            print("Missing derived observations will be calculated.")
        if not self.UV_sensor:
            print("All WeeWX UV fields will be set to None.")
        if not self.solar_sensor:
            print("All WeeWX radiation fields will be set to None.")
        if options.date or options.date_from:
            print("Observations timestamped after %s and "
                  "up to and" % timestamp_to_string(self.first_ts))
            print("including %s will be imported." %
                  timestamp_to_string(self.last_ts))
        if self.dry_run:
            print(
                "This is a dry run, imported data will not be saved to archive."
            )
Example #53
0
    def __init__(self, config_dict, config_path, wu_config_dict, import_config_path, options, log):

        # call our parents __init__
        super(WUSource, self).__init__(config_dict,
                                       wu_config_dict,
                                       options,
                                       log)

        # save our import config path
        self.import_config_path = import_config_path
        # save our import config dict
        self.wu_config_dict = wu_config_dict

        # get our WU station ID
        try:
            self.station_id = wu_config_dict['station_id']
        except KeyError:
            raise weewx.ViolatedPrecondition("Weather Underground station ID not specified in '%s'." % import_config_path)

        # wind dir bounds
        _wind_direction = option_as_list(wu_config_dict.get('wind_direction',
                                                                '0,360'))
        try:
            if float(_wind_direction[0]) <= float(_wind_direction[1]):
                self.wind_dir = [float(_wind_direction[0]),
                                 float(_wind_direction[1])]
            else:
                self.wind_dir = [0, 360]
        except:
            self.wind_dir = [0, 360]

        # some properties we know because of the format of the returned WU data
        # WU returns a fixed format date-time string
        self.raw_datetime_format = '%Y-%m-%d %H:%M:%S'
        # WU only provides hourly rainfall and a daily cumulative rainfall.
        # We use the latter so force 'cumulative' for rain.
        self.rain = 'cumulative'

        # initialise our import field-to-weewx archive field map
        self.map = None
        # For a WU import we might have to import multiple days but we can only
        # get one day at a time from WU. So our start and end properties
        # (counters) are datetime objects and our increment is a timedelta.
        # Get datetime objects for any date or date range specified on the
        # command line, if there wasn't one then default to today.
        self.start = dt.fromtimestamp(startOfDay(self.first_ts))
        self.end = dt.fromtimestamp(startOfDay(self.last_ts))
        # set our increment
        self.increment = datetime.timedelta(days=1)

        # tell the user/log what we intend to do
        _msg = "Observation history for Weather Underground station '%s' will be imported." % self.station_id
        self.wlog.printlog(syslog.LOG_INFO, _msg)
        _msg = "The following options will be used:"
        self.wlog.verboselog(syslog.LOG_DEBUG, _msg)
        _msg = "     config=%s, import-config=%s" % (config_path,
                                                     self.import_config_path)
        self.wlog.verboselog(syslog.LOG_DEBUG, _msg)
        if options.date:
            _msg = "     station=%s, date=%s" % (self.station_id, options.date)
        else:
            # we must have --from and --to
            _msg = "     station=%s, from=%s, to=%s" % (self.station_id,
                                                        options.date_from,
                                                        options.date_to)
        self.wlog.verboselog(syslog.LOG_DEBUG, _msg)
        _msg = "     dry-run=%s, calc-missing=%s" % (self.dry_run,
                                                     self.calc_missing)
        self.wlog.verboselog(syslog.LOG_DEBUG, _msg)
        _msg = "     tranche=%s, interval=%s, wind_direction=%s" % (self.tranche,
                                                                    self.interval,
                                                                    self.wind_dir)
        self.wlog.verboselog(syslog.LOG_DEBUG, _msg)
        _msg = "Using database binding '%s', which is bound to database '%s'" % (self.db_binding_wx,
                                                                                 self.dbm.database_name)
        self.wlog.printlog(syslog.LOG_INFO, _msg)
        _msg = "Destination table '%s' unit system is '%#04x' (%s)." % (self.dbm.table_name,
                                                                        self.archive_unit_sys,
                                                                        unit_nicknames[self.archive_unit_sys])
        self.wlog.printlog(syslog.LOG_INFO, _msg)
        if self.calc_missing:
            print "Missing derived observations will be calculated."
        if options.date or options.date_from:
            print "Observations timestamped after %s and up to and" % (timestamp_to_string(self.first_ts), )
            print "including %s will be imported." % (timestamp_to_string(self.last_ts), )
        if self.dry_run:
            print "This is a dry run, imported data will not be saved to archive."