def dailygen(inputdata): """Internal generator function""" day_start = start count = 0 while day_start <= stop: count += 1 if count % 30 == 0: logger.info("daily: %s", day_start.isoformat(' ')) else: logger.debug("daily: %s", day_start.isoformat(' ')) day_end = day_start + DAY if use_dst: # day might be 23 or 25 hours long day_end = timezone.local_replace( day_end + HOURx3, use_dst=use_dst, hour=day_end_hour) acc.reset() for data in inputdata[day_start:day_end]: acc.add_raw(data) for data in hourly_data[day_start:day_end]: acc.add_hourly(data) new_data = acc.result() if new_data: new_data['start'] = day_start yield new_data day_start = day_end
def reprocess(data_dir, update): if update: logger.warning("Updating status to detect invalid wind_dir") count = 0 raw_data = pywws.storage.RawStore(data_dir) for data in raw_data[:]: count += 1 idx = data['idx'] if count % 10000 == 0: logger.info("update: %s", idx.isoformat(' ')) elif count % 500 == 0: logger.debug("update: %s", idx.isoformat(' ')) if data['wind_dir'] is not None and (data['wind_dir'] & 0x80): data['wind_dir'] = None raw_data[idx] = data raw_data.flush() # delete old format summary files logger.warning('Deleting old summaries') for summary in ['calib', 'hourly', 'daily', 'monthly']: for root, dirs, files in os.walk( os.path.join(data_dir, summary), topdown=False): logger.info(root) for file in files: os.unlink(os.path.join(root, file)) os.rmdir(root) # create data summaries logger.warning('Generating hourly and daily summaries') with pywws.storage.pywws_context(data_dir) as context: pywws.process.process_data(context) return 0
def monthlygen(inputdata, start, local_start): """Internal generator function""" acc = MonthAcc(rain_day_threshold) month_start = start count = 0 while month_start <= stop: count += 1 if count % 12 == 0: logger.info("monthly: %s", month_start.isoformat(' ')) else: logger.debug("monthly: %s", month_start.isoformat(' ')) if local_start.month < 12: local_start = local_start.replace(month=local_start.month+1) else: local_start = local_start.replace( month=1, year=local_start.year+1) month_end = time_zone.local_to_utc(local_start) month_end = time_zone.day_start( month_end, day_end_hour, use_dst=use_dst) acc.reset() for data in inputdata[month_start:month_end]: acc.add_daily(data) new_data = acc.result() if new_data: new_data['start'] = month_start yield new_data month_start = month_end
def reprocess(data_dir, update): with pywws.storage.pywws_context(data_dir) as context: if update: logger.warning("Updating status to detect invalid wind_dir") count = 0 raw_data = context.raw_data for data in raw_data[:]: count += 1 idx = data['idx'] if count % 10000 == 0: logger.info("update: %s", idx.isoformat(' ')) elif count % 500 == 0: logger.debug("update: %s", idx.isoformat(' ')) if data['wind_dir'] is not None and (data['wind_dir'] & 0x80): data['wind_dir'] = None raw_data[idx] = data raw_data.flush() # delete old format summary files logger.warning('Deleting old summaries') context.calib_data.clear() context.hourly_data.clear() context.daily_data.clear() context.monthly_data.clear() # create data summaries logger.warning('Generating hourly and daily summaries') pywws.process.process_data(context) return 0
def dailygen(inputdata): """Internal generator function""" day_start = start count = 0 while day_start <= stop: count += 1 if count % 30 == 0: logger.info("daily: %s", day_start.isoformat(' ')) else: logger.debug("daily: %s", day_start.isoformat(' ')) day_end = day_start + DAY if use_dst: # day might be 23 or 25 hours long day_end = timezone.local_replace(day_end + HOURx3, use_dst=use_dst, hour=day_end_hour) acc.reset() for data in inputdata[day_start:day_end]: acc.add_raw(data) for data in hourly_data[day_start:day_end]: acc.add_hourly(data) new_data = acc.result() if new_data: new_data['start'] = day_start yield new_data day_start = day_end
def monthlygen(inputdata): """Internal generator function""" month_start = start count = 0 while month_start <= stop: count += 1 if count % 12 == 0: logger.info("monthly: %s", month_start.isoformat(' ')) else: logger.debug("monthly: %s", month_start.isoformat(' ')) month_end = month_start + WEEK if month_end.month < 12: month_end = month_end.replace(month=month_end.month + 1) else: month_end = month_end.replace(month=1, year=month_end.year + 1) month_end = month_end - WEEK if use_dst: # month might straddle summer time start or end month_end = timezone.local_replace(month_end + HOURx3, use_dst=use_dst, hour=day_end_hour) acc.reset() for data in inputdata[month_start:month_end]: acc.add_daily(data) new_data = acc.result() if new_data: new_data['start'] = month_start yield new_data month_start = month_end
def calibrate_data(params, raw_data, calib_data): """'Calibrate' raw data, using a user-supplied function.""" start = calib_data.before(datetime.max) if start is None: start = datetime.min start = raw_data.after(start + SECOND) if start is None: return start del calib_data[start:] calibrator = Calib(params, raw_data) count = 0 for data in raw_data[start:]: idx = data['idx'] count += 1 if count % 10000 == 0: logger.info("calib: %s", idx.isoformat(' ')) elif count % 500 == 0: logger.debug("calib: %s", idx.isoformat(' ')) for key in ('rain', 'abs_pressure', 'temp_in'): if data[key] is None: logger.error('Ignoring invalid data at %s', idx.isoformat(' ')) break else: calib_data[idx] = calibrator.calib(data) return start
def monthlygen(inputdata): """Internal generator function""" month_start = start count = 0 while month_start <= stop: count += 1 if count % 12 == 0: logger.info("monthly: %s", month_start.isoformat(' ')) else: logger.debug("monthly: %s", month_start.isoformat(' ')) month_end = month_start + WEEK if month_end.month < 12: month_end = month_end.replace(month=month_end.month+1) else: month_end = month_end.replace(month=1, year=month_end.year+1) month_end = month_end - WEEK if use_dst: # month might straddle summer time start or end month_end = timezone.local_replace( month_end + HOURx3, use_dst=use_dst, hour=day_end_hour) acc.reset() for data in inputdata[month_start:month_end]: acc.add_daily(data) new_data = acc.result() if new_data: new_data['start'] = month_start yield new_data month_start = month_end
def generate_monthly(rain_day_threshold, day_end_hour, use_dst, daily_data, monthly_data, process_from): """Generate monthly summaries from daily data.""" start = monthly_data.before(datetime.max) if start is None: start = datetime.min start = daily_data.after(start + SECOND) if process_from: if start: start = min(start, process_from) else: start = process_from if start is None: return start # set start to start of first day of month (local time) start = timezone.local_replace(start, use_dst=use_dst, day=1, hour=day_end_hour, minute=0, second=0) if day_end_hour >= 12: # month actually starts on the last day of previous month start -= DAY del monthly_data[start:] stop = daily_data.before(datetime.max) if stop is None: return None month_start = start acc = MonthAcc(rain_day_threshold) count = 0 while month_start <= stop: count += 1 if count % 12 == 0: logger.info("monthly: %s", month_start.isoformat(' ')) else: logger.debug("monthly: %s", month_start.isoformat(' ')) month_end = month_start + WEEK if month_end.month < 12: month_end = month_end.replace(month=month_end.month + 1) else: month_end = month_end.replace(month=1, year=month_end.year + 1) month_end = month_end - WEEK if use_dst: # month might straddle summer time start or end month_end = timezone.local_replace(month_end + HOURx3, use_dst=use_dst, hour=day_end_hour) acc.reset() for data in daily_data[month_start:month_end]: acc.add_daily(data) new_data = acc.result() if new_data: new_data['start'] = month_start monthly_data[new_data['idx']] = new_data month_start = month_end return start
def session(self): logger.info("Uploading to web site with FTP") self.ftp = ftplib.FTP() self.ftp.connect(self.site, self.port) logger.debug('welcome message\n' + self.ftp.getwelcome()) self.ftp.login(self.user, self.password) self.ftp.cwd(self.directory) try: yield None finally: self.ftp.close()
def generate_monthly(rain_day_threshold, day_end_hour, use_dst, daily_data, monthly_data, process_from): """Generate monthly summaries from daily data.""" start = monthly_data.before(datetime.max) if start is None: start = datetime.min start = daily_data.after(start + SECOND) if process_from: if start: start = min(start, process_from) else: start = process_from if start is None: return start # set start to start of first day of month (local time) start = timezone.local_replace( start, use_dst=use_dst, day=1, hour=day_end_hour, minute=0, second=0) if day_end_hour >= 12: # month actually starts on the last day of previous month start -= DAY del monthly_data[start:] stop = daily_data.before(datetime.max) month_start = start acc = MonthAcc(rain_day_threshold) count = 0 while month_start <= stop: count += 1 if count % 12 == 0: logger.info("monthly: %s", month_start.isoformat(' ')) else: logger.debug("monthly: %s", month_start.isoformat(' ')) month_end = month_start + WEEK if month_end.month < 12: month_end = month_end.replace(month=month_end.month+1) else: month_end = month_end.replace(month=1, year=month_end.year+1) month_end = month_end - WEEK if use_dst: # month might straddle summer time start or end month_end = timezone.local_replace( month_end + HOURx3, use_dst=use_dst, hour=day_end_hour) acc.reset() for data in daily_data[month_start:month_end]: acc.add_daily(data) new_data = acc.result() if new_data: new_data['start'] = month_start monthly_data[new_data['idx']] = new_data month_start = month_end return start
def generate_daily(day_end_hour, use_dst, calib_data, hourly_data, daily_data, process_from): """Generate daily summaries from calibrated and hourly data.""" start = daily_data.before(datetime.max) if start is None: start = datetime.min start = calib_data.after(start + SECOND) if process_from: if start: start = min(start, process_from) else: start = process_from if start is None: return start # round to start of this day, in local time start = timezone.local_replace(start, use_dst=use_dst, hour=day_end_hour, minute=0, second=0) del daily_data[start:] stop = calib_data.before(datetime.max) day_start = start acc = DayAcc() count = 0 while day_start <= stop: count += 1 if count % 30 == 0: logger.info("daily: %s", day_start.isoformat(' ')) else: logger.debug("daily: %s", day_start.isoformat(' ')) day_end = day_start + DAY if use_dst: # day might be 23 or 25 hours long day_end = timezone.local_replace(day_end + HOURx3, use_dst=use_dst, hour=day_end_hour) acc.reset() for data in calib_data[day_start:day_end]: acc.add_raw(data) for data in hourly_data[day_start:day_end]: acc.add_hourly(data) new_data = acc.result() if new_data: new_data['start'] = day_start daily_data[new_data['idx']] = new_data day_start = day_end return start
def calibgen(inputdata): """Internal generator function""" count = 0 for data in inputdata: idx = data['idx'] count += 1 if count % 10000 == 0: logger.info("calib: %s", idx.isoformat(' ')) elif count % 500 == 0: logger.debug("calib: %s", idx.isoformat(' ')) for key in ('rain', 'abs_pressure', 'temp_in'): if data[key] is None: logger.error('Ignoring invalid data at %s', idx.isoformat(' ')) break else: yield calibrator.calib(data)
def generate_daily(day_end_hour, use_dst, calib_data, hourly_data, daily_data, process_from): """Generate daily summaries from calibrated and hourly data.""" start = daily_data.before(datetime.max) if start is None: start = datetime.min start = calib_data.after(start + SECOND) if process_from: if start: start = min(start, process_from) else: start = process_from if start is None: return start # round to start of this day, in local time start = timezone.local_replace( start, use_dst=use_dst, hour=day_end_hour, minute=0, second=0) del daily_data[start:] stop = calib_data.before(datetime.max) day_start = start acc = DayAcc() count = 0 while day_start <= stop: count += 1 if count % 30 == 0: logger.info("daily: %s", day_start.isoformat(' ')) else: logger.debug("daily: %s", day_start.isoformat(' ')) day_end = day_start + DAY if use_dst: # day might be 23 or 25 hours long day_end = timezone.local_replace( day_end + HOURx3, use_dst=use_dst, hour=day_end_hour) acc.reset() for data in calib_data[day_start:day_end]: acc.add_raw(data) for data in hourly_data[day_start:day_end]: acc.add_hourly(data) new_data = acc.result() if new_data: new_data['start'] = day_start daily_data[new_data['idx']] = new_data day_start = day_end return start
def mqtt_send_data(self, timestamp, prepared_data, ignore_last_update=False): import paho.mqtt.client as mosquitto import time import json topic = prepared_data['topic'] hostname = prepared_data['hostname'] port = prepared_data['port'] client_id = prepared_data['client_id'] retain = prepared_data['retain'] == 'True' auth = prepared_data['auth'] == 'True' multi_topic = prepared_data['multi_topic'] == 'True' # clean up the object del prepared_data['topic'] del prepared_data['hostname'] del prepared_data['port'] del prepared_data['client_id'] del prepared_data['retain'] del prepared_data['auth'] del prepared_data['multi_topic'] mosquitto_client = mosquitto.Client(client_id, protocol=mosquitto.MQTTv31) if auth: logger.debug( "%s:Username and password configured", self.service_name) if(self.password == "unknown"): mosquitto_client.username_pw_set(self.user) else: mosquitto_client.username_pw_set(self.user, self.password) else: logger.debug( "%s:Username and password unconfigured, ignoring", self.service_name) logger.debug( "%s:timestamp: %s. publishing on topic [%s] to hostname [%s] and " + "port [%s] with a client_id [%s] and retain is %s", self.service_name, timestamp.isoformat(' '), topic, hostname, port, client_id, retain) mosquitto_client.connect(hostname, int(port)) mosquitto_client.publish(topic, json.dumps(prepared_data), retain=retain) if multi_topic: #Publish a messages, one for each item in prepared_data to separate Subtopics. for item in prepared_data: if prepared_data[item] == '': prepared_data[item] = 'None' mosquitto_client.publish(topic + "/" + item, prepared_data[item], retain=retain) #Need to make sure the messages have been flushed to the server. mosquitto_client.loop(timeout=0.5) logger.debug("%s:published data: %s", self.service_name, prepared_data) mosquitto_client.disconnect() return True
def hourlygen(inputdata, prev): """Internal generator function""" hour_start = start count = 0 while hour_start <= stop: count += 1 if count % 1008 == 0: logger.info("hourly: %s", hour_start.isoformat(' ')) elif count % 24 == 0: logger.debug("hourly: %s", hour_start.isoformat(' ')) hour_end = hour_start + HOUR acc.reset() for data in inputdata[hour_start:hour_end]: if data['rel_pressure']: pressure_history.append( (data['idx'], data['rel_pressure'])) if prev: err = data['idx'] - prev['idx'] if abs(err - timedelta(minutes=data['delay'])) > TIME_ERR: logger.info('unexpected data interval %s %s', data['idx'].isoformat(' '), str(err)) acc.add_raw(data) prev = data new_data = acc.result() if new_data and (new_data['idx'] - hour_start) >= timedelta(minutes=9): # compute pressure trend new_data['pressure_trend'] = None if new_data['rel_pressure']: target = new_data['idx'] - HOURx3 while (len(pressure_history) >= 2 and abs(pressure_history[0][0] - target) > abs(pressure_history[1][0] - target)): pressure_history.popleft() if (pressure_history and abs(pressure_history[0][0] - target) < HOUR): new_data['pressure_trend'] = ( new_data['rel_pressure'] - pressure_history[0][1]) # store new hourly data yield new_data hour_start = hour_end
def log_data(self, sync=None, clear=False): # get sync config value if sync is None: if self.fixed_block['read_period'] <= 5: sync = int(self.params.get('config', 'logdata sync', '1')) else: sync = int(self.params.get('config', 'logdata sync', '0')) # get address and date-time of last complete logged data logger.info('Synchronising to weather station') range_hi = datetime.max range_lo = datetime.min last_delay = self.ws.get_data(self.ws.current_pos())['delay'] if last_delay == 0: prev_date = datetime.min else: prev_date = datetime.utcnow() for data, last_ptr, logged in self.ws.live_data( logged_only=(sync > 1)): last_date = data['idx'] logger.debug('Reading time %s', last_date.strftime('%H:%M:%S')) if logged: break if sync < 2 and self.ws._station_clock.clock: err = last_date - datetime.fromtimestamp( self.ws._station_clock.clock) last_date -= timedelta(minutes=data['delay'], seconds=err.seconds % 60) logger.debug('log time %s', last_date.strftime('%H:%M:%S')) last_ptr = self.ws.dec_ptr(last_ptr) break if sync < 1: hi = last_date - timedelta(minutes=data['delay']) if last_date - prev_date > timedelta(seconds=50): lo = hi - timedelta(seconds=60) elif data['delay'] == last_delay: lo = hi - timedelta(seconds=60) hi = hi - timedelta(seconds=48) else: lo = hi - timedelta(seconds=48) last_delay = data['delay'] prev_date = last_date range_hi = min(range_hi, hi) range_lo = max(range_lo, lo) err = (range_hi - range_lo) / 2 last_date = range_lo + err logger.debug('est log time %s +- %ds (%s..%s)', last_date.strftime('%H:%M:%S'), err.seconds, lo.strftime('%H:%M:%S'), hi.strftime('%H:%M:%S')) if err < timedelta(seconds=15): last_ptr = self.ws.dec_ptr(last_ptr) break # go back through stored data, until we catch up with what we've already got logger.info('Fetching data') self.status.set('data', 'ptr', '%06x,%s' % (last_ptr, last_date.isoformat(' '))) self.fetch_logged(last_date, last_ptr) if clear: logger.info('Clearing weather station memory') ptr = self.ws.fixed_format['data_count'][0] self.ws.write_data([(ptr, 1), (ptr + 1, 0)])
def log_data(self, sync=None, clear=False): # get sync config value if sync is None: if self.fixed_block['read_period'] <= 5: sync = int(self.params.get('config', 'logdata sync', '1')) else: sync = int(self.params.get('config', 'logdata sync', '0')) # get address and date-time of last complete logged data logger.info('Synchronising to weather station') range_hi = datetime.max range_lo = datetime.min last_delay = self.ws.get_data(self.ws.current_pos())['delay'] if last_delay == 0: prev_date = datetime.min else: prev_date = datetime.utcnow() for data, last_ptr, logged in self.ws.live_data(logged_only=(sync > 1)): last_date = data['idx'] logger.debug('Reading time %s', last_date.strftime('%H:%M:%S')) if logged: break if sync < 2 and self.ws._station_clock.clock: err = last_date - datetime.fromtimestamp( self.ws._station_clock.clock) last_date -= timedelta( minutes=data['delay'], seconds=err.seconds % 60) logger.debug('log time %s', last_date.strftime('%H:%M:%S')) last_ptr = self.ws.dec_ptr(last_ptr) break if sync < 1: hi = last_date - timedelta(minutes=data['delay']) if last_date - prev_date > timedelta(seconds=50): lo = hi - timedelta(seconds=60) elif data['delay'] == last_delay: lo = hi - timedelta(seconds=60) hi = hi - timedelta(seconds=48) else: lo = hi - timedelta(seconds=48) last_delay = data['delay'] prev_date = last_date range_hi = min(range_hi, hi) range_lo = max(range_lo, lo) err = (range_hi - range_lo) / 2 last_date = range_lo + err logger.debug('est log time %s +- %ds (%s..%s)', last_date.strftime('%H:%M:%S'), err.seconds, lo.strftime('%H:%M:%S'), hi.strftime('%H:%M:%S')) if err < timedelta(seconds=15): last_ptr = self.ws.dec_ptr(last_ptr) break # go back through stored data, until we catch up with what we've already got logger.info('Fetching data') self.status.set( 'data', 'ptr', '%06x,%s' % (last_ptr, last_date.isoformat(' '))) self.fetch_logged(last_date, last_ptr) if clear: logger.info('Clearing weather station memory') ptr = self.ws.fixed_format['data_count'][0] self.ws.write_data([(ptr, 1), (ptr+1, 0)])
def hourlygen(inputdata, prev): """Internal generator function""" hour_start = start count = 0 while hour_start <= stop: count += 1 if count % 1008 == 0: logger.info("hourly: %s", hour_start.isoformat(' ')) elif count % 24 == 0: logger.debug("hourly: %s", hour_start.isoformat(' ')) hour_end = hour_start + HOUR acc.reset() for data in inputdata[hour_start:hour_end]: if data['rel_pressure']: pressure_history.append((data['idx'], data['rel_pressure'])) if prev: err = data['idx'] - prev['idx'] if abs(err - timedelta(minutes=data['delay'])) > TIME_ERR: logger.info('unexpected data interval %s %s', data['idx'].isoformat(' '), str(err)) acc.add_raw(data) prev = data new_data = acc.result() if new_data and (new_data['idx'] - hour_start) >= timedelta(minutes=9): # compute pressure trend new_data['pressure_trend'] = None if new_data['rel_pressure']: target = new_data['idx'] - HOURx3 while (len(pressure_history) >= 2 and abs(pressure_history[0][0] - target) > abs(pressure_history[1][0] - target)): pressure_history.popleft() if (pressure_history and abs(pressure_history[0][0] - target) < HOUR): new_data['pressure_trend'] = ( new_data['rel_pressure'] - pressure_history[0][1]) # store new hourly data yield new_data hour_start = hour_end
def aprs_send_data(self, timestamp, prepared_data, ignore_last_update=False): """Upload a weather data record using APRS. The :obj:`prepared_data` parameter contains the data to be uploaded. It should be a dictionary of string keys and string values. :param timestamp: the timestamp of the data to upload. :type timestamp: datetime :param prepared_data: the data to upload. :type prepared_data: dict :param ignore_last_update: don't get or set the 'last update' status.ini entry. :type ignore_last_update: bool :return: success status :rtype: bool """ login = '******' % ( prepared_data['designator'], prepared_data['passcode'], __version__) packet = '%s>APRS,TCPIP*:@%sz%s/%s_%s/%sg%st%sr%sP%sb%sh%s.pywws-%s\n' % ( prepared_data['designator'], prepared_data['idx'], prepared_data['latitude'], prepared_data['longitude'], prepared_data['wind_dir'], prepared_data['wind_ave'], prepared_data['wind_gust'], prepared_data['temp_out'], prepared_data['rain_hour'], prepared_data['rain_day'], prepared_data['rel_pressure'], prepared_data['hum_out'], __version__) logger.debug('%s:packet: "%s"', self.service_name, packet) login = login.encode('ASCII') packet = packet.encode('ASCII') sock = socket.socket() try: sock.connect(self.server) try: response = sock.recv(4096) logger.debug('%s:server software: %s', self.service_name, response.strip()) sock.sendall(login) response = sock.recv(4096) logger.debug('%s:server login ack: %s', self.service_name, response.strip()) sock.sendall(packet) sock.shutdown(socket.SHUT_RDWR) finally: sock.close() except Exception as ex: new_ex = str(ex) if new_ex == self.old_ex: log = logger.debug else: log = logger.error self.old_ex = new_ex log('%s:exc: %s', self.service_name, new_ex) return False if not ignore_last_update: self.set_last_update(timestamp) return True
def generate_hourly(calib_data, hourly_data, process_from): """Generate hourly summaries from calibrated data.""" start = hourly_data.before(datetime.max) if start is None: start = datetime.min start = calib_data.after(start + SECOND) if process_from: if start: start = min(start, process_from) else: start = process_from if start is None: return start # set start of hour in local time (not all time offsets are integer hours) start += timezone.standard_offset start = start.replace(minute=0, second=0) start -= timezone.standard_offset del hourly_data[start:] # preload pressure history, and find last valid rain prev = None pressure_history = deque() last_rain = None for data in calib_data[start - HOURx3:start]: if data['rel_pressure']: pressure_history.append((data['idx'], data['rel_pressure'])) if data['rain'] is not None: last_rain = data['rain'] prev = data # iterate over data in one hour chunks stop = calib_data.before(datetime.max) hour_start = start acc = HourAcc(last_rain) count = 0 while hour_start <= stop: count += 1 if count % 1008 == 0: logger.info("hourly: %s", hour_start.isoformat(' ')) elif count % 24 == 0: logger.debug("hourly: %s", hour_start.isoformat(' ')) hour_end = hour_start + HOUR acc.reset() for data in calib_data[hour_start:hour_end]: if data['rel_pressure']: pressure_history.append((data['idx'], data['rel_pressure'])) if prev: err = data['idx'] - prev['idx'] if abs(err - timedelta(minutes=data['delay'])) > TIME_ERR: logger.info('unexpected data interval %s %s', data['idx'].isoformat(' '), str(err)) acc.add_raw(data) prev = data new_data = acc.result() if new_data and (new_data['idx'] - hour_start) >= timedelta(minutes=9): # compute pressure trend new_data['pressure_trend'] = None if new_data['rel_pressure']: target = new_data['idx'] - HOURx3 while (len(pressure_history) >= 2 and abs(pressure_history[0][0] - target) > abs(pressure_history[1][0] - target)): pressure_history.popleft() if (pressure_history and abs(pressure_history[0][0] - target) < HOUR): new_data['pressure_trend'] = (new_data['rel_pressure'] - pressure_history[0][1]) # store new hourly data hourly_data[new_data['idx']] = new_data hour_start = hour_end return start
def generate_hourly(calib_data, hourly_data, process_from): """Generate hourly summaries from calibrated data.""" start = hourly_data.before(datetime.max) if start is None: start = datetime.min start = calib_data.after(start + SECOND) if process_from: if start: start = min(start, process_from) else: start = process_from if start is None: return start # set start of hour in local time (not all time offsets are integer hours) start += timezone.standard_offset start = start.replace(minute=0, second=0) start -= timezone.standard_offset del hourly_data[start:] # preload pressure history, and find last valid rain prev = None pressure_history = deque() last_rain = None for data in calib_data[start - HOURx3:start]: if data['rel_pressure']: pressure_history.append((data['idx'], data['rel_pressure'])) if data['rain'] is not None: last_rain = data['rain'] prev = data # iterate over data in one hour chunks stop = calib_data.before(datetime.max) hour_start = start acc = HourAcc(last_rain) count = 0 while hour_start <= stop: count += 1 if count % 1008 == 0: logger.info("hourly: %s", hour_start.isoformat(' ')) elif count % 24 == 0: logger.debug("hourly: %s", hour_start.isoformat(' ')) hour_end = hour_start + HOUR acc.reset() for data in calib_data[hour_start:hour_end]: if data['rel_pressure']: pressure_history.append((data['idx'], data['rel_pressure'])) if prev: err = data['idx'] - prev['idx'] if abs(err - timedelta(minutes=data['delay'])) > TIME_ERR: logger.info('unexpected data interval %s %s', data['idx'].isoformat(' '), str(err)) acc.add_raw(data) prev = data new_data = acc.result() if new_data and (new_data['idx'] - hour_start) >= timedelta(minutes=9): # compute pressure trend new_data['pressure_trend'] = None if new_data['rel_pressure']: target = new_data['idx'] - HOURx3 while (len(pressure_history) >= 2 and abs(pressure_history[0][0] - target) > abs(pressure_history[1][0] - target)): pressure_history.popleft() if (pressure_history and abs(pressure_history[0][0] - target) < HOUR): new_data['pressure_trend'] = ( new_data['rel_pressure'] - pressure_history[0][1]) # store new hourly data hourly_data[new_data['idx']] = new_data hour_start = hour_end return start
def aprs_send_data(self, timestamp, prepared_data, ignore_last_update=False): """Upload a weather data record using APRS. The :obj:`prepared_data` parameter contains the data to be uploaded. It should be a dictionary of string keys and string values. :param timestamp: the timestamp of the data to upload. :type timestamp: datetime :param prepared_data: the data to upload. :type prepared_data: dict :param ignore_last_update: don't get or set the 'last update' status.ini entry. :type ignore_last_update: bool :return: success status :rtype: bool """ login = '******' % ( prepared_data['designator'], prepared_data['passcode'], __version__) packet = '%s>APRS,TCPIP*:@%sz%s/%s_%s/%sg%st%sr%sP%sb%sh%s.pywws-%s\n' % ( prepared_data['designator'], prepared_data['idx'], prepared_data['latitude'], prepared_data['longitude'], prepared_data['wind_dir'], prepared_data['wind_ave'], prepared_data['wind_gust'], prepared_data['temp_out'], prepared_data['rain_hour'], prepared_data['rain_day'], prepared_data['rel_pressure'], prepared_data['hum_out'], __version__ ) logger.debug('%s:packet: "%s"', self.service_name, packet) login = login.encode('ASCII') packet = packet.encode('ASCII') sock = socket.socket() try: sock.connect(self.server) try: response = sock.recv(4096) logger.debug('%s:server software: %s', self.service_name, response.strip()) sock.sendall(login) response = sock.recv(4096) logger.debug('%s:server login ack: %s', self.service_name, response.strip()) sock.sendall(packet) sock.shutdown(socket.SHUT_RDWR) finally: sock.close() except Exception as ex: new_ex = str(ex) if new_ex == self.old_ex: log = logger.debug else: log = logger.error self.old_ex = new_ex log('%s:exc: %s', self.service_name, new_ex) return False if not ignore_last_update: self.set_last_update(timestamp) return True
def http_send_data(self, timestamp, prepared_data, ignore_last_update=False): """Upload a weather data record using HTTP. The :obj:`prepared_data` parameter contains the data to be uploaded. It should be a dictionary of string keys and string values. :param timestamp: the timestamp of the data to upload. :type timestamp: datetime :param prepared_data: the data to upload. :type prepared_data: dict :param ignore_last_update: don't get or set the 'last update' status.ini entry. :type ignore_last_update: bool :return: success status :rtype: bool """ coded_data = urlencode(prepared_data) logger.debug('%s:%s', self.service_name, coded_data) new_ex = self.old_ex ex_info = [] success = False try: if self.use_get: request = Request(self.server + '?' + coded_data) else: request = Request(self.server, coded_data.encode('ASCII')) if self.auth_type == 'basic': request.add_header('Authorization', self.auth) if self.http_headers is not None: for header in self.http_headers: request.add_header(header[0], header[1]) rsp = urlopen(request) response = rsp.readlines() rsp.close() if response == self.old_response: log = logger.debug else: log = logger.error self.old_response = response for line in response: log('%s:rsp: %s', self.service_name, line.strip()) for n, expected in enumerate(self.expected_result): if n < len(response): actual = response[n].decode('utf-8') if not re.match(expected, actual): break else: self.old_response = response if not ignore_last_update: self.set_last_update(timestamp) return True return False except HTTPError as ex: if ex.code == 429 and self.service_name == 'metoffice': # UK Met Office server uses 429 to signal duplicate data success = True if sys.version_info >= (2, 7): new_ex = '[%d]%s' % (ex.code, ex.reason) else: new_ex = str(ex) ex_info = str(ex.info()).split('\n') try: for line in ex.readlines(): line = line.decode('utf-8') ex_info.append(re.sub('<.+?>', '', line)) except Exception: pass except URLError as ex: new_ex = str(ex.reason) except Exception as ex: new_ex = str(ex) if new_ex == self.old_ex: log = logger.debug else: log = logger.error self.old_ex = new_ex log('%s:exc: %s', self.service_name, new_ex) for extra in ex_info: extra = extra.strip() if extra: log('info: %s', extra) if success and not ignore_last_update: self.set_last_update(timestamp) return success
def mqtt_send_data(self, timestamp, prepared_data, ignore_last_update=False): import paho.mqtt.client as mosquitto import time import json topic = prepared_data['topic'] hostname = prepared_data['hostname'] port = prepared_data['port'] client_id = prepared_data['client_id'] retain = prepared_data['retain'] == 'True' auth = prepared_data['auth'] == 'True' multi_topic = prepared_data['multi_topic'] == 'True' # clean up the object del prepared_data['topic'] del prepared_data['hostname'] del prepared_data['port'] del prepared_data['client_id'] del prepared_data['retain'] del prepared_data['auth'] del prepared_data['multi_topic'] mosquitto_client = mosquitto.Client(client_id, protocol=mosquitto.MQTTv31) if auth: logger.debug("%s:Username and password configured", self.service_name) if (self.password == "unknown"): mosquitto_client.username_pw_set(self.user) else: mosquitto_client.username_pw_set(self.user, self.password) else: logger.debug("%s:Username and password unconfigured, ignoring", self.service_name) logger.debug( "%s:timestamp: %s. publishing on topic [%s] to hostname [%s] and " + "port [%s] with a client_id [%s] and retain is %s", self.service_name, timestamp.isoformat(' '), topic, hostname, port, client_id, retain) mosquitto_client.connect(hostname, int(port)) mosquitto_client.publish(topic, json.dumps(prepared_data), retain=retain) if multi_topic: #Publish a messages, one for each item in prepared_data to separate Subtopics. for item in prepared_data: if prepared_data[item] == '': prepared_data[item] = 'None' mosquitto_client.publish(topic + "/" + item, prepared_data[item], retain=retain) #Need to make sure the messages have been flushed to the server. mosquitto_client.loop(timeout=0.5) logger.debug("%s:published data: %s", self.service_name, prepared_data) mosquitto_client.disconnect() return True