def LiveLog(data_dir): logger = logging.getLogger('pywws.LiveLog') params = DataStore.params(data_dir) status = DataStore.status(data_dir) # localise application Localisation.SetApplicationLanguage(params) # open data file stores raw_data = DataStore.data_store(data_dir) calib_data = DataStore.calib_store(data_dir) hourly_data = DataStore.hourly_store(data_dir) daily_data = DataStore.daily_store(data_dir) monthly_data = DataStore.monthly_store(data_dir) # create a DataLogger object datalogger = DataLogger(params, status, raw_data) # create a RegularTasks object asynch = eval(params.get('config', 'asynchronous', 'False')) tasks = Tasks.RegularTasks(params, status, raw_data, calib_data, hourly_data, daily_data, monthly_data, asynch=asynch) # get live data try: for data, logged in datalogger.live_data( logged_only=(not tasks.has_live_tasks())): if logged: # process new data Process.Process(params, raw_data, calib_data, hourly_data, daily_data, monthly_data) # do tasks tasks.do_tasks() else: tasks.do_live(data) except Exception, ex: logger.exception(ex)
def get_readings(self): for data in DataStore.data_store(self.data_store)[self.last_ts:]: if data['idx'] <= self.last_ts: continue self.last_ts = data['idx'] for key in self.keys: if not data[key]: continue yield Reading(READING_TYPE, data[key], data['idx'], key)
def Reprocess(data_dir): # delete old format summary files print "Deleting old summaries" for summary in ["calib", "hourly", "daily", "monthly"]: for root, dirs, files in os.walk(os.path.join(data_dir, summary), topdown=False): print root for file in files: os.unlink(os.path.join(root, file)) os.rmdir(root) # create data summaries print "Generating hourly and daily summaries" params = DataStore.params(data_dir) raw_data = DataStore.data_store(data_dir) calib_data = DataStore.calib_store(data_dir) hourly_data = DataStore.hourly_store(data_dir) daily_data = DataStore.daily_store(data_dir) monthly_data = DataStore.monthly_store(data_dir) Process.Process(params, raw_data, calib_data, hourly_data, daily_data, monthly_data) return 0
def Reprocess(data_dir, update): logger = logging.getLogger('pywws-reprocess') raw_data = DataStore.data_store(data_dir) if update: # update old data to copy high nibble of wind_dir to status logger.warning("Updating status to include extra bits from wind_dir") count = 0 for data in raw_data[:]: count += 1 idx = data['idx'] if count % 10000 == 0: logger.info("update: %s", idx.isoformat(' ')) elif count % 500 == 0: logger.debug("update: %s", idx.isoformat(' ')) if data['wind_dir'] is not None: if data['wind_dir'] >= 16: data['status'] |= (data['wind_dir'] & 0xF0) << 4 data['wind_dir'] &= 0x0F raw_data[idx] = data if data['status'] & 0x800: data['wind_dir'] = None raw_data[idx] = data raw_data.flush() # delete old format summary files logger.warning('Deleting old summaries') for summary in ['calib', 'hourly', 'daily', 'monthly']: for root, dirs, files in os.walk( os.path.join(data_dir, summary), topdown=False): logger.info(root) for file in files: os.unlink(os.path.join(root, file)) os.rmdir(root) # create data summaries logger.warning('Generating hourly and daily summaries') params = DataStore.params(data_dir) calib_data = DataStore.calib_store(data_dir) hourly_data = DataStore.hourly_store(data_dir) daily_data = DataStore.daily_store(data_dir) monthly_data = DataStore.monthly_store(data_dir) Process.Process( params, raw_data, calib_data, hourly_data, daily_data, monthly_data) return 0
def Reprocess(data_dir, update): logger = logging.getLogger('pywws.Reprocess') raw_data = DataStore.data_store(data_dir) if update: # update old data to copy high nibble of wind_dir to status logger.warning("Updating status to include extra bits from wind_dir") count = 0 for data in raw_data[:]: count += 1 idx = data['idx'] if count % 10000 == 0: logger.info("update: %s", idx.isoformat(' ')) elif count % 500 == 0: logger.debug("update: %s", idx.isoformat(' ')) if data['wind_dir'] is not None: if data['wind_dir'] >= 16: data['status'] |= (data['wind_dir'] & 0xF0) << 4 data['wind_dir'] &= 0x0F raw_data[idx] = data if data['status'] & 0x800: data['wind_dir'] = None raw_data[idx] = data raw_data.flush() # delete old format summary files logger.warning('Deleting old summaries') for summary in ['calib', 'hourly', 'daily', 'monthly']: for root, dirs, files in os.walk(os.path.join(data_dir, summary), topdown=False): logger.info(root) for file in files: os.unlink(os.path.join(root, file)) os.rmdir(root) # create data summaries logger.warning('Generating hourly and daily summaries') params = DataStore.params(data_dir) calib_data = DataStore.calib_store(data_dir) hourly_data = DataStore.hourly_store(data_dir) daily_data = DataStore.daily_store(data_dir) monthly_data = DataStore.monthly_store(data_dir) Process.Process(params, raw_data, calib_data, hourly_data, daily_data, monthly_data) return 0
def Reprocess(data_dir): # delete old format summary files print 'Deleting old summaries' for summary in ['calib', 'hourly', 'daily', 'monthly']: for root, dirs, files in os.walk(os.path.join(data_dir, summary), topdown=False): print root for file in files: os.unlink(os.path.join(root, file)) os.rmdir(root) # create data summaries print 'Generating hourly and daily summaries' params = DataStore.params(data_dir) raw_data = DataStore.data_store(data_dir) calib_data = DataStore.calib_store(data_dir) hourly_data = DataStore.hourly_store(data_dir) daily_data = DataStore.daily_store(data_dir) monthly_data = DataStore.monthly_store(data_dir) Process.Process(params, raw_data, calib_data, hourly_data, daily_data, monthly_data) return 0
def Hourly(data_dir): # get file locations params = DataStore.params(data_dir) # localise application Localisation.SetApplicationLanguage(params) # open data file stores raw_data = DataStore.data_store(data_dir) calib_data = DataStore.calib_store(data_dir) hourly_data = DataStore.hourly_store(data_dir) daily_data = DataStore.daily_store(data_dir) monthly_data = DataStore.monthly_store(data_dir) # get weather station data # LogData.LogData(params, raw_data) # do the processing Process.Process(params, raw_data, calib_data, hourly_data, daily_data, monthly_data) # do tasks if not Tasks.RegularTasks(params, calib_data, hourly_data, daily_data, monthly_data).do_tasks(): return 1 return 0
def Hourly(data_dir): # get file locations params = DataStore.params(data_dir) # localise application Localisation.SetApplicationLanguage(params) # open data file stores raw_data = DataStore.data_store(data_dir) calib_data = DataStore.calib_store(data_dir) hourly_data = DataStore.hourly_store(data_dir) daily_data = DataStore.daily_store(data_dir) monthly_data = DataStore.monthly_store(data_dir) # get weather station data LogData.LogData(params, raw_data) # do the processing Process.Process( params, raw_data, calib_data, hourly_data, daily_data, monthly_data) # do tasks if not Tasks.RegularTasks( params, calib_data, hourly_data, daily_data, monthly_data ).do_tasks(): return 1 return 0
print >>sys.stderr, 'Error: %s\n' % msg print >>sys.stderr, __usage__.strip() return 1 # process options clear = False sync = None verbose = 0 for o, a in opts: if o in ('-h', '--help'): print __usage__.strip() return 0 elif o in ('-c', '--clear'): clear = True elif o in ('-s', '--sync'): sync = int(a) elif o in ('-v', '--verbose'): verbose += 1 # check arguments if len(args) != 1: print >>sys.stderr, 'Error: 1 argument required\n' print >>sys.stderr, __usage__.strip() return 2 logger = ApplicationLogger(verbose) root_dir = args[0] DataLogger( DataStore.params(root_dir), DataStore.status(root_dir), DataStore.data_store(root_dir)).log_data(sync=sync, clear=clear) if __name__ == "__main__": sys.exit(main())
print >> sys.stderr, 'Error: %s\n' % msg print >> sys.stderr, __usage__.strip() return 1 # process options clear = False sync = None verbose = 0 for o, a in opts: if o in ('-h', '--help'): print __usage__.strip() return 0 elif o in ('-c', '--clear'): clear = True elif o in ('-s', '--sync'): sync = int(a) elif o in ('-v', '--verbose'): verbose += 1 # check arguments if len(args) != 1: print >> sys.stderr, 'Error: 1 argument required\n' print >> sys.stderr, __usage__.strip() return 2 logger = ApplicationLogger(verbose) root_dir = args[0] DataLogger(DataStore.params(root_dir), DataStore.status(root_dir), DataStore.data_store(root_dir)).log_data(sync=sync, clear=clear) if __name__ == "__main__": sys.exit(main())
print >>sys.stderr, 'Error: %s\n' % msg print >>sys.stderr, __usage__.strip() return 1 # process options clear = False sync = None verbose = 0 for o, a in opts: if o in ('-h', '--help'): print __usage__.strip() return 0 elif o in ('-c', '--clear'): clear = True elif o in ('-s', '--sync'): sync = int(a) elif o in ('-v', '--verbose'): verbose += 1 # check arguments if len(args) != 1: print >>sys.stderr, 'Error: 1 argument required\n' print >>sys.stderr, __usage__.strip() return 2 logger = ApplicationLogger(verbose) root_dir = args[0] return LogData( DataStore.params(root_dir), DataStore.status(root_dir), DataStore.data_store(root_dir), sync=sync, clear=clear) if __name__ == "__main__": sys.exit(main())
######## # Main ######## # Global Variables AdaScreenNumber = 0 data = {} forecast_bom_today = "" forecast_bom_tomorrow = "" forecast_file_today = "" forecast_toggle = 0 global_init=True readings = {} # pywws data if config.getboolean('Output','PYWWS_PUBLISH'): ds = DataStore.data_store(config.get('PYWWS','STORAGE')) dstatus = DataStore.status(config.get('PYWWS','STORAGE')) if config.getboolean('Output','ADA_LCD'): AdaLcd.clear() if config.getboolean('Output','SENSEHAT_DISPLAY'): # Set up display PiSenseHat.clear() PiSenseHat.set_rotation(config.get('SenseHat','ROTATION')) if config.getboolean('Sensors','ENOCEAN'): eoCommunicator = eoSerialCommunicator(port=config.get('EnOcean','PORT')) eoCommunicator.start() # Warm up sensors print "Waiting for sensors to settle" for i in range(1,6): Sample() time.sleep(1)
try: opts, args = getopt.getopt(argv[1:], "hv", ['help', 'verbose']) except getopt.error, msg: print >>sys.stderr, 'Error: %s\n' % msg print >>sys.stderr, __usage__.strip() return 1 # process options verbose = 0 for o, a in opts: if o in ('-h', '--help'): print __usage__.strip() return 0 elif o in ('-v', '--verbose'): verbose += 1 # check arguments if len(args) != 1: print >>sys.stderr, 'Error: 1 argument required\n' print >>sys.stderr, __usage__.strip() return 2 logger = ApplicationLogger(verbose) data_dir = args[0] return Process(DataStore.params(data_dir), DataStore.data_store(data_dir), DataStore.calib_store(data_dir), DataStore.hourly_store(data_dir), DataStore.daily_store(data_dir), DataStore.monthly_store(data_dir)) if __name__ == "__main__": sys.exit(main())
if o == '-h' or o == '--help': print __usage__.strip() return 0 elif o == '-n' or o == '--noaction': noaction = True # check arguments if len(args) != 1: print >>sys.stderr, 'Error: 1 argument required\n' print >>sys.stderr, __usage__.strip() return 2 data_dir = args[0] # date & time range of data to be changed, in UTC! start = datetime(2013, 10, 27, 11, 21) stop = datetime(2013, 10, 29, 18, 32) # open data store raw_data = DataStore.data_store(data_dir) # process the data aperture = timedelta(minutes=14, seconds=30) # make list of changes to apply after examining the data changes = [] for data in raw_data[start:stop]: if data['temp_out'] is None: continue # get temperatures at nearby times idx = data['idx'] temp_list = [] for local_data in raw_data[idx-aperture:idx+aperture]: temp = local_data['temp_out'] if temp is not None: temp_list.append(temp) if len(temp_list) < 3:
def LiveLog(data_dir): logger = logging.getLogger('pywws.LiveLog') params = DataStore.params(data_dir) status = DataStore.status(data_dir) # localise application Localisation.SetApplicationLanguage(params) # connect to weather station ws_type = params.get('fixed', 'ws type') if ws_type: params.unset('fixed', 'ws type') params.set('config', 'ws type', ws_type) ws_type = params.get('config', 'ws type', '1080') ws = WeatherStation.weather_station( ws_type=ws_type, params=params, status=status) fixed_block = CheckFixedBlock(ws, params, status, logger) if not fixed_block: logger.error("Invalid data from weather station") return 3 # open data file stores raw_data = DataStore.data_store(data_dir) calib_data = DataStore.calib_store(data_dir) hourly_data = DataStore.hourly_store(data_dir) daily_data = DataStore.daily_store(data_dir) monthly_data = DataStore.monthly_store(data_dir) # create a RegularTasks object tasks = Tasks.RegularTasks( params, status, calib_data, hourly_data, daily_data, monthly_data) # get time of last logged data two_minutes = timedelta(minutes=2) last_stored = raw_data.before(datetime.max) if last_stored == None: last_stored = datetime.min if datetime.utcnow() < last_stored: raise ValueError('Computer time is earlier than last stored data') last_stored += two_minutes # get live data hour = timedelta(hours=1) next_hour = datetime.utcnow().replace( minute=0, second=0, microsecond=0) + hour next_ptr = None for data, ptr, logged in ws.live_data( logged_only=(not tasks.has_live_tasks())): now = data['idx'] if logged: if ptr == next_ptr: # data is contiguous with last logged value raw_data[now] = data else: # catch up missing data Catchup(ws, logger, raw_data, now, ptr) next_ptr = ws.inc_ptr(ptr) # process new data Process.Process(params, status, raw_data, calib_data, hourly_data, daily_data, monthly_data) # do tasks tasks.do_tasks() if now >= next_hour: next_hour += hour fixed_block = CheckFixedBlock(ws, params, status, logger) if not fixed_block: logger.error("Invalid data from weather station") return 3 # save any unsaved data raw_data.flush() else: tasks.do_live(data) return 0
# process options for o, a in opts: if o == '-h' or o == '--help': print __usage__.strip() return 0 # check arguments if len(args) != 1: print >> sys.stderr, 'Error: 1 argument required\n' print >> sys.stderr, __usage__.strip() return 2 data_dir = args[0] # date & time range of data to be changed, in UTC! start = datetime(2013, 10, 26, 15, 23) stop = datetime(2013, 10, 30, 12, 47) # open data store raw_data = DataStore.data_store(data_dir) # change the data for data in raw_data[start:stop]: data['rain'] -= 263.1 raw_data[data['idx']] = data # make sure it's saved raw_data.flush() # clear calibrated data that needs to be regenerated calib_data = DataStore.calib_store(data_dir) del calib_data[start:] calib_data.flush() # done return 0 if __name__ == "__main__":
for o, a in opts: if o == '--help': usage() return 0 # check arguments if len(args) != 2: print >>sys.stderr, 'Error: 2 arguments required\n' print >>sys.stderr, __usage__.strip() return 2 # process arguments in_name = args[0] out_name = args[1] # open input in_file = open(in_name, 'r') # open data file store ds = DataStore.data_store(out_name) # get time to go forward to first_stored = ds.after(datetime.min) if first_stored == None: first_stored = datetime.max # copy any missing data last_date = None count = 0 for line in in_file: items = line.split(',') local_date = DataStore.safestrptime(items[2].strip(), '%Y-%m-%d %H:%M:%S') local_date = local_date.replace(tzinfo=TimeZone.Local) date = local_date.astimezone(TimeZone.utc) if last_date and date < last_date: date = date + timedelta(hours=1) print "Corrected DST ambiguity %s %s -> %s" % (
def LiveLog(data_dir): logger = logging.getLogger('pywws.LiveLog') params = DataStore.params(data_dir) # localise application Localisation.SetApplicationLanguage(params) # connect to weather station ws_type = params.get('config', 'ws type') if ws_type: params._config.remove_option('config', 'ws type') params.set('fixed', 'ws type', ws_type) ws_type = params.get('fixed', 'ws type', '1080') ws = WeatherStation.weather_station(ws_type=ws_type) fixed_block = CheckFixedBlock(ws, params, logger) if not fixed_block: logger.error("Invalid data from weather station") return 3 # open data file stores raw_data = DataStore.data_store(data_dir) calib_data = DataStore.calib_store(data_dir) hourly_data = DataStore.hourly_store(data_dir) daily_data = DataStore.daily_store(data_dir) monthly_data = DataStore.monthly_store(data_dir) # create a RegularTasks object tasks = Tasks.RegularTasks( params, calib_data, hourly_data, daily_data, monthly_data) # get time of last logged data two_minutes = timedelta(minutes=2) last_stored = raw_data.before(datetime.max) if last_stored == None: last_stored = datetime.min if datetime.utcnow() < last_stored: raise ValueError('Computer time is earlier than last stored data') last_stored += two_minutes # get live data hour = timedelta(hours=1) next_hour = datetime.utcnow().replace( minute=0, second=0, microsecond=0) + hour next_ptr = None for data, ptr, logged in ws.live_data( logged_only=(not tasks.has_live_tasks())): now = data['idx'] if logged: if ptr == next_ptr: # data is contiguous with last logged value raw_data[now] = data else: # catch up missing data Catchup(ws, logger, raw_data, now, ptr) next_ptr = ws.inc_ptr(ptr) # process new data Process.Process(params, raw_data, calib_data, hourly_data, daily_data, monthly_data) # do tasks tasks.do_tasks() if now >= next_hour: next_hour += hour fixed_block = CheckFixedBlock(ws, params, logger) if not fixed_block: logger.error("Invalid data from weather station") return 3 params.flush() else: tasks.do_live(data) return 0
for o, a in opts: if o == '--help': usage() return 0 # check arguments if len(args) != 2: print >>sys.stderr, 'Error: 2 arguments required\n' print >>sys.stderr, __doc__.strip() return 2 # process arguments in_name = args[0] out_name = args[1] # open input in_file = open(in_name, 'r') # open data file store ds = DataStore.data_store(out_name) # get time to go forward to first_stored = ds.after(datetime.min) if first_stored == None: first_stored = datetime.max # copy any missing data last_date = None count = 0 for line in in_file: items = line.split(',') local_date = DataStore.safestrptime(items[2].strip(), '%Y-%m-%d %H:%M:%S') local_date = local_date.replace(tzinfo=TimeZone.Local) date = local_date.astimezone(TimeZone.utc) if last_date and date < last_date: date = date + timedelta(hours=1) print "Corrected DST ambiguity %s %s -> %s" % (