def LiveLog(data_dir): logger = logging.getLogger('pywws.LiveLog') params = DataStore.params(data_dir) status = DataStore.status(data_dir) # localise application Localisation.SetApplicationLanguage(params) # open data file stores raw_data = DataStore.data_store(data_dir) calib_data = DataStore.calib_store(data_dir) hourly_data = DataStore.hourly_store(data_dir) daily_data = DataStore.daily_store(data_dir) monthly_data = DataStore.monthly_store(data_dir) # create a DataLogger object datalogger = DataLogger(params, status, raw_data) # create a RegularTasks object asynch = eval(params.get('config', 'asynchronous', 'False')) tasks = Tasks.RegularTasks(params, status, raw_data, calib_data, hourly_data, daily_data, monthly_data, asynch=asynch) # get live data try: for data, logged in datalogger.live_data( logged_only=(not tasks.has_live_tasks())): if logged: # process new data Process.Process(params, raw_data, calib_data, hourly_data, daily_data, monthly_data) # do tasks tasks.do_tasks() else: tasks.do_live(data) except Exception, ex: logger.exception(ex)
def get_pywws_data(dir_data): ydat = {'temp_out_max_hi':float('-inf'),'temp_out_min_lo':float('inf'), 'hum_out_max':float('-inf'),'hum_out_min':float('inf'),'rel_pressure_max':float('-inf'), 'rel_pressure_min':float('inf'),'wind_gust':float('-inf'),'rain':0.0, 'temp_in_max_hi':float('-inf'),'temp_in_min_lo':float('inf'),'hum_in_max':float('-inf'), 'hum_in_min':float('inf')} ahora = datetime.utcnow() try: dat = DataStore.calib_store(dir_data) adat = dat[dat.nearest(ahora)] gdat = dat[dat.nearest(ahora)-timedelta(hours=24):] dat = DataStore.hourly_store(dir_data) hdat = dat[dat.nearest(ahora)-timedelta(hours=24):] dat = DataStore.daily_store(dir_data) ddat = dat[dat.nearest(ahora)] dat = DataStore.monthly_store(dir_data) mdat = dat[dat.nearest(ahora)] f1=datetime(ahora.year,1,1,0,0,0)#primer momento del año for d in dat[dat.after(f1):]: for k in ydat.keys(): if 'min' in k: ydat[k] = min(ydat[k],d[k]) elif 'rain' in k: ydat[k] = ydat[k]+d[k] else: ydat[k] = max(ydat[k],d[k]) except: return None return {'a':adat,'g':gdat,'h':hdat,'d':ddat,'m':mdat,'y':ydat}
def Reprocess(data_dir, update): logger = logging.getLogger('pywws-reprocess') raw_data = DataStore.data_store(data_dir) if update: # update old data to copy high nibble of wind_dir to status logger.warning("Updating status to include extra bits from wind_dir") count = 0 for data in raw_data[:]: count += 1 idx = data['idx'] if count % 10000 == 0: logger.info("update: %s", idx.isoformat(' ')) elif count % 500 == 0: logger.debug("update: %s", idx.isoformat(' ')) if data['wind_dir'] is not None: if data['wind_dir'] >= 16: data['status'] |= (data['wind_dir'] & 0xF0) << 4 data['wind_dir'] &= 0x0F raw_data[idx] = data if data['status'] & 0x800: data['wind_dir'] = None raw_data[idx] = data raw_data.flush() # delete old format summary files logger.warning('Deleting old summaries') for summary in ['calib', 'hourly', 'daily', 'monthly']: for root, dirs, files in os.walk( os.path.join(data_dir, summary), topdown=False): logger.info(root) for file in files: os.unlink(os.path.join(root, file)) os.rmdir(root) # create data summaries logger.warning('Generating hourly and daily summaries') params = DataStore.params(data_dir) calib_data = DataStore.calib_store(data_dir) hourly_data = DataStore.hourly_store(data_dir) daily_data = DataStore.daily_store(data_dir) monthly_data = DataStore.monthly_store(data_dir) Process.Process( params, raw_data, calib_data, hourly_data, daily_data, monthly_data) return 0
def Reprocess(data_dir): # delete old format summary files print "Deleting old summaries" for summary in ["calib", "hourly", "daily", "monthly"]: for root, dirs, files in os.walk(os.path.join(data_dir, summary), topdown=False): print root for file in files: os.unlink(os.path.join(root, file)) os.rmdir(root) # create data summaries print "Generating hourly and daily summaries" params = DataStore.params(data_dir) raw_data = DataStore.data_store(data_dir) calib_data = DataStore.calib_store(data_dir) hourly_data = DataStore.hourly_store(data_dir) daily_data = DataStore.daily_store(data_dir) monthly_data = DataStore.monthly_store(data_dir) Process.Process(params, raw_data, calib_data, hourly_data, daily_data, monthly_data) return 0
def Reprocess(data_dir, update): logger = logging.getLogger('pywws.Reprocess') raw_data = DataStore.data_store(data_dir) if update: # update old data to copy high nibble of wind_dir to status logger.warning("Updating status to include extra bits from wind_dir") count = 0 for data in raw_data[:]: count += 1 idx = data['idx'] if count % 10000 == 0: logger.info("update: %s", idx.isoformat(' ')) elif count % 500 == 0: logger.debug("update: %s", idx.isoformat(' ')) if data['wind_dir'] is not None: if data['wind_dir'] >= 16: data['status'] |= (data['wind_dir'] & 0xF0) << 4 data['wind_dir'] &= 0x0F raw_data[idx] = data if data['status'] & 0x800: data['wind_dir'] = None raw_data[idx] = data raw_data.flush() # delete old format summary files logger.warning('Deleting old summaries') for summary in ['calib', 'hourly', 'daily', 'monthly']: for root, dirs, files in os.walk(os.path.join(data_dir, summary), topdown=False): logger.info(root) for file in files: os.unlink(os.path.join(root, file)) os.rmdir(root) # create data summaries logger.warning('Generating hourly and daily summaries') params = DataStore.params(data_dir) calib_data = DataStore.calib_store(data_dir) hourly_data = DataStore.hourly_store(data_dir) daily_data = DataStore.daily_store(data_dir) monthly_data = DataStore.monthly_store(data_dir) Process.Process(params, raw_data, calib_data, hourly_data, daily_data, monthly_data) return 0
def Hourly(data_dir): # get file locations params = DataStore.params(data_dir) # localise application Localisation.SetApplicationLanguage(params) # open data file stores raw_data = DataStore.data_store(data_dir) calib_data = DataStore.calib_store(data_dir) hourly_data = DataStore.hourly_store(data_dir) daily_data = DataStore.daily_store(data_dir) monthly_data = DataStore.monthly_store(data_dir) # get weather station data # LogData.LogData(params, raw_data) # do the processing Process.Process(params, raw_data, calib_data, hourly_data, daily_data, monthly_data) # do tasks if not Tasks.RegularTasks(params, calib_data, hourly_data, daily_data, monthly_data).do_tasks(): return 1 return 0
def Reprocess(data_dir): # delete old format summary files print 'Deleting old summaries' for summary in ['calib', 'hourly', 'daily', 'monthly']: for root, dirs, files in os.walk(os.path.join(data_dir, summary), topdown=False): print root for file in files: os.unlink(os.path.join(root, file)) os.rmdir(root) # create data summaries print 'Generating hourly and daily summaries' params = DataStore.params(data_dir) raw_data = DataStore.data_store(data_dir) calib_data = DataStore.calib_store(data_dir) hourly_data = DataStore.hourly_store(data_dir) daily_data = DataStore.daily_store(data_dir) monthly_data = DataStore.monthly_store(data_dir) Process.Process(params, raw_data, calib_data, hourly_data, daily_data, monthly_data) return 0
def Hourly(data_dir): # get file locations params = DataStore.params(data_dir) # localise application Localisation.SetApplicationLanguage(params) # open data file stores raw_data = DataStore.data_store(data_dir) calib_data = DataStore.calib_store(data_dir) hourly_data = DataStore.hourly_store(data_dir) daily_data = DataStore.daily_store(data_dir) monthly_data = DataStore.monthly_store(data_dir) # get weather station data LogData.LogData(params, raw_data) # do the processing Process.Process( params, raw_data, calib_data, hourly_data, daily_data, monthly_data) # do tasks if not Tasks.RegularTasks( params, calib_data, hourly_data, daily_data, monthly_data ).do_tasks(): return 1 return 0
argv = sys.argv try: opts, args = getopt.getopt(argv[1:], "h", ['help']) except getopt.error, msg: print >>sys.stderr, 'Error: %s\n' % msg print >>sys.stderr, __usage__.strip() return 1 # process options for o, a in opts: if o == '-h' or o == '--help': print __usage__.strip() return 0 # check arguments if len(args) != 4: print >>sys.stderr, 'Error: 4 arguments required\n' print >>sys.stderr, __usage__.strip() return 2 logger = ApplicationLogger(1) params = DataStore.params(args[0]) status = DataStore.status(args[0]) Localisation.SetApplicationLanguage(params) return GraphPlotter( params, status, DataStore.calib_store(args[0]), DataStore.hourly_store(args[0]), DataStore.daily_store(args[0]), DataStore.monthly_store(args[0]), args[1] ).DoPlot(args[2], args[3]) if __name__ == "__main__": sys.exit(main())
argv = sys.argv try: opts, args = getopt.getopt(argv[1:], "h", ['help']) except getopt.error, msg: print >>sys.stderr, 'Error: %s\n' % msg print >>sys.stderr, __usage__.strip() return 1 # process options for o, a in opts: if o == '-h' or o == '--help': print __usage__.strip() return 0 # check arguments if len(args) != 4: print >>sys.stderr, 'Error: 4 arguments required\n' print >>sys.stderr, __usage__.strip() return 2 logger = ApplicationLogger(2) params = DataStore.params(args[0]) status = DataStore.status(args[0]) Localisation.SetApplicationLanguage(params) return GraphPlotter( params, status, DataStore.calib_store(args[0]), DataStore.hourly_store(args[0]), DataStore.daily_store(args[0]), DataStore.monthly_store(args[0]), args[1] ).DoPlot(GraphFileReader(args[2]), args[3]) if __name__ == "__main__": sys.exit(main())
def LiveLog(data_dir): logger = logging.getLogger('pywws.LiveLog') params = DataStore.params(data_dir) status = DataStore.status(data_dir) # localise application Localisation.SetApplicationLanguage(params) # connect to weather station ws_type = params.get('fixed', 'ws type') if ws_type: params.unset('fixed', 'ws type') params.set('config', 'ws type', ws_type) ws_type = params.get('config', 'ws type', '1080') ws = WeatherStation.weather_station( ws_type=ws_type, params=params, status=status) fixed_block = CheckFixedBlock(ws, params, status, logger) if not fixed_block: logger.error("Invalid data from weather station") return 3 # open data file stores raw_data = DataStore.data_store(data_dir) calib_data = DataStore.calib_store(data_dir) hourly_data = DataStore.hourly_store(data_dir) daily_data = DataStore.daily_store(data_dir) monthly_data = DataStore.monthly_store(data_dir) # create a RegularTasks object tasks = Tasks.RegularTasks( params, status, calib_data, hourly_data, daily_data, monthly_data) # get time of last logged data two_minutes = timedelta(minutes=2) last_stored = raw_data.before(datetime.max) if last_stored == None: last_stored = datetime.min if datetime.utcnow() < last_stored: raise ValueError('Computer time is earlier than last stored data') last_stored += two_minutes # get live data hour = timedelta(hours=1) next_hour = datetime.utcnow().replace( minute=0, second=0, microsecond=0) + hour next_ptr = None for data, ptr, logged in ws.live_data( logged_only=(not tasks.has_live_tasks())): now = data['idx'] if logged: if ptr == next_ptr: # data is contiguous with last logged value raw_data[now] = data else: # catch up missing data Catchup(ws, logger, raw_data, now, ptr) next_ptr = ws.inc_ptr(ptr) # process new data Process.Process(params, status, raw_data, calib_data, hourly_data, daily_data, monthly_data) # do tasks tasks.do_tasks() if now >= next_hour: next_hour += hour fixed_block = CheckFixedBlock(ws, params, status, logger) if not fixed_block: logger.error("Invalid data from weather station") return 3 # save any unsaved data raw_data.flush() else: tasks.do_live(data) return 0
print >>sys.stderr, __usage__.strip() return 1 # process options for o, a in opts: if o in ('-h', '--help'): print __usage__.strip() return 0 # check arguments if len(args) != 1: print >>sys.stderr, "Error: 1 argument required" print >>sys.stderr, __usage__.strip() return 2 data_dir = args[0] params = DataStore.params(data_dir) Localisation.SetApplicationLanguage(params) hourly_data = DataStore.hourly_store(data_dir) idx = hourly_data.before(datetime.max) print 'Zambretti (current):', Zambretti(params, hourly_data[idx]) idx = idx.replace(tzinfo=utc).astimezone(Local) if idx.hour < 8 or (idx.hour == 8 and idx.minute < 30): idx -= timedelta(hours=24) idx = idx.replace(hour=9, minute=0, second=0) idx = hourly_data.nearest(idx.astimezone(utc).replace(tzinfo=None)) lcl = idx.replace(tzinfo=utc).astimezone(Local) print 'Zambretti (at %s):' % lcl.strftime('%H:%M %Z'), Zambretti( params, hourly_data[idx]) return 0 if __name__ == "__main__": sys.exit(main())
def LiveLog(data_dir): logger = logging.getLogger('pywws.LiveLog') params = DataStore.params(data_dir) # localise application Localisation.SetApplicationLanguage(params) # connect to weather station ws_type = params.get('config', 'ws type') if ws_type: params._config.remove_option('config', 'ws type') params.set('fixed', 'ws type', ws_type) ws_type = params.get('fixed', 'ws type', '1080') ws = WeatherStation.weather_station(ws_type=ws_type) fixed_block = CheckFixedBlock(ws, params, logger) if not fixed_block: logger.error("Invalid data from weather station") return 3 # open data file stores raw_data = DataStore.data_store(data_dir) calib_data = DataStore.calib_store(data_dir) hourly_data = DataStore.hourly_store(data_dir) daily_data = DataStore.daily_store(data_dir) monthly_data = DataStore.monthly_store(data_dir) # create a RegularTasks object tasks = Tasks.RegularTasks( params, calib_data, hourly_data, daily_data, monthly_data) # get time of last logged data two_minutes = timedelta(minutes=2) last_stored = raw_data.before(datetime.max) if last_stored == None: last_stored = datetime.min if datetime.utcnow() < last_stored: raise ValueError('Computer time is earlier than last stored data') last_stored += two_minutes # get live data hour = timedelta(hours=1) next_hour = datetime.utcnow().replace( minute=0, second=0, microsecond=0) + hour next_ptr = None for data, ptr, logged in ws.live_data( logged_only=(not tasks.has_live_tasks())): now = data['idx'] if logged: if ptr == next_ptr: # data is contiguous with last logged value raw_data[now] = data else: # catch up missing data Catchup(ws, logger, raw_data, now, ptr) next_ptr = ws.inc_ptr(ptr) # process new data Process.Process(params, raw_data, calib_data, hourly_data, daily_data, monthly_data) # do tasks tasks.do_tasks() if now >= next_hour: next_hour += hour fixed_block = CheckFixedBlock(ws, params, logger) if not fixed_block: logger.error("Invalid data from weather station") return 3 params.flush() else: tasks.do_live(data) return 0
if argv is None: argv = sys.argv try: opts, args = getopt.getopt(argv[1:], "", ['help']) except getopt.error, msg: print >> sys.stderr, 'Error: %s\n' % msg print >> sys.stderr, __usage__.strip() return 1 # check arguments if len(args) != 3: print >> sys.stderr, 'Error: 3 arguments required\n' print >> sys.stderr, __usage__.strip() return 2 # process options for o, a in opts: if o == '--help': print __usage__.strip() return 0 logger = ApplicationLogger(1) params = DataStore.params(args[0]) Localisation.SetApplicationLanguage(params) return Template(params, DataStore.calib_store(args[0]), DataStore.hourly_store(args[0]), DataStore.daily_store(args[0]), DataStore.monthly_store(args[0])).make_file( args[1], args[2]) if __name__ == "__main__": sys.exit(main())
try: opts, args = getopt.getopt(argv[1:], "hv", ['help', 'verbose']) except getopt.error, msg: print >>sys.stderr, 'Error: %s\n' % msg print >>sys.stderr, __usage__.strip() return 1 # process options verbose = 0 for o, a in opts: if o in ('-h', '--help'): print __usage__.strip() return 0 elif o in ('-v', '--verbose'): verbose += 1 # check arguments if len(args) != 1: print >>sys.stderr, 'Error: 1 argument required\n' print >>sys.stderr, __usage__.strip() return 2 logger = ApplicationLogger(verbose) data_dir = args[0] return Process(DataStore.params(data_dir), DataStore.data_store(data_dir), DataStore.calib_store(data_dir), DataStore.hourly_store(data_dir), DataStore.daily_store(data_dir), DataStore.monthly_store(data_dir)) if __name__ == "__main__": sys.exit(main())