def LiveLog(data_dir): logger = logging.getLogger('pywws.LiveLog') params = DataStore.params(data_dir) status = DataStore.status(data_dir) # localise application Localisation.SetApplicationLanguage(params) # open data file stores raw_data = DataStore.data_store(data_dir) calib_data = DataStore.calib_store(data_dir) hourly_data = DataStore.hourly_store(data_dir) daily_data = DataStore.daily_store(data_dir) monthly_data = DataStore.monthly_store(data_dir) # create a DataLogger object datalogger = DataLogger(params, status, raw_data) # create a RegularTasks object asynch = eval(params.get('config', 'asynchronous', 'False')) tasks = Tasks.RegularTasks(params, status, raw_data, calib_data, hourly_data, daily_data, monthly_data, asynch=asynch) # get live data try: for data, logged in datalogger.live_data( logged_only=(not tasks.has_live_tasks())): if logged: # process new data Process.Process(params, raw_data, calib_data, hourly_data, daily_data, monthly_data) # do tasks tasks.do_tasks() else: tasks.do_live(data) except Exception, ex: logger.exception(ex)
def Reprocess(data_dir): # delete old format summary files print "Deleting old summaries" for summary in ["calib", "hourly", "daily", "monthly"]: for root, dirs, files in os.walk(os.path.join(data_dir, summary), topdown=False): print root for file in files: os.unlink(os.path.join(root, file)) os.rmdir(root) # create data summaries print "Generating hourly and daily summaries" params = DataStore.params(data_dir) raw_data = DataStore.data_store(data_dir) calib_data = DataStore.calib_store(data_dir) hourly_data = DataStore.hourly_store(data_dir) daily_data = DataStore.daily_store(data_dir) monthly_data = DataStore.monthly_store(data_dir) Process.Process(params, raw_data, calib_data, hourly_data, daily_data, monthly_data) return 0
def Reprocess(data_dir, update): logger = logging.getLogger('pywws-reprocess') raw_data = DataStore.data_store(data_dir) if update: # update old data to copy high nibble of wind_dir to status logger.warning("Updating status to include extra bits from wind_dir") count = 0 for data in raw_data[:]: count += 1 idx = data['idx'] if count % 10000 == 0: logger.info("update: %s", idx.isoformat(' ')) elif count % 500 == 0: logger.debug("update: %s", idx.isoformat(' ')) if data['wind_dir'] is not None: if data['wind_dir'] >= 16: data['status'] |= (data['wind_dir'] & 0xF0) << 4 data['wind_dir'] &= 0x0F raw_data[idx] = data if data['status'] & 0x800: data['wind_dir'] = None raw_data[idx] = data raw_data.flush() # delete old format summary files logger.warning('Deleting old summaries') for summary in ['calib', 'hourly', 'daily', 'monthly']: for root, dirs, files in os.walk( os.path.join(data_dir, summary), topdown=False): logger.info(root) for file in files: os.unlink(os.path.join(root, file)) os.rmdir(root) # create data summaries logger.warning('Generating hourly and daily summaries') params = DataStore.params(data_dir) calib_data = DataStore.calib_store(data_dir) hourly_data = DataStore.hourly_store(data_dir) daily_data = DataStore.daily_store(data_dir) monthly_data = DataStore.monthly_store(data_dir) Process.Process( params, raw_data, calib_data, hourly_data, daily_data, monthly_data) return 0
def Reprocess(data_dir, update): logger = logging.getLogger('pywws.Reprocess') raw_data = DataStore.data_store(data_dir) if update: # update old data to copy high nibble of wind_dir to status logger.warning("Updating status to include extra bits from wind_dir") count = 0 for data in raw_data[:]: count += 1 idx = data['idx'] if count % 10000 == 0: logger.info("update: %s", idx.isoformat(' ')) elif count % 500 == 0: logger.debug("update: %s", idx.isoformat(' ')) if data['wind_dir'] is not None: if data['wind_dir'] >= 16: data['status'] |= (data['wind_dir'] & 0xF0) << 4 data['wind_dir'] &= 0x0F raw_data[idx] = data if data['status'] & 0x800: data['wind_dir'] = None raw_data[idx] = data raw_data.flush() # delete old format summary files logger.warning('Deleting old summaries') for summary in ['calib', 'hourly', 'daily', 'monthly']: for root, dirs, files in os.walk(os.path.join(data_dir, summary), topdown=False): logger.info(root) for file in files: os.unlink(os.path.join(root, file)) os.rmdir(root) # create data summaries logger.warning('Generating hourly and daily summaries') params = DataStore.params(data_dir) calib_data = DataStore.calib_store(data_dir) hourly_data = DataStore.hourly_store(data_dir) daily_data = DataStore.daily_store(data_dir) monthly_data = DataStore.monthly_store(data_dir) Process.Process(params, raw_data, calib_data, hourly_data, daily_data, monthly_data) return 0
def Reprocess(data_dir): # delete old format summary files print 'Deleting old summaries' for summary in ['calib', 'hourly', 'daily', 'monthly']: for root, dirs, files in os.walk(os.path.join(data_dir, summary), topdown=False): print root for file in files: os.unlink(os.path.join(root, file)) os.rmdir(root) # create data summaries print 'Generating hourly and daily summaries' params = DataStore.params(data_dir) raw_data = DataStore.data_store(data_dir) calib_data = DataStore.calib_store(data_dir) hourly_data = DataStore.hourly_store(data_dir) daily_data = DataStore.daily_store(data_dir) monthly_data = DataStore.monthly_store(data_dir) Process.Process(params, raw_data, calib_data, hourly_data, daily_data, monthly_data) return 0
def Hourly(data_dir): # get file locations params = DataStore.params(data_dir) # localise application Localisation.SetApplicationLanguage(params) # open data file stores raw_data = DataStore.data_store(data_dir) calib_data = DataStore.calib_store(data_dir) hourly_data = DataStore.hourly_store(data_dir) daily_data = DataStore.daily_store(data_dir) monthly_data = DataStore.monthly_store(data_dir) # get weather station data # LogData.LogData(params, raw_data) # do the processing Process.Process(params, raw_data, calib_data, hourly_data, daily_data, monthly_data) # do tasks if not Tasks.RegularTasks(params, calib_data, hourly_data, daily_data, monthly_data).do_tasks(): return 1 return 0
def Hourly(data_dir): # get file locations params = DataStore.params(data_dir) # localise application Localisation.SetApplicationLanguage(params) # open data file stores raw_data = DataStore.data_store(data_dir) calib_data = DataStore.calib_store(data_dir) hourly_data = DataStore.hourly_store(data_dir) daily_data = DataStore.daily_store(data_dir) monthly_data = DataStore.monthly_store(data_dir) # get weather station data LogData.LogData(params, raw_data) # do the processing Process.Process( params, raw_data, calib_data, hourly_data, daily_data, monthly_data) # do tasks if not Tasks.RegularTasks( params, calib_data, hourly_data, daily_data, monthly_data ).do_tasks(): return 1 return 0
argv = sys.argv try: opts, args = getopt.getopt(argv[1:], "h", ['help']) except getopt.error, msg: print >>sys.stderr, 'Error: %s\n' % msg print >>sys.stderr, __usage__.strip() return 1 # process options for o, a in opts: if o == '-h' or o == '--help': print __usage__.strip() return 0 # check arguments if len(args) != 4: print >>sys.stderr, 'Error: 4 arguments required\n' print >>sys.stderr, __usage__.strip() return 2 logger = ApplicationLogger(2) params = DataStore.params(args[0]) status = DataStore.status(args[0]) Localisation.SetApplicationLanguage(params) return GraphPlotter( params, status, DataStore.calib_store(args[0]), DataStore.hourly_store(args[0]), DataStore.daily_store(args[0]), DataStore.monthly_store(args[0]), args[1] ).DoPlot(GraphFileReader(args[2]), args[3]) if __name__ == "__main__": sys.exit(main())
break self.disconnect() return OK def main(argv=None): if argv is None: argv = sys.argv try: opts, args = getopt.getopt(argv[1:], "h", ['help']) except getopt.error, msg: print >>sys.stderr, 'Error: %s\n' % msg print >>sys.stderr, __usage__.strip() return 1 # process options for o, a in opts: if o in ('-h', '--help'): print __usage__.strip() return 0 # check arguments if len(args) < 2: print >>sys.stderr, "Error: at least 2 arguments required" print >>sys.stderr, __usage__.strip() return 2 logger = ApplicationLogger(1) if Upload(DataStore.params(args[0])).upload(args[1:]): return 0 return 3 if __name__ == "__main__": sys.exit(main())
return 1 access_token = dict(urlparse.parse_qsl(content)) params.set('twitter', 'key', access_token['oauth_token']) params.set('twitter', 'secret', access_token['oauth_token_secret']) print 'Success! Authorisation data has been stored in %s' % params._path return 0 def main(argv=None): if argv is None: argv = sys.argv try: opts, args = getopt.getopt(argv[1:], "h", ['help']) except getopt.error, msg: print >>sys.stderr, 'Error: %s\n' % msg print >>sys.stderr, __usage__.strip() return 1 # process options for o, a in opts: if o in ('-h', '--help'): print __usage__.strip() return 0 # check arguments if len(args) != 1: print >>sys.stderr, "Error: 1 argument required" print >>sys.stderr, __usage__.strip() return 2 return TwitterAuth(DataStore.params(args[0])) if __name__ == "__main__": sys.exit(main())
opts, args = getopt.getopt( argv[1:], "hcv", ['help', 'catchup', 'verbose']) except getopt.error, msg: print >>sys.stderr, 'Error: %s\n' % msg print >>sys.stderr, __usage__.strip() return 1 # process options catchup = False verbose = 0 for o, a in opts: if o == '-h' or o == '--help': print __usage__.strip() return 0 elif o == '-c' or o == '--catchup': catchup = True elif o == '-v' or o == '--verbose': verbose += 1 # check arguments if len(args) != 2: print >>sys.stderr, "Error: 2 arguments required" print >>sys.stderr, __usage__.strip() return 2 logger = ApplicationLogger(verbose) return ToService( DataStore.params(args[0]), DataStore.status(args[0]), DataStore.calib_store(args[0]), args[1]).Upload( catchup=catchup, ignore_last_update=not catchup) if __name__ == "__main__": sys.exit(main())
print >>sys.stderr, 'Error: %s\n' % msg print >>sys.stderr, __usage__.strip() return 1 # process options clear = False sync = None verbose = 0 for o, a in opts: if o in ('-h', '--help'): print __usage__.strip() return 0 elif o in ('-c', '--clear'): clear = True elif o in ('-s', '--sync'): sync = int(a) elif o in ('-v', '--verbose'): verbose += 1 # check arguments if len(args) != 1: print >>sys.stderr, 'Error: 1 argument required\n' print >>sys.stderr, __usage__.strip() return 2 logger = ApplicationLogger(verbose) root_dir = args[0] return LogData( DataStore.params(root_dir), DataStore.status(root_dir), DataStore.data_store(root_dir), sync=sync, clear=clear) if __name__ == "__main__": sys.exit(main())
def LiveLog(data_dir): logger = logging.getLogger('pywws.LiveLog') params = DataStore.params(data_dir) status = DataStore.status(data_dir) # localise application Localisation.SetApplicationLanguage(params) # connect to weather station ws_type = params.get('fixed', 'ws type') if ws_type: params.unset('fixed', 'ws type') params.set('config', 'ws type', ws_type) ws_type = params.get('config', 'ws type', '1080') ws = WeatherStation.weather_station( ws_type=ws_type, params=params, status=status) fixed_block = CheckFixedBlock(ws, params, status, logger) if not fixed_block: logger.error("Invalid data from weather station") return 3 # open data file stores raw_data = DataStore.data_store(data_dir) calib_data = DataStore.calib_store(data_dir) hourly_data = DataStore.hourly_store(data_dir) daily_data = DataStore.daily_store(data_dir) monthly_data = DataStore.monthly_store(data_dir) # create a RegularTasks object tasks = Tasks.RegularTasks( params, status, calib_data, hourly_data, daily_data, monthly_data) # get time of last logged data two_minutes = timedelta(minutes=2) last_stored = raw_data.before(datetime.max) if last_stored == None: last_stored = datetime.min if datetime.utcnow() < last_stored: raise ValueError('Computer time is earlier than last stored data') last_stored += two_minutes # get live data hour = timedelta(hours=1) next_hour = datetime.utcnow().replace( minute=0, second=0, microsecond=0) + hour next_ptr = None for data, ptr, logged in ws.live_data( logged_only=(not tasks.has_live_tasks())): now = data['idx'] if logged: if ptr == next_ptr: # data is contiguous with last logged value raw_data[now] = data else: # catch up missing data Catchup(ws, logger, raw_data, now, ptr) next_ptr = ws.inc_ptr(ptr) # process new data Process.Process(params, status, raw_data, calib_data, hourly_data, daily_data, monthly_data) # do tasks tasks.do_tasks() if now >= next_hour: next_hour += hour fixed_block = CheckFixedBlock(ws, params, status, logger) if not fixed_block: logger.error("Invalid data from weather station") return 3 # save any unsaved data raw_data.flush() else: tasks.do_live(data) return 0
except getopt.error, msg: print >>sys.stderr, 'Error: %s\n' % msg print >>sys.stderr, __usage__.strip() return 1 # process options for o, a in opts: if o in ('-h', '--help'): print __usage__.strip() return 0 # check arguments if len(args) != 1: print >>sys.stderr, "Error: 1 argument required" print >>sys.stderr, __usage__.strip() return 2 data_dir = args[0] params = DataStore.params(data_dir) Localisation.SetApplicationLanguage(params) hourly_data = DataStore.hourly_store(data_dir) idx = hourly_data.before(datetime.max) print 'Zambretti (current):', Zambretti(params, hourly_data[idx]) idx = idx.replace(tzinfo=utc).astimezone(Local) if idx.hour < 8 or (idx.hour == 8 and idx.minute < 30): idx -= timedelta(hours=24) idx = idx.replace(hour=9, minute=0, second=0) idx = hourly_data.nearest(idx.astimezone(utc).replace(tzinfo=None)) lcl = idx.replace(tzinfo=utc).astimezone(Local) print 'Zambretti (at %s):' % lcl.strftime('%H:%M %Z'), Zambretti( params, hourly_data[idx]) return 0 if __name__ == "__main__":
params.set('twitter', 'key', access_token['oauth_token']) params.set('twitter', 'secret', access_token['oauth_token_secret']) print 'Success! Authorisation data has been stored in %s' % params._path return 0 def main(argv=None): if argv is None: argv = sys.argv try: opts, args = getopt.getopt(argv[1:], "h", ['help']) except getopt.error, msg: print >> sys.stderr, 'Error: %s\n' % msg print >> sys.stderr, __usage__.strip() return 1 # process options for o, a in opts: if o in ('-h', '--help'): print __usage__.strip() return 0 # check arguments if len(args) != 1: print >> sys.stderr, "Error: 1 argument required" print >> sys.stderr, __usage__.strip() return 2 return TwitterAuth(DataStore.params(args[0])) if __name__ == "__main__": sys.exit(main())
argv = sys.argv try: opts, args = getopt.getopt(argv[1:], "h", ['help']) except getopt.error, msg: print >>sys.stderr, 'Error: %s\n' % msg print >>sys.stderr, __usage__.strip() return 1 # process options for o, a in opts: if o == '-h' or o == '--help': print __usage__.strip() return 0 # check arguments if len(args) != 4: print >>sys.stderr, 'Error: 4 arguments required\n' print >>sys.stderr, __usage__.strip() return 2 logger = ApplicationLogger(1) params = DataStore.params(args[0]) status = DataStore.status(args[0]) Localisation.SetApplicationLanguage(params) return GraphPlotter( params, status, DataStore.calib_store(args[0]), DataStore.hourly_store(args[0]), DataStore.daily_store(args[0]), DataStore.monthly_store(args[0]), args[1] ).DoPlot(args[2], args[3]) if __name__ == "__main__": sys.exit(main())
try: opts, args = getopt.getopt(argv[1:], "hv", ['help', 'verbose']) except getopt.error, msg: print >>sys.stderr, 'Error: %s\n' % msg print >>sys.stderr, __usage__.strip() return 1 # process options verbose = 0 for o, a in opts: if o in ('-h', '--help'): print __usage__.strip() return 0 elif o in ('-v', '--verbose'): verbose += 1 # check arguments if len(args) != 1: print >>sys.stderr, 'Error: 1 argument required\n' print >>sys.stderr, __usage__.strip() return 2 logger = ApplicationLogger(verbose) data_dir = args[0] return Process(DataStore.params(data_dir), DataStore.data_store(data_dir), DataStore.calib_store(data_dir), DataStore.hourly_store(data_dir), DataStore.daily_store(data_dir), DataStore.monthly_store(data_dir)) if __name__ == "__main__": sys.exit(main())
def LiveLog(data_dir): logger = logging.getLogger('pywws.LiveLog') params = DataStore.params(data_dir) # localise application Localisation.SetApplicationLanguage(params) # connect to weather station ws_type = params.get('config', 'ws type') if ws_type: params._config.remove_option('config', 'ws type') params.set('fixed', 'ws type', ws_type) ws_type = params.get('fixed', 'ws type', '1080') ws = WeatherStation.weather_station(ws_type=ws_type) fixed_block = CheckFixedBlock(ws, params, logger) if not fixed_block: logger.error("Invalid data from weather station") return 3 # open data file stores raw_data = DataStore.data_store(data_dir) calib_data = DataStore.calib_store(data_dir) hourly_data = DataStore.hourly_store(data_dir) daily_data = DataStore.daily_store(data_dir) monthly_data = DataStore.monthly_store(data_dir) # create a RegularTasks object tasks = Tasks.RegularTasks( params, calib_data, hourly_data, daily_data, monthly_data) # get time of last logged data two_minutes = timedelta(minutes=2) last_stored = raw_data.before(datetime.max) if last_stored == None: last_stored = datetime.min if datetime.utcnow() < last_stored: raise ValueError('Computer time is earlier than last stored data') last_stored += two_minutes # get live data hour = timedelta(hours=1) next_hour = datetime.utcnow().replace( minute=0, second=0, microsecond=0) + hour next_ptr = None for data, ptr, logged in ws.live_data( logged_only=(not tasks.has_live_tasks())): now = data['idx'] if logged: if ptr == next_ptr: # data is contiguous with last logged value raw_data[now] = data else: # catch up missing data Catchup(ws, logger, raw_data, now, ptr) next_ptr = ws.inc_ptr(ptr) # process new data Process.Process(params, raw_data, calib_data, hourly_data, daily_data, monthly_data) # do tasks tasks.do_tasks() if now >= next_hour: next_hour += hour fixed_block = CheckFixedBlock(ws, params, logger) if not fixed_block: logger.error("Invalid data from weather station") return 3 params.flush() else: tasks.do_live(data) return 0
try: opts, args = getopt.getopt( argv[1:], "hcv", ['help', 'catchup', 'verbose']) except getopt.error, msg: print >>sys.stderr, 'Error: %s\n' % msg print >>sys.stderr, __usage__.strip() return 1 # process options catchup = False verbose = 0 for o, a in opts: if o == '-h' or o == '--help': print __usage__.strip() return 0 elif o == '-c' or o == '--catchup': catchup = True elif o == '-v' or o == '--verbose': verbose += 1 # check arguments if len(args) != 2: print >>sys.stderr, "Error: 2 arguments required" print >>sys.stderr, __usage__.strip() return 2 logger = ApplicationLogger(verbose) return ToService( DataStore.params(args[0]), DataStore.status(args[0]), DataStore.calib_store(args[0]), args[1]).Upload(catchup=catchup) if __name__ == "__main__": sys.exit(main())
return OK def main(argv=None): if argv is None: argv = sys.argv try: opts, args = getopt.getopt(argv[1:], "h", ['help']) except getopt.error, msg: print >> sys.stderr, 'Error: %s\n' % msg print >> sys.stderr, __usage__.strip() return 1 # process options for o, a in opts: if o in ('-h', '--help'): print __usage__.strip() return 0 # check arguments if len(args) < 2: print >> sys.stderr, "Error: at least 2 arguments required" print >> sys.stderr, __usage__.strip() return 2 logger = ApplicationLogger(1) if Upload(DataStore.params(args[0])).upload(args[1:]): return 0 return 3 if __name__ == "__main__": sys.exit(main())
['help', 'catchup', 'verbose']) except getopt.error, msg: print >> sys.stderr, 'Error: %s\n' % msg print >> sys.stderr, __usage__.strip() return 1 # process options catchup = False verbose = 0 for o, a in opts: if o == '-h' or o == '--help': print __usage__.strip() return 0 elif o == '-c' or o == '--catchup': catchup = True elif o == '-v' or o == '--verbose': verbose += 1 # check arguments if len(args) != 2: print >> sys.stderr, "Error: 2 arguments required" print >> sys.stderr, __usage__.strip() return 2 logger = ApplicationLogger(verbose) return ToService(DataStore.params(args[0]), DataStore.status(args[0]), DataStore.calib_store(args[0]), args[1]).Upload(catchup=catchup, ignore_last_update=not catchup) if __name__ == "__main__": sys.exit(main())
print >> sys.stderr, 'Error: %s\n' % msg print >> sys.stderr, __usage__.strip() return 1 # process options clear = False sync = None verbose = 0 for o, a in opts: if o in ('-h', '--help'): print __usage__.strip() return 0 elif o in ('-c', '--clear'): clear = True elif o in ('-s', '--sync'): sync = int(a) elif o in ('-v', '--verbose'): verbose += 1 # check arguments if len(args) != 1: print >> sys.stderr, 'Error: 1 argument required\n' print >> sys.stderr, __usage__.strip() return 2 logger = ApplicationLogger(verbose) root_dir = args[0] DataLogger(DataStore.params(root_dir), DataStore.status(root_dir), DataStore.data_store(root_dir)).log_data(sync=sync, clear=clear) if __name__ == "__main__": sys.exit(main())