def LiveLog(data_dir):
logger = logging.getLogger('pywws.LiveLog')
params = DataStore.params(data_dir)
status = DataStore.status(data_dir)
# localise application
Localisation.SetApplicationLanguage(params)
# open data file stores
raw_data = DataStore.data_store(data_dir)
calib_data = DataStore.calib_store(data_dir)
hourly_data = DataStore.hourly_store(data_dir)
daily_data = DataStore.daily_store(data_dir)
monthly_data = DataStore.monthly_store(data_dir)
# create a DataLogger object
datalogger = DataLogger(params, status, raw_data)
# create a RegularTasks object
asynch = eval(params.get('config', 'asynchronous', 'False'))
tasks = Tasks.RegularTasks(params, status, raw_data, calib_data,
hourly_data, daily_data, monthly_data,
asynch=asynch)
# get live data
try:
for data, logged in datalogger.live_data(
logged_only=(not tasks.has_live_tasks())):
if logged:
# process new data
Process.Process(params, raw_data, calib_data,
hourly_data, daily_data, monthly_data)
# do tasks
tasks.do_tasks()
else:
tasks.do_live(data)
except Exception, ex:
logger.exception(ex)
def get_readings(self):
for data in DataStore.data_store(self.data_store)[self.last_ts:]:
if data['idx'] <= self.last_ts:
continue
self.last_ts = data['idx']
for key in self.keys:
if not data[key]:
continue
yield Reading(READING_TYPE, data[key], data['idx'], key)
def Reprocess(data_dir):
# delete old format summary files
print "Deleting old summaries"
for summary in ["calib", "hourly", "daily", "monthly"]:
for root, dirs, files in os.walk(os.path.join(data_dir, summary), topdown=False):
print root
for file in files:
os.unlink(os.path.join(root, file))
os.rmdir(root)
# create data summaries
print "Generating hourly and daily summaries"
params = DataStore.params(data_dir)
raw_data = DataStore.data_store(data_dir)
calib_data = DataStore.calib_store(data_dir)
hourly_data = DataStore.hourly_store(data_dir)
daily_data = DataStore.daily_store(data_dir)
monthly_data = DataStore.monthly_store(data_dir)
Process.Process(params, raw_data, calib_data, hourly_data, daily_data, monthly_data)
return 0
def Reprocess(data_dir, update):
logger = logging.getLogger('pywws-reprocess')
raw_data = DataStore.data_store(data_dir)
if update:
# update old data to copy high nibble of wind_dir to status
logger.warning("Updating status to include extra bits from wind_dir")
count = 0
for data in raw_data[:]:
count += 1
idx = data['idx']
if count % 10000 == 0:
logger.info("update: %s", idx.isoformat(' '))
elif count % 500 == 0:
logger.debug("update: %s", idx.isoformat(' '))
if data['wind_dir'] is not None:
if data['wind_dir'] >= 16:
data['status'] |= (data['wind_dir'] & 0xF0) << 4
data['wind_dir'] &= 0x0F
raw_data[idx] = data
if data['status'] & 0x800:
data['wind_dir'] = None
raw_data[idx] = data
raw_data.flush()
# delete old format summary files
logger.warning('Deleting old summaries')
for summary in ['calib', 'hourly', 'daily', 'monthly']:
for root, dirs, files in os.walk(
os.path.join(data_dir, summary), topdown=False):
logger.info(root)
for file in files:
os.unlink(os.path.join(root, file))
os.rmdir(root)
# create data summaries
logger.warning('Generating hourly and daily summaries')
params = DataStore.params(data_dir)
calib_data = DataStore.calib_store(data_dir)
hourly_data = DataStore.hourly_store(data_dir)
daily_data = DataStore.daily_store(data_dir)
monthly_data = DataStore.monthly_store(data_dir)
Process.Process(
params,
raw_data, calib_data, hourly_data, daily_data, monthly_data)
return 0
def Reprocess(data_dir, update):
logger = logging.getLogger('pywws.Reprocess')
raw_data = DataStore.data_store(data_dir)
if update:
# update old data to copy high nibble of wind_dir to status
logger.warning("Updating status to include extra bits from wind_dir")
count = 0
for data in raw_data[:]:
count += 1
idx = data['idx']
if count % 10000 == 0:
logger.info("update: %s", idx.isoformat(' '))
elif count % 500 == 0:
logger.debug("update: %s", idx.isoformat(' '))
if data['wind_dir'] is not None:
if data['wind_dir'] >= 16:
data['status'] |= (data['wind_dir'] & 0xF0) << 4
data['wind_dir'] &= 0x0F
raw_data[idx] = data
if data['status'] & 0x800:
data['wind_dir'] = None
raw_data[idx] = data
raw_data.flush()
# delete old format summary files
logger.warning('Deleting old summaries')
for summary in ['calib', 'hourly', 'daily', 'monthly']:
for root, dirs, files in os.walk(os.path.join(data_dir, summary),
topdown=False):
logger.info(root)
for file in files:
os.unlink(os.path.join(root, file))
os.rmdir(root)
# create data summaries
logger.warning('Generating hourly and daily summaries')
params = DataStore.params(data_dir)
calib_data = DataStore.calib_store(data_dir)
hourly_data = DataStore.hourly_store(data_dir)
daily_data = DataStore.daily_store(data_dir)
monthly_data = DataStore.monthly_store(data_dir)
Process.Process(params, raw_data, calib_data, hourly_data, daily_data,
monthly_data)
return 0
def Reprocess(data_dir):
# delete old format summary files
print 'Deleting old summaries'
for summary in ['calib', 'hourly', 'daily', 'monthly']:
for root, dirs, files in os.walk(os.path.join(data_dir, summary),
topdown=False):
print root
for file in files:
os.unlink(os.path.join(root, file))
os.rmdir(root)
# create data summaries
print 'Generating hourly and daily summaries'
params = DataStore.params(data_dir)
raw_data = DataStore.data_store(data_dir)
calib_data = DataStore.calib_store(data_dir)
hourly_data = DataStore.hourly_store(data_dir)
daily_data = DataStore.daily_store(data_dir)
monthly_data = DataStore.monthly_store(data_dir)
Process.Process(params, raw_data, calib_data, hourly_data, daily_data,
monthly_data)
return 0
def Hourly(data_dir):
# get file locations
params = DataStore.params(data_dir)
# localise application
Localisation.SetApplicationLanguage(params)
# open data file stores
raw_data = DataStore.data_store(data_dir)
calib_data = DataStore.calib_store(data_dir)
hourly_data = DataStore.hourly_store(data_dir)
daily_data = DataStore.daily_store(data_dir)
monthly_data = DataStore.monthly_store(data_dir)
# get weather station data
# LogData.LogData(params, raw_data)
# do the processing
Process.Process(params, raw_data, calib_data, hourly_data, daily_data,
monthly_data)
# do tasks
if not Tasks.RegularTasks(params, calib_data, hourly_data, daily_data,
monthly_data).do_tasks():
return 1
return 0
def Hourly(data_dir):
# get file locations
params = DataStore.params(data_dir)
# localise application
Localisation.SetApplicationLanguage(params)
# open data file stores
raw_data = DataStore.data_store(data_dir)
calib_data = DataStore.calib_store(data_dir)
hourly_data = DataStore.hourly_store(data_dir)
daily_data = DataStore.daily_store(data_dir)
monthly_data = DataStore.monthly_store(data_dir)
# get weather station data
LogData.LogData(params, raw_data)
# do the processing
Process.Process(
params, raw_data, calib_data, hourly_data, daily_data, monthly_data)
# do tasks
if not Tasks.RegularTasks(
params, calib_data, hourly_data, daily_data, monthly_data
).do_tasks():
return 1
return 0
def LiveLog(data_dir):
logger = logging.getLogger('pywws.LiveLog')
params = DataStore.params(data_dir)
status = DataStore.status(data_dir)
# localise application
Localisation.SetApplicationLanguage(params)
# open data file stores
raw_data = DataStore.data_store(data_dir)
calib_data = DataStore.calib_store(data_dir)
hourly_data = DataStore.hourly_store(data_dir)
daily_data = DataStore.daily_store(data_dir)
monthly_data = DataStore.monthly_store(data_dir)
# create a DataLogger object
datalogger = DataLogger(params, status, raw_data)
# create a RegularTasks object
asynch = eval(params.get('config', 'asynchronous', 'False'))
tasks = Tasks.RegularTasks(params,
status,
raw_data,
calib_data,
hourly_data,
daily_data,
monthly_data,
asynch=asynch)
# get live data
try:
for data, logged in datalogger.live_data(
logged_only=(not tasks.has_live_tasks())):
if logged:
# process new data
Process.Process(params, raw_data, calib_data, hourly_data,
daily_data, monthly_data)
# do tasks
tasks.do_tasks()
else:
tasks.do_live(data)
except Exception, ex:
logger.exception(ex)
print >>sys.stderr, 'Error: %s\n' % msg
print >>sys.stderr, __usage__.strip()
return 1
# process options
clear = False
sync = None
verbose = 0
for o, a in opts:
if o in ('-h', '--help'):
print __usage__.strip()
return 0
elif o in ('-c', '--clear'):
clear = True
elif o in ('-s', '--sync'):
sync = int(a)
elif o in ('-v', '--verbose'):
verbose += 1
# check arguments
if len(args) != 1:
print >>sys.stderr, 'Error: 1 argument required\n'
print >>sys.stderr, __usage__.strip()
return 2
logger = ApplicationLogger(verbose)
root_dir = args[0]
DataLogger(
DataStore.params(root_dir), DataStore.status(root_dir),
DataStore.data_store(root_dir)).log_data(sync=sync, clear=clear)
if __name__ == "__main__":
sys.exit(main())
print >> sys.stderr, 'Error: %s\n' % msg
print >> sys.stderr, __usage__.strip()
return 1
# process options
clear = False
sync = None
verbose = 0
for o, a in opts:
if o in ('-h', '--help'):
print __usage__.strip()
return 0
elif o in ('-c', '--clear'):
clear = True
elif o in ('-s', '--sync'):
sync = int(a)
elif o in ('-v', '--verbose'):
verbose += 1
# check arguments
if len(args) != 1:
print >> sys.stderr, 'Error: 1 argument required\n'
print >> sys.stderr, __usage__.strip()
return 2
logger = ApplicationLogger(verbose)
root_dir = args[0]
DataLogger(DataStore.params(root_dir), DataStore.status(root_dir),
DataStore.data_store(root_dir)).log_data(sync=sync, clear=clear)
if __name__ == "__main__":
sys.exit(main())
print >>sys.stderr, 'Error: %s\n' % msg
print >>sys.stderr, __usage__.strip()
return 1
# process options
clear = False
sync = None
verbose = 0
for o, a in opts:
if o in ('-h', '--help'):
print __usage__.strip()
return 0
elif o in ('-c', '--clear'):
clear = True
elif o in ('-s', '--sync'):
sync = int(a)
elif o in ('-v', '--verbose'):
verbose += 1
# check arguments
if len(args) != 1:
print >>sys.stderr, 'Error: 1 argument required\n'
print >>sys.stderr, __usage__.strip()
return 2
logger = ApplicationLogger(verbose)
root_dir = args[0]
return LogData(
DataStore.params(root_dir), DataStore.status(root_dir),
DataStore.data_store(root_dir), sync=sync, clear=clear)
if __name__ == "__main__":
sys.exit(main())
########
# Main
########
# Global Variables
AdaScreenNumber = 0
data = {}
forecast_bom_today = ""
forecast_bom_tomorrow = ""
forecast_file_today = ""
forecast_toggle = 0
global_init=True
readings = {}
# pywws data
if config.getboolean('Output','PYWWS_PUBLISH'):
ds = DataStore.data_store(config.get('PYWWS','STORAGE'))
dstatus = DataStore.status(config.get('PYWWS','STORAGE'))
if config.getboolean('Output','ADA_LCD'):
AdaLcd.clear()
if config.getboolean('Output','SENSEHAT_DISPLAY'):
# Set up display
PiSenseHat.clear()
PiSenseHat.set_rotation(config.get('SenseHat','ROTATION'))
if config.getboolean('Sensors','ENOCEAN'):
eoCommunicator = eoSerialCommunicator(port=config.get('EnOcean','PORT'))
eoCommunicator.start()
# Warm up sensors
print "Waiting for sensors to settle"
for i in range(1,6):
Sample()
time.sleep(1)
try:
opts, args = getopt.getopt(argv[1:], "hv", ['help', 'verbose'])
except getopt.error, msg:
print >>sys.stderr, 'Error: %s\n' % msg
print >>sys.stderr, __usage__.strip()
return 1
# process options
verbose = 0
for o, a in opts:
if o in ('-h', '--help'):
print __usage__.strip()
return 0
elif o in ('-v', '--verbose'):
verbose += 1
# check arguments
if len(args) != 1:
print >>sys.stderr, 'Error: 1 argument required\n'
print >>sys.stderr, __usage__.strip()
return 2
logger = ApplicationLogger(verbose)
data_dir = args[0]
return Process(DataStore.params(data_dir),
DataStore.data_store(data_dir),
DataStore.calib_store(data_dir),
DataStore.hourly_store(data_dir),
DataStore.daily_store(data_dir),
DataStore.monthly_store(data_dir))
if __name__ == "__main__":
sys.exit(main())
if o == '-h' or o == '--help':
print __usage__.strip()
return 0
elif o == '-n' or o == '--noaction':
noaction = True
# check arguments
if len(args) != 1:
print >>sys.stderr, 'Error: 1 argument required\n'
print >>sys.stderr, __usage__.strip()
return 2
data_dir = args[0]
# date & time range of data to be changed, in UTC!
start = datetime(2013, 10, 27, 11, 21)
stop = datetime(2013, 10, 29, 18, 32)
# open data store
raw_data = DataStore.data_store(data_dir)
# process the data
aperture = timedelta(minutes=14, seconds=30)
# make list of changes to apply after examining the data
changes = []
for data in raw_data[start:stop]:
if data['temp_out'] is None:
continue
# get temperatures at nearby times
idx = data['idx']
temp_list = []
for local_data in raw_data[idx-aperture:idx+aperture]:
temp = local_data['temp_out']
if temp is not None:
temp_list.append(temp)
if len(temp_list) < 3:
def LiveLog(data_dir):
logger = logging.getLogger('pywws.LiveLog')
params = DataStore.params(data_dir)
status = DataStore.status(data_dir)
# localise application
Localisation.SetApplicationLanguage(params)
# connect to weather station
ws_type = params.get('fixed', 'ws type')
if ws_type:
params.unset('fixed', 'ws type')
params.set('config', 'ws type', ws_type)
ws_type = params.get('config', 'ws type', '1080')
ws = WeatherStation.weather_station(
ws_type=ws_type, params=params, status=status)
fixed_block = CheckFixedBlock(ws, params, status, logger)
if not fixed_block:
logger.error("Invalid data from weather station")
return 3
# open data file stores
raw_data = DataStore.data_store(data_dir)
calib_data = DataStore.calib_store(data_dir)
hourly_data = DataStore.hourly_store(data_dir)
daily_data = DataStore.daily_store(data_dir)
monthly_data = DataStore.monthly_store(data_dir)
# create a RegularTasks object
tasks = Tasks.RegularTasks(
params, status, calib_data, hourly_data, daily_data, monthly_data)
# get time of last logged data
two_minutes = timedelta(minutes=2)
last_stored = raw_data.before(datetime.max)
if last_stored == None:
last_stored = datetime.min
if datetime.utcnow() < last_stored:
raise ValueError('Computer time is earlier than last stored data')
last_stored += two_minutes
# get live data
hour = timedelta(hours=1)
next_hour = datetime.utcnow().replace(
minute=0, second=0, microsecond=0) + hour
next_ptr = None
for data, ptr, logged in ws.live_data(
logged_only=(not tasks.has_live_tasks())):
now = data['idx']
if logged:
if ptr == next_ptr:
# data is contiguous with last logged value
raw_data[now] = data
else:
# catch up missing data
Catchup(ws, logger, raw_data, now, ptr)
next_ptr = ws.inc_ptr(ptr)
# process new data
Process.Process(params, status, raw_data, calib_data,
hourly_data, daily_data, monthly_data)
# do tasks
tasks.do_tasks()
if now >= next_hour:
next_hour += hour
fixed_block = CheckFixedBlock(ws, params, status, logger)
if not fixed_block:
logger.error("Invalid data from weather station")
return 3
# save any unsaved data
raw_data.flush()
else:
tasks.do_live(data)
return 0
# process options
for o, a in opts:
if o == '-h' or o == '--help':
print __usage__.strip()
return 0
# check arguments
if len(args) != 1:
print >> sys.stderr, 'Error: 1 argument required\n'
print >> sys.stderr, __usage__.strip()
return 2
data_dir = args[0]
# date & time range of data to be changed, in UTC!
start = datetime(2013, 10, 26, 15, 23)
stop = datetime(2013, 10, 30, 12, 47)
# open data store
raw_data = DataStore.data_store(data_dir)
# change the data
for data in raw_data[start:stop]:
data['rain'] -= 263.1
raw_data[data['idx']] = data
# make sure it's saved
raw_data.flush()
# clear calibrated data that needs to be regenerated
calib_data = DataStore.calib_store(data_dir)
del calib_data[start:]
calib_data.flush()
# done
return 0
if __name__ == "__main__":
for o, a in opts:
if o == '--help':
usage()
return 0
# check arguments
if len(args) != 2:
print >>sys.stderr, 'Error: 2 arguments required\n'
print >>sys.stderr, __usage__.strip()
return 2
# process arguments
in_name = args[0]
out_name = args[1]
# open input
in_file = open(in_name, 'r')
# open data file store
ds = DataStore.data_store(out_name)
# get time to go forward to
first_stored = ds.after(datetime.min)
if first_stored == None:
first_stored = datetime.max
# copy any missing data
last_date = None
count = 0
for line in in_file:
items = line.split(',')
local_date = DataStore.safestrptime(items[2].strip(), '%Y-%m-%d %H:%M:%S')
local_date = local_date.replace(tzinfo=TimeZone.Local)
date = local_date.astimezone(TimeZone.utc)
if last_date and date < last_date:
date = date + timedelta(hours=1)
print "Corrected DST ambiguity %s %s -> %s" % (
def LiveLog(data_dir):
logger = logging.getLogger('pywws.LiveLog')
params = DataStore.params(data_dir)
# localise application
Localisation.SetApplicationLanguage(params)
# connect to weather station
ws_type = params.get('config', 'ws type')
if ws_type:
params._config.remove_option('config', 'ws type')
params.set('fixed', 'ws type', ws_type)
ws_type = params.get('fixed', 'ws type', '1080')
ws = WeatherStation.weather_station(ws_type=ws_type)
fixed_block = CheckFixedBlock(ws, params, logger)
if not fixed_block:
logger.error("Invalid data from weather station")
return 3
# open data file stores
raw_data = DataStore.data_store(data_dir)
calib_data = DataStore.calib_store(data_dir)
hourly_data = DataStore.hourly_store(data_dir)
daily_data = DataStore.daily_store(data_dir)
monthly_data = DataStore.monthly_store(data_dir)
# create a RegularTasks object
tasks = Tasks.RegularTasks(
params, calib_data, hourly_data, daily_data, monthly_data)
# get time of last logged data
two_minutes = timedelta(minutes=2)
last_stored = raw_data.before(datetime.max)
if last_stored == None:
last_stored = datetime.min
if datetime.utcnow() < last_stored:
raise ValueError('Computer time is earlier than last stored data')
last_stored += two_minutes
# get live data
hour = timedelta(hours=1)
next_hour = datetime.utcnow().replace(
minute=0, second=0, microsecond=0) + hour
next_ptr = None
for data, ptr, logged in ws.live_data(
logged_only=(not tasks.has_live_tasks())):
now = data['idx']
if logged:
if ptr == next_ptr:
# data is contiguous with last logged value
raw_data[now] = data
else:
# catch up missing data
Catchup(ws, logger, raw_data, now, ptr)
next_ptr = ws.inc_ptr(ptr)
# process new data
Process.Process(params, raw_data, calib_data,
hourly_data, daily_data, monthly_data)
# do tasks
tasks.do_tasks()
if now >= next_hour:
next_hour += hour
fixed_block = CheckFixedBlock(ws, params, logger)
if not fixed_block:
logger.error("Invalid data from weather station")
return 3
params.flush()
else:
tasks.do_live(data)
return 0
for o, a in opts:
if o == '--help':
usage()
return 0
# check arguments
if len(args) != 2:
print >>sys.stderr, 'Error: 2 arguments required\n'
print >>sys.stderr, __doc__.strip()
return 2
# process arguments
in_name = args[0]
out_name = args[1]
# open input
in_file = open(in_name, 'r')
# open data file store
ds = DataStore.data_store(out_name)
# get time to go forward to
first_stored = ds.after(datetime.min)
if first_stored == None:
first_stored = datetime.max
# copy any missing data
last_date = None
count = 0
for line in in_file:
items = line.split(',')
local_date = DataStore.safestrptime(items[2].strip(), '%Y-%m-%d %H:%M:%S')
local_date = local_date.replace(tzinfo=TimeZone.Local)
date = local_date.astimezone(TimeZone.utc)
if last_date and date < last_date:
date = date + timedelta(hours=1)
print "Corrected DST ambiguity %s %s -> %s" % (
