def LiveLog(data_dir):
logger = logging.getLogger('pywws.LiveLog')
params = DataStore.params(data_dir)
status = DataStore.status(data_dir)
# localise application
Localisation.SetApplicationLanguage(params)
# open data file stores
raw_data = DataStore.data_store(data_dir)
calib_data = DataStore.calib_store(data_dir)
hourly_data = DataStore.hourly_store(data_dir)
daily_data = DataStore.daily_store(data_dir)
monthly_data = DataStore.monthly_store(data_dir)
# create a DataLogger object
datalogger = DataLogger(params, status, raw_data)
# create a RegularTasks object
asynch = eval(params.get('config', 'asynchronous', 'False'))
tasks = Tasks.RegularTasks(params, status, raw_data, calib_data,
hourly_data, daily_data, monthly_data,
asynch=asynch)
# get live data
try:
for data, logged in datalogger.live_data(
logged_only=(not tasks.has_live_tasks())):
if logged:
# process new data
Process.Process(params, raw_data, calib_data,
hourly_data, daily_data, monthly_data)
# do tasks
tasks.do_tasks()
else:
tasks.do_live(data)
except Exception, ex:
logger.exception(ex)
def get_pywws_data(dir_data):
ydat = {'temp_out_max_hi':float('-inf'),'temp_out_min_lo':float('inf'),
'hum_out_max':float('-inf'),'hum_out_min':float('inf'),'rel_pressure_max':float('-inf'),
'rel_pressure_min':float('inf'),'wind_gust':float('-inf'),'rain':0.0,
'temp_in_max_hi':float('-inf'),'temp_in_min_lo':float('inf'),'hum_in_max':float('-inf'),
'hum_in_min':float('inf')}
ahora = datetime.utcnow()
try:
dat = DataStore.calib_store(dir_data)
adat = dat[dat.nearest(ahora)]
gdat = dat[dat.nearest(ahora)-timedelta(hours=24):]
dat = DataStore.hourly_store(dir_data)
hdat = dat[dat.nearest(ahora)-timedelta(hours=24):]
dat = DataStore.daily_store(dir_data)
ddat = dat[dat.nearest(ahora)]
dat = DataStore.monthly_store(dir_data)
mdat = dat[dat.nearest(ahora)]
f1=datetime(ahora.year,1,1,0,0,0)#primer momento del año
for d in dat[dat.after(f1):]:
for k in ydat.keys():
if 'min' in k:
ydat[k] = min(ydat[k],d[k])
elif 'rain' in k:
ydat[k] = ydat[k]+d[k]
else:
ydat[k] = max(ydat[k],d[k])
except:
return None
return {'a':adat,'g':gdat,'h':hdat,'d':ddat,'m':mdat,'y':ydat}
def Reprocess(data_dir, update):
logger = logging.getLogger('pywws-reprocess')
raw_data = DataStore.data_store(data_dir)
if update:
# update old data to copy high nibble of wind_dir to status
logger.warning("Updating status to include extra bits from wind_dir")
count = 0
for data in raw_data[:]:
count += 1
idx = data['idx']
if count % 10000 == 0:
logger.info("update: %s", idx.isoformat(' '))
elif count % 500 == 0:
logger.debug("update: %s", idx.isoformat(' '))
if data['wind_dir'] is not None:
if data['wind_dir'] >= 16:
data['status'] |= (data['wind_dir'] & 0xF0) << 4
data['wind_dir'] &= 0x0F
raw_data[idx] = data
if data['status'] & 0x800:
data['wind_dir'] = None
raw_data[idx] = data
raw_data.flush()
# delete old format summary files
logger.warning('Deleting old summaries')
for summary in ['calib', 'hourly', 'daily', 'monthly']:
for root, dirs, files in os.walk(
os.path.join(data_dir, summary), topdown=False):
logger.info(root)
for file in files:
os.unlink(os.path.join(root, file))
os.rmdir(root)
# create data summaries
logger.warning('Generating hourly and daily summaries')
params = DataStore.params(data_dir)
calib_data = DataStore.calib_store(data_dir)
hourly_data = DataStore.hourly_store(data_dir)
daily_data = DataStore.daily_store(data_dir)
monthly_data = DataStore.monthly_store(data_dir)
Process.Process(
params,
raw_data, calib_data, hourly_data, daily_data, monthly_data)
return 0
def Reprocess(data_dir):
# delete old format summary files
print "Deleting old summaries"
for summary in ["calib", "hourly", "daily", "monthly"]:
for root, dirs, files in os.walk(os.path.join(data_dir, summary), topdown=False):
print root
for file in files:
os.unlink(os.path.join(root, file))
os.rmdir(root)
# create data summaries
print "Generating hourly and daily summaries"
params = DataStore.params(data_dir)
raw_data = DataStore.data_store(data_dir)
calib_data = DataStore.calib_store(data_dir)
hourly_data = DataStore.hourly_store(data_dir)
daily_data = DataStore.daily_store(data_dir)
monthly_data = DataStore.monthly_store(data_dir)
Process.Process(params, raw_data, calib_data, hourly_data, daily_data, monthly_data)
return 0
def Reprocess(data_dir, update):
logger = logging.getLogger('pywws.Reprocess')
raw_data = DataStore.data_store(data_dir)
if update:
# update old data to copy high nibble of wind_dir to status
logger.warning("Updating status to include extra bits from wind_dir")
count = 0
for data in raw_data[:]:
count += 1
idx = data['idx']
if count % 10000 == 0:
logger.info("update: %s", idx.isoformat(' '))
elif count % 500 == 0:
logger.debug("update: %s", idx.isoformat(' '))
if data['wind_dir'] is not None:
if data['wind_dir'] >= 16:
data['status'] |= (data['wind_dir'] & 0xF0) << 4
data['wind_dir'] &= 0x0F
raw_data[idx] = data
if data['status'] & 0x800:
data['wind_dir'] = None
raw_data[idx] = data
raw_data.flush()
# delete old format summary files
logger.warning('Deleting old summaries')
for summary in ['calib', 'hourly', 'daily', 'monthly']:
for root, dirs, files in os.walk(os.path.join(data_dir, summary),
topdown=False):
logger.info(root)
for file in files:
os.unlink(os.path.join(root, file))
os.rmdir(root)
# create data summaries
logger.warning('Generating hourly and daily summaries')
params = DataStore.params(data_dir)
calib_data = DataStore.calib_store(data_dir)
hourly_data = DataStore.hourly_store(data_dir)
daily_data = DataStore.daily_store(data_dir)
monthly_data = DataStore.monthly_store(data_dir)
Process.Process(params, raw_data, calib_data, hourly_data, daily_data,
monthly_data)
return 0
def Hourly(data_dir):
# get file locations
params = DataStore.params(data_dir)
# localise application
Localisation.SetApplicationLanguage(params)
# open data file stores
raw_data = DataStore.data_store(data_dir)
calib_data = DataStore.calib_store(data_dir)
hourly_data = DataStore.hourly_store(data_dir)
daily_data = DataStore.daily_store(data_dir)
monthly_data = DataStore.monthly_store(data_dir)
# get weather station data
# LogData.LogData(params, raw_data)
# do the processing
Process.Process(params, raw_data, calib_data, hourly_data, daily_data,
monthly_data)
# do tasks
if not Tasks.RegularTasks(params, calib_data, hourly_data, daily_data,
monthly_data).do_tasks():
return 1
return 0
def Reprocess(data_dir):
# delete old format summary files
print 'Deleting old summaries'
for summary in ['calib', 'hourly', 'daily', 'monthly']:
for root, dirs, files in os.walk(os.path.join(data_dir, summary),
topdown=False):
print root
for file in files:
os.unlink(os.path.join(root, file))
os.rmdir(root)
# create data summaries
print 'Generating hourly and daily summaries'
params = DataStore.params(data_dir)
raw_data = DataStore.data_store(data_dir)
calib_data = DataStore.calib_store(data_dir)
hourly_data = DataStore.hourly_store(data_dir)
daily_data = DataStore.daily_store(data_dir)
monthly_data = DataStore.monthly_store(data_dir)
Process.Process(params, raw_data, calib_data, hourly_data, daily_data,
monthly_data)
return 0
def Hourly(data_dir):
# get file locations
params = DataStore.params(data_dir)
# localise application
Localisation.SetApplicationLanguage(params)
# open data file stores
raw_data = DataStore.data_store(data_dir)
calib_data = DataStore.calib_store(data_dir)
hourly_data = DataStore.hourly_store(data_dir)
daily_data = DataStore.daily_store(data_dir)
monthly_data = DataStore.monthly_store(data_dir)
# get weather station data
LogData.LogData(params, raw_data)
# do the processing
Process.Process(
params, raw_data, calib_data, hourly_data, daily_data, monthly_data)
# do tasks
if not Tasks.RegularTasks(
params, calib_data, hourly_data, daily_data, monthly_data
).do_tasks():
return 1
return 0
def LiveLog(data_dir):
logger = logging.getLogger('pywws.LiveLog')
params = DataStore.params(data_dir)
status = DataStore.status(data_dir)
# localise application
Localisation.SetApplicationLanguage(params)
# open data file stores
raw_data = DataStore.data_store(data_dir)
calib_data = DataStore.calib_store(data_dir)
hourly_data = DataStore.hourly_store(data_dir)
daily_data = DataStore.daily_store(data_dir)
monthly_data = DataStore.monthly_store(data_dir)
# create a DataLogger object
datalogger = DataLogger(params, status, raw_data)
# create a RegularTasks object
asynch = eval(params.get('config', 'asynchronous', 'False'))
tasks = Tasks.RegularTasks(params,
status,
raw_data,
calib_data,
hourly_data,
daily_data,
monthly_data,
asynch=asynch)
# get live data
try:
for data, logged in datalogger.live_data(
logged_only=(not tasks.has_live_tasks())):
if logged:
# process new data
Process.Process(params, raw_data, calib_data, hourly_data,
daily_data, monthly_data)
# do tasks
tasks.do_tasks()
else:
tasks.do_live(data)
except Exception, ex:
logger.exception(ex)
argv = sys.argv
try:
opts, args = getopt.getopt(argv[1:], "h", ['help'])
except getopt.error, msg:
print >>sys.stderr, 'Error: %s\n' % msg
print >>sys.stderr, __usage__.strip()
return 1
# process options
for o, a in opts:
if o == '-h' or o == '--help':
print __usage__.strip()
return 0
# check arguments
if len(args) != 4:
print >>sys.stderr, 'Error: 4 arguments required\n'
print >>sys.stderr, __usage__.strip()
return 2
logger = ApplicationLogger(1)
params = DataStore.params(args[0])
status = DataStore.status(args[0])
Localisation.SetApplicationLanguage(params)
return GraphPlotter(
params, status,
DataStore.calib_store(args[0]), DataStore.hourly_store(args[0]),
DataStore.daily_store(args[0]), DataStore.monthly_store(args[0]),
args[1]
).DoPlot(args[2], args[3])
if __name__ == "__main__":
sys.exit(main())
argv = sys.argv
try:
opts, args = getopt.getopt(argv[1:], "h", ['help'])
except getopt.error, msg:
print >>sys.stderr, 'Error: %s\n' % msg
print >>sys.stderr, __usage__.strip()
return 1
# process options
for o, a in opts:
if o == '-h' or o == '--help':
print __usage__.strip()
return 0
# check arguments
if len(args) != 4:
print >>sys.stderr, 'Error: 4 arguments required\n'
print >>sys.stderr, __usage__.strip()
return 2
logger = ApplicationLogger(2)
params = DataStore.params(args[0])
status = DataStore.status(args[0])
Localisation.SetApplicationLanguage(params)
return GraphPlotter(
params, status,
DataStore.calib_store(args[0]), DataStore.hourly_store(args[0]),
DataStore.daily_store(args[0]), DataStore.monthly_store(args[0]),
args[1]
).DoPlot(GraphFileReader(args[2]), args[3])
if __name__ == "__main__":
sys.exit(main())
def LiveLog(data_dir):
logger = logging.getLogger('pywws.LiveLog')
params = DataStore.params(data_dir)
status = DataStore.status(data_dir)
# localise application
Localisation.SetApplicationLanguage(params)
# connect to weather station
ws_type = params.get('fixed', 'ws type')
if ws_type:
params.unset('fixed', 'ws type')
params.set('config', 'ws type', ws_type)
ws_type = params.get('config', 'ws type', '1080')
ws = WeatherStation.weather_station(
ws_type=ws_type, params=params, status=status)
fixed_block = CheckFixedBlock(ws, params, status, logger)
if not fixed_block:
logger.error("Invalid data from weather station")
return 3
# open data file stores
raw_data = DataStore.data_store(data_dir)
calib_data = DataStore.calib_store(data_dir)
hourly_data = DataStore.hourly_store(data_dir)
daily_data = DataStore.daily_store(data_dir)
monthly_data = DataStore.monthly_store(data_dir)
# create a RegularTasks object
tasks = Tasks.RegularTasks(
params, status, calib_data, hourly_data, daily_data, monthly_data)
# get time of last logged data
two_minutes = timedelta(minutes=2)
last_stored = raw_data.before(datetime.max)
if last_stored == None:
last_stored = datetime.min
if datetime.utcnow() < last_stored:
raise ValueError('Computer time is earlier than last stored data')
last_stored += two_minutes
# get live data
hour = timedelta(hours=1)
next_hour = datetime.utcnow().replace(
minute=0, second=0, microsecond=0) + hour
next_ptr = None
for data, ptr, logged in ws.live_data(
logged_only=(not tasks.has_live_tasks())):
now = data['idx']
if logged:
if ptr == next_ptr:
# data is contiguous with last logged value
raw_data[now] = data
else:
# catch up missing data
Catchup(ws, logger, raw_data, now, ptr)
next_ptr = ws.inc_ptr(ptr)
# process new data
Process.Process(params, status, raw_data, calib_data,
hourly_data, daily_data, monthly_data)
# do tasks
tasks.do_tasks()
if now >= next_hour:
next_hour += hour
fixed_block = CheckFixedBlock(ws, params, status, logger)
if not fixed_block:
logger.error("Invalid data from weather station")
return 3
# save any unsaved data
raw_data.flush()
else:
tasks.do_live(data)
return 0
#!/usr/bin/env python from pywws import DataStore from pywws import Process from pywws import Tasks import os import sys try: data_dir = sys.argv[1] except: data_dir = "/apps/weather/weather_data/" # open configuration files params = DataStore.params(data_dir) status = DataStore.status(data_dir) # open data file stores raw_data = DataStore.data_store(data_dir) calib_data = DataStore.calib_store(data_dir) hourly_data = DataStore.hourly_store(data_dir) daily_data = DataStore.daily_store(data_dir) monthly_data = DataStore.monthly_store(data_dir) # Process data Process.Process(params,raw_data, calib_data, hourly_data, daily_data, monthly_data) # Do tasks (calculate aggregates, populate templates, draw graphs) Tasks.RegularTasks(params, status, raw_data, calib_data, hourly_data, daily_data, monthly_data).do_tasks()
def LiveLog(data_dir):
logger = logging.getLogger('pywws.LiveLog')
params = DataStore.params(data_dir)
# localise application
Localisation.SetApplicationLanguage(params)
# connect to weather station
ws_type = params.get('config', 'ws type')
if ws_type:
params._config.remove_option('config', 'ws type')
params.set('fixed', 'ws type', ws_type)
ws_type = params.get('fixed', 'ws type', '1080')
ws = WeatherStation.weather_station(ws_type=ws_type)
fixed_block = CheckFixedBlock(ws, params, logger)
if not fixed_block:
logger.error("Invalid data from weather station")
return 3
# open data file stores
raw_data = DataStore.data_store(data_dir)
calib_data = DataStore.calib_store(data_dir)
hourly_data = DataStore.hourly_store(data_dir)
daily_data = DataStore.daily_store(data_dir)
monthly_data = DataStore.monthly_store(data_dir)
# create a RegularTasks object
tasks = Tasks.RegularTasks(
params, calib_data, hourly_data, daily_data, monthly_data)
# get time of last logged data
two_minutes = timedelta(minutes=2)
last_stored = raw_data.before(datetime.max)
if last_stored == None:
last_stored = datetime.min
if datetime.utcnow() < last_stored:
raise ValueError('Computer time is earlier than last stored data')
last_stored += two_minutes
# get live data
hour = timedelta(hours=1)
next_hour = datetime.utcnow().replace(
minute=0, second=0, microsecond=0) + hour
next_ptr = None
for data, ptr, logged in ws.live_data(
logged_only=(not tasks.has_live_tasks())):
now = data['idx']
if logged:
if ptr == next_ptr:
# data is contiguous with last logged value
raw_data[now] = data
else:
# catch up missing data
Catchup(ws, logger, raw_data, now, ptr)
next_ptr = ws.inc_ptr(ptr)
# process new data
Process.Process(params, raw_data, calib_data,
hourly_data, daily_data, monthly_data)
# do tasks
tasks.do_tasks()
if now >= next_hour:
next_hour += hour
fixed_block = CheckFixedBlock(ws, params, logger)
if not fixed_block:
logger.error("Invalid data from weather station")
return 3
params.flush()
else:
tasks.do_live(data)
return 0
if argv is None:
argv = sys.argv
try:
opts, args = getopt.getopt(argv[1:], "", ['help'])
except getopt.error, msg:
print >> sys.stderr, 'Error: %s\n' % msg
print >> sys.stderr, __usage__.strip()
return 1
# check arguments
if len(args) != 3:
print >> sys.stderr, 'Error: 3 arguments required\n'
print >> sys.stderr, __usage__.strip()
return 2
# process options
for o, a in opts:
if o == '--help':
print __usage__.strip()
return 0
logger = ApplicationLogger(1)
params = DataStore.params(args[0])
Localisation.SetApplicationLanguage(params)
return Template(params, DataStore.calib_store(args[0]),
DataStore.hourly_store(args[0]),
DataStore.daily_store(args[0]),
DataStore.monthly_store(args[0])).make_file(
args[1], args[2])
if __name__ == "__main__":
sys.exit(main())
try:
opts, args = getopt.getopt(argv[1:], "hv", ['help', 'verbose'])
except getopt.error, msg:
print >>sys.stderr, 'Error: %s\n' % msg
print >>sys.stderr, __usage__.strip()
return 1
# process options
verbose = 0
for o, a in opts:
if o in ('-h', '--help'):
print __usage__.strip()
return 0
elif o in ('-v', '--verbose'):
verbose += 1
# check arguments
if len(args) != 1:
print >>sys.stderr, 'Error: 1 argument required\n'
print >>sys.stderr, __usage__.strip()
return 2
logger = ApplicationLogger(verbose)
data_dir = args[0]
return Process(DataStore.params(data_dir),
DataStore.data_store(data_dir),
DataStore.calib_store(data_dir),
DataStore.hourly_store(data_dir),
DataStore.daily_store(data_dir),
DataStore.monthly_store(data_dir))
if __name__ == "__main__":
sys.exit(main())
