def read_time_range(self, node_id, time_col, cols, begin, *, end=None):
"""Retrieve records in the given time period.
If end is not specified, end = the moment this is called.
"""
assert type(cols) is list, 'cols must be a list of string'
assert time_col in self.get_list_of_columns(node_id),\
'no such time_col: {}'.format(time_col)
if end is None:
end = datetime.utcnow()
if type(begin) is float:
end = dt2ts(end)
assert end > begin, '"begin" came after "end"? just swap the two'
time_range = 'WHERE {time_col} BETWEEN "{begin}" AND "{end}"'.\
format(time_col=time_col, begin=begin, end=end)
# SQLite doesn't have its own datetime type. Datetime ranking by string comparison
# somehow seems hackish as it relies on comformity to the ISO8601 format.
cmd = 'SELECT {} FROM `{}` {time_range} ORDER BY {time_col} DESC'.\
format(','.join(cols),
node_id,
time_range=time_range,
time_col=time_col)
return self._execute(cmd)
def parse_SeaFET(line):
try:
if len(line) > 197: # the doc says so
return None
m = line.strip().split(',')
if check_line(line.strip()):
d = dict(zip(tags,m))
for k in tags:
if k not in ['CHECK SUM','STATUS','HEADER','DATE']:
d[k] = float(d[k])
#else:
# del d[k]
# "instrument time"
d['Timestamp'] = dt2ts(datetime.strptime(d['DATE'],'%Y%j') + timedelta(hours=d['TIME']))
return d
if 8 == len(m) and m[0] in ['kph1','kph2','kph3']:
return {'tag':m[0],'tx_id':float(m[1]),'bad_char_count':float(m[2]),
'ticker':float(m[3]),'last_transmitted':float(m[4]),
'last_received':float(m[5]),'Vcc':float(m[6]),'Vbatt':m[7]}
#elif 3 == len(m) and m[0].startswith('kph'):
#return {'tag':m[0],'ticker':int(m[1]),'Vbatt':float(m[2])}
except:
traceback.print_exc()
#pass
return None
def hourly():
# past Nd days, up to the beginning of this hour
end = datetime.utcnow().replace(minute=0).replace(second=0).replace(
microsecond=0)
x = [end - timedelta(hours=i) for i in range(Nd * 24, 0, -1)]
conn = MySQLdb.connect(host='localhost', user='******', charset='utf8mb4')
cur = conn.cursor()
cur.execute(f"""select nodeid from uhcm.devices order by nodeid""")
nodes = [x[0] for x in list(cur.fetchall())]
D = {}
for node in nodes:
c = [0 for _ in x]
for k, xx in enumerate(x):
cmd = f"""select count(*) from uhcm.`{node}`
where ReceptionTime between %s and %s"""
cur.execute(cmd, (
dt2ts(xx),
dt2ts(xx + timedelta(hours=1)),
))
c[k] = cur.fetchone()[0]
D[node] = c
#print(node, json.dumps(d[node], separators=(',',':')))
ts = [int(dt2ts(xx)) for xx in x]
# diff(2)
ts = np.diff(ts, n=2, prepend=[0, 0]).tolist()
for node in nodes:
D[node] = np.diff(D[node], n=2, prepend=[0, 0]).tolist()
D = {
'ts': ts,
'count': D,
}
return D
def p(node, var, time_col, begin, end, outputdir):
r = store.read_time_range(node, time_col, [time_col, var], begin, end)
assert r is not None
print('\t' + var)
# strip all "None" and "NaN"
tmp = zip(r[time_col], r[var])
tmp = filter(lambda x: x[1] is not None, tmp)
tmp = list(filter(lambda x: not numpy.isnan(x[1]), tmp))
if len(tmp) <= 0:
logging.info('No data to plot')
return
# ... and if there's still stuff left to plot,
x, y = zip(*tmp)
var_description = get_description(node, var)
title = '{} ({} of {})'.format(var_description, var, node)
y, unit = poh_preferred_unit_conversion(node, var, y)
if unit is None:
ylabel = '(unitless)'
else:
ylabel = unit
# remove spikes if any
y = medfilt(y, 21)
plot_time_series(x,y,\
join(outputdir,var + '.png'),\
title=title,\
ylabel=ylabel,\
linelabel=var)
plot_config = {
'time_begin':
min(x),
'time_end':
max(x),
'plot_generated_at':
dt2ts(),
'data_point_count':
len(list(filter(lambda yy: yy is not None and not math.isnan(yy), y))),
'unit':
unit,
'description':
var_description
}
with open(join(outputdir, var + '.json'), 'w') as f:
json.dump(plot_config, f, separators=(',', ':'))
def parse_Seabird(m):
try:
if re.match('^seabird\d+.*', m):
m = m.split(',')
if 8 == len(m):
return {
'tag': m[0],
'tx_id': float(m[1]),
'bad_char_count': float(m[2]),
'ticker': float(m[3]),
'last_transmitted': float(m[4]),
'last_received': float(m[5]),
'Vcc': float(m[6]),
'Vbatt': m[7]
}
# LEGACY
if 3 == len(m):
return {'tag': m[0], 'ticker': int(m[1]), 'Vbatt': float(m[2])}
else:
if m.startswith('#'): # HACK - added 20170106
m = m[1:] # for some reason a '#' prefix the CTD messages
root = ET.fromstring(m)
if root[0][0].text == 'Sea-Bird' and\
root[0][1].text == '16plus':
d = {}
d['sn'] = root[0][2].text
for e in root[1]:
if not e.tag == 'dt':
d[e.tag] = float(e.text.strip())
else:
d[e.tag] = e.text.strip()
d['Timestamp'] = dt2ts(
datetime.strptime(e.text.strip(),
'%Y-%m-%dT%H:%M:%S'))
d['conductivity_seabird'] = d['c1']
d['pressure_seabird'] = d['p1']
d['salinity_seabird'] = d['sal']
d['temperature_seabird'] = d['t1']
d['v0_seabird'] = d['v0']
return d
except:
#traceback.print_exc()
pass
return None
for site in sites:
print(site)
plot_dir = join(root_plot_dir, site)
if not exists(plot_dir):
makedirs(plot_dir)
if args.node:
list_of_nodes = [args.node]
else:
list_of_nodes = get_list_of_devices(site=site)
#assert len(list_of_nodes)
query_end = dt2ts(datetime.utcnow())
for node in list_of_nodes:
print(node)
# auto-select a column as time
if args.var:
list_variables = [args.var]
else:
list_variables = get_list_of_disp_vars(node)
if len(list_variables) == 0:
logging.warning('Nothing to plot for {}'.format(node))
continue
columns = store.get_list_of_columns(node)
if len(columns) == 0:
def logtsraw(line):
dt = datetime.utcnow()
ts = dt2ts(dt)
rawf.write('{}\t{}\t{}\n'.format(dt.isoformat(), ts, line.strip()))
rawf.flush()
def test_dtts(self):
from datetime import datetime
dt = datetime.utcnow()
self.assertEqual(ts2dt(dt2ts(dt)), dt)
def logtsraw(line):
dt = datetime.utcnow()
ts = dt2ts(dt)
rawf = open(XBEE_LOG_FILE, 'a+', 1)
rawf.write('{}\t{}\t{}\n'.format(dt.isoformat(), ts, line.strip()))
rawf.flush()
def site_node_variable(site, node, variable):
"""Examples:
http://192.168.0.20:5000/poh/data/node-009/d2w.json?minutes=1
The "site" argument is ignored.
"""
logger.debug((site, node, variable))
b,m = validate_id(node)
if not b:
return m
try:
proxy = xmlrpc.client.ServerProxy('http://127.0.0.1:8000/')
if node not in proxy.get_list_of_tables():
return 'No such node: {}'.format(escape(node))
if variable not in proxy.get_list_of_columns(node):
return 'No such variable: {}'.format(escape(variable))
begin = request.args.get('begin')
end = request.args.get('end')
minutes = request.args.get('minutes')
max_count = request.args.get('max_count')
variable = str(variable) # storage.py doesn't like unicode variable names... TODO
unit = get_unit(node, variable)
desc = get_description(node, variable)
bounds = get_range(node, variable)
assert None not in bounds
assert bounds[0] <= bounds[1]
bounds = [b if b not in [float('-inf'),float('inf')] else None for b in bounds]
d = {'unit':unit,
'description':desc,
'bounds':bounds,
'samples':None}
if begin is not None:
begin = float(begin)
if end is None:
end = dt2ts() # assumption: database time too is in UTC
else:
end = float(end)
'''if begin >= end:
# don't actually need this check - sql will just give you [] in this case.
errmsg = 'require: begin < end'
logger.error(errmsg)
return json.dumps({'error':errmsg},separators=(',',':'))
else:'''
logger.debug('from {} to {}'.format(begin, end))
r = proxy.query_time_range(node, [time_col, variable], begin, end, time_col)
else:
if minutes is None:
minutes = 24*60
minutes = float(minutes)
logger.debug('minutes={}'.format(minutes))
r = proxy.get_last_N_minutes(node, variable, minutes)
# deal with max_count
#if 'error' not in r and max_count is not None:
if max_count is not None:
max_count = int(max_count)
if max_count > 0:
assert 2 == len(r.keys()) # a time column and a variable column
#time_col = list(set(r.keys()) - set([variable]))[0]
tmp = list(zip(r[time_col], r[variable]))
tmp = proxy.condense(tmp, max_count)
tmp = list(zip(*tmp))
r = {time_col:tmp[0], variable:tmp[1]}
d['samples'] = r
return json.dumps(d, separators=(',',':'))
except:
traceback.print_exc()
return "it's beyond my paygrade"
traceback.print_exc()
continue
assert set(V) <= set(columns)
assert len(V) > 0
time_col = auto_time_col(columns)
# what a hack...
# - - - - -
# WEEKLY
# - - - - -
outputdir = join(plot_dir, node, 'weekly')
if not exists(outputdir):
makedirs(outputdir)
end = dt2ts(datetime.utcnow())
begin = dt2ts(datetime.utcnow() - timedelta(hours=24 * 7))
assert end > begin
print(node + ', weekly')
for var in V:
try:
p(node, var, time_col, begin, end, outputdir)
except KeyboardInterrupt:
break
except:
traceback.print_exc()
# - - - - -
# MONTHLY
# - - - - -