def proc(func, colname, endTS, startTS, hour_interval, fixpoint):
col = q.GetSensorList(colname)
#end
if endTS == '':
window, config = rtw.getwindow()
else:
end = pd.to_datetime(endTS)
end_year=end.year
end_month=end.month
end_day=end.day
end_hour=end.hour
end_minute=end.minute
if end_minute<30:end_minute=0
else:end_minute=30
end=datetime.combine(date(end_year,end_month,end_day),time(end_hour,end_minute,0))
window, config = rtw.getwindow(end)
if startTS != '':
#start
start = pd.to_datetime(startTS)
start_year=start.year
start_month=start.month
start_day=start.day
start_hour=start.hour
start_minute=start.minute
if start_minute<30:start_minute=0
else:start_minute=30
window.start=datetime.combine(date(start_year,start_month,start_day),time(start_hour,start_minute,0))
#offsetstart
window.offsetstart = window.start - timedelta(days=(config.io.num_roll_window_ops*window.numpts-1)/48.)
if func == 'colpos' or func == 'vcdgen':
#colpos interval
if hour_interval == '':
if int((window.end-window.start).total_seconds() / (3600 * 24)) <= 5:
hour_interval = 4
else:
hour_interval = 24
config.io.col_pos_interval = str(hour_interval) + 'H'
config.io.num_col_pos = int((window.end-window.start).total_seconds() / (3600 * hour_interval)) + 1
if func == 'displacement' or func == 'colpos':
comp_vel = False
else:
comp_vel = True
monitoring = g.genproc(col[0], window, config, fixpoint, comp_vel=comp_vel)
num_nodes = monitoring.colprops.nos
seg_len = monitoring.colprops.seglen
if comp_vel == True:
monitoring_vel = monitoring.disp_vel.reset_index()[['ts', 'id', 'depth', 'xz', 'xy', 'vel_xz', 'vel_xy']]
else:
monitoring_vel = monitoring.disp_vel.reset_index()[['ts', 'id', 'depth', 'xz', 'xy']]
monitoring_vel = monitoring_vel.loc[(monitoring_vel.ts >= window.start)&(monitoring_vel.ts <= window.end)]
return monitoring_vel, window, config, num_nodes, seg_len
def proc(func, colname, endTS, startTS, hour_interval, fixpoint):
col = q.GetSensorList(colname)
#end
if endTS == '':
window, config = rtw.getwindow()
else:
end = pd.to_datetime(endTS)
end_year=end.year
end_month=end.month
end_day=end.day
end_hour=end.hour
end_minute=end.minute
if end_minute<30:end_minute=0
else:end_minute=30
end=datetime.combine(date(end_year,end_month,end_day),time(end_hour,end_minute,0))
window, config = rtw.getwindow(end)
if startTS != '':
#start
start = pd.to_datetime(startTS)
start_year=start.year
start_month=start.month
start_day=start.day
start_hour=start.hour
start_minute=start.minute
if start_minute<30:start_minute=0
else:start_minute=30
window.start=datetime.combine(date(start_year,start_month,start_day),time(start_hour,start_minute,0))
#offsetstart
window.offsetstart = window.start - timedelta(days=(config.io.num_roll_window_ops*window.numpts-1)/48.)
if func == 'colpos' or func == 'vcdgen':
#colpos interval
if hour_interval == '':
if int((window.end-window.start).total_seconds() / (3600 * 24)) <= 5:
hour_interval = 4
else:
hour_interval = 24
config.io.col_pos_interval = str(hour_interval) + 'H'
config.io.num_col_pos = int((window.end-window.start).total_seconds() / (3600 * hour_interval)) + 1
if func == 'displacement' or func == 'colpos':
comp_vel = False
else:
comp_vel = True
monitoring = g.genproc(col[0], window, config, fixpoint, comp_vel=comp_vel)
num_nodes = monitoring.colprops.nos
seg_len = monitoring.colprops.seglen
if comp_vel == True:
monitoring_vel = monitoring.vel.reset_index()[['ts', 'id', 'xz', 'xy', 'vel_xz', 'vel_xy']]
else:
monitoring_vel = monitoring.vel.reset_index()[['ts', 'id', 'xz', 'xy']]
monitoring_vel = monitoring_vel.loc[(monitoring_vel.ts >= window.start)&(monitoring_vel.ts <= window.end)]
return monitoring_vel, window, config, num_nodes, seg_len
def tsm_plot(tsm_name, end, shift_datetime):
query = "SELECT max(timestamp) AS ts FROM %s" % tsm_name
try:
ts = pd.to_datetime(qdb.GetDBDataFrame(query)['ts'].values[0])
if ts < shift_datetime:
return
except:
return
if ts > end:
ts = end
window, config = rtw.getwindow(ts)
col = qdb.GetSensorList(tsm_name)
monitoring = gen.genproc(col[0],
window,
config,
fixpoint=config.io.column_fix)
plotter.main(monitoring,
window,
config,
realtime=False,
non_event_path=False)
def main():
with open('GSMAlert.txt', 'w') as w:
w.write('')
window,config = rtw.getwindow()
props = q.GetRainProps('rain_props')
PublicAlert = pd.DataFrame({'timestamp': [window.end]*len(props), 'site': props['name'].values, 'source': ['public']*len(props), 'alert': [np.nan]*len(props), 'updateTS': [window.end]*len(props), 'palert_source': [np.nan]*len(props), 'internal_alert': [np.nan]*len(props), 'validity': [np.nan]*len(props), 'sensor_alert': [[]]*len(props), 'rain_alert': [np.nan]*len(props), 'ground_alert': [np.nan]*len(props), 'retriggerTS': [[]]*len(props), 'tech_info': [{}]*len(props)})
PublicAlert = PublicAlert[['timestamp', 'site', 'source', 'alert', 'updateTS', 'palert_source', 'internal_alert', 'validity', 'sensor_alert', 'rain_alert', 'ground_alert', 'retriggerTS', 'tech_info']]
Site_Public_Alert = PublicAlert.groupby('site')
PublicAlert = Site_Public_Alert.apply(SitePublicAlert, window=window)
PublicAlert = PublicAlert[['timestamp', 'site', 'alert', 'internal_alert', 'palert_source', 'validity', 'sensor_alert', 'rain_alert', 'ground_alert', 'retriggerTS', 'tech_info']]
PublicAlert = PublicAlert.rename(columns = {'palert_source': 'source'})
PublicAlert = PublicAlert.sort_values(['alert', 'site'], ascending = [False, True])
PublicAlert.to_csv('PublicAlert.txt', header=True, index=None, sep='\t', mode='w')
PublicAlert['timestamp'] = PublicAlert['timestamp'].apply(lambda x: str(x))
PublicAlert['validity'] = PublicAlert['validity'].apply(lambda x: str(x))
public_json = PublicAlert.to_json(orient="records")
invdf = pd.read_csv('InvalidAlert.txt', sep = ':')
invdf['timestamp'] = invdf['timestamp'].apply(lambda x: str(x))
inv_json = invdf.to_json(orient="records")
df_json = dict({'alerts': public_json, 'invalids': inv_json})
df_json = '[' + str(df_json).replace("\\\'", '').replace('\'', '').replace('alerts:', '"alerts":').replace('invalids:', '"invalids":') + ']'
with open('PublicAlert.json', 'w') as w:
w.write(df_json)
return PublicAlert
def get_tsm_data(tsm_name, start, end, plot_type, node_lst):
col = qdb.GetSensorList(tsm_name)[0]
window, config = rtw.getwindow(pd.to_datetime(end))
window.start = pd.to_datetime(start)
window.offsetstart = window.start - timedelta(days=(config.io.num_roll_window_ops*window.numpts-1)/48.)
if plot_type == 'cml':
config.io.to_smooth = 1
config.io.to_fill = 1
else:
config.io.to_smooth = 1
config.io.to_fill = 1
monitoring = proc.genproc(col, window, config, 'bottom', comp_vel=False)
df = monitoring.disp_vel.reset_index()[['ts', 'id', 'xz', 'xy']]
df = df.loc[(df.ts >= window.start)&(df.ts <= window.end)]
df = df.sort_values('ts')
if plot_type == 'cml':
xzd_plotoffset = 0
if node_lst != 'all':
df = df[df.id.isin(node_lst)]
df = plotter.cum_surf(df, xzd_plotoffset, col.nos)
else:
node_df = df.groupby('id', as_index=False)
df = node_df.apply(zeroed, column='xz')
df['zeroed_xz'] = df['zeroed_xz'] * 100
node_df = df.groupby('id', as_index=False)
df = node_df.apply(zeroed, column='xy')
df['zeroed_xy'] = df['zeroed_xy'] * 100
return df
def main(site, end):
window, config = rtw.getwindow(end)
monwinTS = pd.date_range(start=window.end - timedelta(hours=3),
end=window.end,
freq='30Min')
trending_alert = pd.DataFrame({
'site': [np.nan] * len(monwinTS),
'alert': [np.nan] * len(monwinTS),
'timestamp': monwinTS,
'source': [np.nan] * len(monwinTS)
})
trending_alert = trending_alert[['timestamp', 'site', 'source', 'alert']]
col = q.GetSensorList(site)
monitoring = g.genproc(col[0], window, config, config.io.column_fix)
lgd = q.GetLastGoodDataFromDb(monitoring.colprops.name)
trending_alertTS = trending_alert.groupby('timestamp')
output = trending_alertTS.apply(trending_alertgen,
window=window,
config=config,
monitoring=monitoring,
lgd=lgd)
site_level_alert = output.loc[output.timestamp == window.end]
site_level_alert['updateTS'] = [window.end]
return site_level_alert
def main(name='',custom_end = ''):
if name == '':
name = sys.argv[1].lower()
start = datetime.now()
print "=========================== {} {} =========================".format(str(name), custom_end)
window,config = rtw.getwindow(end = custom_end )
col = q.GetSensorList(name)
monitoring = g.genproc(col[0], window, config, config.io.column_fix)
lgd = q.GetLastGoodDataFromDb(monitoring.colprops.name)
monitoring_vel = monitoring.vel[window.start:window.end]
monitoring_vel = monitoring_vel.reset_index().sort_values('ts',ascending=True)
nodal_dv = monitoring_vel.groupby('id')
alert = nodal_dv.apply(node_alert2, colname=monitoring.colprops.name, num_nodes=monitoring.colprops.nos, T_disp=config.io.t_disp, T_velL2=config.io.t_vell2, T_velL3=config.io.t_vell3, k_ac_ax=config.io.k_ac_ax, lastgooddata=lgd,window=window,config=config)
alert = column_alert(alert, config.io.num_nodes_to_check, config.io.k_ac_ax)
not_working = q.GetNodeStatus(1).loc[q.GetNodeStatus(1).site == name].node.values
for i in not_working:
alert = alert.loc[alert.id != i]
if 'L3' in list(alert.col_alert.values):
site_alert = 'L3'
elif 'L2' in list(alert.col_alert.values):
site_alert = 'L2'
else:
site_alert = min(getmode(list(alert.col_alert.values)))
column_level_alert = pd.DataFrame({'timestamp': [window.end], 'site': [monitoring.colprops.name], 'source': ['sensor'], 'alert': [site_alert], 'updateTS': [window.end]})
print column_level_alert
if site_alert in ('L2', 'L3'):
A.main(monitoring.colprops.name,custom_end)
else:
alert_toDB(column_level_alert, 'column_level_alert', window)
write_site_alert(monitoring.colprops.name, window)
#######################
query = "SELECT * FROM senslopedb.site_level_alert WHERE site = '%s' and source = 'public' ORDER BY updateTS DESC LIMIT 1" %monitoring.colprops.name[0:3]
public_alert = q.GetDBDataFrame(query)
if public_alert.alert.values[0] != 'A0' or RoundTime(pd.to_datetime(public_alert.timestamp.values[0])) == RoundTime(window.end):
plot_time = ['07:30:00', '19:30:00']
if str(window.end.time()) in plot_time:
print "Plotter.main(monitoring, window, config)"
elif RoundTime(pd.to_datetime(public_alert.timestamp.values[0])) == RoundTime(window.end):
print "Plotter.main(monitoring, window, config)"
#######################
print 'run time =', datetime.now()-start
return column_level_alert,monitoring
def disp(date_end, sensor, date_start):
#str.....sila lahat
end = pd.to_datetime(date_end) #inputs specified time
col = q.GetSensorList(sensor) #inputs the name of the sensor
start = (date_start) #inputs monitoring window
window, config = rtw.getwindow(end)
window.start = pd.to_datetime(start)
while True:
start = date_start
try:
window.start = window.end - timedelta(int(start))
break
except:
try:
window.start = pd.to_datetime(start)
break
except:
print 'datetime format or integer only'
continue
window.offsetstart = window.start - timedelta(
days=(config.io.num_roll_window_ops * window.numpts - 1) /
48.) #fixes the time (offsets) for the 3 day monitoring
#somsdata = q.GetSomsData(sensor, window.offsetstart, end)
column_fix = 'bottom' #i dont know the use
config.io.column_fix = column_fix #i dont know the use
#getdispdata = q.GetRawAccelData #i dont know yet!!!!!!!!!!!!!!!!
monitoring = g.genproc(col[0],
window,
config,
config.io.column_fix,
comp_vel=True)
monitoring_vel = monitoring.disp_vel.reset_index()[[
'ts', 'id', 'depth', 'xz', 'xy', 'vel_xz', 'vel_xy'
]] #ColumnPlotter.py line 597
monitoring_vel.sort_values(
['ts', 'id'],
inplace=True) #sorts values ts and id in plance not random!
#monitoring_vel = monitoring_vel.sort_values(['ts','id'],inplace = True)same as monitoring_vel.sort_values(['ts','id'],inplace
#monitoring_vel.to_csv("{} {} to {}.csv".format(col[0].name,end.strftime('%Y-%m-%d_%H-%M'),window.start.strftime('%Y-%m-%d_%H-%M')))#save the data in csv file
return monitoring_vel
def tsm_plot(tsm_name, end, shift_datetime):
query = "SELECT max(timestamp) AS ts FROM %s" %tsm_name
try:
ts = pd.to_datetime(qdb.GetDBDataFrame(query)['ts'].values[0])
if ts < shift_datetime:
return
except:
return
if ts > end:
ts = end
window, config = rtw.getwindow(ts)
col = qdb.GetSensorList(tsm_name)
monitoring = gen.genproc(col[0], window, config,
fixpoint=config.io.column_fix)
plotter.main(monitoring, window, config, realtime=False,
non_event_path=False)
def main(site, end):
window,config = rtw.getwindow(end)
monwinTS = pd.date_range(start = window.end - timedelta(hours=3), end = window.end, freq = '30Min')
trending_alert = pd.DataFrame({'site': [np.nan]*len(monwinTS), 'alert': [np.nan]*len(monwinTS), 'timestamp': monwinTS, 'source': [np.nan]*len(monwinTS)})
trending_alert = trending_alert[['timestamp', 'site', 'source', 'alert']]
col = q.GetSensorList(site)
monitoring = g.genproc(col[0], window, config, config.io.column_fix)
lgd = q.GetLastGoodDataFromDb(monitoring.colprops.name)
trending_alertTS = trending_alert.groupby('timestamp', as_index=False)
output = trending_alertTS.apply(trending_alertgen, window=window, config=config, monitoring=monitoring, lgd=lgd)
output = output.reset_index(drop=True)
site_level_alert = output.loc[output.timestamp == window.end]
site_level_alert['updateTS'] = [window.end]
return site_level_alert
def sensor_data(date_end, sensor, date_start):
end = pd.to_datetime(date_end) #inputs specified time
col = q.GetSensorList(sensor) #inputs the name of the sensor
start = (date_start) #inputs monitoring window
window, config = rtw.getwindow(end)
window.start = pd.to_datetime(start)
while True:
start = date_start
try:
window.start = window.end - timedelta(int(start))
break
except:
try:
window.start = pd.to_datetime(start)
break
except:
print 'datetime format or integer only'
continue
window.offsetstart = window.start - timedelta(days=(config.io.num_roll_window_ops*window.numpts-1)/48.)
column_fix = 'bottom'
config.io.column_fix = column_fix
monitoring = g.genproc(col[0], window, config, config.io.column_fix, comp_vel = True)
monitoring_vel = monitoring.disp_vel.reset_index()[['ts', 'id', 'depth', 'xz', 'xy', 'vel_xz', 'vel_xy']]
monitoring_vel.sort_values(['ts','id'],inplace = True)
return monitoring_vel
#if __name__ == '__main__':
#
# start = '2017-01-01'
# end = '2017-12-30'
# site = 'laysam'
#
# df = soms_data(start,end,site)
def get_tsm_data(tsm_name, start, end, plot_type, node_lst):
col = qdb.GetSensorList(tsm_name)[0]
window, config = rtw.getwindow(pd.to_datetime(end))
window.start = pd.to_datetime(start)
window.offsetstart = window.start - timedelta(
days=(config.io.num_roll_window_ops * window.numpts - 1) / 48.)
if plot_type == 'cml':
config.io.to_smooth = 1
config.io.to_fill = 1
else:
config.io.to_smooth = 1
config.io.to_fill = 1
monitoring = proc.genproc(col, window, config, 'bottom', comp_vel=False)
df = monitoring.disp_vel.reset_index()[['ts', 'id', 'xz', 'xy']]
df = df.loc[(df.ts >= window.start) & (df.ts <= window.end)]
df = df.sort_values('ts')
if plot_type == 'cml':
xzd_plotoffset = 0
if node_lst != 'all':
df = df[df.id.isin(node_lst)]
df = plotter.cum_surf(df, xzd_plotoffset, col.nos)
else:
node_df = df.groupby('id', as_index=False)
df = node_df.apply(zeroed, column='xz')
df['zeroed_xz'] = df['zeroed_xz'] * 100
node_df = df.groupby('id', as_index=False)
df = node_df.apply(zeroed, column='xy')
df['zeroed_xy'] = df['zeroed_xy'] * 100
return df
def main(name='', end='', end_mon=False):
start = datetime.now()
if name == '':
name = sys.argv[1].lower()
if end == '':
try:
end = pd.to_datetime(sys.argv[2])
if end > start + timedelta(hours=0.5):
print 'invalid timestamp'
return
except:
end = datetime.now()
else:
end = pd.to_datetime(end)
window, config = rtw.getwindow(end)
col = q.GetSensorList(name)
monitoring = g.genproc(col[0], window, config, config.io.column_fix)
lgd = q.GetLastGoodDataFromDb(monitoring.colprops.name)
monitoring_vel = monitoring.disp_vel[window.start:window.end]
monitoring_vel = monitoring_vel.reset_index().sort_values('ts',
ascending=True)
nodal_dv = monitoring_vel.groupby('id')
alert = nodal_dv.apply(node_alert2,
colname=monitoring.colprops.name,
num_nodes=monitoring.colprops.nos,
T_disp=config.io.t_disp,
T_velL2=config.io.t_vell2,
T_velL3=config.io.t_vell3,
k_ac_ax=config.io.k_ac_ax,
lastgooddata=lgd,
window=window,
config=config)
alert['col_alert'] = -1
col_alert = pd.DataFrame({
'id': range(1, monitoring.colprops.nos + 1),
'col_alert': [-1] * monitoring.colprops.nos
})
node_col_alert = col_alert.groupby('id', as_index=False)
node_col_alert.apply(column_alert,
alert=alert,
num_nodes_to_check=config.io.num_nodes_to_check,
k_ac_ax=config.io.k_ac_ax,
T_velL2=config.io.t_vell2,
T_velL3=config.io.t_vell3)
alert['node_alert'] = alert['node_alert'].map({
-1: 'ND',
0: 'L0',
1: 'L2',
2: 'L3'
})
alert['col_alert'] = alert['col_alert'].map({
-1: 'ND',
0: 'L0',
1: 'L2',
2: 'L3'
})
not_working = q.GetNodeStatus(1).loc[q.GetNodeStatus(1).site ==
name].node.values
for i in not_working:
alert = alert.loc[alert.id != i]
if 'L3' in list(alert.col_alert.values):
site_alert = 'L3'
elif 'L2' in list(alert.col_alert.values):
site_alert = 'L2'
else:
site_alert = min(getmode(list(alert.col_alert.values)))
column_level_alert = pd.DataFrame({
'timestamp': [window.end],
'site': [monitoring.colprops.name],
'source': ['noadjfilt'],
'alert': [site_alert],
'updateTS': [window.end]
})
if site_alert in ('L2', 'L3'):
column_level_alert = A.main(monitoring.colprops.name, window.end)
alert_toDB(column_level_alert, 'column_level_alert', window)
write_site_alert(monitoring.colprops.name, window)
print column_level_alert
print 'run time =', datetime.now() - start
return column_level_alert
def main(name='', end='', end_mon=False):
start = datetime.now()
if name == '':
name = sys.argv[1].lower()
if end == '':
try:
end = pd.to_datetime(sys.argv[2])
if end > start + timedelta(hours=0.5):
print 'invalid timestamp'
return
except:
end = datetime.now()
else:
end = pd.to_datetime(end)
window,config = rtw.getwindow(end)
col = q.GetSensorList(name)
monitoring = g.genproc(col[0], window, config, config.io.column_fix)
lgd = q.GetLastGoodDataFromDb(monitoring.colprops.name)
monitoring_vel = monitoring.disp_vel[window.start:window.end]
monitoring_vel = monitoring_vel.reset_index().sort_values('ts',ascending=True)
nodal_dv = monitoring_vel.groupby('id')
alert = nodal_dv.apply(node_alert2, colname=monitoring.colprops.name, num_nodes=monitoring.colprops.nos, T_disp=config.io.t_disp, T_velL2=config.io.t_vell2, T_velL3=config.io.t_vell3, k_ac_ax=config.io.k_ac_ax, lastgooddata=lgd,window=window,config=config)
alert['col_alert'] = -1
col_alert = pd.DataFrame({'id': range(1, monitoring.colprops.nos+1), 'col_alert': [-1]*monitoring.colprops.nos})
node_col_alert = col_alert.groupby('id', as_index=False)
node_col_alert.apply(column_alert, alert=alert, num_nodes_to_check=config.io.num_nodes_to_check, k_ac_ax=config.io.k_ac_ax, T_velL2=config.io.t_vell2, T_velL3=config.io.t_vell3)
alert['node_alert']=alert['node_alert'].map({-1:'ND',0:'L0',1:'L2',2:'L3'})
alert['col_alert']=alert['col_alert'].map({-1:'ND',0:'L0',1:'L2',2:'L3'})
not_working = q.GetNodeStatus(1).loc[q.GetNodeStatus(1).site == name].node.values
for i in not_working:
alert = alert.loc[alert.id != i]
if 'L3' in list(alert.col_alert.values):
site_alert = 'L3'
elif 'L2' in list(alert.col_alert.values):
site_alert = 'L2'
else:
site_alert = min(getmode(list(alert.col_alert.values)))
column_level_alert = pd.DataFrame({'timestamp': [window.end], 'site': [monitoring.colprops.name], 'source': ['noadjfilt'], 'alert': [site_alert], 'updateTS': [window.end]})
if site_alert in ('L2', 'L3'):
column_level_alert = A.main(monitoring.colprops.name, window.end)
alert_toDB(column_level_alert, 'column_level_alert', window)
write_site_alert(monitoring.colprops.name, window)
print column_level_alert
print 'run time =', datetime.now()-start
return column_level_alert
##########################################################
###INPUTS
colname = 'pngta'
node = 8
axis = 'xz'
k = 3 #degree of spline
c = 1 #factor of error
#Step 1: Get dataframe for xz and xy using RealTimePlotter Code
col = q.GetSensorList(colname)
start = '2017-01-09 7:00:00'
end = '2017-01-10 09:00:00'
window, config = rtw.getwindow(pd.to_datetime(end))
config.io.to_smooth = 0
window.start = pd.to_datetime(start).to_datetime()
window.numpts = int(7)
window.offsetstart = window.start - timedelta(
days=(config.io.num_roll_window_ops * window.numpts - 1) / 48.)
out_path = 'C:\Users\Win8\Documents\Dynaslope\\Data Analysis\\Filters\\Acceleration Velocity\\'
out_path = out_path + 'Underground\k {} Gaussian num_pts {}\\{}\\{}\\'.format(
k, window.numpts, colname, str(node))
out_path1 = out_path + 'stats\\'
out_path2 = out_path + 'overall trend\\'
out_path3 = out_path + 'v vs a time evolution\\'
for paths in [out_path, out_path1, out_path2, out_path3]:
def main(name='', end=datetime.now(), end_mon=False):
if name == '':
name = sys.argv[1].lower()
window, config = rtw.getwindow(end)
col = q.GetSensorList(name)
monitoring = g.genproc(col[0], window, config, config.io.column_fix)
lgd = q.GetLastGoodDataFromDb(monitoring.colprops.name)
monitoring_vel = monitoring.vel[window.start:window.end]
monitoring_vel = monitoring_vel.reset_index().sort_values('ts',
ascending=True)
nodal_dv = monitoring_vel.groupby('id')
alert = nodal_dv.apply(node_alert2,
colname=monitoring.colprops.name,
num_nodes=monitoring.colprops.nos,
T_disp=config.io.t_disp,
T_velL2=config.io.t_vell2,
T_velL3=config.io.t_vell3,
k_ac_ax=config.io.k_ac_ax,
lastgooddata=lgd,
window=window,
config=config)
alert = column_alert(alert, config.io.num_nodes_to_check,
config.io.k_ac_ax)
not_working = q.GetNodeStatus(1).loc[q.GetNodeStatus(1).site ==
name].node.values
for i in not_working:
alert = alert.loc[alert.id != i]
if 'L3' in list(alert.col_alert.values):
site_alert = 'L3'
elif 'L2' in list(alert.col_alert.values):
site_alert = 'L2'
else:
site_alert = min(getmode(list(alert.col_alert.values)))
column_level_alert = pd.DataFrame({
'timestamp': [window.end],
'site': [monitoring.colprops.name],
'source': ['sensor'],
'alert': [site_alert],
'updateTS': [window.end]
})
if site_alert in ('L2', 'L3'):
column_level_alert = A.main(monitoring.colprops.name, window.end)
alert_toDB(column_level_alert, 'column_level_alert', window)
print column_level_alert
write_site_alert(monitoring.colprops.name, window)
#######################
if monitoring.colprops.name == 'mesta':
colname = 'msu'
elif monitoring.colprops.name == 'messb':
colname = 'msl'
else:
colname = monitoring.colprops.name[0:3]
query = "SELECT * FROM senslopedb.site_level_alert WHERE site = '%s' and source = 'public' and timestamp <= '%s' and updateTS >= '%s' ORDER BY updateTS DESC LIMIT 1" % (
colname, window.end, window.end - timedelta(hours=0.5))
public_alert = q.GetDBDataFrame(query)
if public_alert.alert.values[0] != 'A0':
plot_time = ['07:30:00', '19:30:00']
if str(window.end.time()) in plot_time or end_mon:
plotter.main(monitoring,
window,
config,
plotvel_start=window.end - timedelta(hours=3),
plotvel_end=window.end,
realtime=False)
elif RoundTime(pd.to_datetime(
public_alert.timestamp.values[0])) == RoundTime(window.end):
plotter.main(monitoring,
window,
config,
plotvel_start=window.end - timedelta(hours=3),
plotvel_end=window.end,
realtime=False)
#######################
return column_level_alert
def output_file_path(site, plot_type, monitoring_end=False, positive_trigger=False, end=datetime.now()):
output_path = os.path.abspath(os.path.join(os.path.dirname(__file__), '../..'))
window,config = rtw.getwindow(pd.to_datetime(end))
if window.end.time() >= time(8, 0) and window.end.time() < time(20, 0):
shift_start = window.end.strftime('%d %b %Y AM')
elif window.end.time() > time(20, 0):
shift_start = window.end.strftime('%d %b %Y PM')
else:
shift_start = (window.end - timedelta(1)).strftime('%d %b %Y PM')
if site != 'all':
if site == 'bat':
site = 'bto'
elif site == 'man':
site = 'mng'
elif site == 'pan':
site = 'png'
elif site == 'pob':
site= 'jor'
elif site == 'tag':
site = 'tga'
# 3 most recent non-A0 public alert
query = "SELECT * FROM senslopedb.site_level_alert"
query += " WHERE site = '%s'" %site
query += " AND source = 'public'"
query += " AND (updateTS <= '%s'" %window.end
query += " OR (updateTS >= '%s'" %window.end
query += " AND timestamp <= '%s'))" %window.end
query += " ORDER BY timestamp DESC LIMIT 4"
public_alert = q.GetDBDataFrame(query)
if plot_type == 'rainfall':
monitoring_output_path = output_path + config.io.rainfallplotspath
elif plot_type == 'subsurface':
monitoring_output_path = output_path + config.io.subsurfaceplotspath
elif plot_type == 'surficial':
monitoring_output_path = output_path + config.io.surficialplotspath
elif plot_type == 'trending_surficial':
monitoring_output_path = output_path + config.io.trendingsurficialplotspath
elif plot_type == 'eq':
monitoring_output_path = output_path + config.io.eqplotspath
else:
monitoring_output_path = output_path + config.io.outputfilepath
print 'unrecognized plot type; print to %s' %(monitoring_output_path)
try:
if positive_trigger and public_alert['alert'].values[0] == 'A0':
event_path = output_path + config.io.outputfilepath + 'EventMonitoring/' \
+ (shift_start + '/' + site + '/').upper()
elif (public_alert['alert'].values[0] == 'A0' and not monitoring_end) \
or (not monitoring_end and public_alert['alert'].values[0] != 'A0' \
and plot_type == 'rainfall' and window.end.time() not in [time(7, 30), time(19, 30)]):
event_path = None
else:
event_path = output_path + config.io.outputfilepath + 'EventMonitoring/' \
+ (shift_start + '/' + site + '/').upper()
except:
event_path = None
for i in set([monitoring_output_path, event_path]) - set([None]):
if not os.path.exists(str(i)):
os.makedirs(str(i))
file_path = {'event': event_path, 'monitoring_output': monitoring_output_path}
return file_path
def mon_main():
while True:
plot_all_data = raw_input(
'plot from start to end of data? (Y/N): ').lower()
if plot_all_data == 'y' or plot_all_data == 'n':
break
# plots segment of data
if plot_all_data == 'n':
while True:
monitoring_window = raw_input(
'plot with 3 day monitoring window? (Y/N): ').lower()
if monitoring_window == 'y' or monitoring_window == 'n':
break
# plots with 3 day monitoring window
if monitoring_window == 'y':
while True:
try:
col = q.GetSensorList(raw_input('sensor name: '))
break
except:
print 'sensor name is not in the list'
continue
while True:
test_specific_time = raw_input(
'test specific time? (Y/N): ').lower()
if test_specific_time == 'y' or test_specific_time == 'n':
break
while True:
try:
if test_specific_time == 'y':
end = pd.to_datetime(
raw_input(
'plot end timestamp (format: 2016-12-31 23:30): '
))
window, config = rtw.getwindow(end)
elif test_specific_time == 'n':
window, config = rtw.getwindow()
break
except:
print 'invalid datetime format'
continue
column_fix = raw_input(
'column fix for colpos (top/bottom); default for monitoring is fix bottom: '
).lower()
if column_fix != 'top':
column_fix = 'bottom'
config.io.column_fix = column_fix
monitoring = g.genproc(col[0],
window,
config,
config.io.column_fix,
realtime=True)
plotter.main(monitoring,
window,
config,
plotvel_start=window.end - timedelta(hours=3),
plotvel_end=window.end) #, plot_inc=False)
# plots with customizable monitoring window
elif monitoring_window == 'n':
while True:
try:
col = q.GetSensorList(raw_input('sensor name: '))
break
except:
print 'sensor name is not in the list'
continue
while True:
try:
end = pd.to_datetime(
raw_input(
'plot end timestamp (format: 2016-12-31 23:30): '))
window, config = rtw.getwindow(end)
break
except:
print 'invalid datetime format'
continue
while True:
start = raw_input(
'monitoring window (in days) or datetime (format: 2016-12-31 23:30): '
)
try:
window.start = window.end - timedelta(int(start))
break
except:
try:
window.start = pd.to_datetime(start)
break
except:
print 'datetime format or integer only'
continue
window.offsetstart = window.start - timedelta(
days=(config.io.num_roll_window_ops * window.numpts - 1) / 48.)
while True:
try:
col_pos_interval = int(
raw_input(
'interval between column position dates, in days: '
))
break
except:
print 'enter an integer'
continue
config.io.col_pos_interval = str(col_pos_interval) + 'D'
config.io.num_col_pos = int((window.end - window.start).days /
col_pos_interval + 1)
column_fix = raw_input(
'column fix for colpos (top/bottom); default for monitoring is fix bottom: '
).lower()
if column_fix != 'top':
column_fix = 'bottom'
config.io.column_fix = column_fix
while True:
show_all_legend = raw_input(
'show all legend in column position plot? (Y/N): ').lower(
)
if show_all_legend == 'y' or show_all_legend == 'n':
break
if show_all_legend == 'y':
show_part_legend = False
elif show_all_legend == 'n':
while True:
try:
show_part_legend = int(
raw_input('every nth legend to show: '))
if show_part_legend <= config.io.num_col_pos:
break
else:
print 'integer should be less than number of column position dates to plot:', config.io.num_col_pos
continue
except:
print 'enter an integer'
continue
while True:
plotvel = raw_input('plot velocity? (Y/N): ').lower()
if plotvel == 'y' or plotvel == 'n':
break
if plotvel == 'y':
plotvel = True
else:
plotvel = False
monitoring = g.genproc(col[0],
window,
config,
config.io.column_fix,
comp_vel=plotvel)
plotter.main(monitoring,
window,
config,
plotvel=plotvel,
show_part_legend=show_part_legend,
plotvel_end=window.end,
plotvel_start=window.start,
plot_inc=False,
comp_vel=plotvel)
# plots from start to end of data
elif plot_all_data == 'y':
while True:
try:
col = q.GetSensorList(raw_input('sensor name: '))
break
except:
print 'sensor name is not in the list'
continue
while True:
try:
col_pos_interval = int(
raw_input(
'interval between column position dates, in days: '))
break
except:
print 'enter an integer'
continue
query = "(SELECT * FROM senslopedb.%s where timestamp > '2010-01-01 00:00' ORDER BY timestamp LIMIT 1)" % col[
0].name
query += " UNION ALL"
query += " (SELECT * FROM senslopedb.%s ORDER BY timestamp DESC LIMIT 1)" % col[
0].name
start_end = q.GetDBDataFrame(query)
end = pd.to_datetime(start_end['timestamp'].values[1])
window, config = rtw.getwindow(end)
start_dataTS = pd.to_datetime(start_end['timestamp'].values[0])
start_dataTS_Year = start_dataTS.year
start_dataTS_month = start_dataTS.month
start_dataTS_day = start_dataTS.day
start_dataTS_hour = start_dataTS.hour
start_dataTS_minute = start_dataTS.minute
if start_dataTS_minute < 30: start_dataTS_minute = 0
else: start_dataTS_minute = 30
window.offsetstart = datetime.combine(
date(start_dataTS_Year, start_dataTS_month, start_dataTS_day),
time(start_dataTS_hour, start_dataTS_minute, 0))
window.numpts = int(1 +
config.io.roll_window_length / config.io.data_dt)
window.start = window.offsetstart + timedelta(
days=(config.io.num_roll_window_ops * window.numpts - 1) / 48.)
config.io.col_pos_interval = str(col_pos_interval) + 'D'
config.io.num_col_pos = int((window.end - window.start).days /
col_pos_interval + 1)
column_fix = raw_input(
'column fix for colpos (top/bottom); default for monitoring is fix bottom: '
).lower()
if column_fix != 'top':
column_fix = 'bottom'
config.io.column_fix = column_fix
while True:
show_all_legend = raw_input(
'show all legend in column position plot? (Y/N): ').lower()
if show_all_legend == 'y' or show_all_legend == 'n':
break
if show_all_legend == 'y':
show_part_legend = False
elif show_all_legend == 'n':
while True:
try:
show_part_legend = int(
raw_input('every nth legend to show: '))
if show_part_legend <= config.io.num_col_pos:
break
else:
print 'integer should be less than number of column position dates to plot:', config.io.num_col_pos
continue
except:
print 'enter an integer'
continue
while True:
plotvel = raw_input('plot velocity? (Y/N): ').lower()
if plotvel == 'y' or plotvel == 'n':
break
if plotvel == 'y':
plotvel = True
else:
plotvel = False
monitoring = g.genproc(col[0],
window,
config,
config.io.column_fix,
comp_vel=plotvel)
plotter.main(monitoring,
window,
config,
plotvel=plotvel,
show_part_legend=show_part_legend,
plot_inc=False,
comp_vel=plotvel)