def tsm_analysis(df, window, sc):
try:
end = max(df.ts)
start = min(df.ts)
window.end = end
window.start = start - timedelta(days=3)
window.offsetstart = window.start - timedelta(days=(
sc['subsurface']['num_roll_window_ops'] * window.numpts - 1) / 48.)
tsm_name = df['tsm_name'].values[0]
print(tsm_name)
tsm_props = qdb.get_tsm_list(tsm_name)[0]
data = proc.proc_data(tsm_props,
window,
sc,
realtime=True,
comp_vel=True).tilt.reset_index()
node_data = data.groupby('node_id', as_index=False)
tsm_kinematics = node_data.apply(node_disp).reset_index(drop=True)
node_kinematics = tsm_kinematics.groupby('node_id', as_index=False)
adj_node_movt = node_kinematics.apply(
adj_movt, data=tsm_kinematics,
num_nodes=tsm_props.nos).reset_index(drop=True)
percent_movt = pd.merge(df,
adj_node_movt,
how='inner',
on=['ts', 'tsm_name', 'node_id'])
return percent_movt[[
'ts', 'tsm_name', 'node_id', 'top_disp', 'bottom_disp', 'top_vel',
'bottom_vel', 'na_status'
]]
except:
pass
def get_tsm_data(tsm_name, start, end, plot_type, node_lst):
tsm_props = qdb.get_tsm_list(tsm_name)[0]
window, sc = rtw.get_window(end)
window.start = pd.to_datetime(start)
window.offsetstart = window.start - timedelta(
(sc['subsurface']['num_roll_window_ops'] * window.numpts - 1) / 48.)
data = proc.proc_data(tsm_props,
window,
sc,
comp_vel=False,
analysis=False)
df = data.tilt.reset_index()[['ts', 'node_id', 'xz', 'xy']]
df = df.loc[(df.ts >= window.start) & (df.ts <= window.end)]
df = df.sort_values('ts')
if plot_type == 'cml':
xzd_plotoffset = 0
if node_lst != 'all':
df = df[df.node_id.isin(node_lst)]
df = plotter.cum_surf(df, xzd_plotoffset, tsm_props.nos)
else:
node_df = df.groupby('node_id', as_index=False)
df = node_df.apply(zeroed, column='xz')
df['zeroed_xz'] = df['zeroed_xz'] * 100
node_df = df.groupby('node_id', as_index=False)
df = node_df.apply(zeroed, column='xy')
df['zeroed_xy'] = df['zeroed_xy'] * 100
return df
def proc_data(tsm_name, endTS, startTS, sc, hour_interval, fixpoint):
tsm_props = qdb.get_tsm_list(tsm_name)[0]
#end
if endTS == '':
window, config = rtw.get_window()
else:
end = pd.to_datetime(endTS)
end_year = end.year
end_month = end.month
end_day = end.day
end_hour = end.hour
end_minute = end.minute
if end_minute < 30: end_minute = 0
else: end_minute = 30
end = datetime.combine(date(end_year, end_month, end_day),
time(end_hour, end_minute, 0))
window, config = rtw.get_window(end)
if startTS != '':
#start
start = pd.to_datetime(startTS)
start_year = start.year
start_month = start.month
start_day = start.day
start_hour = start.hour
start_minute = start.minute
if start_minute < 30: start_minute = 0
else: start_minute = 30
window.start = datetime.combine(
date(start_year, start_month, start_day),
time(start_hour, start_minute, 0))
#offsetstart
window.offsetstart = window.start - timedelta(days=(
sc['subsurface']['num_roll_window_ops'] * window.numpts - 1) / 48.)
#colpos interval
if hour_interval == '':
if int((window.end - window.start).total_seconds() / (3600 * 24)) <= 5:
hour_interval = 4
else:
hour_interval = 24
sc['subsurface']['col_pos_interval'] = str(hour_interval) + 'H'
sc['subsurface']['num_col_pos'] = int(
(window.end - window.start).total_seconds() /
(3600 * hour_interval)) + 1
if fixpoint in ['top', 'bottom']:
sc['subsurface']['column_fic'] = fixpoint
data = proc.proc_data(tsm_props, window, sc)
tilt = data.tilt[window.start:window.end]
tilt = tilt.reset_index().sort_values('ts', ascending=True)
tilt = tilt[['ts', 'node_id', 'xz', 'xy', 'vel_xz', 'vel_xy']]
return tilt, window, sc, tsm_props
def tsm_analysis(df, window, sc):
try:
end = max(df.ts)
start = min(df.ts)
window.end = end
window.start = start - timedelta(days=3)
window.offsetstart = window.start - timedelta(days=(sc['subsurface']
['num_roll_window_ops']*window.numpts-1)/48.)
tsm_name = df['tsm_name'].values[0]
print(tsm_name)
tsm_props = qdb.get_tsm_list(tsm_name)[0]
data = proc.proc_data(tsm_props, window, sc, realtime=True, comp_vel=True).tilt.reset_index() # proc using lowess
## check treshold exceedance
percent_movt = pd.merge(df, data, how='inner', on=['ts', 'tsm_name', 'node_id'])
return percent_movt.loc[:, ['ts', 'tsm_name', 'node_id', 'na_status', 'pred']]
except:
pass
def main():
# asks for tsm name
while True:
props = qdb.get_tsm_list(input('sensor name: '))
if len(props) == 1:
break
else:
qdb.print_out('sensor name is not in the list')
continue
tsm_props = props[0]
# asks if to plot from date activated (or oldest data) to most recent data
while True:
input_text = 'plot from start to end of data? (Y/N): '
plot_all_data = input(input_text).lower()
if plot_all_data == 'y' or plot_all_data == 'n':
break
# asks if to specify end timestamp of monitoring window
if plot_all_data == 'n':
while True:
input_text = 'specify end timestamp of monitoring window? (Y/N): '
test_specific_time = input(input_text).lower()
if test_specific_time == 'y' or test_specific_time == 'n':
break
# ask for timestamp of end of monitoring window defaults to datetime.now
if test_specific_time == 'y':
while True:
try:
input_text = 'plot end timestamp (format: 2016-12-31 23:30): '
end = pd.to_datetime(input(input_text))
break
except:
print('invalid datetime format')
continue
else:
end = datetime.now()
# monitoring window and server configurations
window, sc = rtw.get_window(end)
# asks if to plot with 3-day monitoring window
while True:
input_text = 'plot with 3-day monitoring window? (Y/N): '
three_day_window = input(input_text).lower()
if three_day_window == 'y' or three_day_window == 'n':
break
# asks start of monitoring window defaults to values in server config
if three_day_window == 'n':
while True:
input_text = 'start of monitoring window (in days) '
input_text += 'or datetime (format: 2016-12-31 23:30): '
start = input(input_text)
try:
window.start = window.end - timedelta(int(start))
break
except:
try:
window.start = pd.to_datetime(start)
break
except:
print('datetime format or integer only')
continue
# computes offsetstart from given start timestamp
window.offsetstart = window.start - timedelta(
days=(sc['subsurface']['num_roll_window_ops'] * window.numpts -
1) / 48.)
else:
# check date of activation
query = "SELECT date_activated FROM tsm_sensors"
query += " WHERE tsm_name = '%s'" % tsm_props.tsm_name
try:
date_activated = qdb.db.df_read(query).values[0][0]
except:
date_activated = pd.to_datetime('2010-01-01')
#compute for start to end timestamp of data
query = "(SELECT * FROM tilt_%s" % tsm_props.tsm_name
query += " where ts > '%s' ORDER BY ts LIMIT 1)" % date_activated
query += " UNION ALL"
query += " (SELECT * FROM tilt_%s" % tsm_props.tsm_name
query += " ORDER BY ts DESC LIMIT 1)"
start_end = qdb.db.df_read(query)
end = pd.to_datetime(start_end['ts'].values[1])
window, sc = rtw.get_window(end)
start_dataTS = pd.to_datetime(start_end['ts'].values[0])
start_dataTS_Year = start_dataTS.year
start_dataTS_month = start_dataTS.month
start_dataTS_day = start_dataTS.day
start_dataTS_hour = start_dataTS.hour
start_dataTS_minute = start_dataTS.minute
if start_dataTS_minute < 30: start_dataTS_minute = 0
else: start_dataTS_minute = 30
window.offsetstart = datetime.combine(
date(start_dataTS_Year, start_dataTS_month, start_dataTS_day),
time(start_dataTS_hour, start_dataTS_minute, 0))
# computes offsetstart from given start timestamp
window.start = window.offsetstart + timedelta(days=(
sc['subsurface']['num_roll_window_ops'] * window.numpts - 1) / 48.)
# asks if to plot velocity and asks for interval and legends to show in
# column position plots if to plot all data or if monitoring window not
# equal to 3 days
if plot_all_data == 'y' or three_day_window == 'n':
# asks for interval between column position plots
while True:
try:
input_text = 'interval between column position plots, in hours: '
col_pos_interval = int(input(input_text))
break
except:
qdb.print_out('enter an integer')
continue
# computes for interval and number of column position plots
sc['subsurface']['col_pos_interval'] = str(col_pos_interval) + 'H'
sc['subsurface']['num_col_pos'] = int(
(window.end - window.start).total_seconds() /
(3600 * col_pos_interval) + 1)
# asks if to plot all legends
while True:
input_text = 'show all legends in column position plot? (Y/N): '
show_all_legend = input(input_text).lower()
if show_all_legend == 'y' or show_all_legend == 'n':
break
if show_all_legend == 'y':
show_part_legend = False
# asks which legends to show
elif show_all_legend == 'n':
while True:
try:
show_part_legend = int(input('every nth legend to show: '))
if show_part_legend <= sc['subsurface']['num_col_pos']:
break
else:
input_text = 'integer should be less than '
input_text += 'the number of colpos dates to plot: '
input_text += '%s' % (sc['subsurface']['num_col_pos'])
qdb.print_out(input_text)
continue
except:
qdb.print_out('enter an integer')
continue
while True:
plotvel = input('plot velocity? (Y/N): ').lower()
if plotvel == 'y' or plotvel == 'n':
break
if plotvel == 'y':
plotvel = True
else:
plotvel = False
three_day_window = False
else:
plotvel = True
show_part_legend = True
three_day_window = True
# asks which point to fix in column position plots
while True:
input_text = 'column fix for colpos (top/bottom). '
input_text += 'press enter to skip; '
input_text += 'default for monitoring is fix bottom: '
column_fix = input(input_text).lower()
if column_fix in ['top', 'bottom', '']:
break
if column_fix == '':
column_fix = 'bottom'
sc['subsurface']['column_fix'] = column_fix
# mirror xz and/or xy colpos
while True:
try:
mirror_xz = bool(int(input('mirror image of xz colpos? (0/1): ')))
break
except:
print('Invalid. 1 for mirror image of xz colpos else 0')
continue
while True:
try:
mirror_xy = bool(int(input('mirror image of xy colpos? (0/1): ')))
break
except:
print('Invalid. 1 for mirror image of xy colpos else 0')
continue
while True:
print_disp_vel = input(
'print displacement and velocity to csv? (Y/N): ').lower()
if print_disp_vel == 'y' or print_disp_vel == 'n':
break
data = proc.proc_data(tsm_props,
window,
sc,
realtime=True,
comp_vel=plotvel,
analysis=False)
if print_disp_vel == 'y':
tilt = data.tilt.reset_index()
accel = data.accel_data.reset_index()
df = pd.merge(tilt,
accel,
on=['ts', 'node_id', 'tsm_name'],
how='outer').sort_values(['ts', 'node_id'])
df.to_csv('{}_{}-{}.csv'.format(tsm_props.tsm_name,
window.start.strftime('%Y%m%d%H%M'),
window.end.strftime('%Y%m%d%H%M')),
index=False)
plotter.main(data,
tsm_props,
window,
sc,
plotvel=plotvel,
show_part_legend=show_part_legend,
realtime=True,
plot_inc=False,
three_day_window=three_day_window,
mirror_xz=mirror_xz,
mirror_xy=mirror_xy)
return data
def main(tsm_name='', end='', end_mon=False):
run_start = datetime.now()
qdb.print_out(run_start)
qdb.print_out(tsm_name)
if tsm_name == '':
tsm_name = sys.argv[1].lower()
if end == '':
try:
end = pd.to_datetime(sys.argv[2])
except:
end = datetime.now()
else:
end = pd.to_datetime(end)
window, sc = rtw.get_window(end)
tsm_props = qdb.get_tsm_list(tsm_name)[0]
data = proc.proc_data(tsm_props, window, sc)
tilt = data.tilt[window.start:window.end]
lgd = data.lgd
tilt = tilt.reset_index().sort_values('ts', ascending=True)
if lgd.empty:
qdb.print_out('%s: no data' % tsm_name)
return
nodal_tilt = tilt.groupby('node_id', as_index=False)
alert = nodal_tilt.apply(lib.node_alert,
colname=tsm_props.tsm_name,
num_nodes=tsm_props.nos,
disp=float(sc['subsurface']['disp']),
vel2=float(sc['subsurface']['vel2']),
vel3=float(sc['subsurface']['vel3']),
k_ac_ax=float(sc['subsurface']['k_ac_ax']),
lastgooddata=lgd,
window=window,
sc=sc).reset_index(drop=True)
alert.loc[:, 'col_alert'] = -1
col_alert = pd.DataFrame({
'node_id': range(1, tsm_props.nos + 1),
'col_alert': [-1] * tsm_props.nos
})
node_col_alert = col_alert.groupby('node_id', as_index=False)
node_col_alert.apply(lib.column_alert,
alert=alert,
num_nodes_to_check=int(
sc['subsurface']['num_nodes_to_check']),
k_ac_ax=float(sc['subsurface']['k_ac_ax']),
vel2=float(sc['subsurface']['vel2']),
vel3=float(sc['subsurface']['vel3']))
valid_nodes_alert = alert.loc[~alert.node_id.isin(data.inv)]
if max(valid_nodes_alert['col_alert'].values) > 0:
pos_alert = valid_nodes_alert[valid_nodes_alert.col_alert > 0]
site_alert = trend.main(pos_alert, tsm_props.tsm_id, window.end,
data.inv)
else:
site_alert = max(
lib.get_mode(list(valid_nodes_alert['col_alert'].values)))
tsm_alert = pd.DataFrame({
'ts': [window.end],
'tsm_id': [tsm_props.tsm_id],
'alert_level': [site_alert],
'ts_updated': [window.end]
})
qdb.alert_to_db(tsm_alert, 'tsm_alerts')
qdb.write_op_trig(tsm_props.site_id, window.end)
qdb.print_out(tsm_alert)
qdb.print_out('run time = ' + str(datetime.now() - run_start))
return tilt
def main(alert):
site_id = alert.site_id
site = alert.site_code
ts = alert.ts_last_retrigger
source_id = alert.source_id
alert_id = alert.stat_id
#### Open config files
sc = mem.get('server_config')
output_path = os.path.abspath(
os.path.join(os.path.dirname(__file__), '../..'))
# OutputFP= os.path.abspath(os.path.join(os.path.dirname(__file__), '../..')) #os.path.dirname(os.path.realpath(__file__))+'/{} {}/'.format(site, ts.strftime("%Y-%m-%d %H%M"))
OutputFP = '{}/olivia_plots/' + '{} {} {}/'.format(
output_path + sc['fileio']['output_path'], alert_id, site,
ts.strftime("%Y-%m-%d %H%M"))
OutputFP = OutputFP.replace("\\", "/")
if not os.path.exists(OutputFP):
os.makedirs(OutputFP)
else:
return False
if source_id == 1:
ts_before = ts.round('4H') - td(hours=4)
queryalert = """SELECT na_id,ts,t.tsm_id,tsm_name,node_id,disp_alert,vel_alert
FROM node_alerts
inner join tsm_sensors as t
on t.tsm_id=node_alerts.tsm_id
where site_id={} and (ts between '{}' and '{}')
order by tsm_name, node_id, ts desc""".format(
alert.site_id, ts_before, ts)
dfalert = db.df_read(queryalert, connection="analysis").groupby(
['tsm_id', 'node_id']).first().reset_index()
print("ok")
# plot colpos & disp vel
tsm_props = qdb.get_tsm_list(dfalert.tsm_name[0])[0]
window, sc = rtw.get_window(ts)
data = proc.proc_data(tsm_props, window, sc)
plotter.main(data, tsm_props, window, sc, plot_inc=False)
plot_path_sensor = output_path + sc['fileio']['realtime_path']
for img in os.listdir(plot_path_sensor):
shutil.move("{}/{}".format(plot_path_sensor, img), OutputFP)
# plot node data
for i in dfalert.index:
print(dfalert.tsm_name[i], dfalert.node_id[i], dfalert.ts[i])
xyz.xyzplot(dfalert.tsm_id[i], dfalert.node_id[i], dfalert.ts[i],
OutputFP)
elif source_id == 3:
rain.main(site_code=site, end=ts, write_to_db=False, print_plot=True)
plot_path_rain = output_path + sc['fileio']['rainfall_path']
for img in os.listdir(plot_path_rain):
shutil.move("{}/{}".format(plot_path_rain, img), OutputFP)
elif source_id == 2:
# print("marker")
# query_alert = ("SELECT marker_id FROM marker_alerts "
# "where ts = '{}' and alert_level >0".format(ts))
# dfalert=db.df_read(query_alert,connection = "analysis")
#for m_id in dfalert.marker_id:
marker.generate_surficial_alert(site_id=site_id,
ts=ts) #, marker_id=m_id)
plot_path_meas = output_path + sc['fileio']['surficial_meas_path']
plot_path_trend = output_path + sc['fileio']['surficial_trending_path']
for img in os.listdir(plot_path_meas):
shutil.move("{}/{}".format(plot_path_meas, img), OutputFP)
for img in os.listdir(plot_path_trend):
shutil.move("{}/{}".format(plot_path_trend, img), OutputFP)
return OutputFP
"on monitoring_event_alerts.event_id = monitoring_events.event_id "
"inner join monitoring_releases "
"on monitoring_event_alerts.event_alert_id = monitoring_releases.event_alert_id "
"inner join public_alert_symbols "
"on public_alert_symbols.pub_sym_id = monitoring_event_alerts.pub_sym_id "
"where monitoring_event_alerts.pub_sym_id >= 2 and validity > Now() and data_ts >= NOW()-INTERVAL 4 hour "
"AND site_code in ('msu','msl') "
"order by alert_symbol desc")
cur_alert = db.df_read(query, connection="website")
# remove repeating site_code
cur_alert = cur_alert.groupby("site_code").first().reset_index()
if len(cur_alert) > 0:
# plot colpos & disp vel
tsm_props = qdb.get_tsm_list("msuta")[0]
window, sc = rtw.get_window(pd.to_datetime(dt.now()))
data = proc.proc_data(tsm_props, window, sc)
plotter.main(data, tsm_props, window, sc,
plot_inc=False) #, output_path=OutputFP)
plot_path_sensor = output_path + sc['fileio']['realtime_path']
try:
for img in os.listdir(plot_path_sensor):
if re.search("colpos", img):
cmd = "{} {}/upload_image.py --conversation-id {} --image '{}/{}'".format(
python_path, file_path, mesolong_plots,
plot_path_sensor, img)
os.system(cmd)