def pre_proc(tsm_name, n):
sensor = 'tilt_' + tsm_name
start, end, nodes, ts = percent_movement(tsm_name, n)
df = data_query(start, end, sensor)
filtered = filt.apply_filters(df)
data = accel_to_lin_xz_xy(filtered, seg_len=1.5)
data['td'] = data.ts.values - data.ts.values[0]
data['td'] = data['td'].apply(lambda x: x / np.timedelta64(1, 'D'))
data = data[data['node_id'].isin(nodes)]
data['xz'] = data.groupby('node_id')['xz'].transform(lambda v: v.ffill())
data['xy'] = data.groupby('node_id')['xy'].transform(lambda v: v.ffill())
# data['xy'].fillna(method='ffill', inplace = True)
# data['xz'].fillna(method='ffill', inplace = True)
return data, nodes, ts
# actual = np.array(actual)
return accel
def confusion_mat(actual, predicted):
tn, fp, fn, tp = confusion_matrix(actual, predicted).ravel()
TPR = (tp / (tp + fn))
FPR = (fp / (fp + tn))
return TPR, FPR
data = data(start, end, sensor)
filtered = filt.apply_filters(data)
data = accel_to_lin_xz_xy(data, seg_len)
#end,start,offsetstart = get_rt_window(3, 7,1,pd.to_datetime(data.tail(1).ts.values[0]))
#data = fill_smooth(data.set_index('ts'),offsetstart,end,7,0,1)
#data = data.reset_index()
data['td'] = data.ts.values - data.ts.values[0]
data['td'] = data['td'].apply(lambda x:x / np.timedelta64(1,'D'))
n=25
#n = data.node_id.max() + 1
cols = np.arange(1,n+1)
def proc_data(tsm_props,
window,
sc,
realtime=False,
comp_vel=True,
analysis=True):
monitoring = qdb.get_raw_accel_data(tsm_name=tsm_props.tsm_name,
from_time=window.offsetstart,
to_time=window.end,
analysis=analysis)
monitoring = monitoring.loc[monitoring.node_id <= tsm_props.nos]
monitoring = filt.apply_filters(monitoring)
#identify the node ids with no data at start of monitoring window
no_init_val = no_initial_data(monitoring, tsm_props.nos,
window.offsetstart)
#get last good data prior to the monitoring window (LGDPM)
if len(no_init_val) != 0:
lgdpm = qdb.get_single_lgdpm(tsm_props.tsm_name,
no_init_val,
window.offsetstart,
analysis=analysis)
lgdpm = filt.apply_filters(lgdpm)
lgdpm = lgdpm.sort_index(ascending=False).drop_duplicates('node_id')
monitoring = monitoring.append(lgdpm, sort=False)
invalid_nodes = qdb.get_node_status(tsm_props.tsm_id,
ts=window.offsetstart)
monitoring = monitoring.loc[~monitoring.node_id.isin(invalid_nodes)]
lgd = get_last_good_data(monitoring)
#assigns timestamps from LGD to be timestamp of offsetstart
monitoring.loc[(monitoring.ts < window.offsetstart) |
(pd.isnull(monitoring.ts)), ['ts']] = window.offsetstart
monitoring = accel_to_lin_xz_xy(monitoring, tsm_props.seglen)
monitoring = monitoring.drop_duplicates(['ts', 'node_id'])
monitoring = monitoring.set_index('ts')
monitoring = monitoring[['tsm_name', 'node_id', 'xz', 'xy']]
nodes_noval = no_data(monitoring, tsm_props.nos)
nodes_nodata = pd.DataFrame({
'tsm_name': [tsm_props.tsm_name] * len(nodes_noval),
'node_id':
nodes_noval,
'xy': [np.nan] * len(nodes_noval),
'xz': [np.nan] * len(nodes_noval),
'ts': [window.offsetstart] * len(nodes_noval)
})
nodes_nodata = nodes_nodata.set_index('ts')
monitoring = monitoring.append(nodes_nodata, sort=False)
max_min_df, max_min_cml = err.cml_noise_profiling(monitoring, sc,
tsm_props.nos)
#resamples xz and xy values per node using forward fill
monitoring = monitoring.groupby('node_id', as_index=False).apply(
resample_node, window=window).reset_index(drop=True).set_index('ts')
if not realtime:
to_smooth = int(sc['subsurface']['to_smooth'])
to_fill = int(sc['subsurface']['to_fill'])
else:
to_smooth = int(sc['subsurface']['rt_to_smooth'])
to_fill = int(sc['subsurface']['rt_to_fill'])
tilt = monitoring.groupby('node_id', as_index=False).apply(
fill_smooth,
offsetstart=window.offsetstart,
end=window.end,
roll_window_numpts=window.numpts,
to_smooth=to_smooth,
to_fill=to_fill).reset_index(level='ts').set_index('ts')
if comp_vel == True:
tilt.loc[:, 'td'] = tilt.index.values - \
monitoring.index.values[0]
tilt.loc[:, 'td'] = tilt['td'].apply(lambda x: x / \
np.timedelta64(1,'D'))
nodal_filled_smoothened = tilt.groupby('node_id', as_index=False)
tilt = nodal_filled_smoothened.apply(node_inst_vel,
roll_window_numpts=window.numpts,
start=window.start)
tilt = tilt.drop(['td'], axis=1)
tilt = tilt.sort_values('node_id', ascending=True)
tilt = tilt.reset_index(level='ts').set_index('ts')
return ProcData(invalid_nodes, tilt.sort_index(), lgd, max_min_df,
max_min_cml)
def proc_subsurface(tsm_props, window, sc):
monitoring = qdb.get_raw_accel_data(tsm_name=tsm_props.tsm_name,
from_time=window.offsetstart,
to_time=window.end)
monitoring = monitoring.loc[monitoring.node_id <= tsm_props.nos]
monitoring = filt.apply_filters(monitoring)
monitoring = monitoring.groupby('node_id', as_index=False).apply(
magnitude, tsm_props.seglen)
monitoring = theta_yz(monitoring)
#identify the node ids with no data at start of monitoring window
no_init_val = no_initial_data(monitoring, tsm_props.nos,
window.offsetstart)
#get last good data prior to the monitoring window (LGDPM)
if len(no_init_val) != 0:
lgdpm = qdb.get_single_lgdpm(tsm_props.tsm_name, no_init_val,
window.offsetstart)
lgdpm = filt.apply_filters(lgdpm)
lgdpm = lgdpm.sort_index(ascending=False).drop_duplicates('node_id')
monitoring = monitoring.append(lgdpm, sort=False)
invalid_nodes = qdb.get_node_status(tsm_props.tsm_id,
ts=window.offsetstart)
monitoring = monitoring.loc[~monitoring.node_id.isin(invalid_nodes)]
lgd = get_last_good_data(monitoring)
#assigns timestamps from LGD to be timestamp of offsetstart
monitoring.loc[(monitoring.ts < window.offsetstart) |
(pd.isnull(monitoring.ts)), ['ts']] = window.offsetstart
monitoring = monitoring.drop_duplicates(['ts', 'node_id'])
monitoring = monitoring.set_index('ts')
#monitoring = monitoring.loc[monitoring['node_id'].isin(range(1,8,1))] select nodes
monitoring = monitoring[[
'tsm_name', 'node_id', 'x', 'y', 'z', 'magnitude', 'theta_yz'
]]
nodes_noval = no_data(monitoring, tsm_props.nos)
nodes_nodata = pd.DataFrame({
'tsm_name': [tsm_props.tsm_name] * len(nodes_noval),
'node_id':
nodes_noval,
'magnitude': [np.nan] * len(nodes_noval),
'theta_yz': [np.nan] * len(nodes_noval),
'ts': [window.offsetstart] * len(nodes_noval)
})
nodes_nodata = nodes_nodata.set_index('ts')
monitoring = monitoring.append(nodes_nodata, sort=False)
# print ('\n\n\n####### monitoring #######\n\n\n', monitoring)
max_min_df, max_min_cml = err.cml_noise_profiling(monitoring, sc,
tsm_props.nos)
monitoring = monitoring.groupby('node_id', as_index=False).apply(
proc.resample_node,
window=window).reset_index(drop=True).set_index('ts')
to_smooth = int(sc['subsurface']['to_smooth'])
to_fill = int(sc['subsurface']['to_fill'])
monitoring = monitoring.groupby('node_id', as_index=False).apply(
fill_smooth,
offsetstart=window.offsetstart,
end=window.end,
roll_window_numpts=window.numpts,
to_smooth=to_smooth,
to_fill=to_fill).reset_index(level='ts').set_index('ts')
monitoring.loc[:,
'td'] = monitoring.index.values - monitoring.index.values[0]
monitoring.loc[:, 'td'] = monitoring['td'].apply(lambda x: x / \
np.timedelta64(1,'D'))
monitoring = monitoring.groupby('node_id', as_index=False).apply(
slope_intercept, roll_window_numpts=window.numpts,
start=window.start).reset_index(level='ts').set_index('ts')
monitoring = displacement(monitoring)
monitoring = monitoring.groupby('node_id', as_index=False).apply(
node_inst_vel, roll_window_numpts=window.numpts,
start=window.start).reset_index(level='ts').set_index('ts')
# monitoring = monitoring.groupby('node_id', as_index=False).apply(accel,
# roll_window_numpts=window.numpts,
# start=window.start).reset_index(level='ts').set_index('ts')
tilt = alert_generator(monitoring, 0.05, 0.032)
return ProcData(invalid_nodes, tilt, lgd, max_min_df, max_min_cml)