def soms_data(start_date, end_date, site):
db = q.GetSomsData(siteid = site, fromTime = start_date, toTime = end_date)
df = db[db.msgid == 110]
df.drop(['msgid','mval2'], axis = 1, inplace = True)
return df
def getsomscaldata(column="", gid=0, fdate="", tdate=""):
'''
only for landslide sensors v2 and v3
output: df = series of unfiltered SOMS data (calibrated/normalized) of a specific node of the defined column
param:
column = column name (ex. laysa)
gid = geographic id of node [1-40]
'''
v2=['NAGSAM', 'BAYSBM', 'AGBSBM', 'MCASBM', 'CARSBM', 'PEPSBM','BLCSAM']
v3=[ 'lpasam','lpasbm','laysam','laysbm','imesbm','barscm',
'mngsam','gaasam','gaasbm','hinsam','hinsbm','talsam' ]
df = pd.DataFrame()
if column.upper() in v2:
if column.upper()=='NAGSAM':
msgid = 26
else:
msgid = 112
elif column.lower() in v3: # if version 3
msgid= 113
else:
print 'No data available for ' + column.upper()
return df
try:
df = qs.GetSomsData(siteid=column, fromTime=fdate, toTime=tdate, targetnode=gid, msgid=msgid)
df.index=df.ts
df= df.mval1
except:
print 'No data available for ' + column.upper()
return df
return df
def getsomsrawdata(column="", gid="", fdate="", tdate=""):
'''
only for landslide sensors v2 and v3
output: sraw = series of unfiltered SOMS data (raw) of a specific node of the defined column
param:
column = column name (ex. laysam)
gid = geographic id of node [1-40]
'''
v2=['NAGSAM', 'BAYSBM', 'AGBSBM', 'MCASBM', 'CARSBM', 'PEPSBM','BLCSAM']
v3=[ 'lpasam','lpasbm','laysam','laysbm','imesbm','barscm',
'mngsam','gaasam','gaasbm','hinsam','hinsbm','talsam' ]
df = pd.DataFrame(columns=['sraw', 'scal'])
# print 'getsomsdata: ' + column + ',' + str(gid)
try:
df = qs.GetSomsData(siteid=column, fromTime=fdate, toTime=tdate, targetnode=gid)
except:
print 'No data available for ' + column.upper()
return df
df.index = df.ts
if column.upper() in v2:
if column.upper()=='NAGSAM':
sraw =(((8000000/(df.mval1[(df.msgid==21)]))-(8000000/(df.mval2[(df.msgid==21)])))*4)/10
else:
sraw =(((20000000/(df.mval1[(df.msgid==111)]))-(20000000/(df.mval2[(df.msgid==111)])))*4)/10
elif column.lower() in v3: # if version 3
sraw=df.mval1[(df.msgid==110)]
else: # for returning null series if not in version 2 or 3
sraw=pd.Series()
return sraw
def somsdata(end_time, sen, start_time):
data = q.GetSomsData(sen, start_time, end_time)
#data['ts']=data['ts'].dt.round('30min')
x = data[data.msgid == 110]
x.drop(['mval2'], axis=1, inplace=True)
return x
def soms(site, start, end):
df = q.GetSomsData(site, start, end)
df['ts'] = df['ts'].dt.round('30min')
df = df[df.id < 7]
df = df[df.msgid == 21]
df = df.groupby('ts')['mval1'].mean()
df = pd.DataFrame({'timestamp': df.index, 'mval1': df.values})
return df
def getsomscaldata(column="", gid=0, fdate="", tdate="", if_multi=False):
'''
only for landslide sensors v2 and v3
output: df = series of unfiltered SOMS data (calibrated/normalized) of a specific node of the defined column
param:
column = column name (ex. laysa)
gid = geographic id of node [1-40]
'''
v2 = ['NAGSA', 'BAYSB', 'AGBSB', 'MCASB', 'CARSB', 'PEPSB', 'BLCSA']
v3 = [
'lpasa', 'lpasb', 'laysa', 'laysb', 'imesb', 'barsc', 'messb', 'imusc',
'oslsc', 'mngsa', 'gaasa', 'gaasb', 'hinsa', 'hinsb', 'talsa'
]
df = pd.DataFrame(columns=['sraw', 'scal'])
df = pd.DataFrame()
if column.upper() in v2:
if column.upper() == 'NAGSA':
msgid = 26
else:
msgid = 112
elif column.lower() in v3: # if version 3
msgid = 113
else:
# if (is_debug == True):
# print 'No data available for ' + column.upper()
# return df
# else:
return df
# try:
df = qs.GetSomsData(siteid=column + 'm',
fromTime=fdate,
toTime=tdate,
targetnode=gid,
msgid=msgid)
df.index = df.ts
if if_multi:
df = df[['id', 'mval1']]
else:
df = df[['mval1']]
# except:
# if (is_debug == True):
# print 'No data available for ' + column.upper()
# return df
# else:
# return df
return df
# df['norm'] = (df['r15m'] - df['r15m'].min()) / (df['r15m'].max() - df['r15m'].min())
df['roll_sum_1d'] = df['r15m'].rolling(48).sum()
df['norm'] = (df['roll_sum_1d'] - df['roll_sum_1d'].min()) / (
df['roll_sum_1d'].max() - df['roll_sum_1d'].min())
# df ['roll_sum_3d'] = df['r15m'].rolling(144).sum()
rainfall = df.reset_index(drop=True)
return rainfall
start = '2018-01-15'
end = '2018-10-10'
site = 'blcsbm'
gauge = 'nagtbw'
df = q.GetSomsData(site, start, end)
#df['roll_sum_1d'] = df['mval1'].rolling(48).mean()
#df = df[df.msgid == 21]
#############################################################################Rainfall
rainfall = rainfall(start, end, gauge)
rainfall = rainfall.dropna()
##############################################################################
df['ts'] = df['ts'].dt.round('30min')
n = int(df.id.max()) + 1
fig, (axes) = plt.subplots(n, 1, sharex=True, sharey=False)
fig.subplots_adjust(hspace=0)
fig.suptitle('Data points to be used for ANN ({} to {})'.format(start, end),
fontsize=20)
import os
import sys
os.path.abspath('C:\Users\\Vidal Marvin Gabriel\\Desktop\DYNASLOPE\mycodes') #GroundDataAlertLib - line 19
sys.path.insert(1,os.path.abspath('C:\Users\\Vidal Marvin Gabriel\\Desktop\DYNASLOPE\updews-pycodes\Analysis')) #GroundDataAlertLib - line 21
import querySenslopeDb as q
import matplotlib.pyplot as plt
import pandas as pd
import data_mine_disp as dp
n = 7
##################################(Moist)
end = '2016-09-05'
col = 'imuscm'
start = '2016-07-21'
data = q.GetSomsData(col, start, end)
data['ts']=data['ts'].dt.round('30min')
x = data[data.msgid == 113]
x_x = pd.DataFrame(x)
x_x.drop('mval2', 1)
#x = data[data.index %2 != 0].set_index('ts') #sets only odd numbers
#data[data.index %2 == 0] #sets only even numbers
#data[data.index %2 == 0].reset_index()
#data[data.index %2 == 0].set_index('ts') #sets index into timestamp
nod = x_x[x_x.id == n]
nod ['moist_rate'] = nod['mval1'] - nod['mval1'].shift()
#nod ['rate'] = np.where((nod['mval1'] - nod['mval1'].shift()) >=10, (nod['mval1'] - nod['mval1'].shift()),0)
rate = nod ['moist_rate'].abs()
)) #GroundDataAlertLib - line 21
import querySenslopeDb as q
import matplotlib.pyplot as plt
import pandas as pd
import s_d_db as d
import ColumnPlotter as plter
import regression as rg
import numpy as np
from scipy.interpolate import UnivariateSpline
sen = 'lpasam'
start_time = '2017-01-02'
end_time = '2017-01-24'
#data = q.GetSomsData(sen, start_time, end_time)
data = q.GetSomsData(sen, start_time, end_time)
da = rg.regression2(data)
#da['vol'] = (da['grav'] * 1.3)
da['ts'] = da['ts'].dt.round('30min')
x = da[da.msgid == 110]
x.drop(['mval2', 'msgid'], axis=1, inplace=True)
#x.loc[(x['id'] >= 5), 'vol'] = 0.465651
# data.loc[(x['id'] >= 5), 'vol'] = 0.345373
# data.loc[(data['vol'] >= 0.345373), 'vol'] = 0.345373
# m = data.ix[(data['nid'] >= 5)]
###########################################################################################################
#dfg = x.groupby (['ts','id'])['vol'].mean()
#
#da = pd.DataFrame(dfg)
def getsomsrawdata(column="", gid=0, fdate="", tdate="", if_multi=False):
'''
only for landslide sensors v2 and v3
output: sraw = series of unfiltered SOMS data (raw) of a specific node of the defined column
param:
column = column name (ex. laysam)
gid = geographic id of node [1-40]
'''
v2 = ['NAGSA', 'BAYSB', 'AGBSB', 'MCASB', 'CARSB', 'PEPSB', 'BLCSA']
v3 = [
'lpasa', 'lpasb', 'laysa', 'laysb', 'imesb', 'barsc', 'messb', 'imusc',
'oslsc', 'mngsa', 'gaasa', 'gaasb', 'hinsa', 'hinsb', 'talsa'
]
df = pd.DataFrame(columns=['sraw', 'scal'])
sraw = pd.DataFrame()
# print 'getsomsdata: ' + column + ',' + str(gid)
try:
df = qs.GetSomsData(siteid=column + 'm',
fromTime=fdate,
toTime=tdate,
targetnode=gid)
except:
print 'No data available for ' + column.upper()
return df
df.index = df.ts
if column.upper() in v2:
if column.upper() == 'NAGSA':
if if_multi:
df = df[(df.msgid == 21)]
sraw['mval1'] = (((8000000 / (df.mval1)) -
(8000000 / (df.mval2))) * 4) / 10
sraw['id'] = df[['id']]
else:
sraw['mval'] = (((8000000 / (df.mval1[(df.msgid == 21)])) -
(8000000 /
(df.mval2[(df.msgid == 21)]))) * 4) / 10
else:
if if_multi:
df = df[(df.msgid == 111)]
sraw['mval1'] = (((20000000 / (df.mval1)) -
(20000000 / (df.mval2))) * 4) / 10
sraw['id'] = df[['id']]
else:
sraw['mval1'] = (((20000000 / (df.mval1[(df.msgid == 111)])) -
(20000000 /
(df.mval2[(df.msgid == 111)]))) * 4) / 10
elif column.lower() in v3: # if version 3
if if_multi:
df = df[(df.msgid == 110)]
sraw = df[['id', 'mval1']]
else:
df = df[(df.msgid == 110)]
sraw = df[['mval1']]
else:
sraw = pd.Series()
pass
return sraw
def genproc(col, window, config, fixpoint, realtime=False, comp_vel=True):
monitoring = q.GetSomsData(col, window.offsetstart, window.end)
try:
LastGoodData = q.GetLastGoodData(monitoring, col.nos)
q.PushLastGoodData(LastGoodData, col.name)
LastGoodData = q.GetLastGoodDataFromDb(col.name)
except:
LastGoodData = q.GetLastGoodDataFromDb(col.name)
#identify the node ids with no data at start of monitoring window
NodesNoInitVal = GetNodesWithNoInitialData(monitoring, col.nos,
window.offsetstart)
#get last good data prior to the monitoring window (LGDPM)
if len(NodesNoInitVal) != 0:
lgdpm = q.GetSingleLGDPM(col.name, NodesNoInitVal, window.offsetstart)
if len(lgdpm) != 0:
lgdpm = flt.applyFilters(lgdpm)
lgdpm = lgdpm.sort_index(ascending=False).drop_duplicates('id')
if len(lgdpm) != 0:
monitoring = monitoring.append(lgdpm)
monitoring = monitoring.loc[monitoring.id <= col.nos]
invalid_nodes = q.GetNodeStatus(1)
invalid_nodes = invalid_nodes[invalid_nodes.site ==
col.name]['node'].values
monitoring = monitoring.loc[~monitoring.id.isin(invalid_nodes)]
#assigns timestamps from LGD to be timestamp of offsetstart
monitoring.loc[(monitoring.ts < window.offsetstart) |
(pd.isnull(monitoring.ts)), ['ts']] = window.offsetstart
nodes_noval = GetNodesWithNoData(monitoring, col.nos)
nodes_nodata = pd.DataFrame({
'name': [0] * len(nodes_noval),
'id': nodes_noval,
'ts': [window.offsetstart] * len(nodes_noval)
})
monitoring = monitoring.append(nodes_nodata)
max_min_df, max_min_cml = err.cml_noise_profiling(monitoring, config,
fixpoint, col.nos)
monitoring = monitoring.drop_duplicates(['ts', 'id'])
monitoring = monitoring.set_index('ts')
#resamples xz and xy values per node using forward fill
monitoring = monitoring.groupby('id').apply(
resamplenode, window=window).reset_index(level=1).set_index('ts')
nodal_proc_monitoring = monitoring.groupby('id')
if not realtime:
to_smooth = config.io.to_smooth
to_fill = config.io.to_fill
else:
to_smooth = config.io.rt_to_smooth
to_fill = config.io.rt_to_fill
filled_smoothened = nodal_proc_monitoring.apply(
fill_smooth,
offsetstart=window.offsetstart,
end=window.end,
roll_window_numpts=window.numpts,
to_smooth=to_smooth,
to_fill=to_fill)
filled_smoothened = filled_smoothened[['xz', 'xy', 'x', 'y', 'z',
'name']].reset_index()
filled_smoothened['depth'] = filled_smoothened['x'] / np.abs(
filled_smoothened['x']) * np.sqrt(col.seglen**2 -
filled_smoothened['xz']**2 -
filled_smoothened['xy']**2)
filled_smoothened['depth'] = filled_smoothened['depth'].fillna(
value=col.seglen)
monitoring = filled_smoothened.set_index('ts')
return procdata(col, disp_vel.sort(), max_min_df, max_min_cml)