def set_monitoring_window(roll_window_length, data_dt, rt_window_length, num_roll_window_ops, end): ##DESCRIPTION: ##returns number of data points per rolling window, endpoint of interval, starting point of interval, time interval for real-time monitoring, monitoring window dataframe ##INPUT: ##roll_window_length; float; length of rolling/moving window operations, in hours ##data_dt; float; time interval between data points, in hours ##rt_window_length; float; length of real-time monitoring window, in days ##num_roll_window_ops ##OUTPUT: ##roll_window_numpts, end, start, offsetstart, monwin roll_window_numpts = int(1 + roll_window_length / data_dt) end, start, offsetstart = gf.get_rt_window(rt_window_length, roll_window_numpts, num_roll_window_ops, end) monwin_time = pd.date_range(start=offsetstart, end=end, freq='30Min', name='ts', closed=None) monwin = pd.DataFrame(data=np.nan * np.ones(len(monwin_time)), index=monwin_time) return roll_window_numpts, end, start, offsetstart, monwin
def set_monitoring_window(roll_window_length,data_dt,rt_window_length,num_roll_window_ops): roll_window_numpts=int(1+roll_window_length/data_dt) end, start, offsetstart=gf.get_rt_window(rt_window_length,roll_window_numpts,num_roll_window_ops) monwin_time=pd.date_range(start=offsetstart, end=end, freq='30Min',name='ts', closed=None) monwin=pd.DataFrame(data=np.nan*np.ones(len(monwin_time)), index=monwin_time) return roll_window_numpts, end, start, offsetstart, monwin
def generate_proc(site): print rt_window_length #1. setting date boundaries for real-time monitoring window roll_window_numpts=int(1+roll_window_length/data_dt) end, start, offsetstart=gf.get_rt_window(rt_window_length,roll_window_numpts,num_roll_window_ops) sensorlist=GetSensorList() # generating proc monitoring data for each site print "Generating PROC monitoring data for:" for s in sensorlist: if site == s.name: #2. getting current column properties colname,num_nodes,seg_len= s.name,s.nos,s.seglen print colname print num_nodes print seg_len #3. getting accelerometer data for site 'colname' monitoring=GetRawAccelData(colname,offsetstart) #4. evaluating which data needs to be filtered try: monitoring=applyFilters(monitoring) LastGoodData=GetLastGoodData(monitoring,num_nodes) PushLastGoodData(LastGoodData,colname) LastGoodData = GetLastGoodDataFromDb(colname) print 'Done' except: LastGoodData = GetLastGoodDataFromDb(colname) print 'error' if len(LastGoodData)<num_nodes: print colname, " Missing nodes in LastGoodData" #5. extracting last data outside monitoring window LastGoodData=LastGoodData[(LastGoodData.ts<offsetstart)] #6. appending LastGoodData to monitoring monitoring=monitoring.append(LastGoodData) #7. replacing date of data outside monitoring window with first date of monitoring window monitoring.loc[monitoring.ts < offsetstart, ['ts']] = offsetstart #8. computing corresponding horizontal linear displacements (xz,xy), and appending as columns to dataframe monitoring['xz'],monitoring['xy']=gf.accel_to_lin_xz_xy(seg_len,monitoring.x.values,monitoring.y.values,monitoring.z.values) #9. removing unnecessary columns x,y,z monitoring=monitoring.drop(['x','y','z'],axis=1) #10. setting ts as index # monitoring['id']=monitoring.index.values monitoring=monitoring.set_index('ts') #11. reordering columns monitoring=monitoring[['id','xz','xy']] #12. saving proc monitoring data return monitoring
def set_monitoring_window(roll_window_length,data_dt,rt_window_length,num_roll_window_ops): ##DESCRIPTION: ##returns number of data points per rolling window, endpoint of interval, starting point of interval, time interval for real-time monitoring, monitoring window dataframe ##INPUT: ##roll_window_length; float; length of rolling/moving window operations, in hours ##data_dt; float; time interval between data points, in hours ##rt_window_length; float; length of real-time monitoring window, in days ##num_roll_window_ops ##OUTPUT: ##roll_window_numpts, end, start, offsetstart, monwin roll_window_numpts=int(1+roll_window_length/data_dt) end, start, offsetstart=gf.get_rt_window(rt_window_length,roll_window_numpts,num_roll_window_ops) monwin_time=pd.date_range(start=offsetstart, end=end, freq='30Min',name='ts', closed=None) monwin=pd.DataFrame(data=np.nan*np.ones(len(monwin_time)), index=monwin_time) return roll_window_numpts, end, start, offsetstart, monwin
#INPUT/OUTPUT FILES #file headers colarrange = cfg.get('I/O','alerteval_colarrange').split(',') TestSpecificTime = cfg.getboolean('I/O', 'test_specific_time') if TestSpecificTime: end = pd.to_datetime(cfg.get('I/O','use_specific_time')) else: end = datetime.now() roll_window_numpts=int(1+roll_window_length/data_dt) end, start, offsetstart=gf.get_rt_window(rt_window_length,roll_window_numpts,num_roll_window_ops,end) valid_data = end - timedelta(hours=3) def node_alert(colname, xz_tilt, xy_tilt, xz_vel, xy_vel, num_nodes, T_disp, T_velL2, T_velL3, k_ac_ax): #DESCRIPTION #Evaluates node-level alerts from node tilt and velocity data #INPUT #xz_tilt,xy_tilt, xz_vel, xy_vel: Pandas DataFrame objects, with length equal to real-time window size, and columns for timestamp and individual node values #num_nodes: integer; number of nodes in a column #T_disp, TvelL2, TvelL3: floats; threshold values for displacement, and velocities correspoding to alert levels L2 and L3 #k_ac_ax: float; minimum value of (minimum velocity / maximum velocity) required to consider movement as valid
def generate_proc(site,end): #1. setting date boundaries for real-time monitoring window roll_window_numpts=int(1+roll_window_length/data_dt) end, start, offsetstart=gf.get_rt_window(rt_window_length,roll_window_numpts,num_roll_window_ops,end=end) sensorlist=GetSensorList() # generating proc monitoring data for each site # print "Generating PROC monitoring data for:" for s in sensorlist: if site == s.name: #2. getting current column properties colname,num_nodes,seg_len= s.name,s.nos,s.seglen # print colname # print num_nodes # print seg_len #3. getting accelerometer data for site 'colname' monitoring=GetRawAccelData(colname,offsetstart) monitoring = monitoring.loc[(monitoring.ts >= offsetstart) & (monitoring.ts <= end)] #3.1 identify the node ids with no data at start of monitoring window NodesNoInitVal=GetNodesWithNoInitialData(monitoring,num_nodes,offsetstart) #4: get last good data prior to the monitoring window (LGDPM) lgdpm = pd.DataFrame() for node in NodesNoInitVal: temp = GetSingleLGDPM(site, node, offsetstart.strftime("%Y-%m-%d %H:%M")) lgdpm = lgdpm.append(temp,ignore_index=True) #5 TODO: Resample the dataframe together with the LGDOM monitoring=monitoring.append(lgdpm) #6. evaluating which data needs to be filtered try: monitoring=applyFilters(monitoring) LastGoodData=GetLastGoodData(monitoring,num_nodes) PushLastGoodData(LastGoodData,colname) LastGoodData = GetLastGoodDataFromDb(colname) # print 'Done' except: LastGoodData = GetLastGoodDataFromDb(colname) print 'error' if len(LastGoodData)<num_nodes: print colname, " Missing nodes in LastGoodData" #5. extracting last data outside monitoring window LastGoodData=LastGoodData[(LastGoodData.ts<offsetstart)] #6. appending LastGoodData to monitoring monitoring=monitoring.append(LastGoodData) #7. replacing date of data outside monitoring window with first date of monitoring window monitoring.loc[monitoring.ts < offsetstart, ['ts']] = offsetstart #8. computing corresponding horizontal linear displacements (xz,xy), and appending as columns to dataframe monitoring['xz'],monitoring['xy']=gf.accel_to_lin_xz_xy(seg_len,monitoring.x.values,monitoring.y.values,monitoring.z.values) #9. removing unnecessary columns x,y,z monitoring=monitoring.drop(['x','y','z'],axis=1) monitoring = monitoring.drop_duplicates(['ts', 'id']) #10. setting ts as index # monitoring['id']=monitoring.index.values monitoring=monitoring.set_index('ts') #11. reordering columns monitoring=monitoring[['id','xz','xy']] #12. saving proc monitoring data if PrintProc: monitoring.to_csv(proc_monitoring_path+colname+proc_monitoring_file,sep=',', header=False,mode='w') return monitoring
def generate_proc(): #MAIN #1. setting date boundaries for real-time monitoring window roll_window_numpts = int(1 + roll_window_length / data_dt) end, start, offsetstart = gf.get_rt_window(rt_window_length, roll_window_numpts, num_roll_window_ops) #2. getting all column properties sensors = pd.read_csv(columnproperties_path + columnproperties_file, names=columnproperties_headers, index_col=None) ## print "Generating PROC monitoring data for:" for s in range(len(sensors)): #3. getting current column properties colname, num_nodes, seg_len = sensors['colname'][s], sensors[ 'num_nodes'][s], sensors['seg_len'][s] ## print "\nDATA for ",colname," as of ", end.strftime("%Y-%m-%d %H:%M") try: #4. importing monitoring csv file of current column to dataframe monitoring = pd.read_csv(monitoring_path + colname + monitoring_file, names=monitoring_file_headers, parse_dates=[0], index_col=[1]) #5. extracting data within monitoring window monitoring = monitoring[(monitoring.ts >= offsetstart) & (monitoring.ts <= end)] #6. importing LastGoodData csv file of current column to dataframe LastGoodData = pd.read_csv(LastGoodData_path + colname + LastGoodData_file, names=LastGoodData_file_headers, parse_dates=[0], index_col=[1]) if len(LastGoodData) < num_nodes: print colname, " Missing nodes in LastGoodData" #7. extracting last data outside monitoring window LastGoodData = LastGoodData[(LastGoodData.ts < offsetstart)] ## print "\n",colname ## print LastGoodData #8. appending LastGoodData to monitoring monitoring = monitoring.append(LastGoodData) ## print monitoring.tail(num_nodes+1) #9. replacing date of data outside monitoring window with first date of monitoring window monitoring.loc[monitoring.ts < offsetstart, ['ts']] = offsetstart #10. computing corresponding horizontal linear displacements (xz,xy), and appending as columns to dataframe monitoring['xz'], monitoring['xy'] = gf.accel_to_lin_xz_xy( seg_len, monitoring.x.values, monitoring.y.values, monitoring.z.values) #11. removing unnecessary columns x,y,z monitoring = monitoring.drop(['x', 'y', 'z'], axis=1) #12. setting ts as index monitoring['id'] = monitoring.index.values monitoring = monitoring.set_index('ts') #13. reordering columns monitoring = monitoring[['id', 'xz', 'xy', 'm']] ## print "\n",colname ## print monitoring.tail(20) monitoring.to_csv(proc_monitoring_path + colname + proc_monitoring_file, sep=',', header=False, mode='w') ## print " ",colname except: ## print " ",colname, "...FAILED" continue
def generate_proc(colname, num_nodes, seg_len): #1. setting date boundaries for real-time monitoring window roll_window_numpts=int(1+roll_window_length/data_dt) end, start, offsetstart=gf.get_rt_window(rt_window_length,roll_window_numpts,num_roll_window_ops) # generating proc monitoring data for each site print "Generating PROC monitoring data for:" print colname print num_nodes print seg_len #3. getting accelerometer data for site 'colname' monitoring=GetRawAccelData(colname,offsetstart) monitoring = monitoring.loc[(monitoring.ts >= offsetstart) & (monitoring.ts <= end)] #3.1 identify the node ids with no data at start of monitoring window NodesNoInitVal=GetNodesWithNoInitialData(monitoring,num_nodes,offsetstart) #4: get last good data prior to the monitoring window (LGDPM) lgdpm = pd.DataFrame() for node in NodesNoInitVal: temp = GetSingleLGDPM(colname, node, offsetstart.strftime("%Y-%m-%d %H:%M")) lgdpm = lgdpm.append(temp,ignore_index=True) #5 TODO: Resample the dataframe together with the LGDOM monitoring=monitoring.append(lgdpm) #6. evaluating which data needs to be filtered try: monitoring=applyFilters(monitoring) LastGoodData=GetLastGoodData(monitoring,num_nodes) PushLastGoodData(LastGoodData,colname) LastGoodData = GetLastGoodDataFromDb(colname) print 'Done' except: LastGoodData = GetLastGoodDataFromDb(colname) print 'error' if len(LastGoodData)<num_nodes: print colname, " Missing nodes in LastGoodData" #5. extracting last data outside monitoring window LastGoodData=LastGoodData[(LastGoodData.ts<offsetstart)] #6. appending LastGoodData to monitoring monitoring=monitoring.append(LastGoodData) #7. replacing date of data outside monitoring window with first date of monitoring window monitoring.loc[monitoring.ts < offsetstart, ['ts']] = offsetstart #8. computing corresponding horizontal linear displacements (xz,xy), and appending as columns to dataframe monitoring['xz'],monitoring['xy']=gf.accel_to_lin_xz_xy(seg_len,monitoring.x.values,monitoring.y.values,monitoring.z.values) #9. removing unnecessary columns x,y,z monitoring=monitoring.drop(['x','y','z'],axis=1) monitoring = monitoring.drop_duplicates(['ts', 'id']) #10. setting ts as index # monitoring['id']=monitoring.index.values monitoring=monitoring.set_index('ts') #11. reordering columns monitoring=monitoring[['id','xz','xy']] #12. saving proc monitoring data if PrintProc: monitoring.to_csv(proc_monitoring_path+colname+proc_monitoring_file,sep=',', header=False,mode='w') return monitoring