def web_plotter(site_code, end, days): """ Created by Kevin For integration and refactoring in the future """ start_time = datetime.now() qdb.print_out(start_time) site_code = [site_code] dt_end = pd.to_datetime(end) sc = mem.server_config() rtw = sc['rainfall']['roll_window_length'] ts_end, start, offsetstart = get_rt_window( float(days), float(rtw), end=dt_end) gauges = rainfall_gauges() if site_code != '': gauges = gauges[gauges.site_code.isin(site_code)] gauges['site_id'] = gauges['site_id'].apply(lambda x: float(x)) site_props = gauges.groupby('site_id') plot_data = site_props.apply(rp.main, offsetstart=offsetstart, tsn=end, save_plot=False, sc=sc, start=start, output_path="", end=ts_end).reset_index(drop=True) json_plot_data = plot_data.to_json(orient="records") qdb.print_out("runtime = %s" % (datetime.now() - start_time)) return json_plot_data
def main(end_ts=datetime.now()): start_ts = pd.to_datetime(end_ts) - timedelta(2) print(start_ts) surficial_triggers = qdb.get_surficial_trigger(start_ts, end_ts) if len(surficial_triggers) == 0: qdb.print_out("No surficial trigger to process") for index, surficial in surficial_triggers.iterrows(): ts_updated = surficial['ts_updated'] public_ts_start = round_data_ts(ts_updated) alert_level = surficial['alert_level'] alert_symbol = surficial['alert_symbol'] alert_status = surficial['alert_status'] site_id = surficial['site_id'] site_code = surficial['site_code'] if (alert_symbol == 'lt'): if (alert_status == 1): qdb.print_out("Found valid lt surficial trigger for " + \ "%s at %s" %(site_code.upper(), ts_updated)) trigger_sym_id = qdb.get_trigger_sym_id(2, 'surficial') df = pd.DataFrame({ 'ts': [ts_updated], 'site_id': [site_id], 'trigger_sym_id': [trigger_sym_id], 'ts_updated': [ts_updated] }) qdb.alert_to_db(df, 'operational_triggers', lt_overwrite=False) qdb.print_out(" > Added l2 trigger on operational triggers") # Process only l2 and l3 with alert status of -1 (invalid) elif (alert_status == -1): valid_cotriggers = qdb.get_valid_cotriggers( site_id, public_ts_start) dont_delete = False # Check if it has co-triggers on start of event # tho highly unlikely if len(valid_cotriggers) != 0: for index, valid in valid_cotriggers.iterrows(): # Don't delete public alert entry if there # is a co-trigger that's equal or # greater of alert level if (valid['alert_level'] >= alert_level): qdb.print_out( "%s has valid co-trigger: deleting will NOT commence" % (site_code.upper())) dont_delete = True break if dont_delete == False: qdb.delete_public_alert(site_id, public_ts_start) qdb.print_out("Deleted {} public alert of {}".format( public_ts_start, site_code))
def get_last_good_data(df): if df.empty: qdb.print_out("Error: Empty dataframe inputted") return pd.DataFrame() # groupby node_id dfa = df.groupby('node_id') # extract the latest timestamp per node_id, drop the index dflgd = dfa.apply(lambda x: x[x.ts == x.ts.max()]) dflgd = dflgd.reset_index(level=1, drop=True) return dflgd
def main(): """Writes in rainfall_gauges information on available rain gauges for rainfall alert analysis """ start = datetime.now() qdb.print_out(start) if qdb.does_table_exist('rainfall_gauges') == False: #Create a rainfall_gauges table if it doesn't exist yet qdb.create_rainfall_gauges() senslope = mem.get('df_dyna_rain_gauges') senslope = senslope.loc[senslope.has_rain == 1, :] senslope.loc[:, 'data_source'] = 'senslope' noah = noah_gauges() all_gauges = senslope.append(noah, sort=False) all_gauges.loc[:, 'gauge_name'] = all_gauges.loc[:, 'gauge_name'].apply( lambda x: str(x)) all_gauges.loc[:, 'date_activated'] = pd.to_datetime( all_gauges.loc[:, 'date_activated']) written_gauges = mem.get('df_rain_gauges') not_written = set(all_gauges['gauge_name']) \ - set(written_gauges['gauge_name']) new_gauges = all_gauges.loc[all_gauges.gauge_name.isin(not_written), :] new_gauges = new_gauges.loc[new_gauges.date_deactivated.isnull(), :] new_gauges = new_gauges.loc[:, [ 'gauge_name', 'data_source', 'longitude', 'latitude', 'date_activated' ]] if len(new_gauges) != 0: data_table = sms.DataTable('rainfall_gauges', new_gauges) db.df_write(data_table) deactivated = written_gauges.loc[ ~written_gauges.date_deactivated.isnull(), :] deactivated_gauges = all_gauges.loc[(~all_gauges.date_deactivated.isnull()) \ & (~all_gauges.gauge_name.isin(not_written))\ & (~all_gauges.gauge_name.isin(deactivated.gauge_name)), :] date_deactivated = pd.to_datetime( deactivated_gauges.loc[:, 'date_deactivated']) deactivated_gauges.loc[:, 'date_deactivated'] = date_deactivated deactivated_gauges = deactivated_gauges.loc[:, [ 'gauge_name', 'data_source', 'longitude', 'latitude', 'date_activated' ]] if len(deactivated_gauges) != 0: data_table = sms.DataTable('rainfall_gauges', deactivated_gauges) db.df_write(data_table) qdb.print_out('runtime = %s' % (datetime.now() - start))
def plot_column_positions(df, tsm_props, window, sc, show_part_legend, max_min_cml=''): #============================================================================== # # DESCRIPTION # returns plot of xz and xy absolute displacements of each node # # INPUT # colname; array; list of sites # x; dataframe; vertical displacements # xz; dataframe; horizontal linear displacements along the planes defined by xa-za # xy; dataframe; horizontal linear displacements along the planes defined by xa-ya #============================================================================== try: fig=plt.figure() ax_xz=fig.add_subplot(121) ax_xy=fig.add_subplot(122,sharex=ax_xz,sharey=ax_xz) ax_xz=nonrepeat_colors(ax_xz,len(set(df['ts'].values)),color='plasma') ax_xy=nonrepeat_colors(ax_xy,len(set(df['ts'].values)),color='plasma') colposTS = pd.DataFrame({'ts': sorted(set(df.ts)), 'index': range(len(set(df.ts)))}) dfts = df.groupby('ts') dfts.apply(subplot_colpos, ax_xz=ax_xz, ax_xy=ax_xy, show_part_legend=show_part_legend, sc=sc, colposTS=colposTS) # try: # max_min_cml = max_min_cml.apply(lambda x: x*1000) # xl = df.loc[(df.ts == window.end)&(df.node_id <= tsm_props.nos)&(df.node_id >= 1)]['x'].values[::-1] # ax_xz.fill_betweenx(xl, max_min_cml['xz_maxlist'].values, max_min_cml['xz_minlist'].values, where=max_min_cml['xz_maxlist'].values >= max_min_cml['xz_minlist'].values, facecolor='0.7',linewidth=0) # ax_xy.fill_betweenx(xl, max_min_cml['xy_maxlist'].values, max_min_cml['xy_minlist'].values, where=max_min_cml['xy_maxlist'].values >= max_min_cml['xy_minlist'].values, facecolor='0.7',linewidth=0) # except: # qdb.print_out('error in plotting noise env') for tick in list(ax_xz.xaxis.get_minor_ticks()) \ + list(ax_xy.xaxis.get_minor_ticks()) \ + list(ax_xz.xaxis.get_major_ticks()) \ + list(ax_xy.xaxis.get_major_ticks()): tick.label.set_rotation('vertical') plt.subplots_adjust(top=0.92, bottom=0.15, left=0.10, right=0.73) plt.suptitle(tsm_props.tsm_name) ax_xz.grid(True) ax_xy.grid(True) except: qdb.print_out(tsm_props.tsm_name + " ERROR in plotting column position") return ax_xz,ax_xy
def download_rainfall_noah(noah_id, fdate, tdate): """Downloads rainfall data of noah_id from fdate to tdate. Args: noah_id (int): Device id of noah data. fdate (timestamp): Timestamp start of data to be downloaded. tdate (timestamp): Timestamp end of data to be downloaded. Returns: dataframe: Rainfall data of noah_id from fdate to tdate if with data else empty dataframe. """ #Reduce latest_ts by 1 day as a work around for GMT to local conversion offset_date = (pd.to_datetime(fdate) - timedelta(1)).strftime("%Y-%m-%d") sc = mem.server_config() url = (sc['rainfall']['noah_data'] + '/%s/from/%s/to/%s') % (noah_id, offset_date, tdate) try: req = requests.get(url, auth=(sc['rainfall']['noah_user'], sc['rainfall']['noah_password'])) except: qdb.print_out("Can't get request. Please check internet connection") return pd.DataFrame() try: df = pd.DataFrame(req.json()["data"]) except: qdb.print_out("error: %s" % noah_id) return pd.DataFrame() try: #rename dateTimeRead into ts and rain_value into rain df = df.rename(columns={'rain_value': 'rain', 'dateTimeRead': 'ts'}) df = df.drop_duplicates('ts') df['ts'] = df['ts'].apply(lambda x: pd.to_datetime(str(x)[0:19])) df['rain'] = df['rain'].apply(lambda x: float(x)) df = df.sort_values('ts') #remove the entries that are less than fdate df = df[df.ts > fdate] return df[['ts', 'rain']] except: return pd.DataFrame()
def main(): """Updates data of NOAH rain gauges. """ start_time = datetime.now() qdb.print_out(start_time) #get the list of rainfall NOAH rain gauge IDs gauges = mem.get('df_rain_props') gauges = gauges[gauges.data_source == 'noah'].drop_duplicates('rain_id') noah_gauges = gauges.groupby('rain_id') noah_gauges.apply(update_single_table) qdb.print_out('runtime = %s' % (datetime.now() - start_time))
def main(site_code=''): """Writes in rainfall_priorities information on nearest rain gauges from the project sites for rainfall alert analysis """ start = datetime.now() qdb.print_out(start) coord = mem.get('df_sites') if site_code == '': try: site_code = sys.argv[1].lower() site_code = site_code.replace(' ', '').split(',') except: pass else: site_code = site_code.replace(' ', '').split(',') if site_code != '': coord = coord.loc[coord.site_code.isin(site_code), :] coord = coord.loc[coord.active == 1, ['site_id', 'latitude', 'longitude']] rg_coord = mem.get('df_rain_gauges') rg_coord = rg_coord[rg_coord.date_deactivated.isnull()] site_coord = coord.groupby('site_id', as_index=False) nearest_rg = site_coord.apply(get_distance, rg_coord=rg_coord) nearest_rg['distance'] = np.round(nearest_rg.distance,2) nearest_rg = nearest_rg.reset_index(drop=True) if qdb.does_table_exist('rainfall_priorities') == False: #Create a NOAH table if it doesn't exist yet qdb.create_rainfall_priorities() to_mysql(nearest_rg) qdb.print_out('runtime = %s' %(datetime.now() - start))
def update_table_data(noah_id, gauge_name, fdate, tdate, noah_gauges): """Updates data of table gauge_name from fdate to tdate. Args: noah_id (int): Device id of noah data. gauge_name (str): Name of table containing rainfall data of noah_id. fdate (timestamp): Timestamp start of data to be downloaded. tdate (timestamp): Timestamp end of data to be downloaded. noah_gauges (dataframe): Rain gauge properties- id, name, data source. """ noah_data = download_rainfall_noah(noah_id, fdate, tdate) cur_ts = datetime.now() if noah_data.empty: qdb.print_out(" no data...") #Insert an entry with values: [timestamp, -1] as a marker #-1 values should not be included in computation of cml rainfall if pd.to_datetime(tdate) <= cur_ts: place_holder_data = pd.DataFrame({"ts": [tdate], "rain": [-1.0]}) data_table = sms.DataTable(gauge_name, place_holder_data) db.df_write(data_table) else: #Insert the new data on the noahid table data_table = sms.DataTable(gauge_name, noah_data) db.df_write(data_table) #The table is already up to date if pd.to_datetime(tdate) > cur_ts: return else: #call this function again until the maximum recent timestamp is hit update_single_table(noah_gauges)
def update_single_table(noah_gauges): """Updates data of table gauge_name. Args: noah_gauges (dataframe): Rain gauge properties- id, name, data source. """ noah_id = noah_gauges['gauge_name'].values[0] gauge_name = 'rain_noah_%s' % noah_id #check if table gauge_name exists if qdb.does_table_exist(gauge_name) == False: #Create a NOAH table if it doesn't exist yet qdb.print_out("Creating NOAH table '%s'" % gauge_name) qdb.create_NOAH_table(gauge_name) else: qdb.print_out('%s exists' % gauge_name) #Find the latest timestamp for noah_id (which is also the start date) latest_ts = qdb.get_latest_ts(gauge_name) if (latest_ts == '') or (latest_ts == None): #assign a starting date if table is currently empty latest_ts = datetime.now() - timedelta(3) else: latest_ts = latest_ts.strftime("%Y-%m-%d %H:%M:%S") qdb.print_out(" Start timestamp: %s" % latest_ts) #Generate end time end_ts = (pd.to_datetime(latest_ts) + timedelta(1)).strftime("%Y-%m-%d") qdb.print_out(" End timestamp: %s" % end_ts) #Download data for noah_id update_table_data(noah_id, gauge_name, latest_ts, end_ts, noah_gauges)
def main(site_code='', end='', Print=True, write_to_db=True, print_plot=False, save_plot=True, days='', is_command_line_run=True): """Computes alert and plots rainfall data. Args: site_code (list): Site codes to compute rainfall analysis for. Optional. Defaults to empty string which will compute alert and plot for all sites. Print (bool): To print plot and summary of alerts. Optional. Defaults to True. end (datetime): Timestamp of alert and plot to be computed. Optional. Defaults to current timestamp. Returns: str: Json format of cumulative rainfall and alert per site. """ start_time = datetime.now() qdb.print_out(start_time) if site_code == '': if is_command_line_run: site_code = sys.argv[1].lower() site_code = site_code.replace(' ', '').split(',') else: site_code = site_code.replace(' ', '').split(',') if end == '': try: end = pd.to_datetime(sys.argv[2]) except: end = datetime.now() else: end = pd.to_datetime(end) output_path = os.path.abspath(os.path.join(os.path.dirname(__file__), '../../..')) sc = mem.server_config() #creates directory if it doesn't exist if (sc['rainfall']['print_plot'] or sc['rainfall']['print_summary_alert']) and Print: if not os.path.exists(output_path+sc['fileio']['rainfall_path']): os.makedirs(output_path+sc['fileio']['rainfall_path']) # setting monitoring window if days != '': sc['rainfall']['rt_window_length'] = days end, start, offsetstart = get_rt_window(float(sc['rainfall']['rt_window_length']), float(sc['rainfall']['roll_window_length']), end=end) tsn=end.strftime("%Y-%m-%d_%H-%M-%S") # 4 nearest rain gauges of each site with threshold and distance from site gauges = rainfall_gauges() if site_code != '': gauges = gauges[gauges.site_code.isin(site_code)] gauges['site_id'] = gauges['site_id'].apply(lambda x: float(x)) trigger_symbol = mem.get('df_trigger_symbols') trigger_symbol = trigger_symbol[trigger_symbol.trigger_source == 'rainfall'] trigger_symbol['trigger_sym_id'] = trigger_symbol['trigger_sym_id'].apply(lambda x: float(x)) site_props = gauges.groupby('site_id') summary = site_props.apply(ra.main, end=end, sc=sc, trigger_symbol=trigger_symbol, write_to_db=write_to_db) summary = summary.reset_index(drop=True)[['site_id', 'site_code', '1D cml', 'half of 2yr max', '3D cml', '2yr max', 'DataSource', 'alert']] if Print == True: if sc['rainfall']['print_summary_alert']: summary.to_csv(output_path+sc['fileio']['rainfall_path'] + 'SummaryOfRainfallAlertGenerationFor'+tsn+'.csv', sep=',', mode='w', index=False) if sc['rainfall']['print_plot'] or print_plot: rain_data = site_props.apply(rp.main, offsetstart=offsetstart, tsn=tsn, save_plot=save_plot, sc=sc, start=start, output_path=output_path, end=end).reset_index(drop=True) summary = pd.merge(summary, rain_data, on='site_id', validate='1:1') summary_json = summary.to_json(orient="records") qdb.print_out("runtime = %s" %(datetime.now()-start_time)) return summary_json
def main(): # asks for tsm name while True: props = qdb.get_tsm_list(input('sensor name: ')) if len(props) == 1: break else: qdb.print_out('sensor name is not in the list') continue tsm_props = props[0] # asks if to plot from date activated (or oldest data) to most recent data while True: input_text = 'plot from start to end of data? (Y/N): ' plot_all_data = input(input_text).lower() if plot_all_data == 'y' or plot_all_data == 'n': break # asks if to specify end timestamp of monitoring window if plot_all_data == 'n': while True: input_text = 'specify end timestamp of monitoring window? (Y/N): ' test_specific_time = input(input_text).lower() if test_specific_time == 'y' or test_specific_time == 'n': break # ask for timestamp of end of monitoring window defaults to datetime.now if test_specific_time == 'y': while True: try: input_text = 'plot end timestamp (format: 2016-12-31 23:30): ' end = pd.to_datetime(input(input_text)) break except: print('invalid datetime format') continue else: end = datetime.now() # monitoring window and server configurations window, sc = rtw.get_window(end) # asks if to plot with 3-day monitoring window while True: input_text = 'plot with 3-day monitoring window? (Y/N): ' three_day_window = input(input_text).lower() if three_day_window == 'y' or three_day_window == 'n': break # asks start of monitoring window defaults to values in server config if three_day_window == 'n': while True: input_text = 'start of monitoring window (in days) ' input_text += 'or datetime (format: 2016-12-31 23:30): ' start = input(input_text) try: window.start = window.end - timedelta(int(start)) break except: try: window.start = pd.to_datetime(start) break except: print('datetime format or integer only') continue # computes offsetstart from given start timestamp window.offsetstart = window.start - timedelta( days=(sc['subsurface']['num_roll_window_ops'] * window.numpts - 1) / 48.) else: # check date of activation query = "SELECT date_activated FROM tsm_sensors" query += " WHERE tsm_name = '%s'" % tsm_props.tsm_name try: date_activated = qdb.db.df_read(query).values[0][0] except: date_activated = pd.to_datetime('2010-01-01') #compute for start to end timestamp of data query = "(SELECT * FROM tilt_%s" % tsm_props.tsm_name query += " where ts > '%s' ORDER BY ts LIMIT 1)" % date_activated query += " UNION ALL" query += " (SELECT * FROM tilt_%s" % tsm_props.tsm_name query += " ORDER BY ts DESC LIMIT 1)" start_end = qdb.db.df_read(query) end = pd.to_datetime(start_end['ts'].values[1]) window, sc = rtw.get_window(end) start_dataTS = pd.to_datetime(start_end['ts'].values[0]) start_dataTS_Year = start_dataTS.year start_dataTS_month = start_dataTS.month start_dataTS_day = start_dataTS.day start_dataTS_hour = start_dataTS.hour start_dataTS_minute = start_dataTS.minute if start_dataTS_minute < 30: start_dataTS_minute = 0 else: start_dataTS_minute = 30 window.offsetstart = datetime.combine( date(start_dataTS_Year, start_dataTS_month, start_dataTS_day), time(start_dataTS_hour, start_dataTS_minute, 0)) # computes offsetstart from given start timestamp window.start = window.offsetstart + timedelta(days=( sc['subsurface']['num_roll_window_ops'] * window.numpts - 1) / 48.) # asks if to plot velocity and asks for interval and legends to show in # column position plots if to plot all data or if monitoring window not # equal to 3 days if plot_all_data == 'y' or three_day_window == 'n': # asks for interval between column position plots while True: try: input_text = 'interval between column position plots, in hours: ' col_pos_interval = int(input(input_text)) break except: qdb.print_out('enter an integer') continue # computes for interval and number of column position plots sc['subsurface']['col_pos_interval'] = str(col_pos_interval) + 'H' sc['subsurface']['num_col_pos'] = int( (window.end - window.start).total_seconds() / (3600 * col_pos_interval) + 1) # asks if to plot all legends while True: input_text = 'show all legends in column position plot? (Y/N): ' show_all_legend = input(input_text).lower() if show_all_legend == 'y' or show_all_legend == 'n': break if show_all_legend == 'y': show_part_legend = False # asks which legends to show elif show_all_legend == 'n': while True: try: show_part_legend = int(input('every nth legend to show: ')) if show_part_legend <= sc['subsurface']['num_col_pos']: break else: input_text = 'integer should be less than ' input_text += 'the number of colpos dates to plot: ' input_text += '%s' % (sc['subsurface']['num_col_pos']) qdb.print_out(input_text) continue except: qdb.print_out('enter an integer') continue while True: plotvel = input('plot velocity? (Y/N): ').lower() if plotvel == 'y' or plotvel == 'n': break if plotvel == 'y': plotvel = True else: plotvel = False three_day_window = False else: plotvel = True show_part_legend = True three_day_window = True # asks which point to fix in column position plots while True: input_text = 'column fix for colpos (top/bottom). ' input_text += 'press enter to skip; ' input_text += 'default for monitoring is fix bottom: ' column_fix = input(input_text).lower() if column_fix in ['top', 'bottom', '']: break if column_fix == '': column_fix = 'bottom' sc['subsurface']['column_fix'] = column_fix # mirror xz and/or xy colpos while True: try: mirror_xz = bool(int(input('mirror image of xz colpos? (0/1): '))) break except: print('Invalid. 1 for mirror image of xz colpos else 0') continue while True: try: mirror_xy = bool(int(input('mirror image of xy colpos? (0/1): '))) break except: print('Invalid. 1 for mirror image of xy colpos else 0') continue while True: print_disp_vel = input( 'print displacement and velocity to csv? (Y/N): ').lower() if print_disp_vel == 'y' or print_disp_vel == 'n': break data = proc.proc_data(tsm_props, window, sc, realtime=True, comp_vel=plotvel, analysis=False) if print_disp_vel == 'y': tilt = data.tilt.reset_index() accel = data.accel_data.reset_index() df = pd.merge(tilt, accel, on=['ts', 'node_id', 'tsm_name'], how='outer').sort_values(['ts', 'node_id']) df.to_csv('{}_{}-{}.csv'.format(tsm_props.tsm_name, window.start.strftime('%Y%m%d%H%M'), window.end.strftime('%Y%m%d%H%M')), index=False) plotter.main(data, tsm_props, window, sc, plotvel=plotvel, show_part_legend=show_part_legend, realtime=True, plot_inc=False, three_day_window=three_day_window, mirror_xz=mirror_xz, mirror_xy=mirror_xy) return data
def plot_disp_vel(noise_df, df0off, cs_df, colname, window, sc, plotvel, xzd_plotoffset, num_nodes, velplot, plot_inc, inc_df=''): #============================================================================== # # DESCRIPTION: # returns plot of xz & xy displacements per node, xz & xy velocities per node # # INPUT: # xz; array of floats; linear displacements along the planes defined by xa-za # xy; array of floats; linear displacements along the planes defined by xa-ya # xz_vel; array of floats; velocity along the planes defined by xa-za # xy_vel; array of floats; velocity along the planes defined by xa-ya #============================================================================== if plotvel: vel_xz, vel_xy, L2_xz, L2_xy, L3_xz, L3_xy = velplot df0off = df0off.set_index('ts') fig=plt.figure() try: if plotvel: #creating subplots ax_xzd=fig.add_subplot(141) ax_xyd=fig.add_subplot(142,sharex=ax_xzd,sharey=ax_xzd) ax_xzd.grid(True) ax_xyd.grid(True) ax_xzv=fig.add_subplot(143) ax_xzv.invert_yaxis() ax_xyv=fig.add_subplot(144,sharex=ax_xzv,sharey=ax_xzv) else: #creating subplots ax_xzd=fig.add_subplot(121) ax_xyd=fig.add_subplot(122,sharex=ax_xzd,sharey=ax_xzd) ax_xzd.grid(True) ax_xyd.grid(True) except: if plotvel: #creating subplots ax_xzv=fig.add_subplot(121) ax_xzv.invert_yaxis() ax_xyv=fig.add_subplot(122,sharex=ax_xzv,sharey=ax_xzv) try: #plotting cumulative (surface) displacments ax_xzd.plot(cs_df.index, cs_df['xz'].values, color='0.4', linewidth=0.5) ax_xyd.plot(cs_df.index, cs_df['xy'].values, color='0.4', linewidth=0.5) ax_xzd.fill_between(cs_df.index, cs_df.xz, xzd_plotoffset*(num_nodes), color='0.8') ax_xyd.fill_between(cs_df.index, cs_df.xy, xzd_plotoffset*(num_nodes), color='0.8') except: qdb.print_out('Error in plotting cumulative surface displacement') try: #assigning non-repeating colors to subplots axis ax_xzd=nonrepeat_colors(ax_xzd,num_nodes) ax_xyd=nonrepeat_colors(ax_xyd,num_nodes) except: qdb.print_out('Error in assigning non-repeating colors in displacement') if plotvel: ax_xzv=nonrepeat_colors(ax_xzv,num_nodes) ax_xyv=nonrepeat_colors(ax_xyv,num_nodes) #plotting displacement for xz curax=ax_xzd plot_disp(curax, 'xz', df0off) plot_noise_env(curax, 'xz', noise_df) plot_annotation(curax, 'xz', df0off, inc_df, plot_inc) curax.set_title('displacement\n downslope', fontsize='medium') curax.set_ylabel('displacement scale, m') #plotting displacement for xy curax=ax_xyd plot_disp(curax, 'xy', df0off) plot_noise_env(curax, 'xy', noise_df) plot_annotation(curax, 'xy', df0off, inc_df, plot_inc) curax.set_title('displacement\n across slope', fontsize='medium') if plotvel: #plotting velocity for xz curax=ax_xzv vel_xz.plot(ax=curax, marker='.', legend=False) L2_xz = L2_xz.sort_values('ts', ascending = True).set_index('ts') nodal_L2_xz = L2_xz.groupby('node_id') nodal_L2_xz.apply(lambda x: x['node_id'].plot(marker='^', ms=8, mfc='y', lw=0,ax = curax)) L3_xz = L3_xz.sort_values('ts', ascending = True).set_index('ts') nodal_L3_xz = L3_xz.groupby('node_id') nodal_L3_xz.apply(lambda x: x['node_id'].plot(marker='^', ms=10, mfc='r', lw=0,ax = curax)) y = sorted(range(1, num_nodes+1), reverse = True) x = (vel_xz.index)[1] z = sorted(range(1, num_nodes+1), reverse = True) for i,j in zip(y,z): curax.annotate(str(int(j)), xy=(x,i), xytext = (5,-2.5), textcoords='offset points',size = 'x-small') curax.set_ylabel('node ID') curax.set_title('velocity alerts\n downslope', fontsize='medium') #plotting velocity for xy curax=ax_xyv vel_xy.plot(ax=curax, marker='.', legend=False) L2_xy = L2_xy.sort_values('ts', ascending = True).set_index('ts') nodal_L2_xy = L2_xy.groupby('node_id') nodal_L2_xy.apply(lambda x: x['node_id'].plot(marker='^', ms=8, mfc='y', lw=0,ax = curax)) L3_xy = L3_xy.sort_values('ts', ascending = True).set_index('ts') nodal_L3_xy = L3_xy.groupby('node_id') nodal_L3_xy.apply(lambda x: x['node_id'].plot(marker='^', ms=10, mfc='r', lw=0,ax = curax)) y = range(1, num_nodes+1) x = (vel_xy.index)[1] z = range(1, num_nodes+1) for i,j in zip(y,z): curax.annotate(str(int(j)), xy=(x,i), xytext = (5,-2.5), textcoords='offset points',size = 'x-small') curax.set_title('velocity alerts\n across slope', fontsize='medium') # rotating xlabel for tick in list(ax_xzd.xaxis.get_minor_ticks()) \ + list(ax_xyd.xaxis.get_minor_ticks()) \ + list(ax_xzd.xaxis.get_major_ticks()) \ + list(ax_xyd.xaxis.get_major_ticks()): tick.label.set_rotation('vertical') if plotvel: for tick in list(ax_xzv.xaxis.get_minor_ticks()) \ + list(ax_xyv.xaxis.get_minor_ticks()) \ + list(ax_xzv.xaxis.get_major_ticks()) \ + list(ax_xyv.xaxis.get_major_ticks()): tick.label.set_rotation('vertical') try: dfmt = md.DateFormatter('%Y-%m-%d\n%H:%M') ax_xzd.xaxis.set_major_formatter(dfmt) ax_xyd.xaxis.set_major_formatter(dfmt) except: qdb.print_out('Error in setting date format of x-label in disp subplots') fig.set_tight_layout(True) fig.subplots_adjust(top=0.85) fig.suptitle(colname)
def main(tsm_name='', end='', end_mon=False): run_start = datetime.now() qdb.print_out(run_start) qdb.print_out(tsm_name) if tsm_name == '': tsm_name = sys.argv[1].lower() if end == '': try: end = pd.to_datetime(sys.argv[2]) except: end = datetime.now() else: end = pd.to_datetime(end) window, sc = rtw.get_window(end) tsm_props = qdb.get_tsm_list(tsm_name)[0] data = proc.proc_data(tsm_props, window, sc) tilt = data.tilt[window.start:window.end] lgd = data.lgd tilt = tilt.reset_index().sort_values('ts', ascending=True) if lgd.empty: qdb.print_out('%s: no data' % tsm_name) return nodal_tilt = tilt.groupby('node_id', as_index=False) alert = nodal_tilt.apply(lib.node_alert, colname=tsm_props.tsm_name, num_nodes=tsm_props.nos, disp=float(sc['subsurface']['disp']), vel2=float(sc['subsurface']['vel2']), vel3=float(sc['subsurface']['vel3']), k_ac_ax=float(sc['subsurface']['k_ac_ax']), lastgooddata=lgd, window=window, sc=sc).reset_index(drop=True) alert.loc[:, 'col_alert'] = -1 col_alert = pd.DataFrame({ 'node_id': range(1, tsm_props.nos + 1), 'col_alert': [-1] * tsm_props.nos }) node_col_alert = col_alert.groupby('node_id', as_index=False) node_col_alert.apply(lib.column_alert, alert=alert, num_nodes_to_check=int( sc['subsurface']['num_nodes_to_check']), k_ac_ax=float(sc['subsurface']['k_ac_ax']), vel2=float(sc['subsurface']['vel2']), vel3=float(sc['subsurface']['vel3'])) valid_nodes_alert = alert.loc[~alert.node_id.isin(data.inv)] if max(valid_nodes_alert['col_alert'].values) > 0: pos_alert = valid_nodes_alert[valid_nodes_alert.col_alert > 0] site_alert = trend.main(pos_alert, tsm_props.tsm_id, window.end, data.inv) else: site_alert = max( lib.get_mode(list(valid_nodes_alert['col_alert'].values))) tsm_alert = pd.DataFrame({ 'ts': [window.end], 'tsm_id': [tsm_props.tsm_id], 'alert_level': [site_alert], 'ts_updated': [window.end] }) qdb.alert_to_db(tsm_alert, 'tsm_alerts') qdb.write_op_trig(tsm_props.site_id, window.end) qdb.print_out(tsm_alert) qdb.print_out('run time = ' + str(datetime.now() - run_start)) return tilt
if len(valid_cotriggers) != 0: for index, valid in valid_cotriggers.iterrows(): # Don't delete public alert entry if there # is a co-trigger that's equal or # greater of alert level if (valid['alert_level'] >= alert_level): qdb.print_out( "%s has valid co-trigger: deleting will NOT commence" % (site_code.upper())) dont_delete = True break if dont_delete == False: qdb.delete_public_alert(site_id, public_ts_start) qdb.print_out("Deleted {} public alert of {}".format( public_ts_start, site_code)) ############################################################################### if __name__ == "__main__": start_time = datetime.now() qdb.print_out(start_time) main() # test: valit lt of INA # pd.to_datetime('2020-05-17 15:00') qdb.print_out('runtime = %s' % (datetime.now() - start_time))
def main(end_ts=datetime.now()): start_time = datetime.now() qdb.print_out(start_time) start_ts = pd.to_datetime(end_ts) - timedelta(1) surficial_triggers = get_surficial_trigger(start_ts, end_ts) if len(surficial_triggers) == 0: qdb.print_out("================="); qdb.print_out("No surficial trigger (lt, l2, l3) to process") for index, surficial in surficial_triggers.iterrows(): ts_updated = surficial['ts_updated'] public_ts_start = round_data_ts(ts_updated) alert_level = surficial['alert_level'] alert_symbol = surficial['alert_symbol'] alert_status = surficial['alert_status'] site_id = surficial['site_id'] site_code = surficial['site_code'] if (alert_symbol == 'lt'): if (alert_status == 1): qdb.print_out("================="); qdb.print_out("Found valid lt surficial trigger for " + \ "%s at %s" %(site_code.upper(), ts_updated)) qdb.print_out(" > Added l2 trigger on operational triggers") insert_l2_operational_trigger(ts_updated, site_id) # Process only l2 and l3 with alert status of -1 (invalid) elif (alert_status == -1): valid_cotriggers = get_valid_cotriggers(site_id, public_ts_start) dont_delete = False # Check if it has co-triggers on start of event # tho highly unlikely if len(valid_cotriggers) != 0: for index, valid in valid_cotriggers.iterrows(): # Don't delete public alert entry if there # is a co-trigger that's equal or # greater of alert level if (valid['alert_level'] >= alert_level): qdb.print_out("================="); qdb.print_out("%s has valid co-trigger: deleting will NOT commence" %(site_code.upper())) dont_delete = True break if dont_delete == False: qdb.print_out("================="); qdb.print_out("Deleting public alert for site " + \ "%s (%s) at %s" %(site_code.upper(), site_id, public_ts_start)) delete_invalid_public_alert_entry(site_id, public_ts_start) # update ts_updated of latest entry for that site to current time # using round_data_ts(datetime.now()) update_ts_last_site_public_alert(site_id, round_data_ts(datetime.now())) qdb.print_out("================="); qdb.print_out('runtime = %s' %(datetime.now() - start_time))