def WriteRawSmsToDb(msglist, sensor_nums): query = "INSERT INTO smsinbox (timestamp,sim_num,sms_msg,read_status,web_flag) VALUES " for m in msglist: if sensor_nums.find(m.simnum[-10:]) == -1: # if re.search(m.simnum[-10:],sensor_nums): web_flag = 'W' print m.data[:20] if cfg.config().mode.script_mode == 'gsmserver': ret = dsll.sendReceivedGSMtoDEWS(str(m.dt.replace("/", "-")), m.simnum, m.data) #if the SMS Message was sent successfully to the web socket server then, # change web_flag to 'WS' which means "Websocket Server Sent" if ret == 0: web_flag = 'WSS' else: web_flag = 'S' query += "('%s','%s','%s','UNREAD','%s')," % (str( m.dt.replace("/", "-")), str( m.simnum), str(m.data.replace("'", "\"")), web_flag) # query += "('" + str(m.dt.replace("/","-")) + "','" + str(m.simnum) + "','" # query += str(m.data.replace("'","\"")) + "','UNREAD')," # just to remove the trailing ',' query = query[:-1] # print query dbio.commitToDb(query, "WriteRawSmsToDb", instance='GSM')
def checkAlertMessage(): c = cfg.config() dbio.createTable("runtimelog", "runtime") server.logRuntimeStatus("alert", "checked") alertfile = cfg.config().fileio.allalertsfile f = open(alertfile, 'r') alertmsg = f.read() f.close() print alertmsg if not alertmsg: print '>> No alert msg read.' return # write alert message to db server.writeAlertToDb(alertmsg)
def CheckAlertMessages(): c = cfg.config() alllines = '' print c.fileio.allalertsfile if os.path.isfile(c.fileio.allalertsfile) and os.path.getsize( c.fileio.allalertsfile) > 0: f = open(c.fileio.allalertsfile, 'r') alllines = f.read() f.close() else: print '>> Error in reading file', alllines return alllines
def getAllowedPrefixes(network): c = cfg.config() if network.upper() == 'SMART': prefix_list = c.simprefix.smart.split(',') else: prefix_list = c.simprefix.globe.split(',') extended_prefix_list = [] for p in prefix_list: extended_prefix_list.append("639" + p) extended_prefix_list.append("09" + p) return extended_prefix_list
def SendAlertGsm(network, alertmsg): c = cfg.config() try: if network == 'GLOBE': numlist = c.simprefix.globe.split(",") else: numlist = c.simprefix.smart.split(",") # f = open(allalertsfile,'r') # alllines = f.read() # f.close() for n in numlist: gsmio.sendMsg(alertmsg, n) except IndexError: print "Error sending all_alerts.txt"
def SendMessagesFromDb(network, limit=10): c = cfg.config() if not c.mode.sendmsg: return allmsgs = dbio.getAllOutboxSmsFromDb("UNSENT", network, limit) if len(allmsgs) <= 0: # print ">> No messages in outbox" return print ">> Sending messagess from db" msglist = [] for item in allmsgs: smsItem = gsmio.sms(item[0], str(item[2]), str(item[3]), str(item[1])) msglist.append(smsItem) allmsgs = msglist send_success_list = [] fail_success_list = [] allowed_prefixes = getAllowedPrefixes(network) # cycle through all messages for msg in allmsgs: # get recepient numbers in list recepient_list = msg.simnum.split(",") for num in recepient_list: try: num_prefix = re.match("^ *((0)|(63))9\d\d", num).group() num_prefix = num_prefix.strip() except: continue # check if recepient number in allowed prefixed list if num_prefix in allowed_prefixes: ret = gsmio.sendMsg(msg.data, num.strip()) if ret == 0: send_success_list.append(msg.num) else: fail_success_list.append(msg.num) else: print "Number not in prefix list", num_prefix dbio.setSendStatus("FAIL", fail_success_list) dbio.setSendStatus("SENT", send_success_list) #Get all outbox messages with send_status "SENT" and attempt to send # chatterbox acknowledgements # send_status will be changed to "SENT-WSS" if successful dsll.sendAllAckSentGSMtoDEWS()
def logRuntimeStatus(script_name, status): if (status == 'alive'): ts = dt.today() diff = (ts.minute % 10) * 60 + ts.second ts = ts - td(seconds=diff) logtimestamp = ts.strftime("%Y-%m-%d %H:%M:00") else: logtimestamp = dt.today().strftime("%Y-%m-%d %H:%M:00") print ">> Logging runtime '" + status + "' at " + logtimestamp query = """insert ignore into runtimelog (timestamp,script_name,status) values ('%s','%s','%s') """ % (logtimestamp, script_name, status) dbio.commitToDb(query, 'logRuntimeStatus', cfg.config().mode.logtoinstance)
def main(): lockscript.get_lock('alertgenexec') print dt.today().strftime("%c") c = cfg.config() ongoing = [] mc = memcache.Client(['127.0.0.1:11211'], debug=0) proc_limit = c.io.proc_limit while True: alertgenlist = mc.get('alertgenlist') print alertgenlist if alertgenlist is None: break if len(alertgenlist) == 0: break col = alertgenlist.pop() mc.set('alertgenlist', []) mc.set('alertgenlist', alertgenlist) command = "~/anaconda2/bin/python %s %s" % (c.fileio.alertgenscript, col) print "Running", col, "alertgen" if lockscript.get_lock('alertgen for %s' % col, exitifexist=False): p = subprocess.Popen(command, stdout=subprocess.PIPE, shell=True, stderr=subprocess.STDOUT) else: continue while countAlertAnalysisInstances() > proc_limit: time.sleep(5) print '.',
def CheckMessageSource(msg): c = cfg.config() identity = dbio.checkNumberIfExists(msg.simnum, 'community') if identity: smsmsg = "From: %s %s of %s\n" % (identity[0][1], identity[0][0], identity[0][2]) smsmsg += msg.data server.WriteOutboxMessageToDb(smsmsg, c.smsalert.communitynum) return elif dbio.checkNumberIfExists(msg.simnum, 'dewsl'): print ">> From senslope staff" return name = dbio.checkNumberIfExists(msg.simnum, 'sensor') if name: print ">> From sensor", name[0][0] else: print "From unknown number ", msg.simnum
def main(): try: if sys.argv[1] == 'test': writetodb = False else: writetodb = True except: writetodb = True c = cfg.config() dbio.createTable("runtimelog", "runtime") server.logRuntimeStatus("alert", "checked") print '>> Checking for alert sms' alertmsg = server.CheckAlertMessages() print alertmsg if alertmsg: # server.WriteOutboxMessageToDb(alertmsg,c.smsalert.smartnum) # server.WriteOutboxMessageToDb(alertmsg,c.smsalert.globenum) query = """select nickname, numbers from dewslcontacts where grouptags like '%alert%'""" contacts = dbio.querydatabase(query, 'checkalert') # print contacts query = "INSERT INTO smsoutbox (timestamp_written,recepients,sms_msg,send_status) VALUES " tsw = dt.today().strftime("%Y-%m-%d %H:%M:%S") for item in contacts: message = 'SENSOR ALERT:\n%s' % (alertmsg) message = message.replace("ALERT", "AL3RT") query += "('%s','%s','%s','UNSENT')," % (tsw, item[1], message) query = query[:-1] if writetodb: dbio.commitToDb(query, 'checkalertmsg', 'GSM') else: print query print 'done' else: print '>> No alert msg read.'
#11. reordering columns monitoring=monitoring[['id','xz','xy']] return monitoring,monwin def time_site(target,df_sa): if (target < len(df_sa)): site = df_sa['site'].iloc[target] t_time = df_sa.index[target] return site,t_time else: print "Error. Target > len(df_sa)" io = cfg.config() num_roll_window_ops = io.io.num_roll_window_ops roll_window_length = io.io.roll_window_length data_dt = io.io.data_dt rt_window_length = io.io.rt_window_length roll_window_numpts=int(1+roll_window_length/data_dt) col_pos_interval = io.io.col_pos_interval col_pos_num = io.io.num_col_pos to_fill = io.io.to_fill to_smooth = io.io.to_smooth output_path = (__file__) output_file_path = (__file__) proc_file_path = (__file__) CSVFormat = '.csv'
def WriteEQAlertMessageToDb(alertmsg): c = cfg.config() WriteOutboxMessageToDb(alertmsg, c.smsalert.globenum) WriteOutboxMessageToDb(alertmsg, c.smsalert.smartnum)
import gsmSerialio as gsmio import numpy as np from time import localtime, strftime import pandas as pd import cfgfileio as cfg df = 0 casedf = 0 siteHealthdf = 0 statdf = 0 globaldf = pd.DataFrame() c = cfg.config() def readDataframe(): # global globaldf localdf = 0 db, cur = dbio.SenslopeDBConnect('local') query = '''SELECT logger_health.logger_id, name, model_id, sim_num, health_case from logger_health inner join loggers on logger_health.logger_id= loggers.logger_id inner join logger_contacts on logger_health.logger_id= logger_contacts.logger_id where logger_health.health_case !=5 and logger_health.health_id IN (select max(logger_health.health_id) from logger_health group by logger_id)''' try: localdf = psql.read_sql(query, db) except pd.io.sql.DatabaseError, e: localdf = 0 return localdf
def ProcessAllMessages(allmsgs, network): c = cfg.config() read_success_list = [] read_fail_list = [] cur_num = 0 try: while allmsgs: isMsgProcSuccess = True print '\n\n*******************************************************' #gets per text message msg = allmsgs.pop(0) # msg.data = msg.data.upper() cur_num = msg.num msgname = checkNameOfNumber(msg.simnum) ##### Added for V1 sensors removes unnecessary characters pls see function PreProcessColumnV1(data) if re.search("\*FF", msg.data): ProcessPiezometer(msg.data, msg.simnum) # elif re.search("[A-Z]{4}DUE\*[A-F0-9]+\*\d+T?$",msg.data): elif re.search("[A-Z]{4}DUE\*[A-F0-9]+\*.*", msg.data): msg.data = PreProcessColumnV1(msg.data) ProcessColumn(msg.data, msg.dt, msg.simnum) elif re.search("EQINFO", msg.data.upper()): isMsgProcSuccess = ProcessEarthquake(msg) elif re.search("^PSIR ", msg.data.upper()): isMsgProcSuccess = qsi.ProcessServerInfoRequest(msg) elif re.search("^SENDGM ", msg.data.upper()): isMsgProcSuccess = qsi.ServerMessaging(msg) elif re.search("^ACK \d+ .+", msg.data.upper()): isMsgProcSuccess = amsg.processAckToAlert(msg) elif re.search("^ *(R(O|0)*U*TI*N*E )|(EVE*NT )", msg.data.upper()): try: gm = gndmeas.getGndMeas(msg.data) RecordGroundMeasurements(gm) # server.WriteOutboxMessageToDb("READ-SUCCESS: \n" + msg.data,c.smsalert.communitynum) server.WriteOutboxMessageToDb(c.reply.successen, msg.simnum) except ValueError as e: print str(e) errortype = re.search( "(WEATHER|DATE|TIME|GROUND MEASUREMENTS|NAME)", str(e).upper()).group(0) print ">> Error in manual ground measurement SMS", errortype server.WriteOutboxMessageToDb( "READ-FAIL: (%s)\n%s" % (errortype, msg.data), c.smsalert.communitynum) server.WriteOutboxMessageToDb(str(e), msg.simnum) except: server.WriteOutboxMessageToDb( "READ-FAIL: (Unhandled) \n" + msg.data, c.smsalert.communitynum) elif re.search("^[A-Z]{4,5}\*[xyabcXYABC]\*[A-F0-9]+\*[0-9]+T?$", msg.data): try: dlist = ProcTwoAccelColData(msg.data, msg.simnum, msg.dt) if dlist: if len(dlist[0][0]) == 6: WriteSomsDataToDb(dlist, msg.dt) else: WriteTwoAccelDataToDb(dlist, msg.dt) invokeProcessInBgnd( "python ~/masynckaiser/client/bin/invoke-masync-CtoS-single.py %s" % dlist[0][0]) except IndexError: print "\n\n>> Error: Possible data type error" print msg.data except ValueError: print ">> Value error detected" elif re.search("[A-Z]{4}\*[A-F0-9]+\*[0-9]+$", msg.data): #ProcessColumn(msg.data) ProcessColumn(msg.data, msg.dt, msg.simnum) #check if message is from rain gauge # elif re.search("^\w{4},[\d\/:,]+,[\d,\.]+$",msg.data): elif re.search("^\w{4},[\d\/:,]+", msg.data): ProcessRain(msg.data, msg.simnum) elif re.search( r'(\w{4})[-](\d{1,2}[.]\d{02}),(\d{01}),(\d{1,2})/(\d{1,2}),#(\d),(\d),(\d{1,2}),(\d)[*](\d{10})', msg.data): ProcessStats(msg.data, msg.dt) elif re.search("ARQ\+[0-9\.\+/\- ]+$", msg.data): ProcessARQWeather(msg.data, msg.simnum) elif msg.data.split('*')[0] == 'COORDINATOR' or msg.data.split( '*')[0] == 'GATEWAY': isMsgProcSuccess = ProcessCoordinatorMsg(msg.data, msg.simnum) elif re.search("^MANUAL RESET", msg.data): server.WriteOutboxMessageToDb("SENSORPOLL SENSLOPE", msg.simnum) isMsgProcSuccess = True else: print '>> Unrecognized message format: ' print 'NUM: ', msg.simnum print 'MSG: ', msg.data CheckMessageSource(msg) isMsgProcSuccess = False if isMsgProcSuccess: read_success_list.append(msg.num) else: read_fail_list.append(msg.num) # method for updating the read_status all messages that have been processed # so that they will not be processed again in another run except: # print all the traceback routine so that the error can be traced print(traceback.format_exc()) print ">> Setting message read_status to fatal error" dbio.setReadStatus("FATAL ERROR", cur_num) return read_success_list, read_fail_list
def worker(first_target, last_target): #load all global variables? summary = pd.DataFrame() s_f = pd.DataFrame() s_a = pd.DataFrame() io = cfg.config() num_roll_window_ops = io.io.num_roll_window_ops roll_window_length = io.io.roll_window_length data_dt = io.io.data_dt rt_window_length = io.io.rt_window_length roll_window_numpts = int(1 + roll_window_length / data_dt) col_pos_interval = io.io.col_pos_interval col_pos_num = io.io.num_col_pos to_fill = io.io.to_fill to_smooth = io.io.to_smooth # output_path = (__file__) # output_file_path = (__file__) # proc_file_path = (__file__) CSVFormat = '.csv' # PrintProc = io.io.printproc T_disp = io.io.t_disp T_velL2 = io.io.t_vell2 T_velL3 = io.io.t_vell3 k_ac_ax = io.io.k_ac_ax num_nodes_to_check = io.io.num_nodes_to_check colarrange = io.io.alerteval_colarrange.split(',') node_status = qdb.GetNodeStatus(1) for i in range(first_target, last_target): # try: sites, custom_end = ffd.aim(i) sensorlist = qdb.GetSensorList(sites) for s in sensorlist: last_col = sensorlist[-1:] last_col = last_col[0] last_col = last_col.name # getting current column properties colname, num_nodes, seg_len = s.name, s.nos, s.seglen # list of working nodes node_list = range(1, num_nodes + 1) not_working = node_status.loc[(node_status.site == colname) & (node_status.node <= num_nodes)] not_working_nodes = not_working['node'].values for i in not_working_nodes: node_list.remove(i) proc_monitoring, monwin = generate_proc(colname, num_nodes, seg_len, custom_end, roll_window_length, data_dt, rt_window_length, num_roll_window_ops) xz_series_list, xy_series_list = create_series_list( proc_monitoring, monwin, colname, num_nodes) # print "create_series_list tapos na" # create, fill and smooth dataframes from series lists xz = create_fill_smooth_df(xz_series_list, num_nodes, monwin, roll_window_numpts, to_fill, to_smooth) xy = create_fill_smooth_df(xy_series_list, num_nodes, monwin, roll_window_numpts, to_fill, to_smooth) # computing instantaneous velocity vel_xz, vel_xy = compute_node_inst_vel(xz, xy, roll_window_numpts) # computing cumulative displacements cs_x, cs_xz, cs_xy = compute_col_pos(xz, xy, monwin.index[-1], col_pos_interval, col_pos_num, seg_len) # processing dataframes for output xz, xy, xz_0off, xy_0off, vel_xz, vel_xy, vel_xz_0off, vel_xy_0off, cs_x, cs_xz, cs_xy, cs_xz_0, cs_xy_0 = df_to_out( colname, xz, xy, vel_xz, vel_xy, cs_x, cs_xz, cs_xy, # proc_file_path, CSVFormat) # Alert generation # alert_out=alert_generation(colname,xz,xy,vel_xz,vel_xy,num_nodes, T_disp, T_velL2, T_velL3, k_ac_ax, # num_nodes_to_check,custom_end,CSVFormat,colarrange) alert_out = alert_generation(colname, xz, xy, vel_xz, vel_xy, num_nodes, T_disp, T_velL2, T_velL3, k_ac_ax, num_nodes_to_check, custom_end, CSVFormat, colarrange) alert_out = alert_out.reset_index(level=['id']) alert_out = alert_out[[ 'id', 'disp_alert', 'vel_alert', 'node_alert', 'col_alert' ]] alert_out = alert_out[(alert_out['vel_alert'] > 0) | (alert_out.node_alert == 'l2')] alert_out = alert_out[alert_out.id == 1] alert_out['site'] = sites summary = pd.concat((summary, alert_out), axis=0) # except: # print "Error recreating alarm." # continue print "--------------------Filtering chenes----------------------" print "--------------------Store yung mga nafilter----------------------" for j in range(0, len(summary)): # try: sites, custom_end = time_site(j, summary) # print "custom_end -------------> %s" %str(custom_end) sensorlist = qdb.GetSensorList(sites) for s in sensorlist: last_col = sensorlist[-1:] last_col = last_col[0] last_col = last_col.name # getting current column properties colname, num_nodes, seg_len = s.name, s.nos, s.seglen # list of working nodes node_list = range(1, num_nodes + 1) not_working = node_status.loc[(node_status.site == colname) & (node_status.node <= num_nodes)] not_working_nodes = not_working['node'].values for i in not_working_nodes: node_list.remove(i) # proc_monitoring,monwin=generate_proc(colname, num_nodes, seg_len, custom_end,f=True) proc_monitoring, monwin = generate_proc(colname, num_nodes, seg_len, custom_end, roll_window_length, data_dt, rt_window_length, num_roll_window_ops, filt=True) xz_series_list, xy_series_list = create_series_list( proc_monitoring, monwin, colname, num_nodes) xz = create_fill_smooth_df(xz_series_list, num_nodes, monwin, roll_window_numpts, to_fill, to_smooth) xy = create_fill_smooth_df(xy_series_list, num_nodes, monwin, roll_window_numpts, to_fill, to_smooth) # computing instantaneous velocity vel_xz, vel_xy = compute_node_inst_vel(xz, xy, roll_window_numpts) # computing cumulative displacements cs_x, cs_xz, cs_xy = compute_col_pos(xz, xy, monwin.index[-1], col_pos_interval, col_pos_num, seg_len) # processing dataframes for output xz, xy, xz_0off, xy_0off, vel_xz, vel_xy, vel_xz_0off, vel_xy_0off, cs_x, cs_xz, cs_xy, cs_xz_0, cs_xy_0 = df_to_out( colname, xz, xy, vel_xz, vel_xy, cs_x, cs_xz, cs_xy, # proc_file_path, CSVFormat) # Alert generation alert_out = alert_generation(colname, xz, xy, vel_xz, vel_xy, num_nodes, T_disp, T_velL2, T_velL3, k_ac_ax, num_nodes_to_check, custom_end, CSVFormat, colarrange) # print alert_out alert_out = alert_out.reset_index(level=['id']) a_out = alert_out.copy() a_out = a_out[[ 'id', 'disp_alert', 'vel_alert', 'node_alert', 'col_alert' ]] a_out = a_out[(a_out['vel_alert'] < 1.0) | (a_out.node_alert == 'l0')] a_out = a_out[a_out.id == 1] a_out['site'] = sites s_f = pd.concat((s_f, a_out), axis=0) b_out = alert_out.copy() b_out = b_out[[ 'id', 'disp_alert', 'vel_alert', 'node_alert', 'col_alert' ]] b_out = b_out[(b_out['vel_alert'] > 0.0) | (b_out.node_alert == 'l2')] b_out = b_out[b_out.id == 1] b_out['site'] = sites s_a = pd.concat((s_a, b_out), axis=0) # except: # print "Error." # continue print "################# Drawing! Dahil drawing ka! ##################" print "################# Idrawing lahat ng nafilter! ##################" for k in range(0, len(s_f)): try: sites, custom_end = time_site(k, s_f) ce = custom_end.strftime("%y_%m_%d__%H_%M") fname = "FILTERED_" + str(sites) + "_" + ce + "_049_049" sensorlist = qdb.GetSensorList(sites) for s in sensorlist: last_col = sensorlist[-1:] last_col = last_col[0] last_col = last_col.name # getting current column properties colname, num_nodes, seg_len = s.name, s.nos, s.seglen # list of working nodes # node_list = range(1, num_nodes + 1) # not_working = node_status.loc[(node_status.site == colname) & (node_status.node <= num_nodes)] # not_working_nodes = not_working['node'].values # for i in not_working_nodes: # node_list.remove(i) # importing proc_monitoring file of current column to dataframe # try: # print "proc_monitoring here: " proc_monitoring = generate_proc(colname, num_nodes, seg_len, custom_end, roll_window_length, data_dt, rt_window_length, num_roll_window_ops, filt=True, for_plots=True) # print proc_monitoring proc_monitoring = proc_monitoring[proc_monitoring.id == 1] ffd.plotter(proc_monitoring, fname=fname) except: print "Error plotting Filtered." for k in range(0, len(s_a)): try: sites, custom_end = time_site(k, s_a) ce = custom_end.strftime("%y_%m_%d__%H_%M") sensorlist = qdb.GetSensorList(sites) for s in sensorlist: last_col = sensorlist[-1:] last_col = last_col[0] last_col = last_col.name # getting current column properties colname, num_nodes, seg_len = s.name, s.nos, s.seglen # list of working nodes # node_list = range(1, num_nodes + 1) # not_working = node_status.loc[(node_status.site == colname) & (node_status.node <= num_nodes)] # not_working_nodes = not_working['node'].values # for i in not_working_nodes: # node_list.remove(i) # importing proc_monitoring file of current column to dataframe # try: # print "proc_monitoring here: " proc_monitoring = generate_proc(colname, num_nodes, seg_len, custom_end, roll_window_length, data_dt, rt_window_length, num_roll_window_ops, f=True, for_plots=True) # print proc_monitoring proc_monitoring = proc_monitoring[proc_monitoring.id == 1] ffd.plotter(proc_monitoring, fname=fname) except: print "Error plotting Alarms."
def RunSenslopeServer(network): minute_of_last_alert = dt.now().minute timetosend = 0 lastAlertMsgSent = '' logruntimeflag = True global checkIfActive checkIfActive = True try: gsm = gsmio.gsmInit(network) except serial.SerialException: print '**NO COM PORT FOUND**' serverstate = 'serial' gsm.close() logRuntimeStatus(network, "com port error") raise ValueError(">> Error: no com port found") dbio.createTable("runtimelog", "runtime", cfg.config().mode.logtoinstance) logRuntimeStatus(network, "startup") dbio.createTable('smsinbox', 'smsinbox', cfg.config().mode.logtoinstance) dbio.createTable('smsoutbox', 'smsoutbox', cfg.config().mode.logtoinstance) sensor_numbers_str = str(getSensorNumbers()) print '**' + network + ' GSM server active**' print time.asctime() while True: m = gsmio.countmsg() if m > 0: allmsgs = gsmio.getAllSms(network) try: WriteRawSmsToDb(allmsgs, sensor_numbers_str) except MySQLdb.ProgrammingError: print ">> Error: May be an empty line.. skipping message storing" deleteMessagesfromGSM() print dt.today().strftime("\n" + network + " Server active as of %A, %B %d, %Y, %X") logRuntimeStatus(network, "alive") trySendingMessages(network) elif m == 0: trySendingMessages(network) gsmio.gsmflush() today = dt.today() if (today.minute % 10 == 0): if checkIfActive: print today.strftime( "\nServer active as of %A, %B %d, %Y, %X") checkIfActive = False else: checkIfActive = True elif m == -1: print 'GSM MODULE MAYBE INACTIVE' serverstate = 'inactive' logRuntimeStatus(network, "gsm inactive") gsmio.resetGsm() elif m == -2: print '>> Error in parsing mesages: No data returned by GSM' gsmio.resetGsm() else: print '>> Error in parsing mesages: Error unknown' gsmio.resetGsm()
def getGndMeas(text): c = cfg.config() print '\n\n*******************************************************' print text # clean the message cleanText = re.sub(" +", " ", text.upper()) cleanText = re.sub("\.+", ".", cleanText) cleanText = re.sub(";", ":", cleanText) cleanText = re.sub("\n", " ", cleanText) cleanText = cleanText.strip() sms_list = re.split(" ", re.sub("[\W]", " ", cleanText)) sms_date = "" sms_time = "" records = [] # check measurement type if sms_list[0][0] == 'R': meas_type = "ROUTINE" else: meas_type = "EVENT" data_field = re.split(" ", cleanText, maxsplit=2)[2] try: date_str = getDateFromSms(data_field) print "Date: " + date_str except ValueError: raise ValueError(c.reply.faildateen) except IOError: raise ValueError(c.reply.failooben) try: time_str = getTimeFromSms(data_field, date_str) print "Time: " + time_str except ValueError: raise ValueError(c.reply.failtimeen) except IOError: raise ValueError(c.reply.failooben) # get all the measurement pairs meas_pattern = "(?<= )[A-Z] *\d{1,3}\.*\d{0,2} *C*M" meas = re.findall(meas_pattern, data_field) # create records list if meas: pass else: raise ValueError(c.reply.failmeasen) # get all the weather information print repr(data_field) try: wrecord = re.search("(?<=" + meas[-1] + " )[A-Z]+", data_field).group(0) recisvalid = False for keyword in [ "ARAW", "ULAN", "BAGYO", "LIMLIM", "AMBON", "ULAP", "SUN", "RAIN", "CLOUD", "DILIM", "HAMOG" ]: if keyword in wrecord: recisvalid = True print "valid" break if not recisvalid: raise AttributeError except AttributeError: raise ValueError(c.reply.failweaen) # get all the name of reporter/s try: observer_name = re.search("(?<=" + wrecord + " ).+$", data_field).group(0) print observer_name except AttributeError: raise ValueError(c.reply.failobven) gnd_records = "" for m in meas: try: crid = m.split(" ", 1)[0] cm = m.split(" ", 1)[1] print cm except IndexError: crid = m[0] cm = m[1:] try: re.search("\d *CM", cm).group(0) cm = float(re.search("\d{1,3}\.*\d{0,2}", cm).group(0)) except AttributeError: cm = float(re.search("\d{1,3}\.*\d{0,2}", cm).group(0)) * 100.0 gnd_records = gnd_records + "('" + date_str + " " + time_str + "','" + sms_list[ 0] + "','" + sms_list[ 1] + "','" + observer_name + "','" + crid + "','" + str( cm) + "','" + wrecord + "')," gnd_records = gnd_records[:-1] site_code = sms_list[1].lower() ts = date_str + " " + time_str command = """~/anaconda2/bin/python %s %s "%s" > ~/scriptlogs/gndalert.txt 2>&1 && ~/anaconda2/bin/python %s %s "%s" > ~/scriptlogs/gndalert2.txt 2>&1 && ~/anaconda2/bin/python %s %s "%s" > ~/scriptlogs/gndalert3.txt 2>&1""" % ( c.fileio.gndalert1, site_code, ts, c.fileio.gndalert2, site_code, ts, c.fileio.gndalert3, site_code, ts) p = subprocess.Popen(command, stdout=subprocess.PIPE, shell=True, stderr=subprocess.STDOUT) return gnd_records
import os, time, serial, re, sys import MySQLdb import datetime import ConfigParser from datetime import datetime as dt from datetime import timedelta as td import senslopedbio as dbio import gsmSerialio as gsmio import multiprocessing import SomsServerParser as SSP import math import cfgfileio as cfg import memcache mc = memcache.Client(['127.0.0.1:11211'], debug=0) if cfg.config().mode.script_mode == 'gsmserver': sys.path.insert(0, cfg.config().fileio.websocketdir) import dewsSocketLeanLib as dsll #--------------------------------------------------------------------------------------------------------------------------- def updateSimNumTable(name, sim_num, date_activated): db, cur = dbio.SenslopeDBConnect('local') while True: try: query = """select sim_num from senslopedb.site_column_sim_nums where name = '%s' """ % (name) a = cur.execute(query) if a:
def worker(first_target,last_target): #load all global variables? summary = pd.DataFrame() s_f = pd.DataFrame() s_a = pd.DataFrame() io = cfg.config() num_roll_window_ops = io.io.num_roll_window_ops roll_window_length = io.io.roll_window_length data_dt = io.io.data_dt rt_window_length = io.io.rt_window_length roll_window_numpts=int(1+roll_window_length/data_dt) col_pos_interval = io.io.col_pos_interval col_pos_num = io.io.num_col_pos to_fill = io.io.to_fill to_smooth = io.io.to_smooth # output_path = (__file__) # output_file_path = (__file__) # proc_file_path = (__file__) CSVFormat = '.csv' # PrintProc = io.io.printproc T_disp = io.io.t_disp T_velL2 = io.io.t_vell2 T_velL3 = io.io.t_vell3 k_ac_ax = io.io.k_ac_ax num_nodes_to_check = io.io.num_nodes_to_check colarrange = io.io.alerteval_colarrange.split(',') node_status = qdb.GetNodeStatus(1) for i in range(first_target,last_target): # try: sites,custom_end = ffd.aim(i) sensorlist = qdb.GetSensorList(sites) for s in sensorlist: last_col=sensorlist[-1:] last_col=last_col[0] last_col=last_col.name # getting current column properties colname,num_nodes,seg_len= s.name,s.nos,s.seglen # list of working nodes node_list = range(1, num_nodes + 1) not_working = node_status.loc[(node_status.site == colname) & (node_status.node <= num_nodes)] not_working_nodes = not_working['node'].values for i in not_working_nodes: node_list.remove(i) proc_monitoring,monwin=generate_proc(colname, num_nodes, seg_len, custom_end,roll_window_length,data_dt,rt_window_length,num_roll_window_ops) xz_series_list,xy_series_list = create_series_list(proc_monitoring,monwin,colname,num_nodes) # print "create_series_list tapos na" # create, fill and smooth dataframes from series lists xz=create_fill_smooth_df(xz_series_list,num_nodes,monwin, roll_window_numpts,to_fill,to_smooth) xy=create_fill_smooth_df(xy_series_list,num_nodes,monwin, roll_window_numpts,to_fill,to_smooth) # computing instantaneous velocity vel_xz, vel_xy = compute_node_inst_vel(xz,xy,roll_window_numpts) # computing cumulative displacements cs_x, cs_xz, cs_xy=compute_col_pos(xz,xy,monwin.index[-1], col_pos_interval, col_pos_num,seg_len) # processing dataframes for output xz,xy,xz_0off,xy_0off,vel_xz,vel_xy, vel_xz_0off, vel_xy_0off,cs_x,cs_xz,cs_xy,cs_xz_0,cs_xy_0 = df_to_out(colname,xz,xy, vel_xz,vel_xy, cs_x,cs_xz,cs_xy, # proc_file_path, CSVFormat) # Alert generation # alert_out=alert_generation(colname,xz,xy,vel_xz,vel_xy,num_nodes, T_disp, T_velL2, T_velL3, k_ac_ax, # num_nodes_to_check,custom_end,CSVFormat,colarrange) alert_out=alert_generation(colname,xz,xy,vel_xz,vel_xy,num_nodes, T_disp, T_velL2, T_velL3, k_ac_ax,num_nodes_to_check,custom_end,CSVFormat,colarrange) alert_out = alert_out.reset_index(level = ['id']) alert_out = alert_out[['id','disp_alert','vel_alert','node_alert','col_alert']] alert_out = alert_out[(alert_out['vel_alert'] > 0 ) | (alert_out.node_alert == 'l2')] alert_out = alert_out[alert_out.id == 1] alert_out['site'] = sites summary = pd.concat((summary,alert_out),axis = 0) # except: # print "Error recreating alarm." # continue print "--------------------Filtering chenes----------------------" print "--------------------Store yung mga nafilter----------------------" for j in range(0,len(summary)): # try: sites,custom_end = time_site(j,summary) # print "custom_end -------------> %s" %str(custom_end) sensorlist = qdb.GetSensorList(sites) for s in sensorlist: last_col=sensorlist[-1:] last_col=last_col[0] last_col=last_col.name # getting current column properties colname,num_nodes,seg_len= s.name,s.nos,s.seglen # list of working nodes node_list = range(1, num_nodes + 1) not_working = node_status.loc[(node_status.site == colname) & (node_status.node <= num_nodes)] not_working_nodes = not_working['node'].values for i in not_working_nodes: node_list.remove(i) # proc_monitoring,monwin=generate_proc(colname, num_nodes, seg_len, custom_end,f=True) proc_monitoring,monwin=generate_proc(colname, num_nodes, seg_len, custom_end,roll_window_length,data_dt,rt_window_length,num_roll_window_ops,filt=True) xz_series_list,xy_series_list = create_series_list(proc_monitoring,monwin,colname,num_nodes) xz=create_fill_smooth_df(xz_series_list,num_nodes,monwin, roll_window_numpts,to_fill,to_smooth) xy=create_fill_smooth_df(xy_series_list,num_nodes,monwin, roll_window_numpts,to_fill,to_smooth) # computing instantaneous velocity vel_xz, vel_xy = compute_node_inst_vel(xz,xy,roll_window_numpts) # computing cumulative displacements cs_x, cs_xz, cs_xy=compute_col_pos(xz,xy,monwin.index[-1], col_pos_interval, col_pos_num,seg_len) # processing dataframes for output xz,xy,xz_0off,xy_0off,vel_xz,vel_xy, vel_xz_0off, vel_xy_0off,cs_x,cs_xz,cs_xy,cs_xz_0,cs_xy_0 = df_to_out(colname,xz,xy, vel_xz,vel_xy, cs_x,cs_xz,cs_xy, # proc_file_path, CSVFormat) # Alert generation alert_out=alert_generation(colname,xz,xy,vel_xz,vel_xy,num_nodes, T_disp, T_velL2, T_velL3, k_ac_ax, num_nodes_to_check,custom_end,CSVFormat,colarrange) # print alert_out alert_out = alert_out.reset_index(level = ['id']) a_out = alert_out.copy() a_out = a_out[['id','disp_alert','vel_alert','node_alert','col_alert']] a_out = a_out[(a_out['vel_alert'] < 1.0 ) | (a_out.node_alert == 'l0')] a_out = a_out[a_out.id == 1] a_out['site'] = sites s_f = pd.concat((s_f,a_out),axis = 0) b_out = alert_out.copy() b_out = b_out[['id','disp_alert','vel_alert','node_alert','col_alert']] b_out = b_out[(b_out['vel_alert'] > 0.0 ) | (b_out.node_alert == 'l2')] b_out = b_out[b_out.id == 1] b_out['site'] = sites s_a = pd.concat((s_a,b_out),axis = 0) # except: # print "Error." # continue print "################# Drawing! Dahil drawing ka! ##################" print "################# Idrawing lahat ng nafilter! ##################" for k in range(0,len(s_f)): try: sites,custom_end = time_site(k,s_f) ce = custom_end.strftime("%y_%m_%d__%H_%M") fname = "FILTERED_" +str(sites) + "_" + ce + "_049_049" sensorlist = qdb.GetSensorList(sites) for s in sensorlist: last_col=sensorlist[-1:] last_col=last_col[0] last_col=last_col.name # getting current column properties colname,num_nodes,seg_len= s.name,s.nos,s.seglen # list of working nodes # node_list = range(1, num_nodes + 1) # not_working = node_status.loc[(node_status.site == colname) & (node_status.node <= num_nodes)] # not_working_nodes = not_working['node'].values # for i in not_working_nodes: # node_list.remove(i) # importing proc_monitoring file of current column to dataframe # try: # print "proc_monitoring here: " proc_monitoring=generate_proc(colname, num_nodes, seg_len, custom_end,roll_window_length,data_dt,rt_window_length,num_roll_window_ops,filt=True,for_plots=True) # print proc_monitoring proc_monitoring = proc_monitoring[proc_monitoring.id == 1] ffd.plotter(proc_monitoring,fname=fname) except: print "Error plotting Filtered." for k in range(0,len(s_a)): try: sites,custom_end = time_site(k,s_a) ce = custom_end.strftime("%y_%m_%d__%H_%M") sensorlist = qdb.GetSensorList(sites) for s in sensorlist: last_col=sensorlist[-1:] last_col=last_col[0] last_col=last_col.name # getting current column properties colname,num_nodes,seg_len= s.name,s.nos,s.seglen # list of working nodes # node_list = range(1, num_nodes + 1) # not_working = node_status.loc[(node_status.site == colname) & (node_status.node <= num_nodes)] # not_working_nodes = not_working['node'].values # for i in not_working_nodes: # node_list.remove(i) # importing proc_monitoring file of current column to dataframe # try: # print "proc_monitoring here: " proc_monitoring=generate_proc(colname, num_nodes, seg_len, custom_end,roll_window_length,data_dt,rt_window_length,num_roll_window_ops,f=True,for_plots=True) # print proc_monitoring proc_monitoring = proc_monitoring[proc_monitoring.id == 1] ffd.plotter(proc_monitoring,fname=fname) except: print "Error plotting Alarms."
def ProcessEarthquake(msg): line = msg.data.upper() print "Processing earthquake data" print line dbio.createTable('earthquake', 'earthquake') #find date if re.search("\d{1,2}\w+201[6789]", line): datestr_init = re.search("\d{1,2}\w+201[6789]", msg.data).group(0) pattern = ["%d%B%Y", "%d%b%Y"] datestr = None for p in pattern: try: datestr = dt.strptime(datestr_init, p).strftime("%Y-%m-%d") break except: print ">> Error in datetime conversion", datestr, "for pattern", p if datestr == None: return False else: print ">> No date string recognized" return False #find time if re.search("\d{1,2}[:\.]\d{1,2} *[AP]M", line): timestr = re.search("\d{1,2}[:\.]\d{1,2} *[AP]M", line).group(0) timestr = timestr.replace(" ", "").replace(".", ":") try: timestr = dt.strptime(timestr, "%I:%M%p").strftime("%H:%M:00") except: print ">> Error in datetime conversion", timestr return False else: print ">> No time string recognized" return False datetimestr = datestr + ' ' + timestr #find magnitude if re.search("((?<=M[SBLVOW]\=)|(?<=M\=)|(?<=MLV\=))\d+\.\d+(?= )", line): magstr = re.search( "((?<=M[SBLVOW]\=)|(?<=M\=)|(?<=MLV\=))\d+\.\d+(?= )", line).group(0) else: print ">> No magnitude string recognized" magstr = 'NULL' #find depth if re.search("(?<=D\=)\d+(?=K*M)", line): depthstr = re.search("(?<=D\=)\d+(?=K*M)", line).group(0) else: print ">> No depth string recognized" depthstr = 'NULL' #find latitude if re.search("\d+[\.\:]\d+(?=N)", line): latstr = re.search("\d+[\.\:]\d+(?=N)", line).group(0) else: print ">> No latitude string recognized" latstr = 'NULL' #find longitude if re.search("\d+[\.\:]\d+(?=E)", line): longstr = re.search("\d+[\.\:]\d+(?=E)", line).group(0) else: print ">> No longitude string recognized" longstr = 'NULL' #find epicenter distance if re.search("(?<=OR )\d+(?=KM)", line): diststr = re.search("(?<=OR )\d+(?=KM)", line).group(0) else: print ">> No distance string recognized" diststr = 'NULL' # find heading if re.search("[NS]\d+[EW]", line): headstr = re.search("[NS]\d+[EW]", line).group(0) else: print ">> No heading string recognized" headstr = 'NULL' # find Municipality if re.search("(?<=OF )[A-Z ]+(?= \()", line): munistr = re.search("(?<=OF )[A-Z ]+(?= \()", line).group(0) else: print ">> No municipality string recognized" munistr = 'NULL' # find province if re.search("(?<=\()[A-Z ]+(?=\))", line): provistr = re.search("(?<=\()[A-Z ]+(?=\))", line).group(0) else: print ">> No province string recognized" provistr = 'NULL' # find issuer if re.search("(?<=\<)[A-Z]+(?=\>)", line): issuerstr = re.search("(?<=\<)[A-Z]+(?=\>)", line).group(0) else: print ">> No issuer string recognized" issuerstr = 'NULL' query = "INSERT INTO senslopedb.earthquake (timestamp, mag, depth, lat, longi, dist, heading, municipality, province, issuer) VALUES ('%s',%s,%s,%s,%s,%s,'%s','%s','%s','%s') ON DUPLICATE KEY UPDATE mag=mag, depth=depth, lat=lat, longi=longi, dist=dist, heading=heading, municipality=municipality, province=province, issuer=issuer;" % ( datetimestr, magstr, depthstr, latstr, longstr, diststr, headstr, munistr, provistr, issuerstr) print query query.replace("'NULL'", "NULL") dbio.commitToDb(query, 'earthquake') # subprocess.Popen(["python",cfg.config().fileio.eqprocfile]) exec_line = "~/anaconda2/bin/python %s > ~/scriptlogs/earthquakescript.txt 2>&1" % ( cfg.config().fileio.eqprocfile) p = subprocess.Popen(exec_line, stdout=subprocess.PIPE, shell=True, stderr=subprocess.STDOUT) return True