def getOStype(agent, ip, key, port): ''' 获取操作系统类型: bit --> getOStype[0] type --> getOStype[1] ''' system_oid = ObjectName('1.3.6.1.2.1.1.1') system_get = ger_snmp_next(agent,ip,key,port,system_oid) system_name = str(system_get[0][0][1]) if 'Windows' in system_name: system_type = 'Windows' if 'x86' in system_name: system_bit = "32bit" else: system_bit = "64bit" elif 'Linux' in system_name: system_type = 'Linux' if 'x86_64' in system_name: system_bit = "64bit" else: system_bit = "32bit" else: system_type = 'Null' Usql = "update osa_ipinfo set oOsType='"+system_type+"' where oIp = '"+ip+"'" cmdtosql.update(Usql) return system_bit,system_type
def Update_osa_serverinfo(data): ''' 更新osa_serverinfo表里的oStatus和oNotiNum字段 ''' time.sleep(round(float(random.randrange(0, 100, 1))/100,2)) sql="UPDATE osa_serverinfo SET oStatus='"+data[3]+"',oNotiNum="+str(data[4])+" WHERE id="+str(data[0]) try: update(sql) except Exception as e: save_log('ERROR','sql:'+sql+'fail! ERROR:'+str(e))
def clear_monitor_item(): ''' @osa_monitors中关于当天项目的已发邮件计数清零 ''' try: sql = "update osa_monitors set oNotifiedNum = 0 " log_debug("clear_monitor_server()执行sql语句:"+sql) cmdtosql.update(sql) except Exception as e: log_error("clear_monitor_server():"+str(e))
def clear_ipinfo_server(): ''' @osa_ipinfo中关于当天对服务器的已发邮件计数清零 ''' try: sql = "update osa_ipinfo set oNotifiedNum = 0" log_debug("clear_ipinfo_server()执行sql语句:"+sql) cmdtosql.update(sql) except Exception as e: log_error("clear_ipinfo_server():"+str(e))
def Update_osa_alarms(data): ''' 更新osa_alarms表里的oNextCheckTime字段 ''' sql= "UPDATE osa_alarms SET oNextCheckTime='"+data[14]+"' WHERE id="+str(data[0]) try: time.sleep(round(float(random.randrange(0, 100, 1))/100,2)) update(sql) except Exception as e: save_log('ERROR','Update_osa_alarms sql:'+sql+'fail! ERROR:'+str(e))
def monitor_item_updateTime(itemlist): ''' @osaMonitor 更新项目下次监控时间 ''' itemlist = list(itemlist) nextTime = monitor_item_nextTime(itemlist) sql = "update osa_monitors set oNextCheckTime='"+nextTime+"' where id="+str(itemlist[0]) try: time.sleep(1) log_debug("monitor_item_updateTime()执行sql语句:"+str(sql)) cmdtosql.update(sql) except Exception as e: log_error("monitor_item_updateTime():"+str(e))
def chooseoRunCycle(): ''' @根据周期,返回数据列表 ''' rinfo = getoRunCycle() rlist = [] for r in rinfo: if r[7] != None: cmdinfo = '' try: cmdinfo = getoCombinCmd(int(r[0]),r[4]) except Exception as e: save_log('ERROR','getoCombinCmd error:'+str(e)) if cmdinfo: save_log('INFO','TASK CMD:'+str(cmdinfo[0][0])) rlist.append(cmdinfo[0][0]) nexttime = getNextRunTime(r) if nexttime == False or not nexttime: Isql = "INSERT INTO `osa_complantask`(oCmdType,oTaskplanid,oRunCycle,oRunDate,oRunTime) select oCmdType,id,oRunCycle,oRunDate,oRunTime from osa_taskplan" try: con = cmdtosql._get_pcon() cur = con.cursor() cur.execute(Isql) except Exception as Ierror: save_log('ERROR','osa_complantask INSERT ERROR:'+str(Ierror)+',sql is: '+Isql) finally: cmdtosql._exit(con, cur) Dsql = "DELETE from osa_taskplan WHERE id = "+str(r[0]) try: con = cmdtosql._get_pcon() cur = con.cursor() cur.execute(Dsql) except Exception as Ierror: save_log('ERROR','osa_taskplan DELETE ERROR:'+str(Ierror)+',sql is: '+Dsql) finally: cmdtosql._exit(con, cur) else: usql = "UPDATE `osa_taskplan` set oStatus = '运行中',oRunNextTime = '"+nexttime+"',oRunLastTime = '"+str(cmdtosql._get_time(flag=1))+"' WHERE id = "+str(r[0]) cmdtosql.update(usql) else: nexttime = getNextRunTime(r) usql = "UPDATE `osa_taskplan` set oStatus = '运行中',oRunNextTime = '"+nexttime+"',oRunLastTime = '"+str(cmdtosql._get_time(flag=1))+"' WHERE id = "+str(r[0]) cmdtosql.update(usql) return rlist
def update_ipinfo_remind(isNoticeNow,isNoticeNext,ipinfo): ''' @服务器恢复通知时根据isNoticeNow,isNoticeNext更新osa_ipinfo表 @思路,当立即发送通知和转下次通知时,标识该ip今天已发送的通知+1 ''' try: if isNoticeNow>0 or isNoticeNext >0: oNum=int(ipinfo[6])+1 sql = "update osa_ipinfo set oIsAlive='1' ,oStatus='正常' ,oFaultTime='OOOO-OO-OO OO:00:00' ,oNotiNum=0 ,oNotifiedNum="+str(oNum)+",oIsEmail='0' where id="+str(ipinfo[0]) else: sql = "update osa_ipinfo set oIsAlive='1' ,oStatus='正常' ,oFaultTime='0000-00-00 00:00:00' ,oNotiNum=0,oIsEmail='0' where id="+str(ipinfo[0]) log_debug("update_ipinfo_remind()执行sql语句:"+str(sql)) cmdtosql.update(sql) except Exception as e: log_error("update_ipinfo_remind():"+str(e))
def send_email_server(): ''' @服务器报警中根据记录中的值发送邮件 ''' result = get_notice_server() if not result or result == False: return False for record in result: users = record[12] level = record[5] ipid = record[1] mailto_users_server(users,level,ipid) #更新osa_collect_alarm sql = "update osa_collect_alarm set oIsNotice=1 , oIsNoticeNext=0 where id ="+record[0] cmdtosql.update(sql)
def send_email_item(): ''' @项目报警中根据记录中的值发送邮件 ''' result = get_notice_server() if not result or result == False: return False for record in result: level = record[5] alarmType = record[6] users = record[12] itemid = record[1] mailto_users_item(users,alarmType,level,itemid) #更新osa_collect_alarm sql = "update osa_monitor_alarm set oIsNotice=1 , oIsNoticeNext=0 where id ="+record[0] cmdtosql.update(sql)
def update_iteminfo_remind(isNoticeNow,isNoticeNext,iteminfo): ''' @osaMonitor 恢复通知时根据isNoticeNow,isNoticeNext更新osa_monitors表 @oNotifiedNum:已发送告警数量,oNotiNum:连续告警次数 @思路,当立即发送通知和转下次通知时,标识该item今天已发送的通知+1 ''' try: oNotifiedNum = "" if isNoticeNow>0 or isNoticeNext >0: oNum=int(iteminfo[19])+1 sql = "update osa_monitors set oStatus='1' ,oFaultTime='0000-00-00 00:00:00' ,oNotiNum=0 ,oNotifiedNum="+str(oNum)+",oRepeatedNum=0,oIsEmail='0' where id="+str(iteminfo[0]) else: sql = "update osa_monitors set oStatus='1' ,oFaultTime='0000-00-00 00:00:00' ,oNotiNum=0 ,oRepeatedNum=0,oIsEmail='0' where id="+str(iteminfo[0]) log_debug("update_iteminfo_remind()执行sql语句:"+str(sql)) cmdtosql.update(sql) except Exception as e: log_error("update_iteminfo_remind()"+str(e))
def NewIpSubThreed(rev, func, type='batch'): ''' @每IP每线程,执行任务,最大同时执行maxthreed个任务 ''' thread_arr = [] if not rev or not func: save_log('ERROR', 'rev or func Error!') return False iplist = batchIpList(rev) try: q = putBatchIp(iplist) except Exception as e: save_log('ERROR', 'BATCH QUEUE is error:' + str(e)) return False #while True: #线程控制,防止同时执行任务过大导致服务器负载突发 #for x in xrange(BATCH['maxthreed']): for x in xrange(q.qsize()): ip = getBatchIp(q) try: tt = threading.Thread(target=func.index, args=[rev, type, ip, x]) thread_arr.append(tt) except Exception as e: save_log('ERROR', 'BATCH Threed is error:' + str(e)) for i in xrange(len(thread_arr)): try: thread_arr[i].setDaemon(0) thread_arr[i].start() except Exception as e: save_log('ERROR', 'BATCH Threed start is error:' + str(e)) for i in xrange(len(thread_arr)): thread_arr[i].join() if q.qsize() == 0: #此次批量操作执行完成,更新状态 id = str(getBatchId(rev)) if type == 'batch': usql = "UPDATE `osa_tasknow` SET oStatus = '执行完成' WHERE id = " + id + "" try: uid = update(usql) except Exception as uerror: save_log( 'ERROR', 'update error,sql is:' + usql + ',error info:' + str(uerror)) save_log('INFO', 'BATCH Threed is ok!') return True return False
def NewIpSubThreed(rev,func,type='batch'): ''' @每IP每线程,执行任务,最大同时执行maxthreed个任务 ''' thread_arr = [] if not rev or not func: save_log('ERROR','rev or func Error!') return False iplist = batchIpList(rev) try: q = putBatchIp(iplist) except Exception as e: save_log('ERROR','BATCH QUEUE is error:'+str(e)) return False #while True: #线程控制,防止同时执行任务过大导致服务器负载突发 #for x in xrange(BATCH['maxthreed']): for x in xrange(q.qsize()): ip = getBatchIp(q) try: tt = threading.Thread(target=func.index,args=[rev,type,ip,x]) thread_arr.append(tt) except Exception as e: save_log('ERROR','BATCH Threed is error:'+str(e)) for i in xrange(len(thread_arr)): try: thread_arr[i].setDaemon(0) thread_arr[i].start() except Exception as e: save_log('ERROR','BATCH Threed start is error:'+str(e)) for i in xrange(len(thread_arr)): thread_arr[i].join() if q.qsize() == 0: #此次批量操作执行完成,更新状态 id = str(getBatchId(rev)) if type == 'batch': usql = "UPDATE `osa_tasknow` SET oStatus = '执行完成' WHERE id = "+id+"" try: uid = update(usql) except Exception as uerror: save_log('ERROR','update error,sql is:'+usql+',error info:'+str(uerror)) save_log('INFO','BATCH Threed is ok!') return True return False
def update_iteminfo_except(isNoticeNow,isNoticeNext,iteminfo): ''' @osaMonitor 服务异常报警更新osa_monitors 记录 ''' try: oNotifiedNum = "" if str(iteminfo[17]) == '1':#表示上次正常,错误时间从现在开始 faultTime = cmdtosql._get_time(1) else:#表示上次异常,错误时间从过去开始 faultTime = iteminfo[18] if isNoticeNow>0 or isNoticeNext >0: oNum = int(iteminfo[19])+1 notiNum = int(iteminfo[20])+1 sql = "update osa_monitors set oStatus='0' ,oFaultTime='"+str(faultTime)+"',oNotifiedNum="+str(oNum)+",oNotiNum="+str(notiNum)+",oIsEmail='1' where id="+str(iteminfo[0]) else: sql = "update osa_monitors set oStatus='0' ,oFaultTime='"+str(faultTime)+"',oIsEmail='1' where id="+str(iteminfo[0]) log_debug("update_iteminfo_except()执行sql语句:"+str(sql)) cmdtosql.update(sql) except Exception as e: log_error("update_iteminfo_except():"+str(e))
def update_ipinfo_exception(isNoticeNow,isNoticeNext,ipinfo,level): ''' @服务异常报警更新osa_ipinfo 记录 ''' try: if ipinfo[2] == '1':#表示上次正常,错误时间从现在开始 faultTime = cmdtosql._get_time(1) else:#表示上次异常,错误时间从过去开始 faultTime = ipinfo[5] if str(level) == '1': status = '失去响应' elif str(level) == '2': status = '其他异常' if isNoticeNow>0 or isNoticeNext >0: oNum = int(ipinfo[6])+1 notiNum = int(ipinfo[7])+1 sql = "update osa_ipinfo set oIsAlive='0' ,oStatus='"+str(status)+"' ,oFaultTime='"+str(faultTime)+"',oNotifiedNum="+str(oNum)+",oNotiNum="+str(notiNum)+",oIsEmail='1' where id="+str(ipinfo[0]) else: sql = "update osa_ipinfo set oIsAlive='0' ,oStatus='"+str(status)+"' ,oFaultTime='"+str(faultTime)+"' where id="+str(ipinfo[0]) log_debug("update_ipinfo_exception()执行sql语句:"+str(sql)) cmdtosql.update(sql) except Exception as e: log_error("update_ipinfo_exception():"+str(e))
def ipSubThreed(rev, func, type='batch'): ''' @每IP每线程,执行任务,最大同时执行maxthreed个任务 ''' if not rev or not func: return False iplist = batchIpList(rev) try: q = putBatchIp(iplist) except Exception as e: save_log('ERROR', 'BATCH QUEUE is error:' + str(e)) while True: if q.qsize() == 0: #此次批量操作执行完成,更新状态 id = str(getBatchId(rev)) if type == 'batch': usql = "UPDATE `osa_tasknow` SET oStatus = '执行完成' WHERE id = " + id + "" try: uid = update(usql) except Exception as uerror: save_log( 'ERROR', 'update error,sql is:' + usql + ',error info:' + str(uerror)) break #线程控制,防止同时执行任务过大导致服务器负载突发 for x in xrange(BATCH['maxthreed']): ip = getBatchIp(q) if ip == None or not ip: break try: tt = threading.Thread(target=func.index, args=[rev, type, ip, x]) tt.setDaemon(0) tt.start() except Exception as e: save_log('ERROR', 'BATCH Threed is error:' + str(e)) time.sleep(300)
def ipSubThreed(rev,func,type='batch'): ''' @每IP每线程,执行任务,最大同时执行maxthreed个任务 ''' if not rev or not func: return False iplist = batchIpList(rev) try: q = putBatchIp(iplist) except Exception as e: save_log('ERROR','BATCH QUEUE is error:'+str(e)) while True: if q.qsize() == 0: #此次批量操作执行完成,更新状态 id = str(getBatchId(rev)) if type == 'batch': usql = "UPDATE `osa_tasknow` SET oStatus = '执行完成' WHERE id = "+id+"" try: uid = update(usql) except Exception as uerror: save_log('ERROR','update error,sql is:'+usql+',error info:'+str(uerror)) break #线程控制,防止同时执行任务过大导致服务器负载突发 for x in xrange(BATCH['maxthreed']): ip = getBatchIp(q) if ip == None or not ip: break try: tt = threading.Thread(target=func.index,args=[rev,type,ip,x]) tt.setDaemon(0) tt.start() except Exception as e: save_log('ERROR','BATCH Threed is error:'+str(e)) time.sleep(300)
def ThreadForEachServer(each_ipinfo,AllUserEmailAddress): ''' 子线程入口函数,检测服务器的状态 ''' if PortIsAlive(each_ipinfo[1],SOCKET['REMOTE_PORT']): if each_ipinfo[3]=='0' or each_ipinfo[4] != '正常': sql="UPDATE osa_ipinfo SET oStatus = '正常',oIsAlive = '1',oIsAliveNum = 0 WHERE id ="+str(each_ipinfo[0]) row = update(sql) CreateAlarmMsg('服务器存活检测',1,each_ipinfo[1],'服务器'+each_ipinfo[1]+"已恢复正常",0) for EmailAddress in AllUserEmailAddress: try: notifyByEmail(subject='[INFO]Notify from OSA',content = "Message level: INFO, Server "+each_ipinfo[1]+"already has recovered!",address = EmailAddress) except Exception as e: save_log('ERROR','send mail to '+EmailAddress+' failed!! ERROR:'+str(e)) sys.exit() sys.exit() else: result = Ping(each_ipinfo[1]) if True == result: if each_ipinfo[3]=='0': oIsAliveNum=int(each_ipinfo[5]) if oIsAliveNum < 2: for EmailAddress in AllUserEmailAddress: try: notifyByEmail(subject='[WARNING]Notify from OSA',content = "Message level: WARNING,Server "+each_ipinfo[1]+":python client service exception now!",address = EmailAddress) except Exception as e: save_log('ERROR','send mail to '+EmailAddress+' failed!! ERROR:'+str(e)) oIsAliveNum = oIsAliveNum + 1 sql = "UPDATE osa_ipinfo SET oStatus = '服务异常',oIsAliveNum="+str(oIsAliveNum)+" WHERE id ="+str(each_ipinfo[0]) row = update(sql) sys.exit() elif oIsAliveNum >= 2: sql = "UPDATE osa_ipinfo SET oStatus = '服务异常' WHERE oIp ='"+each_ipinfo[1]+"'" row = update(sql) sys.exit() if each_ipinfo[3] == '1': CreateAlarmMsg('服务器存活检测',1,each_ipinfo[1],'服务器'+each_ipinfo[1]+"服务异常",2) sql = "UPDATE osa_ipinfo SET oStatus = '服务异常',oIsAlive = '0',oIsAliveNum=1 WHERE id ="+str(each_ipinfo[0]) row = update(sql) for EmailAddress in AllUserEmailAddress: try: notifyByEmail(subject='[WARNING]Notify from OSA',content="Message level: WARNING,Server "+each_ipinfo[1]+":python client service exception now!",address = EmailAddress) except Exception as e: save_log('ERROR','send mail to '+EmailAddress+' failed!! ERROR:'+str(e)) sys.exit() elif False == result: if each_ipinfo[3] == '0': oIsAliveNum=int(each_ipinfo[5]) if oIsAliveNum < 2: for EmailAddress in AllUserEmailAddress: try: notifyByEmail(subject='[CRITICAL]Notify from OSA',content = "Message level: CRITICAL,Server "+each_ipinfo[1]+"doesn't reach!",address = EmailAddress) except Exception as e: save_log('ERROR','send mail to '+EmailAddress+' failed!! ERROR:'+str(e)) oIsAliveNum = oIsAliveNum + 1 sql = "UPDATE osa_ipinfo SET oStatus = '服务器不可达',oIsAliveNum="+str(oIsAliveNum)+" WHERE id ="+str(each_ipinfo[0]) row = update(sql) sys.exit() if oIsAliveNum>=2: sql = "UPDATE osa_ipinfo SET oStatus = '服务器不可达' WHERE oIp ='"+each_ipinfo[1]+"'" row = update(sql) sys.exit() if each_ipinfo[3] == '1': CreateAlarmMsg('服务器存活检测',1,each_ipinfo[1],'服务器'+each_ipinfo[1]+"不可达",1) for EmailAddress in AllUserEmailAddress: try: notifyByEmail(subject='[CRITICAL]Notify from OSA',content = "Message level: CRITICAL ,Server "+each_ipinfo[1]+"doesn't reach!!",address = EmailAddress) except Exception as e: save_log('ERROR','send mail to '+EmailAddress+' failed!! ERROR:'+str(e)) oIsAliveNum=int(each_ipinfo[5]) oIsAliveNum=oIsAliveNum+1 sql = "UPDATE osa_ipinfo SET oStatus = '服务器不可达',oIsAlive = '0',oIsAliveNum="+str(oIsAliveNum)+" WHERE id ="+str(each_ipinfo[0]) row = update(sql) sys.exit() sys.exit()