def pingLan(): if lanIP == dbOneIp and lanIP != dbTwoIp: portOK = chkPort(dbTwoIp, int(3307)) if not portOK: doPing(dbTwoIp, int(3307)) else: if sDb.read(telIP, "lanAlarm") == "OFF": ## 状态OK且开关为"OFF"时,报恢复 sub = "DBTX LanIP is OK: %s" % ps1 msg = "DateTime: %s\n 备库: %s, 网络与端口恢复正常." % (getTimeNow(), dbTwoIp) saveLog.info(msg) sendMail(sub, msg) sDb.update(telIP, "lanAlarm", "ON") elif lanIP != dbOneIp and lanIP == dbTwoIp: portOK = chkPort(dbOneIp, int(3306)) if not portOK: doPing(dbOneIp, int(3306)) else: if sDb.read(telIP, "lanAlarm") == "OFF": sub = "DBTX LanIP is OK: %s" % ps1 msg = "DateTime: %s\n 备库: %s, 网络与端口恢复正常." % (getTimeNow(), dbTwoIp) saveLog.info(msg) sendMail(sub, msg) sDb.update(telIP, "lanAlarm", "ON") else: pass
def getDisk(self): mountPoints, diskTotal = [], 0 #diskTotal累加 parts = psutil.disk_partitions() #分区信息 for p in parts: mp = str(p).split(",")[1].split("=")[1] mountPoints.append(mp) for x in mountPoints: pTotal = psutil.disk_usage(x.strip("'")).total #单个分区容量 diskTotal += pTotal diskTotal = changeUnit(int(diskTotal)) rootSize, rootRate = psutil.disk_usage('/').free, psutil.disk_usage('/').percent #获取/分区的剩余空间和使用率 writeToFile("totalDisk", diskTotal) writeToFile("rootSize", changeUnit(int(rootSize))) writeToFile("rootRate", rootRate) if float(rootRate) > float(85): """ON开关,防止多次重复报警;报警一次之后设为关闭""" if self.sDb.read(self.telIP, "rootAlarm") == "ON": ## 状态为"NO"且开关为"ON"时,报故障 sub = "DBTX ROOT partition Alarm:%s" % self.ps1 msg = "DateTime: %s\n根分区已使用%s%%, 超过85%%, 请警惕!!!" % (getTimeNow(), rootRate) saveLog.warning(msg) sendMail(sub, msg) self.sDb.update(self.telIP, "rootAlarm", "OFF") ## 报警后,将开关置为"OFF" else: if self.sDb.read(self.telIP, "rootAlarm") == "OFF": ## 状态OK且开关为"OFF"时,报恢复 sub = "DBTX ROOT partition is OK:%s" % self.ps1 msg = "DateTime: %s\n根分区可用空间充足%s." % (getTimeNow(), changeUnit(int(rootSize))) saveLog.info(msg) sendMail(sub, msg) self.sDb.update(self.telIP, "rootAlarm", "ON") ## 使用率不足85%时,将开关置为"ON"
def doPing(tIP, dbPort): if dbPort == 3307: mString = "备库-%s-3307端口不通" % tIP else: mString = "主库-%s-3306端口不通" % tIP p = subprocess.Popen("ping -c7 %s" % tIP,shell=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE) pResult = p.stdout.read() lossRate = pResult.split("---")[-1].split(",")[2].split()[0] ## 取丢包率 if float(lossRate[:-1]) > 20: sub = "DBTX LanIP Alarm: %s" % ps1 msg = "DateTime: %s\n%s, ICMP-Ping丢包率:%s%% !!!" % (getTimeNow(), mString, lossRate) else: sub = "DBTX LanIP-Port Alarm: %s" % ps1 msg = "DateTime: %s\n%s, ICMP-Ping正常,可能是MySQL挂了或者防火墙对应端口没有开启 !!!" % (getTimeNow(), mString) saveLog.error(msg) if sDb.read(telIP, "lanAlarm") == "ON": ## 状态为"NO"且开关为"ON"时,报故障 sendMail(sub, msg) sDb.update(telIP, "lanAlarm", "OFF") ## 报警后,将开关置为"OFF"
def getNginxStatus(self): try: if os.path.exists("/usr/local/nginx/logs/nginx.pid"): fObj = open("/usr/local/nginx/logs/nginx.pid") ngxMainPid = fObj.read().strip() fObj.close() else: ngxMainPid = 0 pids = getProcessID("nginx") if ngxMainPid: if not pids or len(pids) < 8: wp = "" res = os.popen("ps -C nginx -o pid,cmd").readlines() for l in res: wp += "\n" + l.strip() if self.sDb.read(telIP, "ngxAlarm") == "ON": ## 状态为"NO"且开关为"ON"时,报故障 sub = "DBTX Nginx Alarm: %s" % ps1 msg = "Datetime: %s\nNginx主进程PID:%s\nWorker_processes:%s\n少于10个!!" % (getTimeNow(), ngxMainPid, wp) saveLog.warning(msg) sendMail(sub, msg) self.sDb.update(telIP, "ngxAlarm", "OFF") else: if self.sDb.read(telIP, "ngxAlarm") == "OFF": ## 状态OK且开关为"OFF"时,报恢复 sub = "DBTX Nginx is OK: %s" % ps1 msg = "Datetime: %s\nNginx主进程恢复开启状态." % (getTimeNow()) saveLog.info(msg) sendMail(sub, msg) self.sDb.update(telIP, "ngxAlarm", "ON") else: if self.sDb.read(telIP, "ngxAlarm") == "ON": ## 状态为"NO"且开关为"ON"时,报故障 sub = "DBTX Nginx Alarm: %s" % ps1 msg = "Datetime: %s\nNginx主进程挂了!!" % (getTimeNow()) saveLog.warning(msg) sendMail(sub, msg) self.sDb.update(telIP, "ngxAlarm", "OFF") except Exception,err: saveLog.warning("getNginxStatus, %s" % str(err))
def appCpuAnalyze(srvPidStatus, srvFlag, sDb): from dbtx.Merge.etc import ps1 corMuser = "******" tNowCpu = topNow() if not sDb.exists(srvFlag, "ct"): ## Redis库中不存在该服区标识,初始化游戏程序占用CPU达到95%的次数 sDb.intAppCpuCt(srvFlag) for app in srvPidStatus: appName = app.split("_")[0] if srvPidStatus[app][app+"_status"] == 0 or "Snapshot" in app: ## 如果app状态异常,不检测CPU,MEM信息,忽略掉Snapshot continue appPid = srvPidStatus[app][app+"_pid"].split() for pid in appPid: ## CellApp,GateApp,CopyApp会有多个进程ID nCpu, appCount = tNowCpu[pid], int(sDb.read(appName, srvFlag)) if float(nCpu) > 95.5: ## CPU使用率都大于95%时,count+1 appCount += 1 sDb.update(appName, srvFlag, appCount) ## 将新的count值更新到Redis库 else: sDb.update(appName, srvFlag, 0) ## 如果没有超过95%,则置为0 if int(appCount) >= 8: if sDb.read(srvFlag, "isFixing") == "OFF": ## 如果维护标识为"OFF" if sDb.read(srvFlag, "appCpuAlarm") == "ON": ## 状态为"NO"且开关为"ON"时,报故障 sub = "DBTX App-CPU Alarm: %s" % ps1 msg = "DateTime: %s\n%s CPU使用率连续五次达到95%%, 进程ID为: %s !!!" % (getTimeNow(), app, pid) saveLog.warning(msg) sendMail(sub, msg, corMuser) ## sDb.updateDb(mulDb, srvFlag, "appCpuAlarm", "OFF") ## 报警后,将开关置为"OFF" sDb.update(appName, srvFlag, 0) ## 报警后,重置count值 if "DbCacheApp_"+srvFlag == app: proc = psutil.Process(int(pid)) shr = proc.get_ext_memory_info()[2] if float(shr) > 1932735283.2: ## DbCacheApp共享内存大于1.8G时,报警 if sDb.read(srvFlag, "isFixing") == "OFF": ## 如果维护标识为"OFF" if sDb.read(srvFlag, "appShrAlarm") == "ON": ## 状态为"NO"且开关为"ON"时,报故障 sub = "DBTX DbCacheApp-SHR Alarm: %s" % ps1 msg = "DateTime: %s\n%s DbCacheApp 共享内存达到红色警戒,当前占用共享内存: %s !!!" % (getTimeNow(), srvFlag, changeUnit(shr)) saveLog.warning(msg) sendMail(sub, msg, corMuser) sDb.update(srvFlag, "appShrAlarm", "OFF") ## 报警后,将开关置为"OFF" else: if sDb.read(srvFlag, "isFixing") == "OFF": ## 如果维护标识为"OFF" if sDb.read(srvFlag, "appShrAlarm") == "OFF": ## 状态OK且开关为"OFF"时,报恢复 sub = "DBTX DbCacheApp-SHR Alarm: %s" % ps1 msg = "DateTime: %s\n%s DbCacheApp 占用共享内存恢复到正常水平: %s ." % (getTimeNow(), srvFlag, changeUnit(shr)) saveLog.warning(msg) sendMail(sub, msg, corMuser) sDb.update(srvFlag, "appShrAlarm", "ON")
def gateAnalyze(srvFlag, nowNu, sDb): from dbtx.Merge.etc import ps1 pubFile = "/opt/dbtx/" + str(srvFlag) + "/config/public_config.xml" pubObj = open(pubFile) fixGateNum = ''.join([ l.strip().split(">")[1].split("<")[0] for l in pubObj.readlines() if "<GateNum>" in l]) pubObj.close() corMuser = "******" if int(nowNu) < int(fixGateNum): if sDb.read(srvFlag, "isFixing") == "OFF": ## 如果维护标识为"OFF" if sDb.read(srvFlag, "appNumAlarm") == "ON": ## 状态为"NO"且开关为"ON"时,报故障 sub = "DBTX GateApps Alarm: %s" % ps1 msg = "DateTime: %s\n%s 预设开启 %s 个GateApp,当前只有 %s 个,挂了 %s 个 !!!" % (getTimeNow(), srvFlag, fixGateNum, nowNu, (int(fixGateNum)-int(nowNu))) saveLog.warning(msg) sendMail(sub, msg, corMuser) sDb.update(srvFlag, "appNumAlarm", "OFF") ## 报警后,将开关置为"OFF" else: if sDb.read(srvFlag, "isFixing") == "OFF": ## 如果维护标识为"OFF" if sDb.read(srvFlag, "appNumAlarm") == "OFF": ## 状态OK且开关为"OFF"时,报恢复 sub = "DBTX GateApps is OK: %s" % ps1 msg = "DateTime: %s\n%s GateApp Numbers正常." % (getTimeNow(), srvFlag) saveLog.warning(msg) sendMail(sub, msg, corMuser) sDb.update(srvFlag, "appNumAlarm", "ON")
redisData["connects_rate"] = (int(redisData["total_connects"]) - int(last_connects)) / float(60) redisData["command_rate"] = (int(redisData["total_commands"]) - int(last_commands)) / float(60) fZbx = str(redisData).strip("{}").replace(",","\n").replace("'","") ## 转换成zabbix需要的格式 fObj = open(rZbx,'w') fObj.write(fZbx) fObj.close() except Exception,err: saveLog.error("getRedisBaseInfo(): %s" % str(err)) ## 分析数据,必要时报警: 1, 使用内存大于1G; 2, 命中率低于20% if int(redisData["used_memory"]) > 1073741824: sDb = opRedis() ## 实例化 Redis库操作类 if sDb.read(telIP, "rRateAlarm") == "ON": sub = "DBTX Redis Memory Alarm: %s" % ps1 msg = "DateTime: %s\nRedis使用内存为: %s, 超过1G,请警惕 !!!" % (getTimeNow(), changeUnit(int(redisData["used_memory"]))) saveLog.warning(msg) sendMail(sub, msg) sDb.update(telIP, "rRateAlarm", "OFF") ## 报警后,将开关置为"OFF" else: sDb = opRedis() ## 实例化 Redis库操作类 if sDb.read(telIP, "rRateAlarm") == "OFF": sub = "DBTX Redis Memory is OK: %s" % ps1 msg = "DateTime: %s\nRedis使用内存情况正常: %s." % (getTimeNow(), changeUnit(int(redisData["used_memory"]))) saveLog.info(msg) sendMail(sub, msg) sDb.update(telIP, "rRateAlarm", "ON") ## if totalHits != 0 and redisData["hit_rate"] < 20: ## sDb = opShelveDb() ## 实例化 shelve库操作类 ## if sDb.readDb(pubDb, telIP, "rRateAlarm") == "ON":
def getServerTime(self): nowTime = getTimeNow() self.data_for_post += "server_time=%s&" % nowTime
nowTime = getTimeNow() self.data_for_post += "server_time=%s&" % nowTime def getDbStatus(self): ## 检查MySQL状态, 1 正常, 0 异常 try: db1Status = chkPort(dbOneIp, db1Port) db2Status = chkPort(dbTwoIp, db2Port) except Exception, err: saveLog.error("getDbStatus() Error Occur : %s" % err) if db1Status: db1_status = 1 if self.sDb.read(telIP, "db1Alarm") == "OFF": ## 状态OK且开关为"OFF"时,报恢复 sub = "DBTX DB1 mysql is OK: %s" % ps1 msg = "DateTime: %s\n主库-%s-3306端口 is up" % (getTimeNow(), dbOneIp) saveLog.info(msg) sendMail(sub, msg) self.sDb.update(telIP, "db1Alarm", "ON") ## statusAnalyze("3306") ## 获取其它性能数据,zabbix成图或报警 else: db1_status = 0 try: if self.sDb.read(telIP, "db1Alarm") == "ON": ## 状态为"NO"且开关为"ON"时,报故障 sub = "DBTX DB1 mysql Alarm: %s" % ps1 msg = "DateTime: %s\n主库-%s-3306端口 DOWN 了,请查看日志 !!!" % (getTimeNow(), dbOneIp) saveLog.warning(msg) sendMail(sub, msg) self.sDb.update(telIP, "db1Alarm", "OFF") ## 报警后,将开关置为"OFF" except Exception, err: saveLog.error("db1Alarm() Error Occur : %s" % str(err))
if fixSwStat == "OFF": ## 如果维护标识为"OFF" if appSwStat == "ON": ## 状态为"NO"且开关为"ON"时,报故障 sub = "DBTX AppService Alarm: %s" % ps1 msg = "DateTime: %s\n%s 挂啦 !!!" % (getTimeNow(), srvStatus['Error']) saveLog.warning(msg) sendMail(sub, msg, mUser) self.sDb.update(srvFlag, "appAlarm", "OFF") except Exception, err: saveLog.warning("app_status is 2,get switch status error occured.") elif srvStatus['Error'] and not srvStatus['Success']: ## 全部处于关闭状态 app_status = 0 try: if fixSwStat == "OFF": ## 如果维护标识为"OFF" if appSwStat == "ON": ## 状态为"NO"且开关为"ON"时,报故障 sub = "DBTX AppService is Down: %s" % ps1 msg = "DateTime: %s\n%s AppService 关闭 !!!" % (getTimeNow(), srvFlag) saveLog.warning(msg) sendMail(sub, msg, mUser) self.sDb.update(srvFlag, "appAlarm", "OFF") except Exception, err: saveLog.warning("app_status is 0,get switch status error occured.") elif not srvStatus['Error'] and srvStatus['Success']: ## 全部处于开启状态 app_status = 1 appCpuAnalyze(srvPidStatus,srvFlag,self.sDb) self.getSrvVer(srvPidStatus) gateAnalyze(srvFlag,GateAppCount,self.sDb) try: if fixSwStat == "OFF": ## 如果维护标识为"OFF" if appSwStat == "OFF": ## 状态OK且开关为"OFF"时,报恢复 sub = "DBTX AppService is OK: %s" % ps1 msg = "DateTime: %s\n%s AppService 恢复开启 !!!" % (getTimeNow(), srvFlag)