def getDbStatus(self):
## 检查MySQL状态, 1 正常, 0 异常
try:
db1Status = chkPort(dbOneIp, db1Port)
db2Status = chkPort(dbTwoIp, db2Port)
except Exception, err:
saveLog.error("getDbStatus() Error Occur : %s" % err)
def outRedis(self, mode=1, page=1, count=100, target_directory=None):
if mode == 1:
if self.redis_o.exists(self.redis_keys[u'index']):
start_index = int((page-1)*count) # redis start index
end_index = int(page*count-1) # redis end index
skipped_directory = 0
for index,directory in enumerate(self.redis_o.lrange(name=self.redis_keys[u'index'], start=start_index, end=end_index)): # select for point reange directory
try:
directory = unicode(directory, encoding='utf8', errors='strict') # to utf8 unicode
except Exception,err:
saveLog.error(u'%s to unicode faild' % directory)
saveLog.error(err)
if not os.path.exists(directory): # confirm directory exists
saveLog.error(u'%s is not exists' % directory)
skipped_directory += 1
directory = u''
self.row = start_index + index # now redis index
try:
main_Image = self.redis_o.hget(name=self.redis_keys[u'mainImage'], key=directory)
main_Image = unicode(main_Image, encoding='utf8', errors='strict') # to utf8 unicode
except Exception,err:
saveLog.error(u'Get %s main Image faild' % directory)
saveLog.error(err)
main_Image = u'tornado_no_main_Image.jpg'
yield directory,main_Image
def inRedis(self, expiretime=600):
if os.path.exists(self.resoure):
time.clock()
saveLog.info(u'Begin init redis')
saveLog.info(u'Resoure Directory:"%s"' % self.resoure)
for key in self.redis_keys.values():
if self.redis_o.exists(key):
ttl = self.redis_o.ttl(key)
self.redis_o.delete(key)
saveLog.info(u'Delete "%s", ttl:%s' % (key, str(ttl)))
for parents_item,Images in self.__traverse(self.resoure):
self.redis_o.lpush(self.redis_keys[u'index'], parents_item)
self.redis_o.hset(self.redis_keys[u'Images'], parents_item, Images)
self.redis_o.hset(self.redis_keys[u'mainImage'], parents_item, Images[0])
self.redis_o.set(self.redis_keys[u'main'], u'True')
use_time = str(time.clock())
saveLog.info(u'Save to redis done.use:%ss' % use_time)
for key in self.redis_keys.values():
if self.redis_o.exists(key):
self.redis_o.expire(key, int(expiretime))
saveLog.info(u'Set "%s", expire time:%s' % (key, str(expiretime)))
return True
else:
saveLog.error(u'"%s" is not exists' % self.resoure)
return False
def BuildConfig(self,md5):
sql = "SELECT sIP,iSSHport,sServer_Version_old,sServer_Version_new,GROUP_CONCAT(DISTINCT sSerFlag) sSerGroup,COUNT(distinct sSerFlag) sSerCount FROM dbtx_maintain_info WHERE iRecId = %d AND dDate = '%s' GROUP BY sIP ;" % (self.id, self.date)
Gamegroups = self.dbo.fetchall() if self.dbo.query(sql) else ()
if Gamegroups:
for group in Gamegroups:
config = self.config % (
str(group['sIP']),
str(group['sServer_Version_old']),
str(group['sServer_Version_new']),
md5,
str(group['sSerGroup']),
)
# 临时目录
tmpDir = os.path.join(TempDir,group['sIP'])
if not os.path.exists(tmpDir):
os.makedirs()
# 复制脚本到目录
for name,shellScript in Scripts.items():
shutil.copy(shellScript,os.path.join(tmpDir,name))
# 创建配置文件
with open(u"D:\\GitHub\\giter\\python\\Update\\tmp\\%s\\dbtx_maintain_config.py" % str(group['sIP']),
'wb+') as f:
f.write(config)
else:
saveLog.error('维护信息为空')
exit(3)
def getTornadoStatus(self):
## 检查Tornado状态, 0 正常, 1异常
tCmd = "sh /data/sh/dbtx_tornadoManage.sh status"
try:
p = subprocess.Popen(tCmd,shell=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
out, err = p.communicate()
except Exception,err:
saveLog.error("getTornadoStatus() error occured: %s" % str(err))
def getRedisStatus(self):
## 检查Redis状态, 0 正常, 1异常
rCmd = "/usr/local/bin/redis-cli -h localhost -p 6379 info"
try:
p = subprocess.Popen(rCmd,shell=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
out, err = p.communicate()
except Exception,err:
saveLog.error("getRedisStatus() error occured: %s" % str(err))
def getSlaveStatus(self):
try:
conn = MySQLdb.connect(host=dbTwoIp,port=db2Port,user=dbUser,passwd=dbPass,charset=dbChar)
cursor = conn.cursor(cursorclass = MySQLdb.cursors.DictCursor)
cursor.execute("show slave status;")
slaveStatus = cursor.fetchone()
self.data_for_post += "Slave_IO_Running=%s&Slave_SQL_Running=%s&" % (slaveStatus["Slave_IO_Running"], slaveStatus["Slave_SQL_Running"])
except Exception, err:
saveLog.error("getSlaveStatus() Error Occur : %s" % str(err))
def getRedisBaseInfo(redisStatus):
'''
获取redis运行状态, 传入redisStatus作为入参,字符类型
'''
try:
from dbtx.Merge.etc import telIP, ps1
redisInfo, redisData = {}, {}
redisStatus = redisStatus.split() ## 转为 列表类型
for kv in redisStatus:
if ":" in kv:
tmpList = kv.split(":")
if tmpList[0] == "used_memory":
redisInfo["used_memory"] = tmpList[1]
if tmpList[0] == "connected_clients":
redisInfo["connected_clients"] = tmpList[1]
if tmpList[0] == "keyspace_misses":
redisInfo["keyspace_misses"] = tmpList[1]
if tmpList[0] == "keyspace_hits":
redisInfo["keyspace_hits"] = tmpList[1]
if tmpList[0] == "total_commands_processed":
redisInfo["total_commands_processed"] = tmpList[1]
if tmpList[0] == "total_connections_received":
redisInfo["total_connections_received"] = tmpList[1]
redisData["used_memory"] = redisInfo["used_memory"]
redisData["connected_clients"] = redisInfo["connected_clients"]
totalHits = float(redisInfo["keyspace_hits"]) + float(redisInfo["keyspace_misses"])
if totalHits == 0:
redisData["hit_rate"] = 0
else:
redisData["hit_rate"] = float(redisInfo["keyspace_hits"]) / totalHits
redisData["total_commands"] = redisInfo["total_commands_processed"]
redisData["total_connects"] = redisInfo["total_connections_received"]
rZbx = DIRs['REDIS_DATA_FOR_ZBX']
if not os.path.exists(rZbx) or os.stat(rZbx).st_size <= 150:
last_commands = 0
last_connects = 0
else:
fObj = open(rZbx)
fc = fObj.readlines()
for line in fc:
if "total_commands" in line:
last_commands = line.split(":")[1].strip()
if "total_connects" in line:
last_connects = line.split(":")[1].strip()
fObj.close()
redisData["connects_rate"] = (int(redisData["total_connects"]) - int(last_connects)) / float(60)
redisData["command_rate"] = (int(redisData["total_commands"]) - int(last_commands)) / float(60)
fZbx = str(redisData).strip("{}").replace(",","\n").replace("'","") ## 转换成zabbix需要的格式
fObj = open(rZbx,'w')
fObj.write(fZbx)
fObj.close()
except Exception,err:
saveLog.error("getRedisBaseInfo(): %s" % str(err))
def getOnline(self):
try:
dbName = self.srvFlag + "_dbtx"
conn = MySQLdb.connect(host=dbTwoIp,port=db2Port,user=dbUser,passwd=dbPass,db=dbName,charset=dbChar)
cursor = conn.cursor(cursorclass = MySQLdb.cursors.DictCursor)
cursor.execute("select count(id) as iTotal from t_player_extend_2 where online=1 group by gate_channel_id order by iTotal;")
res, allOnline, eOnline = cursor.fetchall(), 0, []
for gate in res:
eOnline.append(str(gate['iTotal']))
allOnline += gate['iTotal']
except Exception, err:
saveLog.error("getOnline() Error Occur : %s" % str(err))
def ctCrt(self):
'''
创建CRT文件,并上传到SVN
'''
sAgent, sID, sIP = self.kwargs[1], self.kwargs[2], self.kwargs[0]
if isIP(sIP):
upCmd = "bash %s %s %s %s" % (DIRs['GEN_CRT_SCPT'], sAgent, sID, sIP)
if runCmd(upCmd) != 0: ## 如果命令返回值不为0,表示更新SVN失败
saveLog.error("%s %s %s, Gen CRT file and SVN up error." % (sAgent, sID, sIP))
return
else:
saveLog.info("%s %s %s, Gen CRT file and SVN up success." % (sAgent, sID, sIP))
else:
saveLog.error("invalid ip")
sys.exit(4)
def doPing(tIP, dbPort):
if dbPort == 3307:
mString = "备库-%s-3307端口不通" % tIP
else:
mString = "主库-%s-3306端口不通" % tIP
p = subprocess.Popen("ping -c7 %s" % tIP,shell=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
pResult = p.stdout.read()
lossRate = pResult.split("---")[-1].split(",")[2].split()[0] ## 取丢包率
if float(lossRate[:-1]) > 20:
sub = "DBTX LanIP Alarm: %s" % ps1
msg = "DateTime: %s\n%s, ICMP-Ping丢包率:%s%% !!!" % (getTimeNow(), mString, lossRate)
else:
sub = "DBTX LanIP-Port Alarm: %s" % ps1
msg = "DateTime: %s\n%s, ICMP-Ping正常,可能是MySQL挂了或者防火墙对应端口没有开启 !!!" % (getTimeNow(), mString)
saveLog.error(msg)
if sDb.read(telIP, "lanAlarm") == "ON": ## 状态为"NO"且开关为"ON"时,报故障
sendMail(sub, msg)
sDb.update(telIP, "lanAlarm", "OFF") ## 报警后,将开关置为"OFF"
def genInstCfg(self):
'''
配置文件需要包含的内容:
1. 主库,备库IP;
2. 域名;
3. 代理名称,服区ID;
4. 源服务器IP;
5. 游戏服务端目录名称
'''
theIp, dbOneIp, dbTwoIp, agent, server_id, domain, open_dt = \
self.kwargs[0], self.kwargs[4], self.kwargs[5], self.kwargs[1], self.kwargs[2], self.kwargs[3], self.kwargs[6]
dbo = ClassDb() ## 实例化数据库
dbo.connect(host=dbHost,port=dbPort,user=dbUser,passwd=dbPass,db=dbName,charset="utf8")
sql = "select sIP,sPlatServer from dbtx_plat_server where sPlatName='%s' and iStatus in (1,2) and replace(sPlatServer,'S','')<2000 order by replace(sPlatServer,'S','')+0 DESC limit 1;" % (agent)
dbo.execute(sql)
res = dbo.fetchone()
try:
targSid, file_source_server_ip = res['sPlatServer'], res['sIP']
except Exception,err:
saveLog.error("Get file_source_server_ip error occured, %s" % str(err))
def __exit(self,msg,code=1):
saveLog.error(str(msg))
saveLog.info('操作终止')
exit(int(code))
msg = "DateTime: %s\n主库-%s-3306端口 is up" % (getTimeNow(), dbOneIp)
saveLog.info(msg)
sendMail(sub, msg)
self.sDb.update(telIP, "db1Alarm", "ON")
## statusAnalyze("3306") ## 获取其它性能数据,zabbix成图或报警
else:
db1_status = 0
try:
if self.sDb.read(telIP, "db1Alarm") == "ON": ## 状态为"NO"且开关为"ON"时,报故障
sub = "DBTX DB1 mysql Alarm: %s" % ps1
msg = "DateTime: %s\n主库-%s-3306端口 DOWN 了,请查看日志 !!!" % (getTimeNow(), dbOneIp)
saveLog.warning(msg)
sendMail(sub, msg)
self.sDb.update(telIP, "db1Alarm", "OFF") ## 报警后,将开关置为"OFF"
except Exception, err:
saveLog.error("db1Alarm() Error Occur : %s" % str(err))
if db2Status:
db2_status = 1
if self.sDb.read(telIP, "db2Alarm") == "OFF": ## 状态OK且开关为"OFF"时,报恢复
sub = "DBTX DB2 mysql is OK: %s" % ps1
msg = "DateTime: %s\n备库-%s-3307端口 is up." % (getTimeNow(), dbTwoIp)
saveLog.info(msg)
sendMail(sub, msg)
self.sDb.update(telIP, "db2Alarm", "ON")
self.getSlaveStatus()
## statusAnalyze("3307") ## 获取其它性能数据,zabbix成图或报警
else:
db2_status = 0
self.data_for_post += "Slave_IO_Running=%s&Slave_SQL_Running=%s&" % ("null", "null")
try:
if self.sDb.read(telIP, "db2Alarm") == "ON": ## 状态为"NO"且开关为"ON"时,报故障
skipped_directory += 1
directory = u''
self.row = start_index + index # now redis index
try:
main_Image = self.redis_o.hget(name=self.redis_keys[u'mainImage'], key=directory)
main_Image = unicode(main_Image, encoding='utf8', errors='strict') # to utf8 unicode
except Exception,err:
saveLog.error(u'Get %s main Image faild' % directory)
saveLog.error(err)
main_Image = u'tornado_no_main_Image.jpg'
yield directory,main_Image
else:
saveLog.error(u'key %s is not exists' % self.redis_keys[u'index'])
yield None,None
elif mode == 2:
try:
Images = self.redis_o.hget(name=self.redis_keys[u'Images'], key=target_directory)
Images = list(eval(Images))
except Exception, err:
saveLog.error(u'Get Images from %s faild' % target_directory)
saveLog.error(err)
Images = []
if Images:
for Image in Images:
try:
Image = unicode(Image, encoding='utf8', errors='strict') # to utf8 unicode
except:
def getRaidCard(self):
try:
runCmd("sh /data/sh/monitor/Alive/shell/runRaidCli.sh")
except Exception,err:
saveLog.error("getRaidCard() run runRaidCli.sh error occured: %s" % str(err))
dbo.connect(host=dbHost,port=dbPort,user=dbUser,passwd=dbPass,db=dbName,charset="utf8")
sql = "select sIP,sPlatServer from dbtx_plat_server where sPlatName='%s' and iStatus in (1,2) and replace(sPlatServer,'S','')<2000 order by replace(sPlatServer,'S','')+0 DESC limit 1;" % (agent)
dbo.execute(sql)
res = dbo.fetchone()
try:
targSid, file_source_server_ip = res['sPlatServer'], res['sIP']
except Exception,err:
saveLog.error("Get file_source_server_ip error occured, %s" % str(err))
else:
sql = """select b.sServer_Version from dbtx_plat_server a,dbtx_host_info b
where a.iId=b.iPlatServerId and b.iStatus=1 and a.sPlatName='%s' and a.sPlatServer='%s';""" % (agent, targSid)
dbo.execute(sql)
try:
vRes = dbo.fetchone()['sServer_Version']
except Exception,err:
saveLog.error("Get sServer_Version error occured, %s" % str(err))
vRes = {'sServer_Version': '1.7.1.0_20150723_1'}
server_full_path = "/opt/dbtx/%s_%s/relServer_1.7.1.0_20150723_1/" % (agent, targSid)
else:
server_full_path = "/opt/dbtx/%s_%s/relServer_%s/" % (agent, targSid, vRes)
content = cfgDemo % (dbOneIp, dbTwoIp, agent, server_id, domain, file_source_server_ip, server_full_path, open_dt)
cfgDir = DIRs["IST_CFG_DIR"] + agent + "_" + server_id
if not os.path.exists(cfgDir):
os.makedirs(cfgDir)
shutil.copy(DIRs["IST_SH_DIR"]+"dbtx_install_game.sh", cfgDir)
cfgFile = cfgDir + "/dbtx_install_config.sh"
wToCfg(cfgFile, content)
saveLog.info("Gen install config file success.")
def sigRmAct(self, fPath): ## 删除单个文件/目录
ap = os.path.abspath(fPath)
splPath = ap.split("/")
if os.path.ismount(fPath): ## 判断是否是挂载点,挂载点严禁删除
saveLog.error("This path is a mount point,can't rm !!!")
sys.exit(1)
elif os.path.isdir(fPath): ## 如果是目录
if splPath[1] == "" or splPath[1] == "n*":
saveLog.error("You may deleting root-dir,can't rm !!!")
sys.exit(2)
elif splPath[1] in self.sysDirList:
saveLog.error("This path is in system-dir-lists, can't rm !!!")
sys.exit(2)
elif splPath[1] in self.riskDirList:
saveLog.warning("This path is in risk_dir_lists, move to %s... " % self.tmpDir)
refPath = ap.replace("/","-") ## 重命名,将绝对路径中的"/"替换成"-",以保存完整路径
oTime = str(datetime.datetime.now()).replace(" ","_")
shutil.move(ap, self.tmpDir+"/"+refPath+"_"+oTime)
else:
saveLog.info("Deleting %s... " % ap)
shutil.rmtree(ap)
elif os.path.isfile(fPath): ## 如果是文件
if splPath[1] in self.riskDirList:
if os.path.basename(ap) in self.riskFileList:
saveLog.error("Important document(file),can't rm !!!")
sys.exit(3)
saveLog.warning("This path(file) is in risk_dir_lists, move to %s..." % self.tmpDir)
refPath = ap.replace("/","-") ## 重命名,将绝对路径中的"/"替换成"-",以保存完整路径
oTime = str(datetime.datetime.now()).replace(" ","_")
shutil.move(ap, self.tmpDir+"/"+refPath+"_"+oTime)
elif splPath[1] in self.sysDirList:
saveLog.error("This path is in system-dir-lists, can't rm !!!")
sys.exit(2)
else:
saveLog.info("Deleting %s..." % ap)
os.remove(ap)
elif os.path.islink(fPath): ## 是链接的话,先找出它的真实路径
realPath = os.path.realpath(fPath)
splPath = realPath.split("/")
if splPath[1] in self.riskDirList:
if os.path.basename(ap) in self.riskFileList:
saveLog.error("Important document(link),can't rm !!!")
sys.exit(3)
saveLog.warning("This path(link) is in risk_dir_lists, move to %s..." % self.tmpDir)
refPath = ap.replace("/","-") ## 重命名,将绝对路径中的"/"替换成"-",以保存完整路径
oTime = str(datetime.datetime.now()).replace(" ","_")
shutil.move(ap, self.tmpDir+"/"+refPath+"_"+oTime)
elif splPath[1] in self.sysDirList:
saveLog.error("This path is in system-dir-lists, can't rm !!!")
sys.exit(2)
else:
saveLog.info("Deleting %s..." % ap)
os.remove(ap)
else:
pass
if crossSwStat == "ON": ## 状态为"NO"且开关为"ON"时,报故障
sub = "DBTX CrossConnect Alarm: %s" % ps1
msg = "Datetime: %s\n%s 跨服: %s 至少有一个连接异常" % (getTimeNow(), self.srvFlag, str(",".join([pkDomain, fbDomain])))
#crsMuser = "******"
saveLog.warning(msg)
#sendMail(sub, msg, crsMuser)
self.sDb.update(self.srvFlag, "crossAlarm", "OFF")
except Exception, err:
saveLog.warning("Cross connection is 1,get switch status error occured. %s" % str(err))
else:
crossStatus = 0
if crossSwStat == "OFF":
self.sDb.update(self.srvFlag, "crossAlarm", "ON")
## else:
## saveLog.error("t_cross_copy_channel() Error Occur : %s" % str(res))
## saveLog.error("May be AppServices is not up or t_cross_copy_channel table is not configed.")
## crossStatus = 0 ## 大多数情况,从数据库拿不到数据是因为游戏未开启,所以设为"正常"
except Exception, err:
saveLog.error("CrossInfo() Error Occur : %s" % str(err))
self.data_for_post += "crossStatus=%s&crsConInfo=%s&" % (crossStatus, str(",".join([pkDomain, fbDomain])))
def runCollMul(self):
global ps1, dbTwoIp, dbOneIp
self.getAppStatus()
self.getShuntStatus()
self.getOnline()
## return self.data_for_post
opUrl = "http://121.10.118.218/op_center/dbtx/dbtx_host_info_interface.php"
postToOp(opUrl, self.data_for_post)
