def inRedis(self, expiretime=600):
if os.path.exists(self.resoure):
time.clock()
saveLog.info(u'Begin init redis')
saveLog.info(u'Resoure Directory:"%s"' % self.resoure)
for key in self.redis_keys.values():
if self.redis_o.exists(key):
ttl = self.redis_o.ttl(key)
self.redis_o.delete(key)
saveLog.info(u'Delete "%s", ttl:%s' % (key, str(ttl)))
for parents_item,Images in self.__traverse(self.resoure):
self.redis_o.lpush(self.redis_keys[u'index'], parents_item)
self.redis_o.hset(self.redis_keys[u'Images'], parents_item, Images)
self.redis_o.hset(self.redis_keys[u'mainImage'], parents_item, Images[0])
self.redis_o.set(self.redis_keys[u'main'], u'True')
use_time = str(time.clock())
saveLog.info(u'Save to redis done.use:%ss' % use_time)
for key in self.redis_keys.values():
if self.redis_o.exists(key):
self.redis_o.expire(key, int(expiretime))
saveLog.info(u'Set "%s", expire time:%s' % (key, str(expiretime)))
return True
else:
saveLog.error(u'"%s" is not exists' % self.resoure)
return False
def pingLan():
if lanIP == dbOneIp and lanIP != dbTwoIp:
portOK = chkPort(dbTwoIp, int(3307))
if not portOK:
doPing(dbTwoIp, int(3307))
else:
if sDb.read(telIP, "lanAlarm") == "OFF": ## 状态OK且开关为"OFF"时,报恢复
sub = "DBTX LanIP is OK: %s" % ps1
msg = "DateTime: %s\n 备库: %s, 网络与端口恢复正常." % (getTimeNow(), dbTwoIp)
saveLog.info(msg)
sendMail(sub, msg)
sDb.update(telIP, "lanAlarm", "ON")
elif lanIP != dbOneIp and lanIP == dbTwoIp:
portOK = chkPort(dbOneIp, int(3306))
if not portOK:
doPing(dbOneIp, int(3306))
else:
if sDb.read(telIP, "lanAlarm") == "OFF":
sub = "DBTX LanIP is OK: %s" % ps1
msg = "DateTime: %s\n 备库: %s, 网络与端口恢复正常." % (getTimeNow(), dbTwoIp)
saveLog.info(msg)
sendMail(sub, msg)
sDb.update(telIP, "lanAlarm", "ON")
else:
pass
def getDisk(self):
mountPoints, diskTotal = [], 0 #diskTotal累加
parts = psutil.disk_partitions() #分区信息
for p in parts:
mp = str(p).split(",")[1].split("=")[1]
mountPoints.append(mp)
for x in mountPoints:
pTotal = psutil.disk_usage(x.strip("'")).total #单个分区容量
diskTotal += pTotal
diskTotal = changeUnit(int(diskTotal))
rootSize, rootRate = psutil.disk_usage('/').free, psutil.disk_usage('/').percent #获取/分区的剩余空间和使用率
writeToFile("totalDisk", diskTotal)
writeToFile("rootSize", changeUnit(int(rootSize)))
writeToFile("rootRate", rootRate)
if float(rootRate) > float(85):
"""ON开关，防止多次重复报警；报警一次之后设为关闭"""
if self.sDb.read(self.telIP, "rootAlarm") == "ON": ## 状态为"NO"且开关为"ON"时,报故障
sub = "DBTX ROOT partition Alarm:%s" % self.ps1
msg = "DateTime: %s\n根分区已使用%s%%, 超过85%%, 请警惕!!!" % (getTimeNow(), rootRate)
saveLog.warning(msg)
sendMail(sub, msg)
self.sDb.update(self.telIP, "rootAlarm", "OFF") ## 报警后,将开关置为"OFF"
else:
if self.sDb.read(self.telIP, "rootAlarm") == "OFF": ## 状态OK且开关为"OFF"时,报恢复
sub = "DBTX ROOT partition is OK:%s" % self.ps1
msg = "DateTime: %s\n根分区可用空间充足%s." % (getTimeNow(), changeUnit(int(rootSize)))
saveLog.info(msg)
sendMail(sub, msg)
self.sDb.update(self.telIP, "rootAlarm", "ON") ## 使用率不足85%时,将开关置为"ON"
def ctCrt(self):
'''
创建CRT文件,并上传到SVN
'''
sAgent, sID, sIP = self.kwargs[1], self.kwargs[2], self.kwargs[0]
if isIP(sIP):
upCmd = "bash %s %s %s %s" % (DIRs['GEN_CRT_SCPT'], sAgent, sID, sIP)
if runCmd(upCmd) != 0: ## 如果命令返回值不为0,表示更新SVN失败
saveLog.error("%s %s %s, Gen CRT file and SVN up error." % (sAgent, sID, sIP))
return
else:
saveLog.info("%s %s %s, Gen CRT file and SVN up success." % (sAgent, sID, sIP))
else:
saveLog.error("invalid ip")
sys.exit(4)
def sigRmAct(self, fPath): ## 删除单个文件/目录
ap = os.path.abspath(fPath)
splPath = ap.split("/")
if os.path.ismount(fPath): ## 判断是否是挂载点,挂载点严禁删除
saveLog.error("This path is a mount point,can't rm !!!")
sys.exit(1)
elif os.path.isdir(fPath): ## 如果是目录
if splPath[1] == "" or splPath[1] == "n*":
saveLog.error("You may deleting root-dir,can't rm !!!")
sys.exit(2)
elif splPath[1] in self.sysDirList:
saveLog.error("This path is in system-dir-lists, can't rm !!!")
sys.exit(2)
elif splPath[1] in self.riskDirList:
saveLog.warning("This path is in risk_dir_lists, move to %s... " % self.tmpDir)
refPath = ap.replace("/","-") ## 重命名,将绝对路径中的"/"替换成"-",以保存完整路径
oTime = str(datetime.datetime.now()).replace(" ","_")
shutil.move(ap, self.tmpDir+"/"+refPath+"_"+oTime)
else:
saveLog.info("Deleting %s... " % ap)
shutil.rmtree(ap)
elif os.path.isfile(fPath): ## 如果是文件
if splPath[1] in self.riskDirList:
if os.path.basename(ap) in self.riskFileList:
saveLog.error("Important document(file),can't rm !!!")
sys.exit(3)
saveLog.warning("This path(file) is in risk_dir_lists, move to %s..." % self.tmpDir)
refPath = ap.replace("/","-") ## 重命名,将绝对路径中的"/"替换成"-",以保存完整路径
oTime = str(datetime.datetime.now()).replace(" ","_")
shutil.move(ap, self.tmpDir+"/"+refPath+"_"+oTime)
elif splPath[1] in self.sysDirList:
saveLog.error("This path is in system-dir-lists, can't rm !!!")
sys.exit(2)
else:
saveLog.info("Deleting %s..." % ap)
os.remove(ap)
elif os.path.islink(fPath): ## 是链接的话,先找出它的真实路径
realPath = os.path.realpath(fPath)
splPath = realPath.split("/")
if splPath[1] in self.riskDirList:
if os.path.basename(ap) in self.riskFileList:
saveLog.error("Important document(link),can't rm !!!")
sys.exit(3)
saveLog.warning("This path(link) is in risk_dir_lists, move to %s..." % self.tmpDir)
refPath = ap.replace("/","-") ## 重命名,将绝对路径中的"/"替换成"-",以保存完整路径
oTime = str(datetime.datetime.now()).replace(" ","_")
shutil.move(ap, self.tmpDir+"/"+refPath+"_"+oTime)
elif splPath[1] in self.sysDirList:
saveLog.error("This path is in system-dir-lists, can't rm !!!")
sys.exit(2)
else:
saveLog.info("Deleting %s..." % ap)
os.remove(ap)
else:
pass
def getNginxStatus(self):
try:
if os.path.exists("/usr/local/nginx/logs/nginx.pid"):
fObj = open("/usr/local/nginx/logs/nginx.pid")
ngxMainPid = fObj.read().strip()
fObj.close()
else:
ngxMainPid = 0
pids = getProcessID("nginx")
if ngxMainPid:
if not pids or len(pids) < 8:
wp = ""
res = os.popen("ps -C nginx -o pid,cmd").readlines()
for l in res:
wp += "\n" + l.strip()
if self.sDb.read(telIP, "ngxAlarm") == "ON": ## 状态为"NO"且开关为"ON"时,报故障
sub = "DBTX Nginx Alarm: %s" % ps1
msg = "Datetime: %s\nNginx主进程PID:%s\nWorker_processes:%s\n少于10个!!" % (getTimeNow(), ngxMainPid, wp)
saveLog.warning(msg)
sendMail(sub, msg)
self.sDb.update(telIP, "ngxAlarm", "OFF")
else:
if self.sDb.read(telIP, "ngxAlarm") == "OFF": ## 状态OK且开关为"OFF"时,报恢复
sub = "DBTX Nginx is OK: %s" % ps1
msg = "Datetime: %s\nNginx主进程恢复开启状态." % (getTimeNow())
saveLog.info(msg)
sendMail(sub, msg)
self.sDb.update(telIP, "ngxAlarm", "ON")
else:
if self.sDb.read(telIP, "ngxAlarm") == "ON": ## 状态为"NO"且开关为"ON"时,报故障
sub = "DBTX Nginx Alarm: %s" % ps1
msg = "Datetime: %s\nNginx主进程挂了!!" % (getTimeNow())
saveLog.warning(msg)
sendMail(sub, msg)
self.sDb.update(telIP, "ngxAlarm", "OFF")
except Exception,err:
saveLog.warning("getNginxStatus, %s" % str(err))
dbo.connect(host=dbHost,port=dbPort,user=dbUser,passwd=dbPass,db=dbName,charset="utf8")
sql = "select sIP,sPlatServer from dbtx_plat_server where sPlatName='%s' and iStatus in (1,2) and replace(sPlatServer,'S','')<2000 order by replace(sPlatServer,'S','')+0 DESC limit 1;" % (agent)
dbo.execute(sql)
res = dbo.fetchone()
try:
targSid, file_source_server_ip = res['sPlatServer'], res['sIP']
except Exception,err:
saveLog.error("Get file_source_server_ip error occured, %s" % str(err))
else:
sql = """select b.sServer_Version from dbtx_plat_server a,dbtx_host_info b
where a.iId=b.iPlatServerId and b.iStatus=1 and a.sPlatName='%s' and a.sPlatServer='%s';""" % (agent, targSid)
dbo.execute(sql)
try:
vRes = dbo.fetchone()['sServer_Version']
except Exception,err:
saveLog.error("Get sServer_Version error occured, %s" % str(err))
vRes = {'sServer_Version': '1.7.1.0_20150723_1'}
server_full_path = "/opt/dbtx/%s_%s/relServer_1.7.1.0_20150723_1/" % (agent, targSid)
else:
server_full_path = "/opt/dbtx/%s_%s/relServer_%s/" % (agent, targSid, vRes)
content = cfgDemo % (dbOneIp, dbTwoIp, agent, server_id, domain, file_source_server_ip, server_full_path, open_dt)
cfgDir = DIRs["IST_CFG_DIR"] + agent + "_" + server_id
if not os.path.exists(cfgDir):
os.makedirs(cfgDir)
shutil.copy(DIRs["IST_SH_DIR"]+"dbtx_install_game.sh", cfgDir)
cfgFile = cfgDir + "/dbtx_install_config.sh"
wToCfg(cfgFile, content)
saveLog.info("Gen install config file success.")
## 分析数据,必要时报警: 1, 使用内存大于1G; 2, 命中率低于20%
if int(redisData["used_memory"]) > 1073741824:
sDb = opRedis() ## 实例化 Redis库操作类
if sDb.read(telIP, "rRateAlarm") == "ON":
sub = "DBTX Redis Memory Alarm: %s" % ps1
msg = "DateTime: %s\nRedis使用内存为: %s, 超过1G,请警惕 !!!" % (getTimeNow(), changeUnit(int(redisData["used_memory"])))
saveLog.warning(msg)
sendMail(sub, msg)
sDb.update(telIP, "rRateAlarm", "OFF") ## 报警后,将开关置为"OFF"
else:
sDb = opRedis() ## 实例化 Redis库操作类
if sDb.read(telIP, "rRateAlarm") == "OFF":
sub = "DBTX Redis Memory is OK: %s" % ps1
msg = "DateTime: %s\nRedis使用内存情况正常: %s." % (getTimeNow(), changeUnit(int(redisData["used_memory"])))
saveLog.info(msg)
sendMail(sub, msg)
sDb.update(telIP, "rRateAlarm", "ON")
## if totalHits != 0 and redisData["hit_rate"] < 20:
## sDb = opShelveDb() ## 实例化 shelve库操作类
## if sDb.readDb(pubDb, telIP, "rRateAlarm") == "ON":
## sub = "DBTX Redis HitRate Alarm: %s" % ps1
## msg = "DateTime: %s\nRedis 命中率为: %s%%, 低于20%%,请警惕 !!!" % (getTimeNow(), redisData["hit_rate"])
## saveLog.warning(msg)
#### sendMail(sub, msg)
## sDb.updateDb(pubDb, telIP, "rRateAlarm", "OFF") ## 报警后,将开关置为"OFF"
## else:
## sDb = opShelveDb() ## 实例化 shelve库操作类
## if sDb.readDb(pubDb, telIP, "rRateAlarm") == "OFF":
## sub = "DBTX Redis HitRate is OK: %s" % ps1
def __exit(self,msg,code=1):
saveLog.error(str(msg))
saveLog.info('操作终止')
exit(int(code))
# -*- coding: utf-8 -*-
import saveLog
import os
saveLog.set_logger(filename=u"D:\\GitHub\\giter\\python\\Update\\dbtx_Update.log" ) ## 定义日志文件
saveLog.info("init maintain....")
class MaintainInit:
def __init__(self,date,dbo,iLanguages,andsql,steps):
self.date = date # 日期
self.dbo = dbo # 数据库连接类
self.iLanguage_locked = 1
self.iLanguages = iLanguages
self.andSql = andsql
self.steps = steps
# 从dbtx_maintain_plan表获取今天维护计划
def __TodayMaintainPlans(self):
sql = 'SELECT iId,iLanguage,sVersion,sTargetVersion,DATE(dCreateTime) AS iDate,TIME(dCreateTime) AS iTime,sContents,sDesc FROM dbtx_maintain_plan WHERE iStatus = 1 AND iLanguage = %d AND DATE(dCreateTime) = "%s";' % (int(self.iLanguage_locked),self.date)
maintain_plan = self.dbo.fetchone() if self.dbo.query(sql) else ()
if maintain_plan:
self.maintain_plan = maintain_plan
return True
else:
self.__exit('没有维护计划',2)
return False
# -*- coding: utf-8 -*-
import saveLog
import os,shutil
from Update.etc.config import TempDir,Scripts
saveLog.set_logger(filename=u"D:\\GitHub\\giter\\python\\Update\\dbtx_Update.log" ) ## 定义日志文件
saveLog.info("create config for maintain....")
class MaintainConfig:
def __init__(self,dbo,date,config,id):
self.dbo = dbo
self.date = date
self.config = config
self.id = id
def BuildConfig(self,md5):
sql = "SELECT sIP,iSSHport,sServer_Version_old,sServer_Version_new,GROUP_CONCAT(DISTINCT sSerFlag) sSerGroup,COUNT(distinct sSerFlag) sSerCount FROM dbtx_maintain_info WHERE iRecId = %d AND dDate = '%s' GROUP BY sIP ;" % (self.id, self.date)
Gamegroups = self.dbo.fetchall() if self.dbo.query(sql) else ()
if Gamegroups:
for group in Gamegroups:
config = self.config % (
str(group['sIP']),
str(group['sServer_Version_old']),
str(group['sServer_Version_new']),
md5,
def genMergeCfg(self):
'''
配置文件需要包含的内容:
1. 本机服区ID;
2. 目标合服服区ID;
3. 目标合服机外网IP;
4. 所有参与合服服区ID(用逗号隔开)
'''
dbo = ClassDb() ## 实例化数据库
dbo.connect(host=dbHost,port=dbPort,user=dbUser,passwd=dbPass,db=dbName,charset="utf8")
sPlatName, sSIDRange, sTargetPlat, sTargetSID, iRecId =\
self.kwargs[0], self.kwargs[1], self.kwargs[2], self.kwargs[3], self.kwargs[4]
sIdLists = sSIDRange.split(",")
## 获取合服目标IP
dstMergeSrvFlag = sTargetPlat + "_" + sTargetSID
tSql = "select sIP from dbtx_plat_server where sPlatName='%s' and sPlatServer='%s';" % (sTargetPlat, sTargetSID)
dbo.execute(tSql)
dstMergeSrvIp = dbo.fetchone()['sIP']
## joinListDict = {}
for srv in sIdLists:
srcMergeSrvFlag = sPlatName + "_" + srv
joinMergeSrvs = ','.join([sPlatName + "_" + x for x in sIdLists])
## srcMergeSrvFlag, dstMergeSrvFlag, joinMergeSrvs = srv, sTargetSID, sSIDRange
## thisPlatName, thisSid = srv.split("_")[0], srv.split("_")[1]
## 获取各个参与合服的机器IP
iSql = "select sIP from dbtx_plat_server where sPlatName='%s' and sPlatServer='%s';" % (sPlatName, srv)
dbo.execute(iSql)
tSrvIp = dbo.fetchone()['sIP']
## ## 将 参与合服的服区ID与IP生成一个字典,后续删服区数据时使用
## if srcMergeSrvFlag != dstMergeSrvFlag:
## joinListDict[srcMergeSrvFlag] = tSrvIp
cfg_content = cfgDemo % (srcMergeSrvFlag, dstMergeSrvFlag, dstMergeSrvIp, joinMergeSrvs)
cfgDir = DIRs["MERGE_CFG_DIR"] + srcMergeSrvFlag
if not os.path.exists(cfgDir): ## 创建配置文件目录
os.makedirs(cfgDir)
shutil.copy(DIRs["MERGE_SH_DIR"]+"dbtx_merge_backup.sh", cfgDir)
shutil.copy(DIRs["MERGE_SH_DIR"]+"dbtx_merge_curl.sh", cfgDir)
shutil.copy(DIRs["MERGE_SH_DIR"]+"dbtx_merge_hxdata.sh", cfgDir)
shutil.copy(DIRs["MERGE_SH_DIR"]+"dbtx_merge_getdata.sh", cfgDir)
shutil.copy(DIRs["MERGE_SH_DIR"]+"dbtx_merge_putdata.sh", cfgDir)
shutil.copy(DIRs["MERGE_SH_DIR"]+"dbtx_merge.sql", cfgDir)
shutil.copy(DIRs["MERGE_SH_DIR"]+"dbtx_merge_fanti.sql", cfgDir)
shutil.copy(DIRs["MERGE_SH_DIR"]+"dbtx_merge_yuenan.sql", cfgDir)
shutil.copy(DIRs["MERGE_SH_DIR"]+"dbtx_merge_import.sh", cfgDir)
shutil.copy(DIRs["MERGE_SH_DIR"]+"dbtx_merge_php.sh", cfgDir)
shutil.copy(DIRs["MERGE_SH_DIR"]+"dbtx_merge_active.sh", cfgDir)
cfgFile = cfgDir + "/dbtx_merge_config.sh"
## 内容写入到配置文件
wToCfg(cfgFile, cfg_content)
saveLog.info("Gen merge config file success.")
##
## listFile = DIRs["LOG_DIR"] + "%s_joinServers.py" % getToday()
## wToCfg(listFile, str(joinListDict))
# 插入到 dbtx_merge_opLog 表
stepNums = 9 if srcMergeSrvFlag == dstMergeSrvFlag else 6
for st in range(stepNums):
sSql = "INSERT INTO dbtx_merge_opLog (iRecId,sPlatName,sPlatServer,iStep,iStatus,sComment,tTime) VALUES \
('%s', '%s', '%s', %s, 0, '', '%s');" % (int(iRecId), sPlatName, srv, st, getTimeNow())
dbo.execute(sSql)
## 通过机器人在远程合服IP上创建目录,存放合服脚本
cCmd = "mkdir -p /root/hefu/%s" % getToday()
cDesc, cRemarks, cdcTime = "mkdir today hefu_script_dir.", "mkdir today hefu_script_dir.", getTimeNow()
mkSql = "insert into dbtx_cmd_execute (sCmd,iDependID,eType,sIP,sSSHport,iStatus,sDescription,sRemarks,dCreateTime,sUser,iTimeout) values (\
'%s','0','remote','%s','22','1','%s','%s','%s','autoMerge','600');" % (cCmd, tSrvIp, cDesc, cRemarks, cdcTime)
dbo.execute(mkSql)
## 插入语句,机器人将合服脚本scp推送到远程服务器上
#tCmd = "%s /root/hefu/%s/%s" % (cfgDir, getToday(), srcMergeSrvFlag)
tCmd = "%s /root/hefu/%s" % (cfgDir, getToday())
tDependId = int(dbo.getLastId()) ## 获取上一条"创建目录"SQL的主键ID作为"SCP"SQL的依赖ID,即必须等创建目录完成才能执行SCP
tDesc, tRemarks, dcTime = "Send %s dbtx_merge_config.sh" % srcMergeSrvFlag, "Send %s dbtx_merge_config.sh" % srcMergeSrvFlag, getTimeNow()
dSql = "insert into dbtx_cmd_execute (sCmd,iDependID,eType,sIP,sSSHport,iStatus,sDescription,sRemarks,dCreateTime,sUser,iTimeout) values (\
'%s','%d','scp','%s','22','1','%s','%s','%s','autoMerge','600');" % (tCmd, tDependId, tSrvIp, tDesc, tRemarks, dcTime)
dbo.execute(dSql)
## 插入语句,机器人在远程服务器上执行合服脚本
#tCmd = "%s /root/hefu/%s/%s" % (cfgDir, getToday(), srcMergeSrvFlag)
mCmd = "/usr/bin/python /data/sh/Python/Automatic/mergeCtrStep.py %s %s %s > /dev/null 2>&1 &" % (iRecId, sPlatName, srv)
mDependId = int(dbo.getLastId()) ## 获取上一条"推送配置文件"SQL的主键ID作为"执行合服脚本"SQL的依赖ID,即必须等推送配置文件完成才能执行合服脚本
mDesc, mRemarks, mcTime = "Run %s mergeCtrStep.py" % srcMergeSrvFlag, "Run %s mergeCtrStep.py" % srcMergeSrvFlag, getTimeNow()
mSql = "insert into dbtx_cmd_execute (sCmd,iDependID,eType,sIP,sSSHport,iStatus,sDescription,sRemarks,dCreateTime,sUser,iTimeout) values (\
'%s','%d','remote','%s','22','1','%s','%s','%s','autoMerge','600');" % (mCmd, mDependId, tSrvIp, mDesc, mRemarks, mcTime)
dbo.execute(mSql)
