def check(self):
"""
检测进程是否存在，检测方法：
1.检测pid文件是否存在，如果不存在，则说明进程没有运行，新进程可以运行
2.如果pid文件存在，则读取pid文件中的pid，在所有进程中查找文件中的pid的进程是否存在，
如果进程存在，则新的进程不能运行，直接退出，如果进程不存在，说明可能是由于非正常退出(如：CTRL + C)
造成pid文件未被清理，这种情况下，agent可以启动，先删除失效的pid文件，然后启动新的进程即可。
"""
if not os.path.isfile(self.__pidFile__):
return
pid = 0
try:
file = open(self.__pidFile__, 'rt')
data = file.read()
file.close()
# 获取文件中的pid进程号
pid = int(data)
if not self.isAlive(pid):
try:
os.remove(self.__pidFile__)
return
except:
AgentLog.warning("remove pid file {0} failed.".format(
self.__pidFile__))
except:
pass
AgentLog.info("agent exist, only one process allow running")
sys.exit(1)
def __init__(self,
name_="RabbitMQSender",
host_="localhost",
port_=5672,
exchange_='exchange',
routing_key_='key'):
RabbitMQWrapper.__init__(self, name_, host_, port_, exchange_, '',
routing_key_)
AgentLog.info(
"create sender %s {host:%s, port:%d, exchange:%s, routing_key:%s}"
% (name_, host_, port_, exchange_, routing_key_))
def __init__(self, attr):
AgentLog.info('Start to init Mysql from config')
Database.__init__(self, attr)
try:
self.installPath = attr['home']
self.configFile = attr['config']
self.backupDir = attr['backupdir']
self.socket = self.getSocket()
self.dataDir = self.getDataDir()
self.binlogName = self.getBinlogName()
except (KeyError, AgentAtrrException), e:
raise AgentAtrrException('Mysql init KeyError:%s' % e)
def stop(self):
"""
程序退出时调用，在进程退出时，删除pid文件
"""
AgentLog.info("RDS Agent start to exit")
try:
os.remove(self.__pidFile__)
except:
AgentLog.warning("remove pid file {0} error".format(
self.__pidFile__))
raise AgentException("remove pid file {0} error".format(
self.__pidFile__))
def start(self):
"""
把进程pid写入到对应的pid文件
"""
AgentLog.info("RDS Agent start to run")
try:
file = open(self.__pidFile__, 'wt')
file.write(str(os.getpid()))
file.close()
except:
AgentLog.error("open pid file {0} error, start failed".format(
self.__pidFile__))
raise AgentException("open pid file {0} failed".format(
self.__pidFile__))
def __init__(self, name_='RabbitMQReceiver', host_='localhost', port_=5672, exchange_='exchange', queue_='',
routing_key_='key'):
RabbitMQWrapper.__init__(self, name_, host_, port_, exchange_, queue_, routing_key_)
self.exclusive = False
if queue_ == "":
queue_ = self.channel.queue_declare(exclusive=True).method.queue
self.queue = queue_
self.exclusive = True
else:
self.channel.queue_declare(queue=queue_)
self.channel.queue_bind(exchange=exchange_, queue=queue_, routing_key=routing_key_)
self.doWork = None
AgentLog.info("create receiver %s {host:%s, port:%d, exchange:%s, routing_key:%s}" % (
name_, host_, port_, exchange_, routing_key_))
def __init__(self, threadName, activeReportInterval=5.0, loopInterval=60):
threading.Thread.__init__(self, name=threadName)
self.isRunning = True
# 线程循环间隔时间
self.loopInterval = loopInterval
# 线程每隔一段时间需要上报一次当前状态，activeReportInterval为上报间隔
self.activeReportInterval = activeReportInterval
self.lastReportTime = 0
# 多数线程为循环线程，需要每个一定时间运行一次，通过event.wait()进行等待，
# 采用event.wait而不是sleep的好处是，在线程退出时可以直接通过set命令取消等待，
# 不需要等待sleep结束
self.event = threading.Event()
self.event.clear()
AgentLog.info("thread [{0}] created, report interval: {1}, loop interval: {2}".format( \
self.getName(), activeReportInterval, loopInterval))
def upload_ftp(self, params, srcDir):
remoteHost = params.get('remoteHost', None)
remotePort = params.get('remotePort', None)
remoteUser = params.get('remoteUser', None)
remotePasswd = params.get('remotePassword', None)
backupPath = params.get('backupPath', None)
uploadLimit = params.get('uploadLimit', None)
if remoteHost == None or remotePort == None or remoteUser == None or \
remotePasswd == None or backupPath == None or uploadLimit == None:
return False, 'remote host information errors'
AgentLog.info("start upload backup data to ftp server")
try:
ftp = FTP()
ftp.connect(remoteHost, int(remotePort), 60)
except Exception, e:
return False, 'can not connect to remote host: %s with port: %s, error: %s' % (
remoteHost, remotePort, e)
def createMonitors(self, context, dbName, monitorList):
if isinstance(monitorList, list) is not True:
raise AgentAtrrException(
'MonitorManager.createMonitors invalid monitorList')
AgentLog.info('start to create monitors :%s' % (','.join(monitorList)))
try:
monitorInstances = {}
base = 'from imagent.' + dbName
for monitor in monitorList:
url = base + '.monitor.' + monitor + ' import ' + monitor
exec url
monitorInstances[monitor] = eval(monitor)(context)
return monitorInstances
except AttributeError, e:
raise AgentAtrrException(
'MonitorManager.createMonitors can not get monitor :%s' % e)
def run(self):
AgentLog.info('thread {0} start running'.format(self.getName()))
while self.isRunning:
try:
self.doWork()
if self.lastReportTime == 0:
self.lastReportTime = int(time.time())
# 获取当前时间
currentTime = int(time.time())
# 比较是否已经超时，如果超时，则在日志文件中打印线程alive信息
if currentTime >= self.lastReportTime + self.activeReportInterval:
AgentLog.info(
' thread:[{0}] is alive '.format(self.getName()).ljust(45, ' ').center(75, '#'))
self.lastReportTime = currentTime
except Exception, e:
AgentLog.error(
"thread [{0}] raise Exception: {1}".format(self.getName(), traceback.format_exc()))
# 等待下一次运行
self.event.wait(self.loopInterval)
def upload_scp(self, params, srcDir):
remoteHost = params.get('remoteHost', None)
remotePort = params.get('remotePort', None)
remoteUser = params.get('remoteUser', None)
remotePasswd = params.get('remotePassword', None)
backupPath = params.get('backupPath', None)
uploadLimit = params.get('uploadLimit', None)
if remoteHost == None or remotePort == None or remoteUser == None or \
remotePasswd == None or backupPath == None or uploadLimit == None:
return False, 'remote host information errors'
uploadLimit = long(uploadLimit)
AgentLog.info("start upload backup data to remote server")
cmd = "scp -r -P %s %s %s@%s:%s" % (remotePort, srcDir, remoteUser,
remoteHost, backupPath)
if uploadLimit > 0:
cmd = "scp -r -P %s -l %d %s %s@%s:%s" % (remotePort, uploadLimit *
8, srcDir, remoteUser,
remoteHost, backupPath)
lines = Util.popen(cmd).readlines()
if len(lines) != 0:
return False, ' '.join(lines)
return True, ''
class BaseThread(threading.Thread):
def __init__(self, threadName, activeReportInterval=5.0, loopInterval=60):
threading.Thread.__init__(self, name=threadName)
self.isRunning = True
# 线程循环间隔时间
self.loopInterval = loopInterval
# 线程每隔一段时间需要上报一次当前状态，activeReportInterval为上报间隔
self.activeReportInterval = activeReportInterval
self.lastReportTime = 0
# 多数线程为循环线程，需要每个一定时间运行一次，通过event.wait()进行等待，
# 采用event.wait而不是sleep的好处是，在线程退出时可以直接通过set命令取消等待，
# 不需要等待sleep结束
self.event = threading.Event()
self.event.clear()
AgentLog.info("thread [{0}] created, report interval: {1}, loop interval: {2}".format( \
self.getName(), activeReportInterval, loopInterval))
def run(self):
AgentLog.info('thread {0} start running'.format(self.getName()))
while self.isRunning:
try:
self.doWork()
if self.lastReportTime == 0:
self.lastReportTime = int(time.time())
# 获取当前时间
currentTime = int(time.time())
# 比较是否已经超时，如果超时，则在日志文件中打印线程alive信息
if currentTime >= self.lastReportTime + self.activeReportInterval:
AgentLog.info(
' thread:[{0}] is alive '.format(self.getName()).ljust(45, ' ').center(75, '#'))
self.lastReportTime = currentTime
except Exception, e:
AgentLog.error(
"thread [{0}] raise Exception: {1}".format(self.getName(), traceback.format_exc()))
# 等待下一次运行
self.event.wait(self.loopInterval)
AgentLog.info('thread:{0} exit'.format(self.getName()))
def stop(self):
AgentLog.info("wait thread: {0} to exit".format(self.getName()))
self.isRunning = False
self.event.set()
self.join()
self.clear()
return False, 'remote host information errors'
AgentLog.info("start upload backup data to ftp server")
try:
ftp = FTP()
ftp.connect(remoteHost, int(remotePort), 60)
except Exception, e:
return False, 'can not connect to remote host: %s with port: %s, error: %s' % (
remoteHost, remotePort, e)
try:
ftp.login(remoteUser, remotePasswd)
ftp.cwd(backupPath)
srcDir = srcDir.rstrip('/')
localName = Util.pathSplit(srcDir)[1]
ftp.mkd(localName)
self.uploadDir(ftp, srcDir, localName)
AgentLog.info("upload completely")
except Exception, e:
return False, 'upload error, error:%s' % e
finally:
ftp.quit()
return True, ''
def uploadDir(self, ftp, localDir, remoteDir):
ftp.cwd(remoteDir)
for file in os.listdir(localDir):
src = os.path.join(localDir, file)
if os.path.isfile(src):
ftp.storbinary('STOR ' + file, open(src, 'rb'))
elif os.path.isdir(src):
ftp.mkd(file)
self.uploadDir(ftp, src, file)