def __init__(self) :
""
threading.Thread.__init__(self)
self.logger = LoggerUtil().getLogger()
self.conf = ConfigManage()
self.bs = BusinessAction()
class Server(Daemon):
""
#系统初始化
def __init__(self, config):
self.logger = LoggerUtil().getLogger()
self.config = config
self.running = False
pidfile = self.config['pidfile']
stderr = self.config['stderr']
stdout = self.config['stdout']
stdin = self.config['stdin']
Daemon.__init__(self, pidfile, stderr, stdout, stdin)
#启动任务
def _run(self):
task = TaskBss()
mq = MqBss()
mb = MonitorBss()
mdb = MonitorDbBss()
wb = WarnBss()
wdb = WarnDbBss()
task.start()
mb.start()
mdb.start()
wb.start()
wdb.start()
self.logger.debug("_run is start")
mq.start()
def __init__(self):
""
self.logger = LoggerUtil().getLogger()
self.vmUseStat = VmUseStat()
self.conf = ConfigManage()
self.monitor_keep_max_time = self.conf.get_monitor_conf(
)['monitor_keep_max_time']
class WarnDbBss(threading.Thread):
""
def __init__(self):
""
threading.Thread.__init__(self)
self.logger = LoggerUtil().getLogger()
self.conf = ConfigManage()
self.warn2db_frequency = self.conf.get_monitor_conf(
)['warn2db_frequency']
self.warnDbAction = WarnDbAction()
def process(self):
self.warnDbAction.processVmRunStat()
def run(self):
while True:
try:
time.sleep(self.warn2db_frequency)
self.logger.debug("warndb bss start is runing")
self.process()
except Exception, e:
log = str(e)
self.logger.error(log)
def __init__(self) :
""
self.logger = LoggerUtil().getLogger()
self.vmUseStat = VmUseStat()
self.vmWarnStat = VmWarnStat()
self.resourceStat = ResourceStat()
self.conf = ConfigManage()
self.warn_resource_keep_max_time = self.conf.get_monitor_conf()['warn_resource_keep_max_time']
def __init__(self):
""
threading.Thread.__init__(self)
self.logger = LoggerUtil().getLogger()
self.conf = ConfigManage()
self.warn_frequency = self.conf.get_monitor_conf()['warn_frequency']
self.warnAction = WarnAction()
def __init__(self):
self.logger = LoggerUtil().getLogger()
self.conf = ConfigManage()
self.mq_conf = self.conf.get_mq_conf()
self.mq_addr = self.mq_conf['mq_addr']
self.brodcast_exchange = self.mq_conf['brodcast_exchange']
self.brodcast_type = self.mq_conf['brodcast_type']
self.task_exchange = self.mq_conf['task_exchange']
self.task_type = self.mq_conf['task_type']
self.queue_name = self.mq_conf['queue_name']
def __init__(self, config):
self.logger = LoggerUtil().getLogger()
self.config = config
self.running = False
pidfile = self.config['pidfile']
stderr = self.config['stderr']
stdout = self.config['stdout']
stdin = self.config['stdin']
Daemon.__init__(self, pidfile, stderr, stdout, stdin)
class VmUseStat():
infoList = []
def __init__(self):
self.logger = LoggerUtil().getLogger()
'''info =[{instanceId:"",vmName:"",vmState:"",cpuRate:"",memRate:"",flow:"",eventTime:"",receiveTime:""}] '''
#增加监控的消息
def addVmRunInfo(self, vmRunInfo):
tmp = {}
tmp['vmName'] = vmRunInfo['vmName']
tmp['vmState'] = vmRunInfo['vmState']
keys = vmRunInfo.keys()
if 'instanceId' in keys:
tmp['instanceId'] = vmRunInfo['instanceId']
if 'cpuRate' in keys:
tmp['cpuRate'] = vmRunInfo['cpuRate']
if 'memRate' in keys:
tmp['memRate'] = vmRunInfo['memRate']
if 'netStats' in keys:
if len(vmRunInfo['netStats']) > 0:
tmp['flow'] = vmRunInfo['netStats'][0]['flow']
tmp['eventTime'] = vmRunInfo['eventTime']
now = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time()))
tmp['receiveTime'] = now
for i in range(0, len(VmUseStat.infoList)):
if tmp['vmName'] == VmUseStat.infoList[i]['vmName']:
del VmUseStat.infoList[i]
break
#增加
VmUseStat.infoList.append(tmp)
#获取所有vm的运行信息
def cleanAllVmRunInfo(self, keepTime=None):
self.logger.debug("VmUseStat.infoList was clean")
if keepTime is None:
keepTime = 2 * 60 * 60 #2小时
now = time.strftime('%Y-%m-%d %H:%M:%S',
time.localtime(time.time() - keepTime))
for info in VmUseStat.infoList:
if info['receiveTime'] < now:
del VmUseStat.infoList[0]
else:
break
return VmUseStat.infoList
def getAllVmRunInfo(self):
return VmUseStat.infoList
class Mq():
def __init__(self):
self.logger = LoggerUtil().getLogger()
self.conf = ConfigManage()
self.mq_conf = self.conf.get_mq_conf()
self.mq_addr = self.mq_conf['mq_addr']
self.brodcast_exchange = self.mq_conf['brodcast_exchange']
self.brodcast_type = self.mq_conf['brodcast_type']
self.task_exchange = self.mq_conf['task_exchange']
self.task_type = self.mq_conf['task_type']
self.queue_name = self.mq_conf['queue_name']
def send_task_message(self, routing_key, message):
"""
@summary: mq发送task消息
@param：router_key 路由key
@param: message 消息内容
"""
self.logger.debug("send task:routing_key:%s,message:%s" %
(routing_key, message))
mq = RabbitMq(self.mq_addr)
mq.exchange_declare(exchange_name=self.task_exchange,
mq_type=self.task_type)
mq.mq_send(msg=message,
exchange_name=self.task_exchange,
routing_key=routing_key)
mq.mq_close()
def send_bdct_message(self, routing_key, message):
"""
@summary: mq发送广播消息
@param：router_key 路由key
@param: message 消息内容
"""
mq = RabbitMq(self.mq_addr)
mq.exchange_declare(exchange_name=self.brodcast_exchange,
mq_type=self.brodcast_type)
mq.mq_send(msg=message,
exchange_name=self.brodcast_exchange,
routing_key=routing_key)
mq.mq_close()
class MonitorDbBss(threading.Thread):
""
def __init__(self):
""
threading.Thread.__init__(self)
self.logger = LoggerUtil().getLogger()
self.conf = ConfigManage()
self.monitor_frequency = self.conf.get_monitor_conf(
)['monitor_frequency']
self.monitorDbAction = MonitorDbAction()
def process(self):
try:
self.logger.debug("start is runing")
self.monitorDbAction.processVmRunStat()
except Exception, e:
log = str(e)
self.logger.error(log)
time.sleep(300)
class TaskBss(threading.Thread) :
""
def __init__(self) :
""
threading.Thread.__init__(self)
self.logger = LoggerUtil().getLogger()
self.conf = ConfigManage()
self.bs = BusinessAction()
# 收到请求处理函数
def process_action(self, taskinfo):
""
#业务处理
try :
self.bs.processTask(taskinfo)
except Exception,e:
print e
log = str(e)
self.logger.error(log)
class MonitorBss(threading.Thread):
""
def __init__(self):
""
threading.Thread.__init__(self)
self.logger = LoggerUtil().getLogger()
self.conf = ConfigManage()
self.monitorAction = MonitorAction()
# 收到请求处理函数
def process_action(self, message):
""
self.logger.debug("received monitor mq message")
# 1 返回信息
routing_key = message.delivery_info['routing_key']
exchange = message.delivery_info['exchange']
msg = message.body
action = routing_key.split('.', 1)[1]
#self.logger.info('action: %s ;message: %s'%(action,str(msg)))
js = JsonParser()
json_msg = js.decode(msg)
# 2 json格式报错,日志记录
if json_msg == -1:
self.logger.error('josn format error!')
return
# 3业务处理
try:
self.monitorAction.processMq(action, json_msg)
except Exception, e:
log = str(e)
self.logger.error(log)
class WarnDbAction:
""
tableTime = None
def __init__(self):
""
self.logger = LoggerUtil().getLogger()
self.vmWarnStat = VmWarnStat()
self.conf = ConfigManage()
def __getDb(self):
db_conf = self.conf.get_db_conf()
db = MysqlTools(db_conf)
return db
def processVmRunStat(self):
"""
@summary: 虚拟机和应用资源使用监控信息入库
"""
nowTime = time.localtime(time.time())
now = time.strftime('%Y-%m-%d %H:%M:%S', nowTime)
cur_tableTime = time.strftime('%Y%m', nowTime)
#获取数据操作对象
db = self.__getDb()
if WarnDbAction.tableTime != cur_tableTime:
self.createTable(db, cur_tableTime)
WarnDbAction.tableTime = cur_tableTime
vmWarnInfos = self.vmWarnStat.getAllVmWarnInfo()
vm_warn_param = []
appinstance_warn_param = []
warnKeys = vmWarnInfos.keys()
for key in warnKeys:
info = None
try:
info = vmWarnInfos[key]
except:
pass
if info is None or info['in2db'] == False:
continue
logId = UUIDUtils.getId()
keys = info.keys()
if 'instanceId' in keys and info['instanceId'] is not None and len(
info['instanceId']) > 0:
#app 应用 监控
appinstance_warn_param.append(
(logId, info['instanceId'], info['warnType'],
info['vmState'], info['cpuRate'], info['memRate'],
info['flow'], info['eventTime'], info['receiveTime'],
info['duration'], info['times'], info['maxDuration']))
else:
#vm 监控
vm_warn_param.append(
(logId, info['vmName'], info['warnType'], info['vmState'],
info['cpuRate'], info['memRate'], info['flow'],
info['eventTime'], info['receiveTime'], info['duration'],
info['times'], info['maxDuration']))
try:
if len(vm_warn_param) != 0:
sql = "insert into log_vm_warning" + WarnDbAction.tableTime + "(logId,vmName,warnType,status,cpuRate,memRate,flow,eventTime,receiveTime,createTime,duration,times,maxDuration)values(%s,%s,%s,%s,%s,%s,%s,%s,%s,now(),%s,%s,%s)"
db.insertMany(sql, vm_warn_param)
except Exception, e:
self.logger.error("insert vm warn info to db was error,msg:" +
str(e))
try:
if len(appinstance_warn_param) != 0:
sql = "insert into log_appinstance_warning" + WarnDbAction.tableTime + "(logId,instanceId,warnType,status,cpuRate,memRate,flow,eventTime,receiveTime,createTime,duration,times,maxDuration)values(%s,%s,%s,%s,%s,%s,%s,%s,%s,now(),%s,%s,%s)"
db.insertMany(sql, appinstance_warn_param)
except Exception, e:
self.logger.error(
"insert app instance warn info to db was error,msg:" + str(e))
self.logger.error("param:" + str(appinstance_warn_param))
def __init__(self):
""
self.logger = LoggerUtil().getLogger()
self.vmWarnStat = VmWarnStat()
self.conf = ConfigManage()
def __init__(self) :
""
self.logger = LoggerUtil().getLogger()
self.vmUseStat = VmUseStat()
class MonitorDbAction:
""
tableTime = None
def __init__(self):
""
self.logger = LoggerUtil().getLogger()
self.vmUseStat = VmUseStat()
self.conf = ConfigManage()
self.monitor_keep_max_time = self.conf.get_monitor_conf(
)['monitor_keep_max_time']
def __getDb(self):
db_conf = self.conf.get_db_conf()
db = MysqlTools(db_conf)
return db
def processVmRunStat(self):
"""
@summary: 虚拟机和应用资源使用监控信息入库
"""
nowTime = time.localtime(time.time())
now = time.strftime('%Y-%m-%d %H:%M:%S', nowTime)
cur_tableTime = time.strftime('%Y%m', nowTime)
#获取数据操作对象
db = self.__getDb()
if MonitorDbAction.tableTime != cur_tableTime:
self.createTable(db, cur_tableTime)
MonitorDbAction.tableTime = cur_tableTime
vmRunInfos = self.vmUseStat.cleanAllVmRunInfo(
keepTime=self.monitor_keep_max_time)
vm_running_param = []
appinstance_running_param = []
for info in vmRunInfos:
keys = info.keys()
logId = UUIDUtils.getId()
vmName = info['vmName']
status = info['vmState']
cpuRate = float(0)
memRate = float(0)
flow = long(0)
eventTime = info['eventTime']
receiveTime = info['receiveTime']
if 'cpuRate' in keys:
cpuRate = float(info['cpuRate'])
if 'memRate' in keys:
memRate = float(info['memRate'])
if 'flow' in keys:
flow = long(info['flow'])
if 'instanceId' in keys and info['instanceId'] is not None and len(
info['instanceId']) > 0:
#app 应用 监控
appinstance_running_param.append(
(logId, info['instanceId'], status, cpuRate, memRate, flow,
eventTime, receiveTime))
else:
#vm 监控
vm_running_param.append(
(logId, vmName, status, cpuRate, memRate, flow, eventTime,
receiveTime))
try:
if len(vm_running_param) != 0:
sql = "insert into log_vm_running" + MonitorDbAction.tableTime + "(logId,vmName,status,cpuRate,memRate,flow,eventTime,receiveTime,createTime)values(%s,%s,%s,%s,%s,%s,%s,%s,now())"
db.insertMany(sql, vm_running_param)
except Exception, e:
self.logger.error("insert vm running info to db was error,msg:" +
str(e))
try:
if len(appinstance_running_param) != 0:
sql = "insert into log_appinstance_running" + MonitorDbAction.tableTime + "(logId,instanceId,status,cpuRate,memRate,flow,eventTime,receiveTime,createTime)values(%s,%s,%s,%s,%s,%s,%s,%s,now())"
db.insertMany(sql, appinstance_running_param)
except Exception, e:
self.logger.error(
"insert vm app instance running info to db was error,msg:" +
str(e))
def __init__(self):
self.logger = LoggerUtil().getLogger()
'''info ={"vmName":{instanceId:"",vmName:"","warnType":"highWarn|lowWarn|paused|shutdown|crashed|overtime",vmState:"",cpuRate:"",memRate:"",flow:"",eventTime:"",receiveTime:"",duration:120,times:3,maxDuration:23}} '''
def __init__(self):
self.logger = LoggerUtil().getLogger()
class AppDeploy:
""
def __init__(self):
""
self.logger = LoggerUtil().getLogger()
self.conf = ConfigManage()
self.mq = Mq()
def __getDb(self):
db_conf = self.conf.get_db_conf()
db = MysqlTools(db_conf)
return db
def appDeploy(self, taskinfo):
"""
@summary: 软件部署任务
@param：taskinfo 任务内容
taskStatus :
10 : 等待部署
20 : 部署询问广播
30 : 接收资源应答
40 : 部署申请
90 : 部署成功
91 : 部署失败
"""
taskId = taskinfo['taskId']
taskStatus = taskinfo['taskStatus']
if taskStatus == '10':
self.appDeploy_10(taskinfo)
else:
self.logger.warning('Not support taskinfo,taskId:' + taskId +
',taskStatus:' + taskStatus)
#------appdeploy------#
#任务一
def appDeploy_10(self, taskinfo):
"""
@summary: 待部署任务
@param：taskinfo 任务内容
"""
#发送ask广播action=bdct.app.deploy.ask
#更改taskStatus=20等待资源，dealwith=close
self.logger.debug('appDeploy_10: begin dealwith task')
jsonParser = JsonParser()
taskId = taskinfo['taskId']
content = taskinfo['content']
businessId = taskinfo['businessId']
contentObj = jsonParser.decode(content)
cpu = contentObj['cpu']
mem = contentObj['mem']
disk = contentObj['disk']
serviceId = contentObj['serviceId']
#获取数据操作对象
db = self.__getDb()
router_key = 'bdct.app.deploy.ask'
action = 'app.deploy.ask'
dealwithId = UUIDUtils.getId()
#部署询问广播消息
message = {}
message['action'] = action
message['taskId'] = dealwithId
message['content'] = {}
message['content']['cpu'] = cpu
message['content']['mem'] = mem
message['content']['disk'] = disk
message['content']['serviceId'] = serviceId
message = jsonParser.encode(message)
#写任务处理表库(DealwithInfoModule)
self.logger.debug(
'appDeploy_10: write send ask message to dealwithInfo table')
'dealwithId,taskId,dealwithType,message,status'
dim_param = {}
dim_param['dealwithId'] = dealwithId
dim_param['taskId'] = taskId
dim_param['dealwithType'] = '20'
dim_param['message'] = message
dim_param['status'] = 'success'
print "1999"
dealwithInfoModule = DealwithInfoModule(db)
dealwithInfoModule.insert(dim_param)
self.logger.debug('appDeploy_10: begin change task status')
#开启事物
db.begin()
try:
#修改任务信息表(TaskinfoModule)
self.logger.debug('appDeploy_10: update task status')
'taskStatus=20,dealwith=close'
tim_param = {}
tim_param['taskId'] = taskId
tim_param['taskStatus'] = '20'
tim_param['dealwith'] = 'close'
taskinfoModule = TaskinfoModule(db)
res = taskinfoModule.update(tim_param)
#修改应用实例表(InstanceInfoModule)
self.logger.debug('appDeploy_10: update instance status')
'status=deploy'
iim_param = {}
iim_param['instanceId'] = businessId
iim_param['status'] = 'deploy'
instanceInfoModule = InstanceInfoModule(db)
instanceInfoModule.update(iim_param)
#发送广播消息
self.logger.debug(
'appDeploy_10: send ask message to the bdct of mq')
self.mq.send_bdct_message(router_key, message)
db.end()
self.logger.debug('appDeploy_10: process task is over')
except Exception, e:
db.end(option='rollback')
self.logger.error(
'appDeploy_10: process task error, error message: ' + str(e))
finally:
class TaskProcess:
"Task process list"
def __init__(self):
self.logger = LoggerUtil().getLogger()
self.conf = ConfigManage()
# 得到数据连接
def __getDb(self):
db_conf = self.conf.get_db_conf()
db = MysqlTools(db_conf)
return db
# 任务：增加app实例
def IncreaseInstance(self, appId):
""
self.logger.debug('IncreaseInstance: begin')
# 获取数据对象
db = self.__getDb()
jsonParser = JsonParser()
# 1 查询app信息
appInfoModule = AppInfoModule(db)
appInfo = appInfoModule.getById(appId)
# 准备参数
command = {}
command['domain'] = appInfo['domain']
command['cpu'] = appInfo['cpu']
command['mem'] = appInfo['mem']
command['disk'] = appInfo['disk']
command['serviceId'] = appInfo['serviceId']
command['domain'] = appInfo['domain']
command['appName'] = appInfo['appName']
command['appFileId'] = appInfo['appFileId']
command['appEnv'] = appInfo['appEnv']
command['userEnv'] = appInfo['userEnv']
# 2 新建instance
instanceId = UUIDUtils.getId()
iim_param = {}
iim_param['instanceId'] = instanceId
iim_param['appId'] = appId
iim_param['status'] = 'deploy'
instanceInfoModule = InstanceInfoModule(db)
instanceInfoModule.insert(iim_param)
# 3 新建任务
tim_param = {}
tim_param['taskId'] = UUIDUtils.getId()
tim_param['taskType'] = 'appDeploy'
tim_param['dealwith'] = 'open'
tim_param['taskStatus'] = '10'
tim_param['content'] = str(jsonParser.encode(command))
tim_param['businessId'] = instanceId
taskinfoModule = TaskinfoModule(db)
res = taskinfoModule.insert(tim_param)
self.logger.debug('IncreaseInstance: end')
# 任务：减少app实例
def ReduceInstance(self, instanceId):
""
self.logger.debug('ReduceInstance: begin')
# 1 更改instance状态
iim_param = {}
iim_param['instanceId'] = instanceId
iim_param['status'] = 'droping'
instanceInfoModule = InstanceInfoModule(db)
instanceInfoModule.update(iim_param)
# 2 新建任务
tim_param = {}
tim_param['taskId'] = UUIDUtils.getId()
tim_param['taskType'] = 'appDrop'
tim_param['dealwith'] = 'open'
tim_param['taskStatus'] = '10'
info = {}
info['instanceId'] = instanceId
tim_param['content'] = str(self.js.encode(info))
tim_param['businessId'] = instanceId
taskinfoModule = TaskinfoModule(db)
res = taskinfoModule.insert(tim_param)
self.logger.debug('ReduceInstance: end')
class VmCreate:
""
def __init__(self):
""
self.logger = LoggerUtil().getLogger()
self.conf = ConfigManage()
self.mq = Mq()
def __getDb(self):
db_conf = self.conf.get_db_conf()
db = MysqlTools(db_conf)
return db
def vmCreate(self, taskinfo):
"""
@summary: 软件创建任务
@param：taskinfo 任务内容
taskStatus :
10 : 等待创建
20 : 创建询问广播
30 : 接收资源应答
40 : 创建申请
90 : 创建成功
91 : 创建失败
"""
taskId = taskinfo['taskId']
taskStatus = taskinfo['taskStatus']
if taskStatus == '10':
self.vmCreate_10(taskinfo)
else:
self.logger.warning('Not support taskinfo,taskId:' + taskId +
',taskStatus:' + taskStatus)
#------vmdeploy------#
#任务一
def vmCreate_10(self, taskinfo):
"""
@summary: 待创建任务
@param：taskinfo 任务内容
"""
#发送ask广播action=bdct.vm.deploy.ask
#更改taskStatus=20等待资源，dealwith=close
self.logger.debug('vmCreate_10: begin dealwith task')
jsonParser = JsonParser()
taskId = taskinfo['taskId']
content = taskinfo['content']
businessId = taskinfo['businessId']
contentObj = jsonParser.decode(content)
cpu = contentObj['cpu']
mem = contentObj['mem']
disk = contentObj['size']
imageId = contentObj['imageId']
#获取数据操作对象
db = self.__getDb()
router_key = 'bdct.vm.create.ask'
action = 'vm.create.ask'
dealwithId = UUIDUtils.getId()
# 查询vm类型
vmImage = VmImageModule(db)
ImageInfo = vmImage.getByImageId(imageId)
vmType = ImageInfo['vtype']
#创建询问广播消息
message = {}
message['action'] = action
message['taskId'] = dealwithId
message['content'] = {}
message['content']['cpu'] = cpu
message['content']['mem'] = mem
message['content']['disk'] = disk
message['content']['vmType'] = vmType
message = jsonParser.encode(message)
#写任务处理表库(DealwithInfoModule)
self.logger.debug(
'vmCreate_10: write send ask message to dealwithInfo table')
dim_param = {}
dim_param['dealwithId'] = dealwithId
dim_param['taskId'] = taskId
dim_param['dealwithType'] = '20'
dim_param['message'] = message
dim_param['status'] = 'success'
dealwithInfoModule = DealwithInfoModule(db)
dealwithInfoModule.insert(dim_param)
self.logger.debug('vmCreate_10: begin change task status')
#开启事物
db.begin()
try:
#修改任务信息表(TaskinfoModule)
self.logger.debug('vmCreate_10: update task status')
tim_param = {}
tim_param['taskId'] = taskId
tim_param['taskStatus'] = '20'
tim_param['dealwith'] = 'close'
taskinfoModule = TaskinfoModule(db)
res = taskinfoModule.update(tim_param)
#发送广播消息
self.logger.debug(
'vmCreate_10: send ask message to the bdct of mq')
self.mq.send_bdct_message(router_key, message)
db.end()
self.logger.debug('vmCreate_10: process task is over')
except Exception, e:
db.end(option='rollback')
self.logger.error(
'vmCreate_10: process task error, error message: ' + str(e))
finally:
class AppDControl:
""
def __init__(self):
""
self.logger = LoggerUtil().getLogger()
self.conf = ConfigManage()
self.mq = Mq()
def __getDb(self):
db_conf = self.conf.get_db_conf()
db = MysqlTools(db_conf)
return db
def appControl(self, taskinfo):
"""
@summary: 软件控制任务
@param：taskinfo 任务内容
taskStatus :
10 : 等待启动
20 : 启动中
30 : 启动成功
31 : 启动失败
30 : 待关闭
40 : 关闭中
50 : 关闭成功
51 : 关闭失败
"""
taskId = taskinfo['taskId']
taskStatus = taskinfo['taskStatus']
if taskStatus == '10':
self.appStart(taskinfo)
else:
self.logger.warning('Not support taskinfo,taskId:' + taskId +
',taskStatus:' + taskStatus)
#------appControl------#
#任务一
def appStart(self, taskinfo):
"""
@summary: 软件启动
@param：taskinfo 任务内容
"""
self.logger.debug('appStart: begin dealwith task')
jsonParser = JsonParser()
taskId = taskinfo['taskId']
content = taskinfo['content']
businessId = taskinfo['businessId']
#获取数据操作对象
db = self.__getDb()
#读取应用实例信息表
dealwithInfoModule = DealwithInfoModule(db)
instanceInfoModule = InstanceInfoModule(db)
instanceInfo = instanceInfoModule.getById(businessId)
if not instanceInfo or not instanceInfo['engineId']:
#写任务信息表
self.logger.debug(
'appDeploy_can: update task status taskStatus=40 and dealwith=close'
)
'taskStatus=40,dealwith=close'
tim_param = {}
tim_param['taskId'] = taskId
tim_param['taskStatus'] = '31'
tim_param['dealwith'] = 'close'
taskinfoModule = TaskinfoModule(db)
res = taskinfoModule.update(tim_param)
#写任务处理表(DealwithInfoModule) 启动申请
self.logger.debug(
'appDeploy_can: write deploy message to dealwithinfo table')
message = {}
dealwithId = UUIDUtils.getId()
message['action'] = 'app.start.error'
message['taskId'] = dealwithId
message['content'] = {}
message['instanceId'] = businessId
message['errorCode'] = '2011'
message['errorMessage'] = 'app instance not exist'
message = jsonParser.encode(message)
dim_param = {}
dim_param['dealwithId'] = dealwithId
dim_param['taskId'] = taskId
dim_param['dealwithType'] = '31'
dim_param['message'] = message
dim_param['status'] = 'failure'
dealwithInfoModule.insert(dim_param)
return
engineId = instanceInfo['engineId']
router_key = engineId + ".app.start.apply"
action = ".app.start.apply"
dealwithId = UUIDUtils.getId()
message = {}
message['action'] = action
message['taskId'] = dealwithId
message['content'] = {}
message['content']['instanceId'] = businessId
message = jsonParser.encode(message)
#写任务处理表(DealwithInfoModule) 启动申请消息
dim_param = {}
dim_param['dealwithId'] = dealwithId
dim_param['taskId'] = taskId
dim_param['dealwithType'] = '20'
dim_param['message'] = message
dim_param['status'] = 'success'
dealwithInfoModule.insert(dim_param)
#开启事物
self.logger.debug('appStart: begin change the task status')
db.begin()
try:
#更新任务信息表 状态
self.logger.debug(
'appStart: update task status taskStatus=20 and dealwith=close'
)
'taskStatus=40,dealwith=close'
tim_param = {}
tim_param['taskId'] = taskId
tim_param['taskStatus'] = '20'
tim_param['dealwith'] = 'close'
taskinfoModule = TaskinfoModule(db)
res = taskinfoModule.update(tim_param)
#发送广播消息
self.logger.debug('appStart: send app start request to task of mq')
self.mq.send_task_message(router_key, message)
db.end()
self.logger.debug('appStart: send app start request was over')
except Exception, e:
db.end(option='rollback')
self.logger.debug(
'appStart: process status error, error message: ' + str(e))
finally:
def __init__(self):
""
threading.Thread.__init__(self)
self.logger = LoggerUtil().getLogger()
self.conf = ConfigManage()
self.monitorAction = MonitorAction()
def __init__(self):
self.logger = LoggerUtil().getLogger()
'''info =[{instanceId:"",vmName:"",vmState:"",cpuRate:"",memRate:"",flow:"",eventTime:"",receiveTime:""}] '''
class AppDrop:
""
def __init__(self) :
""
self.logger = LoggerUtil().getLogger()
self.conf = ConfigManage()
self.mq = Mq()
def __getDb(self) :
db_conf = self.conf.get_db_conf()
db = MysqlTools(db_conf)
return db
def appDrop(self, taskinfo):
"""
taskStatus :
10 : 等待删除
20 : 删除MQ消息发送
90 : 删除成功
91 : 删除失败
"""
taskId = taskinfo['taskId']
taskStatus = taskinfo['taskStatus']
if taskStatus == '10':
self.appDrop_10(taskinfo)
else :
self.logger.warning('Not support taskinfo,taskId:'+taskId+',taskStatus:'+taskStatus)
#------appdrop------#
#任务一
def appDrop_10(self, taskinfo):
""
self.logger.debug('appDrop_10: begin drop app ')
jsonParser = JsonParser()
instanceId = taskinfo['businessId']
taskId = taskinfo['taskId']
#获取数据操作对象
db = self.__getDb()
# 查询instance表
instanceInfoModule = InstanceInfoModule(db)
instanceInfo = instanceInfoModule.getById(instanceId)
# 获取engineid
engineId = instanceInfo['engineId']
# 发现消息
action = 'app.drop.apply'
router_key = engineId + ".app.drop.apply"
dealwithId = UUIDUtils.getId()
# 删除应用消息
message = {}
message['action'] = action
message['taskId'] = dealwithId
message['content'] = {}
message['content']['instanceId'] = instanceId
message = jsonParser.encode(message)
#写任务处理表库(DealwithInfoModule)
self.logger.debug('appDrop_10: write drop message to dealwithInfo table')
dim_param = {}
dim_param['dealwithId'] = dealwithId
dim_param['taskId'] = taskId
dim_param['dealwithType'] = '20'
dim_param['message'] = message
dim_param['status'] = 'success'
dealwithInfoModule = DealwithInfoModule(db)
dealwithInfoModule.insert(dim_param)
#开启事物
db.begin()
try :
#修改任务信息表(TaskinfoModule)
self.logger.debug('appDrop_10: update task status')
tim_param = {}
tim_param['taskId'] = taskId
tim_param['taskStatus'] = '20'
tim_param['dealwith'] = 'close'
taskinfoModule = TaskinfoModule(db)
res = taskinfoModule.update(tim_param)
#发送消息
self.logger.debug('appDrop_10: send drop app request to task of mq')
self.mq.send_task_message(router_key, message)
db.end()
self.logger.debug('appDrop_10: process task is over')
except Exception,e:
db.end(option='rollback')
self.logger.error('appDrop_10: process task error, error message: '+str(e))
finally :
def __init__(self):
self.logger = LoggerUtil().getLogger()
self.conf = ConfigManage()
class WarnAction:
""
tableTime = None
appInstance = {}
vmInstance = {}
resourceLoadTime = None
def __init__(self) :
""
self.logger = LoggerUtil().getLogger()
self.vmUseStat = VmUseStat()
self.vmWarnStat = VmWarnStat()
self.resourceStat = ResourceStat()
self.conf = ConfigManage()
self.warn_resource_keep_max_time = self.conf.get_monitor_conf()['warn_resource_keep_max_time']
def __getDb(self) :
db_conf = self.conf.get_db_conf()
db = MysqlTools(db_conf)
return db
def processVmRunStat(self) :
"""
@summary: 虚拟机和应用资源使用监控信息入库
"""
_time = time.time()
nowTime = time.localtime(_time)
now = time.strftime('%Y-%m-%d %H:%M:%S',nowTime)
cur_tableTime = time.strftime('%Y%m',nowTime)
#获取数据操作对象
db = self.__getDb()
self.logger.debug("this is running 1......")
#重新载入
if WarnAction.resourceLoadTime is None or _time - WarnAction.resourceLoadTime > self.warn_resource_keep_max_time :
self.logger.debug("this is running 2......")
self.resourceStat.reloadVmInfo(db)
self.logger.debug("this is running 3......")
self.resourceStat.reloadAppInstanceInfo(db)
self.logger.debug("this is running 4......")
WarnAction.resourceLoadTime = _time
vmRunInfos = self.vmUseStat.getAllVmRunInfo()
rule = '''{
"highWarn":{"cpu":90,"mem":90,"flow":8388608,"duration":3},
"lowWarn":{"cpu":10,"mem":40,"flow":8000,"duration":240},
"overtime":300
}'''
jp = JsonParser()
rule = jp.decode(rule)
self.logger.debug("warn info:"+str(self.vmWarnStat.getAllVmWarnInfo()))
#appInstanceKeys = WarnAction.appInstance.keys()
#vmInstanceKeys = WarnAction.vmInstance.keys()
instanceIds = []
vmNames = []
load_effective_time = 1800
for info in vmRunInfos :
keys = info.keys()
self.logger.debug("this is running 5......")
vmName = info['vmName']
status = info['vmState']
self.logger.debug("this is running 6......")
flag = 'normal'
cpuRate = float(0)
memRate = float(0)
flow = long(0)
instanceId = ''
eventTime = info['eventTime']
receiveTime = info['receiveTime']
if 'cpuRate' in keys:
cpuRate = float(info['cpuRate'])
if 'memRate' in keys:
memRate = float(info['memRate'])
if 'flow' in keys:
flow = long(info['flow'])
vmInfo = None
instanceInfo = None
if 'instanceId' in keys and info['instanceId'] :
instanceInfo = self.resourceStat.getInstanceInfo(info['instanceId'])
else:
vmInfo = self.resourceStat.getVmInfo(vmName)
if vmInfo is None and instanceInfo is None:
continue
#规则信息
ruleKeys = rule.keys()
hightFlag = False
lowFlag = False
hightDuration = 0
lowDuration = 0
if 'overtime' in ruleKeys:
maxDuration = float(rule['overtime'])
overtime = time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(_time - maxDuration))
#过期事件不处理
if receiveTime < overtime :
continue
if 'lowWarn' in ruleKeys:
lowWarn = rule['lowWarn']
lowWarnKeys = lowWarn.keys()
lowFlag = True
if 'cpu' in lowWarnKeys:
if cpuRate >= float(lowWarn['cpu']):
lowFlag = False
if 'mem' in lowWarnKeys:
if memRate >= float(lowWarn['mem']):
lowFlag = False
if 'flow' in lowWarnKeys:
if flow >= long(lowWarn['flow']):
lowFlag = False
if 'duration' in lowWarnKeys:
lowDuration = int(lowWarn['duration'])
if 'highWarn' in ruleKeys:
highWarn = rule['highWarn']
highWarnKeys = highWarn.keys()
hightFlag = True
if 'cpu' in highWarnKeys:
if cpuRate < float(highWarn['cpu']):
hightFlag = False
if 'mem' in highWarnKeys:
if memRate < float(highWarn['mem']):
hightFlag = False
if 'flow' in highWarnKeys:
if flow < long(highWarn['flow']):
hightFlag = False
if 'duration' in highWarnKeys:
hightDuration = int(highWarn['duration'])
#根据运行状态，判断是否告警
maxDuration = 0
if "running" == status:
if hightFlag:
flag = 'highWarn'
maxDuration = hightDuration
if lowFlag:
flag = 'lowWarn'
maxDuration = lowDuration
elif "paused" == status:
flag = 'paused'
if instanceInfo is not None and instanceInfo['status'] != 'running':
flag = 'normal'
elif vmInfo is not None and vmInfo['status'] != 'running':
flag = 'normal'
elif "shutdown" == status:
flag = 'shutdown'
if instanceInfo is not None and instanceInfo['status'] != 'running':
flag = 'normal'
elif vmInfo is not None and vmInfo['status'] != 'running':
flag = 'normal'
elif "crashed" == status:
flag = 'crashed'
#处理过的应用实例和虚拟机
if instanceInfo:
instanceIds.append(instanceInfo['instanceId'])
elif vmInfo:
vmNames.append(vmInfo['vmName'])
#告警处理
if flag == 'normal':
self.vmWarnStat.removeVmRunInfo('vmName')
if instanceInfo:
#主要考虑warnType=overtime的应用，vmName设置的值为instanceId
self.vmWarnStat.removeVmRunInfo(instanceInfo['instanceId'])
else :
self.vmWarnStat.addVmRunInfo(info, flag, maxDuration)
self.logger.debug("this is running 7......")
#没有监控信息的虚拟机告警
vmInfoList = self.resourceStat.getAllVmInfoList()
for info in vmInfoList:
if info['vmName'] not in vmNames:
ruleKeys = rule.keys()
tmp = {}
tmp['vmName'] = info['vmName']
tmp['vmState'] = 'shutdown'
tmp['eventTime'] = now
tmp['receiveTime'] = now
flag = 'overtime'
maxDuration = -1
if 'overtime' in ruleKeys:
maxDuration = float(rule['overtime'])
self.vmWarnStat.addVmRunInfo(tmp, flag, maxDuration)
self.logger.debug("this is running 8......")
#没有监控信息的应用实例告警
appInstanceInfoList = self.resourceStat.getAllAppInstanceInfoList()
for info in appInstanceInfoList:
if info['instanceId'] not in instanceIds:
ruleKeys = rule.keys()
tmp = {}
tmp['instanceId'] = info['instanceId']
tmp['vmName'] = info['instanceId']
tmp['vmState'] = 'shutdown'
tmp['eventTime'] = now
tmp['receiveTime'] = now
flag = 'overtime'
maxDuration = -1
if 'overtime' in ruleKeys:
maxDuration = float(rule['overtime'])
self.vmWarnStat.addVmRunInfo(tmp, flag, maxDuration)
self.logger.debug("this is running 9......")
