def test_get_work(self):
hostName="huamac"
workMonitor = StoreHolder.getServerStore().getWorkMonitor(hostName)
print "---------------%s-----------------"%hostName
for task in workMonitor.taskMonitors:
print task.taskName
print "taskRuntime:%s"%task.taskRuntime
print "taskSucceeded:%s"%task.taskSucceeded
print "taskFailed:%s"%task.taskFailed
print "taskQueueTime:%s"%task.taskQueueTime
print "taskCount:%s"%task.taskCount
print "taskRuntime:%s"%workMonitor.taskRuntime
print "taskSucceeded:%s"%workMonitor.taskSucceeded
print "taskFailed:%s"%workMonitor.taskFailed
print "taskCount:%s"%workMonitor.taskCount
hostName="ubuntu"
print "---------------%s-----------------"%hostName
workMonitor = StoreHolder.getServerStore().getWorkMonitor(hostName)
for task in workMonitor.taskMonitors:
print task.taskName
print "taskRuntime:%s"%task.taskRuntime
print "taskSucceeded:%s"%task.taskSucceeded
print "taskFailed:%s"%task.taskFailed
print "taskQueueTime:%s"%task.taskQueueTime
print "taskCount:%s"%task.taskCount
print "taskRuntime:%s"%workMonitor.taskRuntime
print "taskSucceeded:%s"%workMonitor.taskSucceeded
print "taskFailed:%s"%workMonitor.taskFailed
print "taskCount:%s"%workMonitor.taskCount
print "------------------------------------------------------------------------------------------------"
def workOnlineWatch(data, stat=None, event=None):
if event is not None:
#节点已死
if event.type=="DELETED":
#"/cabbage/works/"+HOST_NAME+"/"+ON_LINE
hostName = event.path.split("/")[3]
work = StoreHolder.getServerStore().getWork(hostName)
work.status = LOST
Logger.info( log,"节点:【%s】IP:【%s】已经死亡！" % (hostName,work.ip) )
StoreHolder.getServerStore().updateWorkStatus(work)
def test_save_Monitor(self):
taskCount = 2
taskSucceeded = 1
taskFailed = 1
taskRuntime = 1
taskName = "task-test"
jobId = "job-test"
hour = "1"
date = "20160913"
hostName = "test"
brokerServer = "test"
taskQueueTime = 1
taskMonitor = TaskMonitor(taskCount=taskCount,
taskSucceeded=taskSucceeded,
taskFailed=taskFailed,
taskRuntime=taskRuntime,
taskName=taskName,
jobId=jobId,
taskQueueTime=taskQueueTime)
jobMonitor = JobMonitor(taskCount=taskCount,
taskSucceeded=taskSucceeded,
taskFailed=taskFailed,
taskRuntime=taskRuntime,
jobId=jobId,
taskMonitors=[taskMonitor],
taskQueueTime=taskQueueTime)
dateMonitor = DateMonitor(taskCount=taskCount,
taskSucceeded=taskSucceeded,
taskFailed=taskFailed,
taskRuntime=taskRuntime,
date=date,
hour=hour)
workMonitor = WorkMonitor(taskCount=taskCount,
taskSucceeded=taskSucceeded,
taskFailed=taskFailed,
taskRuntime=taskRuntime,
taskMonitors=[taskMonitor],
dateMonitors=[dateMonitor],
hostName=hostName)
brokerMonitor = BrokerMonitor(taskCount=taskCount,
taskSucceeded=taskSucceeded,
taskFailed=taskFailed,
taskRuntime=taskRuntime,
brokerServer=brokerServer,
taskQueueTime=taskQueueTime)
monitor = Monitor(jobMonitors=[jobMonitor],
workMonitors=[workMonitor],
brokerMonitors=[brokerMonitor])
StoreHolder.getServerStore().saveMonitor(monitor)
def workWatch(children):
for hostname in children:
if CacheHolder.getCache().hasKey(hostname, WORKS) is False:
work = StoreHolder.getServerStore().getWork(hostname)
CacheHolder.getCache().put(hostname, work,WORKS)
parent="/"+CABBAGE+"/"+WORKS+"/"+hostname
kazooClient.addDataListener(parent+"/"+ON_LINE, workOnlineWatch)
def _initJobs(self):
store = StoreHolder.getRetryStore()
jobs =store.getJobs()
work =store.getWork(HOST_NAME)
queues=work.queues
routes={}
for job in jobs:
if job.status != JOB_DELETE and job.brokerQueue in queues:
#fixbug 动态扩容时，缓存JOB
if not CacheHolder.getCache().hasKey(job.jobId, JOBS):
CacheHolder.getCache().put(job.jobId, job,JOBS)
clientDir = ConfigHolder.getConfig().getProperty(BASE,CLIENT_FILE_DIRECTORY)
path = clientDir+"/"+job.jobId
if not os.path.isdir(path) :
# @FIX BUG 文件不同步
syncJob(job.jobId,store)
self.addScriptJobId(job.jobId)
for taskName in job.tasks:
que= store.getQueue(job.brokerQueue)
routes[taskName]={'queue': que.queueName, 'routing_key': que.routingKey}
celeryconfig.CELERY_ROUTES = routes
def jobReadyWatch(children):
try:
# brokers={}
for jobId in children:
try:
job=StoreHolder.getServerStore().getJob(jobId)
if CacheHolder.getCache().hasKey(jobId, JOBS) is False:
CacheHolder.getCache().put(jobId, job,JOBS)
for taskName in job.tasks:
TaskCacheHolder.getJobCache().put(taskName,job.jobId)
except Exception:
Logger.exception( log)
#偷个懒，只要没有删除的全部放到ROUTER里面去
# if job.status != JOB_DELETE:
#
# for taskName in job.tasks:
# que=StoreHolder.getServerStore().getQueue(job.brokerQueue)
# TaskCacheHolder.getJobCache().put(taskName,job.jobId)
#
# if brokerServer in brokers:
# brokers[brokerServer].update(routes)
# else:
# brokers[brokerServer] = routes
#
# #偷个懒，只要没有删除的全部放到ROUTER里面去
# for broker,routes in brokers.items():
# brokerServer = StoreHolder.getServerStore().getBrokerServer(broker)
# CabbageHolder.getServerCabbages()[brokerServer.hostName].getApp().conf.update(CELERY_ROUTES = routes)
# Logger.info(log,"更新队列服务器【%s】ROUTES【%s】"% (CabbageHolder.getServerCabbagesStr(),str(routes)))
except Exception:
Logger.exception( log)
def _getConnectUri(self):
connectUri = ConfigHolder.getConfig().getProperty(BASE, CONNECT_URI)
work = CacheHolder.getCache().get(HOST_NAME, WORKS)
if work.brokerServer:
brokerServer = StoreHolder.getStore().getBrokerServer(
work.brokerServer)
connectUri = brokerServer.connectUri
return connectUri
def __init__(self, cfgPath=None):
path = ConfigHolder.getConfig(cfgPath=cfgPath).getProperty(
BASE, SERVER_FILE_DIRECTORY)
if not os.path.isdir(path):
os.makedirs(path)
self.kazooClient = ZookeeperClientHolder.getRetryClient()
self.store = StoreHolder.getServerStore()
self.status = None
self._initConifg()
def test_get_broker(self):
brokerServer="cabbage"
brokerMonitor=StoreHolder.getServerStore().getBrokerMonitor(brokerServer)
print "---------------%s-----------------"%brokerServer
print "taskCount:%s"%brokerMonitor.taskCount
print "taskRuntime:%s"%brokerMonitor.taskRuntime
print "taskSucceeded:%s"%brokerMonitor.taskSucceeded
print "taskFailed:%s"%brokerMonitor.taskFailed
print "taskQueueTime:%s"%brokerMonitor.taskQueueTime
print "------------------------------------------------------------------------------------------------"
def test_status(self):
self.store = StoreHolder.getStore()
work = Work()
work.hostName = getHostName()
work.status = OFF_LINE
self.store.updateWorkStatus(work)
time.sleep(10)
work.status = ON_LINE
self.store.updateWorkStatus(work)
time.sleep(20)
work.status = REMOVE
self.store.updateWorkStatus(work)
def test_get_job(self):
jobId="job-29f3a426-267b-4fbd-97d8-919e693e030d"
jobMonitor = StoreHolder.getServerStore().getJobMonitor(jobId)
for task in jobMonitor.taskMonitors:
print task.taskName
print "taskRuntime:%s"%task.taskRuntime
print "taskSucceeded:%s"%task.taskSucceeded
print "taskFailed:%s"%task.taskFailed
print "taskQueueTime:%s"%task.taskQueueTime
print "taskCount:%s"%task.taskCount
print "---------------------%s--------------------"%jobId
print jobMonitor.taskQueueTime
print "------------------------------------------------------------------------------------------------"
def addScriptJobId(self,jobId):
store =None
try:
store = StoreHolder.getStore()
log.info("节点【%s】当前任务【%s】的脚本开始加载。。。。" % (HOST_NAME,jobId))
self.loadJobScript(jobId,store)
work=store.getWork(HOST_NAME)
self.sendBeReady(jobId,work,store)
store.close()
except Exception:
Logger.exception(log)
finally:
if store:
store.close()
def test_get_hour(self):
hostName="huamac"
date="20170407"
hourMonitors=StoreHolder.getServerStore().getWorkDateMonitors(hostName,date)
for hourMonitor in hourMonitors:
print "---------------%s-----------------"%hourMonitor.hour
print "taskCount:%s"%hourMonitor.taskCount
print "taskRuntime:%s"%hourMonitor.taskRuntime
print "taskSucceeded:%s"%hourMonitor.taskSucceeded
print "taskFailed:%s"%hourMonitor.taskFailed
print "taskQueueTime:%s"%hourMonitor.taskQueueTime
print "------------------------------------------------------------------------------------------------"
def getCabbage(self):
connectUri = CabbageHolder._getConnectUri()
work = CacheHolder.getCache().get(HOST_NAME, WORKS)
if work.queues and len(work.queues) > 0:
queues = []
for queueName in work.queues:
brokerQueue = StoreHolder.getStore().getQueue(queueName)
queues.append(
Queue(name=brokerQueue.queueName,
exchange=Exchange(brokerQueue.exchangeName),
routing_key=brokerQueue.routingKey))
celeryconfig.CELERY_QUEUES = tuple(queues)
return Cabbage(broker=str(connectUri))
def save_queue(self):
brokerQueue = BrokerQueue(server=self.testQueue,
queueName=self.testQueue,
exchangeName=self.testQueue,
routingKey=self.testQueue)
store = StoreHolder.getServerStore()
store.saveQueue(brokerQueue)
brokerQueue2 = store.getQueue(self.testQueue)
self.assertEqual(brokerQueue.queueName, brokerQueue2.queueName,
"121212")
print brokerQueue2.server
def save_broker(self):
#cabbage_celery:[email protected]:5672/cabbage_vhost
brokeServer = BrokerServer(port="5672",
ip=self.testIp,
serverType="amqp",
connectUri=self.testUri,
hostName=self.testQueue,
queues=[self.testQueue])
store = StoreHolder.getServerStore()
store.saveBrokerServer(brokeServer)
brokeServer2 = store.getBrokerServer(self.testQueue)
self.assertEqual(brokeServer.hostName, brokeServer2.hostName, "asdfa")
print brokeServer2.connectUri
def _initJobs(self, cabbage):
store = StoreHolder.getRetryStore()
jobs = store.getJobs()
work = store.getWork(HOST_NAME)
queues = work.queues
routes = {}
queues_celery = []
for que in queues:
que = store.getQueue(que)
queues_celery.append(
Queue(que.queueName,
Exchange(que.queueName),
routing_key=que.queueName,
queue_arguments={'x-max-priority': int(que.priority)}))
for job in jobs:
if job.status != JOB_DELETE and job.brokerQueue in queues:
#fixbug 动态扩容时，缓存JOB
if not CacheHolder.getCache().hasKey(job.jobId, JOBS):
CacheHolder.getCache().put(job.jobId, job, JOBS)
clientDir = ConfigHolder.getConfig().getProperty(
BASE, CLIENT_FILE_DIRECTORY)
path = clientDir + "/" + job.jobId
if not os.path.isdir(path):
# @FIX BUG 文件不同步
syncJob(job.jobId, store)
self.addScriptJobId(job.jobId, cabbage)
for taskName in job.tasks:
que = store.getQueue(job.brokerQueue)
routes[taskName] = {
'queue': que.queueName,
'routing_key': que.routingKey
}
log.info(routes)
celeryconfig.CELERY_QUEUES = tuple(queues_celery)
celeryconfig.CELERY_ROUTES = routes
def setUp(self):
self.store = StoreHolder.getStore()
def _create(self):
self.num_created += 1
return StoreHolder.getStore()
def statAciton(taskSentDict, taskReceivedDict, taskFailDict,
taskSucceededDict):
log.info("开始统计系统TASK运行情况 !")
tasks = {}
works = {}
workTasks = {}
# taskSucceeded = 0
# taskRuntime=0
# taskQueueTime =0
# taskFailed=0
for taskName, value in taskSentDict.items():
taskCount = value
tasks.update({
taskName: {
TASK_COUNT: taskCount,
TASK_FAILED: 0,
TASK_SUCCEEDED: 0,
TASK_RUNTIME: 0,
TASK_QUEUE_TIME: 0
}
})
for taskName, taskValues in taskFailDict.items():
taskFailed = 0
for work, value in taskValues.items():
taskFailedTemp = value
if work in workTasks:
if taskName in workTasks[work]:
workTasks[work][taskName][TASK_FAILED] = taskFailedTemp
else:
workTasks[work].update(
{taskName: {
TASK_FAILED: taskFailedTemp
}})
else:
workTasks[work] = {taskName: {TASK_FAILED: taskFailedTemp}}
if work in works:
if TASK_FAILED in works[work]:
works[work][TASK_FAILED] = works[work][
TASK_FAILED] + taskFailedTemp
else:
works[work].update({TASK_FAILED: taskFailedTemp})
else:
works[work] = {TASK_FAILED: taskFailedTemp}
taskFailed += taskFailedTemp
if taskName in tasks:
tasks[taskName].update({TASK_FAILED: taskFailed})
else:
tasks.update({taskName: {TASK_FAILED: taskFailed}})
# print tasks
# print taskSucceededDict
for taskName, taskValues in taskSucceededDict.items():
taskSucceeded = 0
taskRuntime = 0
taskQueueTime = 0
for work, value in taskValues.items():
taskSucceededTemp = value[TASK_COUNT]
taskRuntimeTemp = value[TASK_RUNTIME]
taskQueueTimeTemp = value[TASK_QUEUE_TIME]
if work in workTasks:
workTasks[work].update({
taskName: {
TASK_SUCCEEDED: taskSucceededTemp,
TASK_RUNTIME: taskRuntimeTemp,
TASK_QUEUE_TIME: taskQueueTimeTemp
}
})
else:
workTasks[work] = {
taskName: {
TASK_SUCCEEDED: taskSucceededTemp,
TASK_RUNTIME: taskRuntimeTemp,
TASK_QUEUE_TIME: taskQueueTimeTemp
}
}
if work in works:
tempWork = works[work]
if TASK_SUCCEEDED in tempWork:
tempWork[TASK_SUCCEEDED] = tempWork[
TASK_SUCCEEDED] + taskSucceededTemp
tempWork[TASK_RUNTIME] = tempWork[
TASK_RUNTIME] + taskRuntimeTemp
else:
tempWork.update({
TASK_SUCCEEDED: taskSucceededTemp,
TASK_RUNTIME: taskRuntimeTemp
})
else:
works[work] = {
TASK_SUCCEEDED: taskSucceededTemp,
TASK_RUNTIME: taskRuntimeTemp
}
taskSucceeded += taskSucceededTemp
taskRuntime += taskRuntimeTemp
taskQueueTime += taskQueueTimeTemp
if taskName in tasks:
tasks[taskName].update({
TASK_SUCCEEDED: taskSucceeded,
TASK_RUNTIME: taskRuntime,
TASK_QUEUE_TIME: taskQueueTime
})
else:
tasks[taskName] = {
TASK_SUCCEEDED: taskSucceeded,
TASK_RUNTIME: taskRuntime,
TASK_QUEUE_TIME: taskQueueTime
}
# print tasks
for taskName, value in taskReceivedDict.items():
taskAllReceived = 0
for work, taskCountTemp in value.items():
taskAllReceived += taskCountTemp
if work in workTasks:
if taskName in workTasks[work]:
workTasks[work][taskName][TASK_COUNT] = taskCountTemp
else:
workTasks[work].update(
{taskName: {
TASK_COUNT: taskCountTemp
}})
else:
workTasks[work] = {taskName: {TASK_COUNT: taskCountTemp}}
if work in works:
if TASK_COUNT in works[work]:
works[work][
TASK_COUNT] = works[work][TASK_COUNT] + taskCountTemp
else:
works[work].update({TASK_COUNT: taskCountTemp})
else:
works[work] = {TASK_COUNT: taskCountTemp}
if taskName in tasks:
tasks[taskName].update({TASK_RECEIVED: taskAllReceived})
else:
tasks.update({taskName: {TASK_RECEIVED: taskAllReceived}})
# print tasks
#stat job
jobs = {}
brokers = {}
taskMonitors = {}
for taskName, values in tasks.items():
jobId = TaskCacheHolder.getJobCache().get(taskName)
job = CacheHolder.getCache().get(jobId, JOBS)
broker = job.brokerServer
for key, value in values.items():
if job.brokerServer in brokers:
if key in brokers[broker]:
brokers[broker][key] = brokers[broker][key] + value
else:
brokers[broker][key] = value
else:
brokers[broker] = {key: value}
if jobId in jobs:
if key in jobs[jobId]:
jobs[jobId][key] = jobs[jobId][key] + value
else:
jobs[jobId][key] = value
else:
jobs[jobId] = {key: value}
a = set()
a.add(taskName)
jobs[jobId][TASKS] = a
jobs[jobId][TASKS].add(taskName)
taskMonitors[taskName] = TaskMonitor(
taskCount=values[TASK_COUNT] if TASK_COUNT in values else 0,
taskSucceeded=values[TASK_SUCCEEDED]
if TASK_SUCCEEDED in values else 0,
taskFailed=values[TASK_FAILED] if TASK_FAILED in values else 0,
taskRuntime=values[TASK_RUNTIME] if TASK_RUNTIME in values else 0,
taskName=taskName,
jobId=TaskCacheHolder.getJobCache().get(taskName),
taskQueueTime=values[TASK_QUEUE_TIME]
if TASK_QUEUE_TIME in values else 0,
taskReceived=values[TASK_RECEIVED]
if TASK_RECEIVED in values else 0)
# print jobs
jobMonitors = []
# print "taskMonitors: %s "%taskMonitors
for jobId, values in jobs.items():
jobMonitor = StoreHolder.getServerStore().getJobMonitor(jobId)
if jobMonitor is None:
jobMonitor = JobMonitor(jobId=jobId, taskMonitors=[])
for key, value in values.items():
#jobMonitor
_setValue(jobMonitor, key, value)
#taskMonitor
for taskName in values[TASKS]:
taskMonitorTemp = taskMonitors[taskName]
taskMonitor = None
for taskMonitor in jobMonitor.taskMonitors:
if taskMonitor.taskName == taskName:
taskMonitor = taskMonitor
break
if taskMonitor:
_setValue(taskMonitor, TASK_COUNT, taskMonitorTemp.taskCount)
_setValue(taskMonitor, TASK_RECEIVED,
taskMonitorTemp.taskReceived)
_setValue(taskMonitor, TASK_SUCCEEDED,
taskMonitorTemp.taskSucceeded)
_setValue(taskMonitor, TASK_FAILED, taskMonitorTemp.taskFailed)
_setValue(taskMonitor, TASK_RUNTIME,
taskMonitorTemp.taskRuntime)
_setValue(taskMonitor, TASK_QUEUE_TIME,
taskMonitorTemp.taskQueueTime)
else:
jobMonitor.taskMonitors.append(taskMonitorTemp)
jobMonitors.append(jobMonitor)
workMonitors = []
dateStr = getNowDateStr()
hour = str(getNowHour())
for hostName, values in works.items():
workMonitor = StoreHolder.getServerStore().getWorkMonitor(hostName)
if workMonitor is None:
workMonitor = WorkMonitor(hostName=hostName,
taskMonitors=[],
dateMonitors=[])
dateMonitor = StoreHolder.getServerStore().getWorkHourMonitor(
hostName, dateStr, hour)
if not dateMonitor:
dateMonitor = DateMonitor(date=dateStr, hour=hour)
for key, value in values.items():
#workMonitor
_setValue(workMonitor, key, value)
#dateMonitor
_setValue(dateMonitor, key, value)
print key, value
workMonitor.dateMonitors.append(dateMonitor)
#taskMonitor
wTasks = workTasks[hostName]
# print id(workMonitor)
# print wTasks
for taskName, wvalues in wTasks.items():
wTaskMonitor = None
for tm in workMonitor.taskMonitors:
if tm.taskName == taskName:
wTaskMonitor = tm
break
if wTaskMonitor is None:
wTaskMonitor = TaskMonitor(taskName=taskName)
workMonitor.taskMonitors.append(wTaskMonitor)
for key, value in wvalues.items():
_setValue(wTaskMonitor, key, value)
workMonitors.append(workMonitor)
brokerMonitors = []
for brokerName, values in brokers.items():
brokerMonitor = StoreHolder.getServerStore().getBrokerMonitor(
brokerName)
if not brokerMonitor:
brokerMonitor = BrokerMonitor(brokerServer=brokerName)
dateMonitor = StoreHolder.getServerStore().getBrokerDateMonitor(
brokerName, dateStr)
if dateMonitor is None:
dateMonitor = DateMonitor(date=dateStr)
for key, value in values.items():
_setValue(brokerMonitor, key, value)
_setValue(dateMonitor, key, value)
brokerMonitor.dateMonitors.append(dateMonitor)
brokerMonitors.append(brokerMonitor)
# print "jobMonitors: %s "%jobMonitors
# print "jobs: %s " % jobs
# print "brokers: %s" % brokers
# print "tasks: %s" % tasks
# print "works: %s" % works
# print "workTasks: %s "%workTasks
# #
# print "taskFail: %s"%taskFailDict
# print "taskReceived: %s"%taskReceivedDict
# print "taskSent: %s"%taskSentDict
# print "taskSucceeded: %s"%taskSucceededDict
log.info("结束统计系统TASK运行情况 !")
if len(jobMonitors) > 0 or len(workMonitors) > 0 or len(
brokerMonitors) > 0:
monitor = Monitor(jobMonitors=jobMonitors,
workMonitors=workMonitors,
brokerMonitors=brokerMonitors)
StoreHolder.getServerStore().saveMonitor(monitor)
def test_get_job(self):
jobId = "job-test"
jobMonitor = StoreHolder.getServerStore().getJobMonitor(jobId)
print jobMonitor.taskMonitors[0].taskName
print jobMonitor.taskQueueTime
def test_get_work(self):
hostName = "test"
workMonitor = StoreHolder.getServerStore().getWorkMonitor(hostName)
print workMonitor.taskMonitors[0].taskName
print workMonitor.taskMonitors[0].taskQueueTime
def test_get_broker(self):
brokerServer = "test"
brokerMonitor = StoreHolder.getServerStore().getBrokerMonitor(
brokerServer)
print brokerMonitor.taskCount
print brokerMonitor.taskQueueTime
def _create(self):
return StoreHolder.getRetryStore()
def brokerServerWatch(children):
for brokerName in children:
if not CabbageHolder.getServerCabbages().has_key(brokerName):
brokerServer = StoreHolder.getServerStore().getBrokerServer(brokerName)
zope.event.notify(AddBrokerServerEvent(brokerServer))
zope.event.notify(MonitorBrokerServerEvent(brokerServer))
def worker():
print "worker_pid:%d" % os.getpid()
job = StoreHolder.getStore().getJob("123456789")
print StoreHolder.getStore()
print job.fileName
def updateJobCache(jobId):
job=StoreHolder.getServerStore().getJob(jobId)
CacheHolder.getCache().put(jobId, job,JOBS)
