def jobReadyWatch(children):
try:
# brokers={}
for jobId in children:
try:
job=StoreHolder.getServerStore().getJob(jobId)
if CacheHolder.getCache().hasKey(jobId, JOBS) is False:
CacheHolder.getCache().put(jobId, job,JOBS)
for taskName in job.tasks:
TaskCacheHolder.getJobCache().put(taskName,job.jobId)
except Exception:
Logger.exception( log)
#偷个懒，只要没有删除的全部放到ROUTER里面去
# if job.status != JOB_DELETE:
#
# for taskName in job.tasks:
# que=StoreHolder.getServerStore().getQueue(job.brokerQueue)
# TaskCacheHolder.getJobCache().put(taskName,job.jobId)
#
# if brokerServer in brokers:
# brokers[brokerServer].update(routes)
# else:
# brokers[brokerServer] = routes
#
# #偷个懒，只要没有删除的全部放到ROUTER里面去
# for broker,routes in brokers.items():
# brokerServer = StoreHolder.getServerStore().getBrokerServer(broker)
# CabbageHolder.getServerCabbages()[brokerServer.hostName].getApp().conf.update(CELERY_ROUTES = routes)
# Logger.info(log,"更新队列服务器【%s】ROUTES【%s】"% (CabbageHolder.getServerCabbagesStr(),str(routes)))
except Exception:
Logger.exception( log)
def action(queue):
while True:
try:
jobId = queue.get()
doAction(jobId)
except Exception:
Logger.exception(log)
def runJob(q):
while True:
try:
evnet = q.get()
zope.event.notify(evnet)
except Exception:
Logger.exception(log)
def jobRunStatusWatch(data, stat, event=None):
try:
if data and event is not None:
jobId = event.path.split("/")[3]
zope.event.notify(JobStatusChangeEvent(jobId))
except Exception:
Logger.exception(log)
def handleRequest(conn, addr):
try:
resultMessage = doRequestHandle(conn) #msgInstance.doAction()
#result一般为message类型
if resultMessage:
conn.sendall(MessageCodec().encode(resultMessage))
except Exception:
Logger.exception(log)
def workStatusWatch(data, stat, event=None):
try:
if data and event is not None:
#path=u'/cabbage/works/huamac/status')
hostname = event.path.split("/")[3]
if hostname == HOST_NAME:
zope.event.notify(ClientWorkStatusEvent(data))
except Exception:
Logger.exception(log)
def configWatch(children):
try:
for l in children :
pa =CONFIG_PATH+"/"+l
kazooClient.addDataListener(pa, configOptionDataChange)
data= kazooClient.getData(pa)
ConfigHolder.getConfig().setProperty(BASE,l ,data)
except Exception:
Logger.exception( log)
def workBrokerQueueWatch(children):
try:
data = kazooClient.getData("/" + CABBAGE + "/" + WORKS + "/" +
HOST_NAME + "/" + QUEUES)
# print data
if data and data == DO_NOTHING:
return
zope.event.notify(WorkBrokerQueueChangeEvent(children, isEvent=True))
except Exception:
Logger.exception(log)
def updateTaksSent(self,taskName):
try:
self.lock.acquire()
if taskName in self.taskSent:
self.taskSent[taskName]= self.taskSent.get(taskName)+1
else:
self.taskSent[taskName]=1
except Exception as e:
Logger.exception(log)
finally:
self.lock.release()
def jobAduitStatusWatch(data, stat, event=None):
try:
if data and data == JOB_AUTH_PASS and event is not None:
# example /cabbage/jobs/job-47778319-7a86-4b2b-a43a-5e2e94504350/status
jobId = event.path.split("/")[3]
with storeFactory.store() as store:
job = store.getJob(jobId)
updateJobCache(jobId, job)
zope.event.notify(JobAuditPassEvent(jobId))
except Exception:
Logger.exception(log)
def jobAuditPassHandler(event):
try:
jobId = event.jobId
syncJob(jobId)
#通知子进程进行加载模块
# zope.event.notify(JobNeedLoadEvent(jobId))
from cabbage.process.cabbage_control_holder import CabbageControlHolder
# CabbageControlHolder.getCabbageControl().addJobId(event.jobId)
CabbageControlHolder.getCabbageControl().restartCelery()
except Exception:
Logger.exception(log)
def taskSucceeded(state, event, app):
taskId = event['uuid']
task = state.tasks.get(taskId)
# taskName = task.name if task and hasattr(task,'name') else None
jobId, taskName = _getJobIdAndTaskName(taskId)
queueTime = 0
runtime = 0
# state.
#FIXME 经常找不到TASK.name
#@TODO
log.debug("【%s】 TASK SUCCEEDED !" % (event['uuid']))
try:
# jobId= None
# if hasattr(task,'kwargs') and task.kwargs is not None and JOB_ID in task.kwargs:
# jobId= eval(str(task.kwargs))[JOB_ID]
#
# if jobId is None and taskName:
# jobId = TaskCacheHolder.getJobCache().get(taskName)
#
# if taskName is None or jobId is None:
# jobId,taskName = _getJobIdAndTaskName(taskId)
# print jobId,taskName
job = CacheHolder.getCache().get(jobId, JOBS)
result = AsyncResult(taskId,
app=CabbageHolder.getServerCabbage(
job.brokerServer).getApp())
if not isinstance(result.backend, DisabledBackend):
log.debug("【%s】 TASK SUCCEEDED result【%s】 !" %
(event['uuid'], result.result))
if result.result:
jobResults.addResult(jobId, result.result)
# # print task.started - task.received
# print queueTime
# print "task.started:%s"%task.started
# print "task.received:%s"%task.received
# print "task.runtime:%s" % event['runtime']
# print event
if task and task.started and task.received:
queueTime = task.started - task.received
runtime = event['runtime']
# with storeFactory.store() as store:
# store.deleteTaskId( taskId)
CabbageCounterHolder.getCabbageCounter().updateTaskSucceeded(
taskName, _getHostName(event), runtime, queueTime)
# raise Exception("test")
except Exception as e:
Logger.exception(log)
def taskReceived(state, event, app):
# task = state.tasks.get(event['uuid'])
log.debug("%s" % event)
# print task.name
# print "monitor"
# print event
# print dir(task)
try:
CabbageCounterHolder.getCabbageCounter().updateTaskReceived(
event['name'], _getHostName(event))
except:
Logger.exception(log)
def addScriptJobId(self,jobId):
store =None
try:
store = StoreHolder.getStore()
log.info("节点【%s】当前任务【%s】的脚本开始加载。。。。" % (HOST_NAME,jobId))
self.loadJobScript(jobId,store)
work=store.getWork(HOST_NAME)
self.sendBeReady(jobId,work,store)
store.close()
except Exception:
Logger.exception(log)
finally:
if store:
store.close()
def updateTaskReceived(self,taskName,hostName):
try:
self.lock.acquire()
if taskName in self.taskReceived:
if hostName in self.taskReceived[taskName]:
self.taskReceived[taskName][hostName]=self.taskReceived[taskName][hostName]+1
else:
self.taskReceived[taskName].update({hostName:1})
else:
self.taskReceived[taskName]={hostName:1}
except Exception as e:
Logger.exception(log)
finally:
self.lock.release()
def save(self):
try:
nfsPath=ConfigHolder.getConfig().getProperty(BASE,NFS_DIRECTORY)
dateStr = getNowDateStr()
if self.jobId:
localPath = getLocalFilesPath()
dateStr = getNowDateStr()
hour = getNowHour()
if hour == 0:# 提交前一天的数据
dateStr = formatDate(subDay(getNow(),1),f="%Y%m%d")
localPath = localPath+"/"+self.jobId+"/result/"+dateStr
Logger.info( log, "upload file to nfs. jobId【%s】 date【%s】" % (self.jobId,dateStr))
if not os.path.isdir(localPath):
return
fileNames = os.listdir(localPath)
if len(fileNames) == 0:
return
remoteDire=nfsPath+"/"+self.jobId+"/"+dateStr
if not os.path.isdir(remoteDire):
os.makedirs(remoteDire)
# os.chmod(remoteDire,777)
Logger.info(log,"hour:%s files:%s"%(hour,",".join(fileNames)))
for fileName in fileNames:
if hour != 0:
if int(fileName) >= hour:
continue
newFileName = None
if os.environ[CABBAGE] ==MASTER:
newFileName = HOST_NAME+"_"+LOCAL_IP+"_"+MASTER+"_"+fileName
else:
newFileName = HOST_NAME+"_"+LOCAL_IP+"_"+NODE+"_"+fileName
if os.path.isfile(localPath+"/"+fileName):
shutil.move(localPath+"/"+fileName,remoteDire+"/"+newFileName)
except Exception as e:
Logger.exception(log)
def taskSent(state, event, app):
# task = state.tasks.get(event['uuid'])
# log.info("%s"% task)
# print task.name
# print "monitor"
# print event
# print dir(task)
try:
taskName = event['name']
taskId = event['uuid']
# jobId = eval(str(event['kwargs']))[JOB_ID]
jobId = TaskCacheHolder.getJobCache().get(taskName)
# StoreHolder.getRedisStaticStore().saveTaskId(jobId, taskName, taskId)
CabbageCounterHolder.getCabbageCounter().updateTaksSent(taskName)
except:
Logger.exception(log)
def jobChildWatch(children):
try:
for jobId in children:
jobId = str(jobId)
with storeFactory.store() as store:
job = store.getJob(jobId)
work = store.getWork(HOST_NAME)
if CacheHolder.getCache().hasKey(
jobId, JOBS) is False and job.brokerQueue in work.queues:
'''添加job的状态监控'''
parent = "/" + CABBAGE + "/" + JOBS + "/" + jobId
kazooClient.addDataListener(parent + "/" + STATUS,
jobRunStatusWatch)
kazooClient.addDataListener(parent + "/" + AUDIT_STATUS,
jobAduitStatusWatch)
updateJobCache(jobId, job)
except Exception:
Logger.exception(log)
def updateTaskSucceeded(self,taskName,hostName,runTime,queueTime):
try:
self.lock.acquire()
if taskName in self.taskSucceeded:
if hostName in self.taskSucceeded[taskName]:
m= self.taskSucceeded[taskName][hostName]
m[TASK_COUNT]=m[TASK_COUNT]+1
m[TASK_RUNTIME]=m[TASK_RUNTIME]+runTime
m[TASK_QUEUE_TIME]=m[TASK_QUEUE_TIME]+queueTime
else:
self.taskSucceeded[taskName].update({hostName:{TASK_COUNT:1,TASK_RUNTIME:runTime,TASK_QUEUE_TIME:queueTime}})
else:
self.taskSucceeded[taskName]={hostName:{TASK_COUNT:1,TASK_RUNTIME:runTime,TASK_QUEUE_TIME:queueTime}}
except Exception as e:
Logger.exception(log)
finally:
self.lock.release()
def taskFailed(state, event, app):
eventOutDic = event.copy()
taskName = None
try:
taskId = event['uuid']
task = state.tasks.get(taskId)
#
# taskName = task.name if task and hasattr(task,'name') else None
#
# if hasattr(task,'kwargs') and task.kwargs is not None and JOB_ID in task.kwargs:
# eventOutDic[JOB_ID] = eval(str(task.kwargs))[JOB_ID]
#
# if eventOutDic[JOB_ID] is None:
# eventOutDic[JOB_ID] = TaskCacheHolder.getJobCache().get(taskName)
# if taskName is None or eventOutDic[JOB_ID] is None :
jobId, taskName = _getJobIdAndTaskName(taskId)
eventOutDic[JOB_ID] = jobId
job = CacheHolder.getCache().get(eventOutDic[JOB_ID], JOBS)
brokerServer = job.brokerServer
taskPath = ConfigHolder.getConfig().getProperty(
BASE, TASK_FAILLOG_PATH)
if not os.path.isdir(taskPath):
os.makedirs(taskPath)
dateStr = getNowDateStr()
with open(taskPath + "/" + brokerServer + "_" + dateStr + ".log",
"a+") as writer:
writer.write(str(eventOutDic) + "\n")
# with storeFactory.store() as store:
# store.deleteTaskId(task.id)
# StoreHolder.getRedisStaticStore().deleteTaskId(taskId)
CabbageCounterHolder.getCabbageCounter().updateTaksFail(
taskName, _getHostName(event))
except Exception as e:
Logger.exception(log)
def jobWebWatch(children):
store = storeFactory.getStore()
try:
brokers={}
for jobId in children:
try:
job=store.getJob(jobId)#toreHolder.getStore().getJob(jobId)
if CacheHolder.getCache().hasKey(jobId, JOBS) is False:
CacheHolder.getCache().put(jobId, job,JOBS)
# kazooClient.addDataListener(parent+"/"+STATUS, jobRunStatusWatch)
#偷个懒，只要没有删除的全部放到ROUTER里面去
if job.status != JOB_DELETE:
brokerServer=job.brokerServer
routes={}
for taskName in job.tasks:
que=store.getQueue(job.brokerQueue)
routes[taskName]={'queue': que.queueName, 'routing_key': que.routingKey}
TaskCacheHolder.getJobCache().put(taskName,job.jobId)
if brokerServer in brokers:
brokers[brokerServer].update(routes)
else:
brokers[brokerServer] = routes
except Exception:
Logger.exception( log)
#偷个懒，只要没有删除的全部放到ROUTER里面去
for broker,routes in brokers.items():
brokerServer = store.getBrokerServer(broker)
#修复BUG，导致任务提交的celery队列里面去了
cabbage = Cabbage(hostName=brokerServer.hostName,broker=brokerServer.connectUri)
cabbage.app.conf.update(CELERY_ROUTES = routes)
CabbageHolder.getServerCabbages()[brokerServer.hostName] = cabbage
# CabbageHolder.getServerCabbages()[brokerServer.hostName].getApp().conf.update(CELERY_ROUTES = routes)
Logger.info(log,"更新队列服务器【%s】ROUTES【%s】"% (brokerServer.hostName,str(routes)))
except Exception:
Logger.exception( log)
finally:
storeFactory.returnStroe(store)
def doAction(self,actionFun):
sendDict=None
receivedDict=None
failDict=None
succeedDict=None
try:
self.lock.acquire()
sendDict = self.taskSent.copy()
receivedDict = self.taskReceived.copy()
failDict = self.taskFail.copy()
succeedDict = self.taskSucceeded.copy()
actionFun(sendDict,receivedDict,failDict,succeedDict)
self.taskFail.clear()
self.taskSent.clear()
self.taskSucceeded.clear()
self.taskReceived.clear()
except Exception as e:
Logger.exception(log)
finally:
self.lock.release()
def jobRemoveHandler(event):
try:
jobId = event.jobId
if JobCacheHolder.getJobCache().has_key(jobId):
jobRun = JobCacheHolder.getJobCache().get(jobId)
if jobRun : #停止运行TASK
jobRun.stop()
else:
job =CacheHolder.getCache().get(jobId, JOBS)
for taskName in job.tasks:
CabbageHolder.getServerCabbage(job.brokerServer).revokeByTaskName(taskName)
with storeFactory.store() as store:
store.updateJobStatus(jobId, JOB_DELETE)
#删除缓存让下一个task可以同名
tasks=CacheHolder.getCache().get(jobId, JOBS).tasks
for taskName in tasks:
if TaskCacheHolder.getJobCache().has_key(taskName):
TaskCacheHolder.getJobCache().remove(taskName)
CacheHolder.getCache().remove(jobId, JOBS)
except:
Logger.exception(log)
