def syncJob(jobId, store=None):
jobId = jobId
if store is None:
with storeFactory.store() as store:
work = store.getWork(HOST_NAME)
store.removeJobWorkReady(jobId, work)
else:
work = store.getWork(HOST_NAME)
store.removeJobWorkReady(jobId, work)
job = CacheHolder.getCache().get(jobId, JOBS)
fileNames = []
#不同步主节点文件
# CacheHolder.getCache().put(job.fileName, "", jobId)
# try:
# syncFile(job.fileName,jobId,FileRequestMessage.MAIN)
# except Exception:
# syncFile(job.fileName,jobId,FileRequestMessage.MAIN)
#
# fileNames.append(job.fileName)
if job.attachFiles:
for attachFile in job.attachFiles:
CacheHolder.getCache().put(attachFile.fileName, "", jobId)
try:
syncFile(attachFile.fileName, jobId, FileRequestMessage.ATTACH)
except Exception:
syncFile(attachFile.fileName, jobId, FileRequestMessage.ATTACH)
fileNames.append(attachFile.fileName)
checkAllFileIsReady(fileNames, jobId)
def workStatusHandler(event):
if event:
work = CacheHolder.getCache().get(event.hostName,WORKS)
if work:
work.status=event.status
with storeFactory.store() as store:
store.updateWorkStatus(work)
def run(self):
jobId = self.message.jobId
fileName = self.message.fileName
serverDir = ConfigHolder.getConfig().getProperty(
BASE, SERVER_FILE_DIRECTORY)
jobDir = serverDir + "/" + jobId
filePath = jobDir + "/" + fileName
# job =None
with storeFactory.store() as sotre:
job = sotre.getJob(
jobId
) #StoreHolder.getServerStore().getJob(jobId)#CacheHolder.getCache().get(jobId,JOBS)
msg = FileResponseMessage()
msg.fileName = fileName
msg.jobId = jobId
if self.message.type == self.message.MAIN:
if fileName == job.fileName:
f = open(filePath)
msg.fileContent = base64.encodestring(f.read())
f.close()
return msg
if self.message.type == self.message.ATTACH:
for attachFile in job.attachFiles:
if attachFile.fileName == fileName:
f = open(filePath)
msg.fileContent = base64.encodestring(f.read())
f.close()
break
return msg
def workBrokerQueueChangeHandler(event):
# CacheHolder.getCache().put(QUEUES,event.brokerQueues,WORKS)
with storeFactory.store() as store:
work = store.getWork(HOST_NAME)
CacheHolder.getCache().put(HOST_NAME,work,WORKS)
if event.isEvent and (work.status == ON_LINE):
Logger.info(log,"restart")
CabbageControlHolder.getCabbageControl().restartCelery()
def jobUpdateHandler(event):
jobId = event.jobId
status = event.status
if status == JOB_DELETE:
jobRun = JobCacheHolder.getJobCache().get(jobId)
if jobRun : #停止运行TASK
jobRun.stop()
with storeFactory.store() as store:
store.updateJobStatus(jobId, JOB_DELETE)
#删除缓存让下一个task可以同名
tasks=CacheHolder.getCache().get(jobId, JOBS).tasks
for taskName in tasks:
if TaskCacheHolder.getJobCache().has_key(taskName):
TaskCacheHolder.getJobCache().remove(taskName)
with storeFactory.store() as store:
job=store.getJob(jobId)
CacheHolder.getCache().put(jobId, job,JOBS)
def stop(self):
CabbageControlHolder.getCabbageControl().stopCelery()
work = Work()
work.hostName = HOST_NAME
work.status = REMOVE
self.status = work.status
with storeFactory.store() as store:
store.updateWorkStatus(work)
tornado.ioloop.IOLoop.current().stop()
def jobAduitStatusWatch(data, stat, event=None):
try:
if data and data == JOB_AUTH_PASS and event is not None:
# example /cabbage/jobs/job-47778319-7a86-4b2b-a43a-5e2e94504350/status
jobId = event.path.split("/")[3]
with storeFactory.store() as store:
job = store.getJob(jobId)
updateJobCache(jobId, job)
zope.event.notify(JobAuditPassEvent(jobId))
except Exception:
Logger.exception(log)
def checkAllWorkBeReady(job):
jobId=job.jobId
with storeFactory.store() as store:
readyWorks = store.getJobWorksReadyDone(jobId)
notReadyWorks =[]
isAllBeReady = True
for work in job.works:
hostName = work.hostName
if hostName not in readyWorks:
notReadyWorks.append(hostName)
isAllBeReady=False
return (isAllBeReady,notReadyWorks)
def workBrokerServerChangeHandler(event):
print "------workBrokerServerChangeHandler-----------"
hostName = HOST_NAME
# work = StoreHolder.getStore().getWork(hostName)
with storeFactory.store() as store:
work = store.getWork(hostName)
# CacheHolder.getCache().put(QUEUES,work.queues,WORKS)
CacheHolder.getCache().put(hostName, work, WORKS)
if event.isEvent and (work.status == READY or work.status == ON_LINE):
# parent="/"+CABBAGE+"/"+WORKS+"/"+hostName
# from cabbage.watch.client_jobs_watch import workBrokerServerWatch
# ZookeeperClientHolder.getClient().addDataListener(parent+"/"+BROKER_SERVER, workBrokerServerWatch)
CabbageControlHolder.getCabbageControl().restartCelery()
def doAction(jobId):
if not CacheHolder.getCache().hasKey(jobId, JOBS):
with storeFactory.store() as store:
job = store.getJob(jobId)
CacheHolder.getCache().put(jobId, job, JOBS)
job = CacheHolder.getCache().get(jobId, JOBS)
Logger.debug(log, "upload files. job【%s】" % str(job.asDict()))
if job.resultBackend == None:
return
elif job.resultBackend == NFS:
CabbageNfsBackend(jobId).save()
elif job.resultBackend == HDFS:
CabbageHdfsBackend(jobId).save()
def jobChildWatch(children):
try:
for jobId in children:
jobId = str(jobId)
with storeFactory.store() as store:
job = store.getJob(jobId)
work = store.getWork(HOST_NAME)
if CacheHolder.getCache().hasKey(
jobId, JOBS) is False and job.brokerQueue in work.queues:
'''添加job的状态监控'''
parent = "/" + CABBAGE + "/" + JOBS + "/" + jobId
kazooClient.addDataListener(parent + "/" + STATUS,
jobRunStatusWatch)
kazooClient.addDataListener(parent + "/" + AUDIT_STATUS,
jobAduitStatusWatch)
updateJobCache(jobId, job)
except Exception:
Logger.exception(log)
def jobRunHandler(event):
jobId = event.jobId
params = event.params
ignoreNotPerWork = event.ignoreNotPerWork
if not CacheHolder.getCache().hasKey(jobId, JOBS):
with storeFactory.store() as store:
job=store.getJob(jobId)
CacheHolder.getCache().put(jobId, job,JOBS)
else:
job = CacheHolder.getCache().get(jobId,JOBS)
if ignoreNotPerWork:
runJob(job,params)
else:
(isAllBeReady,works)=checkAllWorkBeReady(job)
if isAllBeReady:
runJob(job,params)
else:
raise Exception("works:%s not be ready"% ",".join(works))
def start(self):
work = Work()
work.hostName = HOST_NAME
work.port = "1024"
work.ip = LOCAL_IP
work.status = READY
with storeFactory.store() as store:
if not store.isExistWork(work):
store.saveWork(work)
CacheHolder.getCache().put(HOST_NAME, work, WORKS)
else:
CacheHolder.getCache().put(HOST_NAME, store.getWork(HOST_NAME),
WORKS)
store.updateWorkStatus(work)
self.runCeleryServer(work, store)
t2 = threading.Thread(target=scheduler)
t2.setDaemon(True)
t2.start()
tornado.ioloop.IOLoop.current().start()
def jobRemoveHandler(event):
try:
jobId = event.jobId
if JobCacheHolder.getJobCache().has_key(jobId):
jobRun = JobCacheHolder.getJobCache().get(jobId)
if jobRun : #停止运行TASK
jobRun.stop()
else:
job =CacheHolder.getCache().get(jobId, JOBS)
for taskName in job.tasks:
CabbageHolder.getServerCabbage(job.brokerServer).revokeByTaskName(taskName)
with storeFactory.store() as store:
store.updateJobStatus(jobId, JOB_DELETE)
#删除缓存让下一个task可以同名
tasks=CacheHolder.getCache().get(jobId, JOBS).tasks
for taskName in tasks:
if TaskCacheHolder.getJobCache().has_key(taskName):
TaskCacheHolder.getJobCache().remove(taskName)
CacheHolder.getCache().remove(jobId, JOBS)
except:
Logger.exception(log)
def clentWorkStatusChangeHandler(event):
from cabbage.client_start import CabbageClientHolder
hostName = HOST_NAME
# work = StoreHolder.getStore().getWork(hostName)
with storeFactory.store() as store:
work = store.getWork(hostName)
CacheHolder.getCache().put(hostName, work, WORKS)
clentStatus = CabbageClientHolder.getClient().status
if event.status == OFF_LINE and clentStatus == ON_LINE:
CabbageControlHolder.getCabbageControl().stopCelery(hostName)
CabbageClientHolder.getClient().status = OFF_LINE
if event.status == ON_LINE and clentStatus == OFF_LINE:
# def run():
CabbageClientHolder.getClient().status = ON_LINE
CabbageControlHolder.getCabbageControl().restartCelery()
# t1 = threading.Thread(target=run)
# t1.setDaemon(True)
# t1.start()
if event.status == REMOVE and clentStatus != REMOVE:
CabbageClientHolder.getClient().stop()
def onLine(self):
work = Work()
work.hostName = HOST_NAME
work.status = ON_LINE
with storeFactory.store() as store:
self.runCeleryServer(work, store)
def updateWorkCache(hostName):
with storeFactory.store() as store:
work = store.getWork(hostName)
CacheHolder.getCache().put(hostName, work, WORKS)
def jobAuditStatusHandler(event):
jobId = event.jobId
status = event.status
with storeFactory.store() as store:
store.updateAuditStatus(jobId,status)