def addBroberServerHandler(event):
if event and event.brokerServer:
brokerServer = event.brokerServer
Logger.info(log,"添加队列服务器【%s】,URI:【%s】"%( brokerServer.hostName,brokerServer.connectUri))
cabbage = Cabbage(hostName=brokerServer.hostName,broker=brokerServer.connectUri)
CabbageHolder.getServerCabbages()[brokerServer.hostName]= cabbage
Logger.debug(log,"添加队列服务器【%s】"% CabbageHolder.getServerCabbagesStr())
def saveJob(self, job):
parent = "/" + CABBAGE + "/" + JOBS + "/" + job.jobId
self.client.create(parent, makepath=True)
Logger.debug(log, parent)
self.client.create(parent + "/" + JOB_NAME, value=job.jobName)
self.client.create(parent + "/" + FILE_PATH, value=job.filePath)
self.client.create(parent + "/" + FILE_NAME, value=job.fileName)
self.client.create(parent + "/" + FILE_TYPE, value=job.fileType)
self.client.create(parent + "/" + STATUS, value=job.status)
self.client.create(parent + "/" + AUDIT_STATUS, value=job.auditStatus)
self.client.create(parent + "/" + RUN_STRATEGY, value=job.runStrategy)
self.client.create(parent + "/" + STRATEGY_VALUE,
value=job.strategyValue)
self.client.create(parent + "/" + ATTACH_FILES)
self.client.create(parent + "/" + REULST_BACKEND,
value=job.resultBackend)
for f in job.attachFiles:
self.client.create(parent + "/" + ATTACH_FILES + "/" + f.fileName,
value=f.filePath,
makepath=True)
self.client.create(parent + "/" + ATTACH_FILES + "/" + f.fileName +
"/" + FILE_TYPE,
value=f.fileType,
makepath=True)
for w in job.works:
self.client.create(parent + "/" + WORKS + "/" + LIST + "/" +
w.hostName,
value=w.port,
makepath=True)
if not self.client.isExistPath(parent + "/" + WORKS + "/" +
READIES):
self.client.create(parent + "/" + WORKS + "/" + READIES)
if job.tasks:
for task in job.tasks:
self.client.create(parent + "/" + TASKS + "/" + task,
makepath=True)
self.client.create("/" + CABBAGE + "/" + JOBS + "/" + RESULTS +
"/" + job.jobId + "/" + task,
makepath=True)
self.client.create(parent + "/" + BROKER_SERVER,
value=job.brokerServer)
self.client.create(parent + "/" + QUEUE, value=job.brokerQueue)
# for q in job.queue:
# self.client.create(parent+"/"+BROKER_SERVER+"/"+QUEUES+"/"+q)
#使用该目录是因为，如果前面的目录没有创建完，集群的节点已经开始监控，导致数据不完整，所以，单独才用该目录来进行节点的监控
self.client.create("/" + CABBAGE + "/" + JOBS + "/" + READIES + "/" +
job.jobId)
def doAction(jobId):
if not CacheHolder.getCache().hasKey(jobId, JOBS):
with storeFactory.store() as store:
job = store.getJob(jobId)
CacheHolder.getCache().put(jobId, job, JOBS)
job = CacheHolder.getCache().get(jobId, JOBS)
Logger.debug(log, "upload files. job【%s】" % str(job.asDict()))
if job.resultBackend == None:
return
elif job.resultBackend == NFS:
CabbageNfsBackend(jobId).save()
elif job.resultBackend == HDFS:
CabbageHdfsBackend(jobId).save()
def save(self):
hdfsPath=ConfigHolder.getConfig().getProperty(BASE,HDFS_ROOT_PATH)
dateStr = getNowDateStr()
if self.jobId:
localPath = getLocalFilesPath()
dateStr = getNowDateStr()
hour = getNowHour()
if hour == 0:# 提交前一天的数据
dateStr = formatDate(subDay(getNow(),1),f="%Y%m%d")
p = localPath+"/"+self.jobId+"/result/"+dateStr
Logger.debug( log, "upload file to hdfs. jobId【%s】 date【%s】" % (self.jobId,dateStr))
if not os.path.isdir(p):
return
fileNames = os.listdir(p)
if len(fileNames) == 0:
return
client =HdfsClientHolder.getHdfsClient()
remoteDire=hdfsPath+"/"+self.jobId
if not client.isDirectory(remoteDire):
client.mkdir(remoteDire)
remoteDire= remoteDire+"/"+dateStr
if not client.isDirectory(remoteDire):
client.mkdir(remoteDire)
Logger.info(log,"hour:%s files:%s"%(hour,",".join(fileNames)))
for fileName in fileNames:
if hour != 0:
if int(fileName) >= hour:
continue
# if os.path.isfile(p+"/"+fileName):
self.uploadToHdfs(client,localPath,self.jobId,hdfsPath,fileName,dateStr)
os.remove(p+"/"+fileName)