def doRun(self,aaa=1,bbb=2,no=None):
if random.randint(0,10) % 2 == 0:
print self.jobId,no
self.addResult(getNow())
return "f**k"
return "hello world!"
def doRun(self,aaa=1,bbb=2,no=None):
self.addResult(getNow())
if random.randint(0,10) % 2 == 0:
print "jobId:%s\tNo:%s" % (self.jobId,no)
return "F**K"
return "hello world!"
def listener(self, event):
if event.exception:
log.error("任务【%s】 任务出错 : %s" % (event.job_id, event.traceback))
else:
log.debug("任务【%s】已经跑完，结束时间 : %s " % (event.job_id, getNow()))
# jobMange = JobManage()
def save(self):
try:
nfsPath=ConfigHolder.getConfig().getProperty(BASE,NFS_DIRECTORY)
dateStr = getNowDateStr()
if self.jobId:
localPath = getLocalFilesPath()
dateStr = getNowDateStr()
hour = getNowHour()
if hour == 0:# 提交前一天的数据
dateStr = formatDate(subDay(getNow(),1),f="%Y%m%d")
localPath = localPath+"/"+self.jobId+"/result/"+dateStr
Logger.info( log, "upload file to nfs. jobId【%s】 date【%s】" % (self.jobId,dateStr))
if not os.path.isdir(localPath):
return
fileNames = os.listdir(localPath)
if len(fileNames) == 0:
return
remoteDire=nfsPath+"/"+self.jobId+"/"+dateStr
if not os.path.isdir(remoteDire):
os.makedirs(remoteDire)
# os.chmod(remoteDire,777)
Logger.info(log,"hour:%s files:%s"%(hour,",".join(fileNames)))
for fileName in fileNames:
if hour != 0:
if int(fileName) >= hour:
continue
newFileName = None
if os.environ[CABBAGE] ==MASTER:
newFileName = HOST_NAME+"_"+LOCAL_IP+"_"+MASTER+"_"+fileName
else:
newFileName = HOST_NAME+"_"+LOCAL_IP+"_"+NODE+"_"+fileName
if os.path.isfile(localPath+"/"+fileName):
shutil.move(localPath+"/"+fileName,remoteDire+"/"+newFileName)
except Exception as e:
Logger.exception(log)
def save(self):
hdfsPath=ConfigHolder.getConfig().getProperty(BASE,HDFS_ROOT_PATH)
dateStr = getNowDateStr()
if self.jobId:
localPath = getLocalFilesPath()
dateStr = getNowDateStr()
hour = getNowHour()
if hour == 0:# 提交前一天的数据
dateStr = formatDate(subDay(getNow(),1),f="%Y%m%d")
p = localPath+"/"+self.jobId+"/result/"+dateStr
Logger.debug( log, "upload file to hdfs. jobId【%s】 date【%s】" % (self.jobId,dateStr))
if not os.path.isdir(p):
return
fileNames = os.listdir(p)
if len(fileNames) == 0:
return
client =HdfsClientHolder.getHdfsClient()
remoteDire=hdfsPath+"/"+self.jobId
if not client.isDirectory(remoteDire):
client.mkdir(remoteDire)
remoteDire= remoteDire+"/"+dateStr
if not client.isDirectory(remoteDire):
client.mkdir(remoteDire)
Logger.info(log,"hour:%s files:%s"%(hour,",".join(fileNames)))
for fileName in fileNames:
if hour != 0:
if int(fileName) >= hour:
continue
# if os.path.isfile(p+"/"+fileName):
self.uploadToHdfs(client,localPath,self.jobId,hdfsPath,fileName,dateStr)
os.remove(p+"/"+fileName)
# test="job-da019109-6296-42ef-b8f5-4e8af4241349"
# test2="job-f1ec097f-9130-4e4e-abf1-30062cef438b"
# test3="job-3f451485-51ac-4912-abe6-786b8d710e87"
both ="job-29f3a426-267b-4fbd-97d8-919e693e030d"
mac="job-138d5bb7-8489-47a8-9b79-994aab068900"
ubuntu="job-6911124b-1f7c-4acc-a155-fb4513b74bc2"
# error="job-9103ee58-1368-4406-879a-463daa8d0d79"
# hdfs="job-00faa376-e9c1-4079-93b2-19bb3db57f4a"
# long_time="job-b81fc441-a1e6-40ef-ab6f-5400e61e6614"
#
# nfs="job-57cebce9-8eff-4853-9ac4-e119cd8235f4"
long_time="job-37679f6c-8d07-4deb-afd9-b66549b78161"
shot_time="job-71d2c8e5-9c06-4e6c-ab81-d105808ca5e0"
url="http://127.0.0.1:2048/runJob"
# url ="http://101.198.156.26:2048/runJob"
begin= getNow()
for i in range(100):
# time.sleep(1)
print i
# requests.post(url, data = {'jobId':mac,"params":i})
requests.post("http://127.0.0.1:2048/runJob", data = {'jobId':shot_time})
# requests.post("http://127.0.0.1:2048/runJob", data = {'jobId':mac})
# requests.post("http://127.0.0.1:2048/runJob", data = {'jobId':ubuntu})
print "begin:【%s】 end:【%s】"%(begin,getNow())