def doTask(self, v_task):
try :
task = cPickle.loads(decompress(v_task))
return compress(cPickle.dumps(self.worker.doTask(task.data),-1))
except Exception, e:
print '>>> traceback <<<'
traceback.print_exc()
print '>>> end of traceback <<<'
logger.error(e)
raise ServiceError(e)
def doTask(self, v_task):
try :
if self.asynchronous :
return self.__doTaskAsyn(v_task)
task = compress(cPickle.dumps(v_task,-1))
decompResult = decompress(self.workerService.doTask(task))
return v_task.id, Task.TASK_FINISHED, cPickle.loads(decompResult)
except Exception, e:
logger.error(e)
return v_task.id, Task.TASK_ERROR,OtherFailure(e.message)
def run_task(task_data):
try:
gc.disable()
(task, ntry) = cPickle.loads(decompress(task_data))
result = task.run(ntry)
return mesos_pb2.TASK_FINISHED, compress(cPickle.dumps(result, -1))
except Exception, e:
logger.error(e)
import traceback
traceback.print_exc()
msg = traceback.format_exc()
return mesos_pb2.TASK_FAILED, msg
def run_task(task_data):
try:
gc.disable()
(task, ntry) = cPickle.loads(decompress(task_data))
result = task.run(ntry)
return mesos_pb2.TASK_FINISHED, compress(cPickle.dumps(result, -1))
except Exception, e:
logger.error(e)
import traceback
traceback.print_exc()
msg = traceback.format_exc()
return mesos_pb2.TASK_FAILED, msg
def task_update(self, t, status, result):
if status != mesos_pb2.TASK_FINISHED:
t.result = Exception(str(result))
else:
t.result = cPickle.loads(decompress(result))
if len(self.warehouse_addrs) > 2:
cur = self.outputdb.cursor()
value = [status, result, t.id]
cur.execute(
"update t_task set status=%s,result=%s,task_end_time=now() where task_id=%s",
value)
self.outputdb.commit()
cur.close()
else:
try:
self.producer.send_messages("%sResult" % (t.warehouse),
cPickle.dumps(t))
except Exception, e:
logger.error(e)
for i in range(0,5) :
j = str(random.randint(0, 9000000)).ljust(7,'0')
taskId = "%s%s" % (datetime.datetime.now().strftime("%Y%m%d%H%M%S%f"),j)
task = Task(taskId,workerClass="helloWorker.HelloWorker",workDir = os.path.dirname(os.path.abspath(__file__)))
task.data = taskId + " world"
bt = time.time()
tasks.append(task)
tasksLen = len(tasks)
wk.schedule(tasks)
i = 1
for e in wk.completionEvents() :
if isinstance(e.reason,Success) :
print i,e.task.id,cPickle.loads(decompress(e.result))
else:
print i,"Failed:" + str(e.reason.message)
i += 1
time.sleep(1)
wk.shutdown()
except KeyboardInterrupt:
print "stopped by KeyboardInterrupt"
sys.exit(1)
# task.resources = {"cpus":1,"mem":100,"gpus":0}
# task.warehouse = wk.options.warehouse
bt = time.time()
tasks.append(task)
tasksLen = len(tasks)
wk.schedule(tasks)
i = 1
if wk.mode=="factory":
check_kafka_events()
else:
for e in wk.completionEvents() :
if isinstance(e.reason,Success) :
print i,e.task.id,cPickle.loads(decompress(e.result))
else:
print i,"Failed:" + str(e.reason.message)
i += 1
print "%d tasks finished in %.1f seconds" % (tasksLen, time.time() - start)
time.sleep(1)
wk.shutdown()
except KeyboardInterrupt:
print "stopped by KeyboardInterrupt"
loopCondition = False
sys.exit(1)
def receiveMaterials(self, v_task):
return self.worker.receiveMaterials(cPickle.loads(decompress(v_task)))