def doTask(self, v_task): try : task = cPickle.loads(decompress(v_task)) return compress(cPickle.dumps(self.worker.doTask(task.data),-1)) except Exception, e: print '>>> traceback <<<' traceback.print_exc() print '>>> end of traceback <<<' logger.error(e) raise ServiceError(e)
def doTask(self, v_task): try : if self.asynchronous : return self.__doTaskAsyn(v_task) task = compress(cPickle.dumps(v_task,-1)) decompResult = decompress(self.workerService.doTask(task)) return v_task.id, Task.TASK_FINISHED, cPickle.loads(decompResult) except Exception, e: logger.error(e) return v_task.id, Task.TASK_ERROR,OtherFailure(e.message)
def run_task(task_data): try: gc.disable() (task, ntry) = cPickle.loads(decompress(task_data)) result = task.run(ntry) return mesos_pb2.TASK_FINISHED, compress(cPickle.dumps(result, -1)) except Exception, e: logger.error(e) import traceback traceback.print_exc() msg = traceback.format_exc() return mesos_pb2.TASK_FAILED, msg
def run_task(task_data): try: gc.disable() (task, ntry) = cPickle.loads(decompress(task_data)) result = task.run(ntry) return mesos_pb2.TASK_FINISHED, compress(cPickle.dumps(result, -1)) except Exception, e: logger.error(e) import traceback traceback.print_exc() msg = traceback.format_exc() return mesos_pb2.TASK_FAILED, msg
def task_update(self, t, status, result): if status != mesos_pb2.TASK_FINISHED: t.result = Exception(str(result)) else: t.result = cPickle.loads(decompress(result)) if len(self.warehouse_addrs) > 2: cur = self.outputdb.cursor() value = [status, result, t.id] cur.execute( "update t_task set status=%s,result=%s,task_end_time=now() where task_id=%s", value) self.outputdb.commit() cur.close() else: try: self.producer.send_messages("%sResult" % (t.warehouse), cPickle.dumps(t)) except Exception, e: logger.error(e)
for i in range(0,5) : j = str(random.randint(0, 9000000)).ljust(7,'0') taskId = "%s%s" % (datetime.datetime.now().strftime("%Y%m%d%H%M%S%f"),j) task = Task(taskId,workerClass="helloWorker.HelloWorker",workDir = os.path.dirname(os.path.abspath(__file__))) task.data = taskId + " world" bt = time.time() tasks.append(task) tasksLen = len(tasks) wk.schedule(tasks) i = 1 for e in wk.completionEvents() : if isinstance(e.reason,Success) : print i,e.task.id,cPickle.loads(decompress(e.result)) else: print i,"Failed:" + str(e.reason.message) i += 1 time.sleep(1) wk.shutdown() except KeyboardInterrupt: print "stopped by KeyboardInterrupt" sys.exit(1)
# task.resources = {"cpus":1,"mem":100,"gpus":0} # task.warehouse = wk.options.warehouse bt = time.time() tasks.append(task) tasksLen = len(tasks) wk.schedule(tasks) i = 1 if wk.mode=="factory": check_kafka_events() else: for e in wk.completionEvents() : if isinstance(e.reason,Success) : print i,e.task.id,cPickle.loads(decompress(e.result)) else: print i,"Failed:" + str(e.reason.message) i += 1 print "%d tasks finished in %.1f seconds" % (tasksLen, time.time() - start) time.sleep(1) wk.shutdown() except KeyboardInterrupt: print "stopped by KeyboardInterrupt" loopCondition = False sys.exit(1)
def receiveMaterials(self, v_task): return self.worker.receiveMaterials(cPickle.loads(decompress(v_task)))