Python compress示例

示例#1

文件： workerlocal.py 项目： ITking666/opencluster

 def __doTaskAsyn(self, v_task):
     try :
         task =  compress(cPickle.dumps(v_task,-1))
         return v_task.id, Task.TASK_FINISHED, self.workerService.doTask(task)
     except Exception, e:
         logger.error(e)
         return v_task.id, Task.TASK_ERROR,OtherFailure(e.message)

示例#2

文件： schedule.py 项目： astroitlab/opencluster

    def create_task(self, offer, t, cpus):
        task = mesos_pb2.TaskInfo()

        task.task_id.value = t.id
        task.slave_id.value = offer.slave_id.value
        task.name = "task(%s/%d)" % (t.id, self.taskNum)
        task.executor.MergeFrom(self.executor)

        task.data = compress(cPickle.dumps((t, t.tried), -1))

        cpu = task.resources.add()
        cpu.name = "cpus"
        cpu.type = 0 # mesos_pb2.Value.SCALAR
        cpu.scalar.value = min(self.cpus, cpus)

        mem = task.resources.add()
        mem.name = "mem"
        mem.type = 0 # mesos_pb2.Value.SCALAR
        mem.scalar.value = self.mem
        #
        # gpu = task.resources.add()
        # gpu.name = "gpus"
        # gpu.type = 0 # mesos_pb2.Value.SCALAR
        # gpu.scalar.value = self.gpus

        return task

示例#3

文件： factorymesos.py 项目： ITking666/opencluster

    def create_task(self, offer, t):
        task = mesos_pb2.TaskInfo()
        task.task_id.value = str(t.id)
        task.slave_id.value = offer.slave_id.value
        task.name = "%s-%s" % (t.id, t.tried)
        task.executor.MergeFrom(self.executor)

        task.data = compress(cPickle.dumps((t, t.tried), -1))

        cpu = task.resources.add()
        cpu.name = "cpus"
        cpu.type = 0  # mesos_pb2.Value.SCALAR
        cpu.scalar.value = t.resources.get("cpus", 1)

        mem = task.resources.add()
        mem.name = "mem"
        mem.type = 0  # mesos_pb2.Value.SCALAR
        mem.scalar.value = t.resources.get("mem", 100)

        # gpu = task.resources.add()
        # gpu.name = "gpus"
        # gpu.type = 0 # mesos_pb2.Value.SCALAR
        # gpu.scalar.value = t.resources.get("gpus",1)

        return task

示例#4

文件： schedule.py 项目： ITking666/opencluster

    def create_task(self, offer, t, cpus):
        task = mesos_pb2.TaskInfo()

        task.task_id.value = t.id
        task.slave_id.value = offer.slave_id.value
        task.name = "task(%s/%d)" % (t.id, self.taskNum)
        task.executor.MergeFrom(self.executor)

        task.data = compress(cPickle.dumps((t, t.tried), -1))

        cpu = task.resources.add()
        cpu.name = "cpus"
        cpu.type = 0  # mesos_pb2.Value.SCALAR
        cpu.scalar.value = min(self.cpus, cpus)

        mem = task.resources.add()
        mem.name = "mem"
        mem.type = 0  # mesos_pb2.Value.SCALAR
        mem.scalar.value = self.mem
        #
        # gpu = task.resources.add()
        # gpu.name = "gpus"
        # gpu.type = 0 # mesos_pb2.Value.SCALAR
        # gpu.scalar.value = self.gpus

        return task

示例#5

 def doTask(self, v_task):
     try :
         task =  cPickle.loads(decompress(v_task))
         return compress(cPickle.dumps(self.worker.doTask(task.data),-1))
     except Exception, e:
         print '>>> traceback <<<'
         traceback.print_exc()
         print '>>> end of traceback <<<'
         logger.error(e)
         raise ServiceError(e)

示例#6

文件： schedule.py 项目： astroitlab/opencluster

def run_task(task, aid):
    logger.debug("Running task %s", task.id)
    try:
        result = task.run(aid)
        # find Worker instance by workerType
        return (task.id, Success(), compress(cPickle.dumps(result,-1)))
    except Exception, e:
        logger.error("error in task %s", task)
        import traceback
        traceback.print_exc()
        return (task.id, OtherFailure("exception:" + str(e)), None)

示例#7

文件： workerlocal.py 项目： ITking666/opencluster

    def doTask(self, v_task):
        try :
            if self.asynchronous :
                return self.__doTaskAsyn(v_task)
            task =  compress(cPickle.dumps(v_task,-1))

            decompResult = decompress(self.workerService.doTask(task))
            return v_task.id, Task.TASK_FINISHED, cPickle.loads(decompResult)
        except Exception, e:
            logger.error(e)
            return v_task.id, Task.TASK_ERROR,OtherFailure(e.message)

示例#8

文件： schedule.py 项目： ITking666/opencluster

def run_task(task, aid):
    logger.debug("Running task %s", task.id)
    try:
        result = task.run(aid)
        # find Worker instance by workerType
        return (task.id, Success(), compress(cPickle.dumps(result, -1)))
    except Exception, e:
        logger.error("error in task %s", task)
        import traceback
        traceback.print_exc()
        return (task.id, OtherFailure("exception:" + str(e)), None)

示例#9

文件： executor.py 项目： astroitlab/opencluster

def run_task(task_data):
    try:
        gc.disable()
        (task, ntry) = cPickle.loads(decompress(task_data))
        result = task.run(ntry)
        return mesos_pb2.TASK_FINISHED, compress(cPickle.dumps(result, -1))
    except Exception, e:
        logger.error(e)
        import traceback
        traceback.print_exc()
        msg = traceback.format_exc()
        return mesos_pb2.TASK_FAILED, msg

示例#10

def run_task(task_data):
    try:
        gc.disable()
        (task, ntry) = cPickle.loads(decompress(task_data))
        result = task.run(ntry)
        return mesos_pb2.TASK_FINISHED, compress(cPickle.dumps(result, -1))
    except Exception, e:
        logger.error(e)
        import traceback
        traceback.print_exc()
        msg = traceback.format_exc()
        return mesos_pb2.TASK_FAILED, msg