Exemple #1
0
  def stealTasks(self, offer, numToSteal=None, minNumTasks=2, stealFromBack=True):
    """
    Give an offer to the queue. If accepted, it will return a
    new task chunk containing the subTasks it stole.
    If rejected, returns None
    """
    popped_tasks = []
    stolenTasksChunk = None

    while self.queue.hasNext():
      task = self.queue.pop()

      if (chunk_utils.numSubTasks(task) > minNumTasks and fitsIn(task, offer)):
        taskCopy = mesos_pb2.TaskInfo()
        taskCopy.CopyFrom(task)
        if numToSteal is None:  #Steal Half
          numToSteal = chunk_utils.numSubTasks(taskCopy) / 2

        stolenTasks = self.stealSubTasks(taskCopy, numToSteal, stealFromBack)
        stolenTasksChunk = chunk_utils.newTaskChunk(offer.slave_id,
                executor=taskCopy.executor, subTasks=stolenTasks)

        popped_tasks.append(taskCopy)
        break

      popped_tasks.append(task)

    for task in popped_tasks:
      self.queue.push(task)

    return stolenTasksChunk
Exemple #2
0
  def resourceOffers(self, driver, offers):
    print "Got %d resource offers" % len(offers)

    # Initialize the tasks.
    if self.all_tasks is None:
      if ENABLE_LEVELS:
        self.all_tasks = []
        for i in xrange(NUM_TASK_CHUNKS):
          tasks = self.initializeTasks(offers, i)
          taskChunk = chunk_utils.newTaskChunk(offers[0].slave_id,
                                               executor=self.executor,
                                               subTasks = tasks)
          self.all_tasks.append(taskChunk)
          taskChunk.task_id.value = "Initial_taskChunk_"+str(i)
          taskChunk.name = "Initial_taskChunk_"+str(i)
      else:
        self.all_tasks = self.initializeTasks(offers)

    taskChunk = chunk_utils.newTaskChunk(offers[0].slave_id,
            executor=self.executor, subTasks = self.all_tasks)

    # dict <offerId, listofTaskChunks>
    dictOffers = task_utils.selectTasksforOffers(offers, [taskChunk],
            len(self.all_tasks), self.num_slaves,
            distribution=self.distribution, isTaskChunk=ENABLE_TASK_CHUNKING)

    for offer in offers:
      subTasks = dictOffers[offer.id.value]
      print "offer_id = %s, numSubTasks" % (offer.id.value), len(subTasks)
      if subTasks:
        offerTotalTasks = 0
        for subTask in subTasks:
          offerTotalTasks += chunk_utils.numSubTasks(subTask)

        self.tasksLaunched += offerTotalTasks
        index = min(offerTotalTasks,len(self.all_tasks))
        del self.all_tasks[:index]

        print "Accepting offer on %s to start task chunk" % offer.hostname
        driver.launchTasks(offer.id, subTasks)
      else:
        print "Rejecting offer {0}".format(offer.id.value)
        driver.launchTasks(offer.id, [])
  def resourceOffers(self, driver, offers):
    print "Got %d resource offers" % len(offers)
    for offer in offers:

      tasks = []
      print "Got resource offer %s" % offer.id.value
      while self.tasksLaunched < TOTAL_TASKS:
        tid = self.tasksLaunched

        task = mesos_pb2.TaskInfo()
        task.task_id.value = str(tid)
        task.name = "task %d" % tid

        cpus = task.resources.add()
        cpus.name = "cpus"
        cpus.type = mesos_pb2.Value.SCALAR
        cpus.scalar.value = TASK_CPUS

        mem = task.resources.add()
        mem.name = "mem"
        mem.type = mesos_pb2.Value.SCALAR
        mem.scalar.value = TASK_MEM

        if not steal_utils.fitsIn(task, offer):
            print "Offer isn't enough to run task. Ignoring."
            break

        self.tasksLaunched += 1
        print "Adding subtask %d to chunk" % tid

        tasks.append(task)
        if len(tasks) > TOTAL_TASKS / 2:
            break

      if tasks:
        taskChunk = chunk_utils.newTaskChunk(offer.slave_id,
                executor=self.executor, subTasks=tasks)
        taskChunk.task_id.value = "chunk_id_{0}".format(self.tasksLaunched)
        taskChunk.name = "taskChunk"
        self.taskChunkId = taskChunk.task_id

        print "Accepting offer on %s to start task chunk" % offer.hostname
        driver.launchTasks(offer.id, [taskChunk])
      else:
        print "Rejecting offer {0}".format(offer.id.value)
        driver.launchTasks(offer.id, [])
 def newTaskChunk(self, numSubTasks):
     taskChunk = chunk_utils.newTaskChunk(self.slave_id,
                              subTasks=self.newTasks(numSubTasks))
     taskChunk.task_id.value = self.nextId("chunk")
     return taskChunk