Beispiel #1
0
def selectTasksforOffers(offers, tasks, ratio, isTaskChunk=False):
  """
  Maps tasks to Offers and returns a dict of <offer, taskChunks>
  """
  taskQueue = steal_utils.TaskQueue(tasks)
  offerQueue = steal_utils.PriorityQueue(offers,
                          sort_key = steal_utils.getOfferSize,
                          mapper = lambda offer: offer.id.value)

  stolenTasksChunks = defaultdict(list)
  
  while offerQueue.hasNext():
    offer = offerQueue.pop()
    if taskChunk:
      stolenTasksChunk = taskQueue.stealTasks(offer, ratio)
    else:
      stolenTasksChunk = taskQueue.stealTasks(offer, 1)
      
    if stolenTasksChunk:
      stolenTasksChunk.name = "task_chunk"
      stolenTasksChunk.task_id.value = self.generateTaskId()
      stolenTasksChunks[offer.id.value].append(stolenTasksChunk)
      
      offerCopy = mesos_pb2.Offer()
      offerCopy.CopyFrom(offer)

      chunk_utils.decrementResources(offerCopy.resources,
                                     stolenTasksChunk.resources)

      if not chunk_utils.isOfferEmpty(offerCopy):
        offerQueue.push(offerCopy)
        
    return stolenTasksChunks
    def selectTasksToSteal(self, driver, offers, pendingTasks):
        """
        Selects from the table of pending tasks a number of sub tasks to
        steal. Returns a mapping of offerId to the list of task (chunks) to run.
        """
        taskQueue = steal_utils.TaskQueue(pendingTasks)
        offerQueue = steal_utils.PriorityQueue(offers,
                sort_key = steal_utils.getOfferSize,
                mapper = lambda offer: offer.id.value)

        stolenTasksChunks = defaultdict(list)
        while offerQueue.hasNext():
            offer = offerQueue.pop()

            stolenTasksChunk = taskQueue.stealTasks(offer)
            if stolenTasksChunk:
                stolenTasksChunk.name = "Stolen task"
                stolenTasksChunk.task_id.value = self.generateTaskId()

                stolenTasksChunks[offer.id.value].append(stolenTasksChunk)

                offerCopy = mesos_pb2.Offer()
                offerCopy.CopyFrom(offer)

                chunk_utils.decrementResources(offerCopy.resources,
                        stolenTasksChunk.resources)
                if not chunk_utils.isOfferEmpty(offerCopy):
                    offerQueue.push(offerCopy)

        return stolenTasksChunks
 def updateOffers(self, offers):
     """
     Reduces the given offers by the resources consumed by the tasks
     launched for this offer.
     """
     for offer in offers:
         consumedResources = self.consumedResources[offer.id.value]
         chunk_utils.decrementResources(offer.resources, consumedResources)
Beispiel #4
0
def fitsIn(task, offer):
  """
  Checks if task resources are less than Offer resources
  """
  offerCopy = mesos_pb2.Offer()
  offerCopy.CopyFrom(offer)

  chunk_utils.decrementResources(offerCopy.resources, task.resources)
  if chunk_utils.isOfferValid(offerCopy):
    return True
  return False
Beispiel #5
0
def selectTasksforOffers(offers, tasks, numTasks, numSlaves, distribution=Distribution.UNIFORM, isTaskChunk=False):
  """
  Maps tasks to Offers and returns a dict of <offer, taskChunks>
  """
  taskQueue = steal_utils.TaskQueue(tasks)
  offerQueue = steal_utils.PriorityQueue(offers,
                          sort_key = steal_utils.getOfferSize,
                                         mapper = lambda offer: offer.id.value)

  createdTasksChunks = defaultdict(list)

  while offerQueue.hasNext():
    offer = offerQueue.pop()
    if isTaskChunk:
      tasks_per_chunk = getTaskChunkSize(distribution, numTasks, numSlaves)
      if tasks_per_chunk is None:
        createdTasksChunk = None
      else:
        createdTasksChunk = taskQueue.stealTasks(offer, tasks_per_chunk, 0, False)
      updateNormList(createdTasksChunk, distribution, tasks_per_chunk)
    else:
      createdTasksChunk = taskQueue.stealTasks(offer, 1, 0, False)

    if createdTasksChunk:
      createdTasksChunk.name = "task_chunk"
      createdTasksChunk.task_id.value = generateTaskId()
      createdTasksChunks[offer.id.value].append(createdTasksChunk)

      offerCopy = mesos_pb2.Offer()
      offerCopy.CopyFrom(offer)

      chunk_utils.decrementResources(offerCopy.resources,
                                     createdTasksChunk.resources)

      if not chunk_utils.isOfferEmpty(offerCopy):
        offerQueue.push(offerCopy)

  return createdTasksChunks