def selectTasksforOffers(offers, tasks, ratio, isTaskChunk=False):
"""
Maps tasks to Offers and returns a dict of <offer, taskChunks>
"""
taskQueue = steal_utils.TaskQueue(tasks)
offerQueue = steal_utils.PriorityQueue(offers,
sort_key = steal_utils.getOfferSize,
mapper = lambda offer: offer.id.value)
stolenTasksChunks = defaultdict(list)
while offerQueue.hasNext():
offer = offerQueue.pop()
if taskChunk:
stolenTasksChunk = taskQueue.stealTasks(offer, ratio)
else:
stolenTasksChunk = taskQueue.stealTasks(offer, 1)
if stolenTasksChunk:
stolenTasksChunk.name = "task_chunk"
stolenTasksChunk.task_id.value = self.generateTaskId()
stolenTasksChunks[offer.id.value].append(stolenTasksChunk)
offerCopy = mesos_pb2.Offer()
offerCopy.CopyFrom(offer)
chunk_utils.decrementResources(offerCopy.resources,
stolenTasksChunk.resources)
if not chunk_utils.isOfferEmpty(offerCopy):
offerQueue.push(offerCopy)
return stolenTasksChunks
def selectTasksToSteal(self, driver, offers, pendingTasks):
"""
Selects from the table of pending tasks a number of sub tasks to
steal. Returns a mapping of offerId to the list of task (chunks) to run.
"""
taskQueue = steal_utils.TaskQueue(pendingTasks)
offerQueue = steal_utils.PriorityQueue(offers,
sort_key = steal_utils.getOfferSize,
mapper = lambda offer: offer.id.value)
stolenTasksChunks = defaultdict(list)
while offerQueue.hasNext():
offer = offerQueue.pop()
stolenTasksChunk = taskQueue.stealTasks(offer)
if stolenTasksChunk:
stolenTasksChunk.name = "Stolen task"
stolenTasksChunk.task_id.value = self.generateTaskId()
stolenTasksChunks[offer.id.value].append(stolenTasksChunk)
offerCopy = mesos_pb2.Offer()
offerCopy.CopyFrom(offer)
chunk_utils.decrementResources(offerCopy.resources,
stolenTasksChunk.resources)
if not chunk_utils.isOfferEmpty(offerCopy):
offerQueue.push(offerCopy)
return stolenTasksChunks
def updateOffers(self, offers):
"""
Reduces the given offers by the resources consumed by the tasks
launched for this offer.
"""
for offer in offers:
consumedResources = self.consumedResources[offer.id.value]
chunk_utils.decrementResources(offer.resources, consumedResources)
def fitsIn(task, offer):
"""
Checks if task resources are less than Offer resources
"""
offerCopy = mesos_pb2.Offer()
offerCopy.CopyFrom(offer)
chunk_utils.decrementResources(offerCopy.resources, task.resources)
if chunk_utils.isOfferValid(offerCopy):
return True
return False
def selectTasksforOffers(offers, tasks, numTasks, numSlaves, distribution=Distribution.UNIFORM, isTaskChunk=False):
"""
Maps tasks to Offers and returns a dict of <offer, taskChunks>
"""
taskQueue = steal_utils.TaskQueue(tasks)
offerQueue = steal_utils.PriorityQueue(offers,
sort_key = steal_utils.getOfferSize,
mapper = lambda offer: offer.id.value)
createdTasksChunks = defaultdict(list)
while offerQueue.hasNext():
offer = offerQueue.pop()
if isTaskChunk:
tasks_per_chunk = getTaskChunkSize(distribution, numTasks, numSlaves)
if tasks_per_chunk is None:
createdTasksChunk = None
else:
createdTasksChunk = taskQueue.stealTasks(offer, tasks_per_chunk, 0, False)
updateNormList(createdTasksChunk, distribution, tasks_per_chunk)
else:
createdTasksChunk = taskQueue.stealTasks(offer, 1, 0, False)
if createdTasksChunk:
createdTasksChunk.name = "task_chunk"
createdTasksChunk.task_id.value = generateTaskId()
createdTasksChunks[offer.id.value].append(createdTasksChunk)
offerCopy = mesos_pb2.Offer()
offerCopy.CopyFrom(offer)
chunk_utils.decrementResources(offerCopy.resources,
createdTasksChunk.resources)
if not chunk_utils.isOfferEmpty(offerCopy):
offerQueue.push(offerCopy)
return createdTasksChunks