def stealTasks(self, offer, numToSteal=None, minNumTasks=2, stealFromBack=True): """ Give an offer to the queue. If accepted, it will return a new task chunk containing the subTasks it stole. If rejected, returns None """ popped_tasks = [] stolenTasksChunk = None while self.queue.hasNext(): task = self.queue.pop() if (chunk_utils.numSubTasks(task) > minNumTasks and fitsIn(task, offer)): taskCopy = mesos_pb2.TaskInfo() taskCopy.CopyFrom(task) if numToSteal is None: #Steal Half numToSteal = chunk_utils.numSubTasks(taskCopy) / 2 stolenTasks = self.stealSubTasks(taskCopy, numToSteal, stealFromBack) stolenTasksChunk = chunk_utils.newTaskChunk(offer.slave_id, executor=taskCopy.executor, subTasks=stolenTasks) popped_tasks.append(taskCopy) break popped_tasks.append(task) for task in popped_tasks: self.queue.push(task) return stolenTasksChunk
def resourceOffers(self, driver, offers): print "Got %d resource offers" % len(offers) # Initialize the tasks. if self.all_tasks is None: if ENABLE_LEVELS: self.all_tasks = [] for i in xrange(NUM_TASK_CHUNKS): tasks = self.initializeTasks(offers, i) taskChunk = chunk_utils.newTaskChunk(offers[0].slave_id, executor=self.executor, subTasks = tasks) self.all_tasks.append(taskChunk) taskChunk.task_id.value = "Initial_taskChunk_"+str(i) taskChunk.name = "Initial_taskChunk_"+str(i) else: self.all_tasks = self.initializeTasks(offers) taskChunk = chunk_utils.newTaskChunk(offers[0].slave_id, executor=self.executor, subTasks = self.all_tasks) # dict <offerId, listofTaskChunks> dictOffers = task_utils.selectTasksforOffers(offers, [taskChunk], len(self.all_tasks), self.num_slaves, distribution=self.distribution, isTaskChunk=ENABLE_TASK_CHUNKING) for offer in offers: subTasks = dictOffers[offer.id.value] print "offer_id = %s, numSubTasks" % (offer.id.value), len(subTasks) if subTasks: offerTotalTasks = 0 for subTask in subTasks: offerTotalTasks += chunk_utils.numSubTasks(subTask) self.tasksLaunched += offerTotalTasks index = min(offerTotalTasks,len(self.all_tasks)) del self.all_tasks[:index] print "Accepting offer on %s to start task chunk" % offer.hostname driver.launchTasks(offer.id, subTasks) else: print "Rejecting offer {0}".format(offer.id.value) driver.launchTasks(offer.id, [])
def resourceOffers(self, driver, offers): print "Got %d resource offers" % len(offers) for offer in offers: tasks = [] print "Got resource offer %s" % offer.id.value while self.tasksLaunched < TOTAL_TASKS: tid = self.tasksLaunched task = mesos_pb2.TaskInfo() task.task_id.value = str(tid) task.name = "task %d" % tid cpus = task.resources.add() cpus.name = "cpus" cpus.type = mesos_pb2.Value.SCALAR cpus.scalar.value = TASK_CPUS mem = task.resources.add() mem.name = "mem" mem.type = mesos_pb2.Value.SCALAR mem.scalar.value = TASK_MEM if not steal_utils.fitsIn(task, offer): print "Offer isn't enough to run task. Ignoring." break self.tasksLaunched += 1 print "Adding subtask %d to chunk" % tid tasks.append(task) if len(tasks) > TOTAL_TASKS / 2: break if tasks: taskChunk = chunk_utils.newTaskChunk(offer.slave_id, executor=self.executor, subTasks=tasks) taskChunk.task_id.value = "chunk_id_{0}".format(self.tasksLaunched) taskChunk.name = "taskChunk" self.taskChunkId = taskChunk.task_id print "Accepting offer on %s to start task chunk" % offer.hostname driver.launchTasks(offer.id, [taskChunk]) else: print "Rejecting offer {0}".format(offer.id.value) driver.launchTasks(offer.id, [])
def newTaskChunk(self, numSubTasks): taskChunk = chunk_utils.newTaskChunk(self.slave_id, subTasks=self.newTasks(numSubTasks)) taskChunk.task_id.value = self.nextId("chunk") return taskChunk