def resourceOffers(self, driver, offers): ''' Offer resources and launch tasks ''' for offer in offers: if all(task.offered for task in self.tasks): driver.declineOffer(offer.id, mesos_pb2.Filters(refuse_seconds=FOREVER)) continue offered_cpus = offered_mem = 0.0 offered_tasks = [] for resource in offer.resources: if resource.name == "cpus": offered_cpus = resource.scalar.value elif resource.name == "mem": offered_mem = resource.scalar.value for task in self.tasks: if task.offered: continue if not (task.cpus <= offered_cpus and task.mem <= offered_mem): continue offered_cpus -= task.cpus offered_mem -= task.mem task.offered = True offered_tasks.append(task.to_task_info(offer, self.addr)) driver.launchTasks(offer.id, offered_tasks, mesos_pb2.Filters())
def resourceOffers(self, driver, offers): with self._lock: log.debug('Got %d resource offers' % len(offers)) self._metrics.resource_offers.add(len(offers)) # Current scheduling algorithm: randomly pick an offer and loop through the list of launchers # until one decides to use this offer to launch a task. # It's possible to launch multiple tasks on the same Mesos slave (in different batches of # offers). for offer in shuffled(offers): task_id = None # 'filters' is set when we need to create a special filter for incompatible roles. filters = None # For each offer, launchers are asked to launch a task in the order they are added (FCFS). for name in self._launchers: launcher = self._launchers[name] try: task_id, _ = launcher.launch(offer) except MySQLClusterLauncher.IncompatibleRoleError as e: log.info("Declining offer %s for %s because '%s'" % (offer.id.value, INCOMPATIBLE_ROLE_OFFER_REFUSE_DURATION, e)) # This "error" is not severe and we expect this to occur frequently only when Mysos # first joins the cluster. For a running Mesos cluster this should be somewhat rare # because we refuse the offer "forever". filters = mesos_pb2.Filters() filters.refuse_seconds = INCOMPATIBLE_ROLE_OFFER_REFUSE_DURATION.as_( Time.SECONDS) self._metrics.offers_incompatible_role.increment() break # No need to check with other launchers. if task_id: self._metrics.tasks_launched.increment() self._tasks[task_id] = launcher.cluster_name # No need to check with other launchers. 'filters' remains unset. break if task_id: break # Some launcher has used this offer. Move on to the next one. if not filters: log.debug( "Declining unused offer %s because no launcher accepted this offer: %s" % (offer.id.value, offer)) # Mesos scheduler Python binding doesn't deal with filters='None' properly. # See https://issues.apache.org/jira/browse/MESOS-2567. filters = mesos_pb2.Filters() self._metrics.offers_unused.increment() self._driver.declineOffer(offer.id, filters)
def submit_tasks(self): prev_q_len = 0 prev_task_len = 0 if (len(self.offers) > 0) and (len(self.taskQueue) > 0): list = self.taskQueue.__copy__() for instance in list: offer = self.select_offer(instance) if offer: instance.offer = offer.offer task = self.new_docker_task(offer.offer, instance) # TODO(gokrokve) this is not thread safe. Should be properly changed instance.taskInfo = task instance.task_id = task.task_id.value tasks = [] tasks.append(task) LOG.info("Removing offer for a task") self.offers.remove(offer) self.resources.delete_offer(offer.offer) self.taskQueue.remove(instance) driver.launchTasks(offer.offer.id, tasks, mesos_pb2.Filters()) self.tasks[task.task_id.value] = instance self.persistent_store.storeByTask(task.task_id.value, instance) if len(self.offers) == 0: #No more offers available. Lets wait for new offers break
def resourceOffers(self, driver, offers): self.printStatistics() #TODO(gokrokve) In Kolla-Mesos approach slaves are tagged with special tag (compute, or controller) #Here we need to filter out offers from computes and never touch controller offers for offer in offers: if len(self.offers) > OFFERS_MAX: driver.declineOffer(offer.id, mesos_pb2.Filters()) else: print "Got resource offer [%s]" % offer.id.value for rsc in offer.resources: LOG.info("Resource %s value: %s" % (rsc.name, rsc.scalar.value)) cpus = next(rsc.scalar.value for rsc in offer.resources if rsc.name == "cpus") mem = next(rsc.scalar.value for rsc in offer.resources if rsc.name == "mem") disk = next(rsc.scalar.value for rsc in offer.resources if rsc.name == "disk") / 1024 print "Offer info: %d cpu %d mem %d disk" % (cpus, mem, disk) self.resources.add_offer(cpus, mem, disk) offerInfo = OfferInfo(cpus, mem, offer) self.offers.append(offerInfo) if self.shuttingDown: print "Shutting down: declining offer on [%s]" % offer.hostname driver.declineOffer(offer.id) continue self.submit_tasks()
def __init__(self, implicitAcknowledgements, executor, cpu_alloc, mem_alloc, refuse_seconds, interval=1): self.implicitAcknowledgements = implicitAcknowledgements self.executor = executor self.tasksLaunched = 0 self.tasksFinished = 0 self.messagesSent = 0 self.messagesReceived = 0 self.appQueue = Queue.Queue() self.appDatadict = {} # resource requirements self._cpu_alloc = cpu_alloc self._mem_alloc = mem_alloc self._offer_timeout = mesos_pb2.Filters() self._offer_timeout.refuse_seconds = refuse_seconds self.clientQueue = Queue.Queue() self.clientDict = {} self.interval = interval thread = threading.Thread(target=self.run, args=()) thread.daemon = True # Daemonize thread thread.start() # Start the execution
def resourceOffers(self, driver, offers): logging.debug("resourceOffers (%s, %s)", driver, offers) for offer in offers: logging.info("Received offer: %s", offer) task = mesos_pb2.TaskInfo() task.task_id.value = "job-{}".format(self.number) task.slave_id.value = offer.slave_id.value task.name = "Job {}".format(self.number) task.command.shell = False task.command.value = "36000" task.container.type = mesos_pb2.ContainerInfo.DOCKER task.container.docker.image = "python:2.7-alpine" task.container.docker.network = task.container.docker.BRIDGE task.container.docker.force_pull_image = False task.container.hostname = task.task_id.value entrypoint = task.container.docker.parameters.add() entrypoint.key = "entrypoint" entrypoint.value = "/bin/sleep" rmparam = task.container.docker.parameters.add() rmparam.key = "rm" rmparam.value = "true" cpus = task.resources.add() cpus.name = "cpus" cpus.type = mesos_pb2.Value.SCALAR cpus.scalar.value = 2 cpus.role = "*" mem = task.resources.add() mem.name = "mem" mem.type = mesos_pb2.Value.SCALAR mem.scalar.value = 2927 mem.role = "*" logging.info("Created a Mesos task for %s", self.number) self.number += 1 driver.launchTasks(offer.id, [task], filters=mesos_pb2.Filters())
def resourceOffers(self, driver, offers): ''' Offer resources and launch tasks ''' for offer in offers: if all(task.offered for task in self.tasks): driver.declineOffer(offer.id, mesos_pb2.Filters(refuse_seconds=FOREVER)) continue offered_cpus = offered_mem = 0.0 offered_gpus = [] offered_tasks = [] for resource in offer.resources: if resource.name == "cpus": offered_cpus = resource.scalar.value elif resource.name == "mem": offered_mem = resource.scalar.value elif resource.name == "gpus": offered_gpus = resource.set.item for task in self.tasks: if task.offered: continue if not (task.cpus <= offered_cpus and task.mem <= offered_mem and task.gpus <= len(offered_gpus)): continue offered_cpus -= task.cpus offered_mem -= task.mem gpus = int(math.ceil(task.gpus)) gpu_uuids = offered_gpus[:gpus] offered_gpus = offered_gpus[gpus:] task.offered = True offered_tasks.append( task.to_task_info( offer, self.addr, gpu_uuids=gpu_uuids)) driver.launchTasks(offer.id, offered_tasks, mesos_pb2.Filters())
def resourceOffers(self, driver, offers): rf = mesos_pb2.Filters() rf.refuse_seconds = 60 if self.tasks_total_len() == 0: for o in offers: driver.launchTasks(o.id, [], rf) return random.shuffle(offers) self.last_offer_time = time.time() for offer in offers: attrs = self.getAttributes(offer) if self.options.group and attrs.get( 'group', 'None') not in self.options.group: driver.launchTasks(offer.id, [], rf) continue cpus, mem, gpus = self.getResources(offer) logger.debug( "got resource offer %s: cpus:%s, mem:%s, gpus:%s at %s", offer.id.value, cpus, mem, gpus, offer.hostname) sid = offer.slave_id.value tasks = [] for priority in range(1, self.priority_size + 1): for t in self.waiting_tasks(priority): if priority > 1 and cpus <= 1: break if cpus >= t.resources.get( "cpus", 1) and mem >= t.resources.get( "mem", 100) and ( t.resources.get("gpus", 0) == 0 or attrs.get('gpus', None) is not None): self.remove_task(t) t.state = mesos_pb2.TASK_STARTING t.state_time = time.time() task = self.create_task(offer, t) tasks.append(task) self.task_launched[t.id] = t self.slaveTasks.setdefault(sid, set()).add(t.id) cpus -= t.resources.get("cpus", 1) mem -= t.resources.get("mem", 100) operation = mesos_pb2.Offer.Operation() operation.type = mesos_pb2.Offer.Operation.LAUNCH operation.launch.task_infos.extend(tasks) driver.acceptOffers([offer.id], [operation]) if len(tasks) > 0: logger.debug("dispatch %d tasks to slave %s", len(tasks), offer.hostname)
def resourceOffers(self, driver, offers): rf = mesos_pb2.Filters() if not self.task_waiting: rf.refuse_seconds = 5 for o in offers: driver.launchTasks(o.id, [], rf) return random.shuffle(offers) self.last_offer_time = time.time() for offer in offers: if self.shuttingdown: print "Shutting down: declining offer on [%s]" % offer.hostname driver.declineOffer(offer.id) continue attrs = self.getAttributes(offer) if self.options.group and attrs.get( 'group', 'None') not in self.options.group: driver.launchTasks(offer.id, [], rf) continue cpus, mem, gpus = self.getResources(offer) logger.debug( "got resource offer %s: cpus:%s, mem:%s, gpus:%s at %s", offer.id.value, cpus, mem, gpus, offer.hostname) logger.debug("attributes,gpus:%s", attrs.get('gpus', None)) sid = offer.slave_id.value tasks = [] while (len(self.task_waiting) > 0 and cpus >= self.cpus and mem >= self.mem and (self.gpus == 0 or attrs.get('gpus', None) is not None)): logger.debug("Accepting resource on slave %s (%s)", offer.slave_id.value, offer.hostname) t = self.task_waiting.pop() t.state = mesos_pb2.TASK_STARTING t.state_time = time.time() task = self.create_task(offer, t, cpus) tasks.append(task) self.task_launched[t.id] = t self.slaveTasks.setdefault(sid, set()).add(t.id) cpus -= self.cpus mem -= self.mem # gpus -= self.gpus operation = mesos_pb2.Offer.Operation() operation.type = mesos_pb2.Offer.Operation.LAUNCH operation.launch.task_infos.extend(tasks) driver.acceptOffers([offer.id], [operation])
def launch_tasks_for_offers(self, driver, offers): """For each offer tries to launch all tasks that can fit in there. Declines offer if no fitting tasks found.""" launched_tasks = [] for offer in offers: with self.constraint_state_lock: try: tasks, new_state = self.tasks_and_state_for_offer( driver, offer, self.constraint_state) if tasks is not None and len(tasks) > 0: operation = mesos_pb2.Offer.Operation() operation.type = mesos_pb2.Offer.Operation.LAUNCH operation.launch.task_infos.extend(tasks) driver.acceptOffers([offer.id], [operation]) for task in tasks: staging_timer = self.staging_timer_for_task( self.staging_timeout, driver, task.task_id.value) self.tasks_with_flags.setdefault( task.task_id.value, MesosTaskParameters( health=None, mesos_task_state=TASK_STAGING, offer=offer, staging_timer=staging_timer, )) staging_timer.start() launched_tasks.extend(tasks) self.constraint_state = new_state else: driver.declineOffer(offer.id) except ConstraintFailAllTasksError: paasta_print( "Offer failed constraints for every task, rejecting 60s" ) filters = mesos_pb2.Filters() filters.refuse_seconds = 60 driver.declineOffer(offer.id, filters) return launched_tasks
def resourceOffers(self, driver, offers): if self.frozen: return if self.within_reconcile_backoff(): paasta_print( "Declining all offers since we started reconciliation too recently" ) for offer in offers: driver.declineOffer(offer.id) else: for idx, offer in enumerate(offers): if offer.slave_id.value in self.blacklisted_slaves: log.critical( "Ignoring offer %s from blacklisted slave %s" % (offer.id.value, offer.slave_id.value)) filters = mesos_pb2.Filters() filters.refuse_seconds = self.blacklist_timeout driver.declineOffer(offer.id, filters) del offers[idx] self.launch_tasks_for_offers(driver, offers)
def resourceOffers(self, driver, offers): rf = mesos_pb2.Filters() if not self.activeJobs: rf.refuse_seconds = 60 * 5 for o in offers: driver.launchTasks(o.id, [], rf) return start = time.time() random.shuffle(offers) cpus = [self.getResource(o.resources, 'cpus') for o in offers] mems = [ self.getResource(o.resources, 'mem') - (o.slave_id.value not in self.slaveTasks and EXECUTOR_MEMORY or 0) for o in offers ] logger.debug("get %d offers (%s cpus, %s mem), %d jobs", len(offers), sum(cpus), sum(mems), len(self.activeJobs)) tasks = {} for job in self.activeJobsQueue: while True: launchedTask = False for i, o in enumerate(offers): sid = o.slave_id.value if self.group and (self.getAttribute( o.attributes, 'group') or 'none') not in self.group: continue if self.slaveFailed.get(sid, 0) >= MAX_FAILED: continue if self.slaveTasks.get(sid, 0) >= self.task_per_node: continue if mems[i] < self.mem or cpus[i] + 1e-4 < self.cpus: continue t = job.slaveOffer(str(o.hostname), cpus[i], mems[i]) if not t: continue task = self.createTask(o, job, t, cpus[i]) tasks.setdefault(o.id.value, []).append(task) logger.debug("dispatch %s into %s", t, o.hostname) tid = task.task_id.value self.jobTasks[job.id].add(tid) self.taskIdToJobId[tid] = job.id self.taskIdToSlaveId[tid] = sid self.slaveTasks[sid] = self.slaveTasks.get(sid, 0) + 1 cpus[i] -= min(cpus[i], t.cpus) mems[i] -= t.mem launchedTask = True if not launchedTask: break used = time.time() - start if used > 10: logger.error("use too much time in slaveOffer: %.2fs", used) rf.refuse_seconds = 5 for o in offers: driver.launchTasks(o.id, tasks.get(o.id.value, []), rf) logger.debug("reply with %d tasks, %s cpus %s mem left", sum(len(ts) for ts in tasks.values()), sum(cpus), sum(mems))
def resourceOffers(self, driver, offers): ''' Offer resources and launch tasks ''' for offer in offers: if all(task.offered for _, task in self.jobs.tasks.iteritems()): driver.declineOffer(offer.id, mesos_pb2.Filters(refuse_seconds=FOREVER)) continue offer_ip = offer.url.address.ip offered_cpus = offered_mem = offered_gpus = 0.0 offered_port_begin = 0 offered_port_end = 0 offered_tasks = [] for resource in offer.resources: if resource.name == 'cpus': offered_cpus = resource.scalar.value elif resource.name == 'mem': offered_mem = resource.scalar.value elif resource.name == 'gpus': offered_gpus = int(resource.scalar.value) elif resource.name == 'ports': offered_port_begin = int(resource.ranges.range[0].begin) offered_port_end = int(resource.ranges.range[0].end) logger.info( 'Offered cpus: {0} offered mem: {1} offered gpus: {2} offerd port: {3}~{4} on {5}' .format(offered_cpus, offered_mem, offered_gpus, offered_port_begin, offered_port_end, offer.url.address.hostname)) for _, task in self.jobs.tasks.iteritems(): if task.offered: continue if not (task.cpus <= offered_cpus and task.mem <= offered_mem and int(math.ceil(task.gpus)) <= offered_gpus and offered_port_begin <= offered_port_end and offered_port_begin != 0): logger.info( '/job:{0}/task{1} does not get all needed resources on offer: {2}' .format(task.job_name, task.task_index, offer.id.value)) continue offered_cpus -= task.cpus offered_mem -= task.mem offered_gpus -= math.ceil(task.gpus) offered_tasks.append( task.to_task_info(offer.slave_id.value, offered_port_begin, True)) node = '/job:{0}/task:{1}'.format(task.job_name, task.task_index).replace( '/', '_').replace(':', '_') addr = str(offer_ip) + ':' + str(offered_port_begin) offered_port_begin += 1 logger.info('Registering in zookeeper {0}, value: {1}'.format( self.job_zk_dir + '/member/' + node, addr)) self.zk.create(self.job_zk_dir + '/member/' + node, addr) if task.job_name == 'ps': self.zk.create(self.job_zk_dir + '/status/' + node, 'false') task.offered = True logger.info( 'Allocating resource for /job:{0}/task:{1} successful'. format(task.job_name, task.task_index)) driver.launchTasks(offer.id, offered_tasks, mesos_pb2.Filters())
def _filters(self, seconds): f = mesos_pb2.Filters() f.refuse_seconds = seconds return f
import dpark.conf as conf from dpark.util import getuser, memory_str_to_mb from dpark import moosefs logger = logging.getLogger('dpark.scheduler') class Task: def __init__(self, id): self.id = id self.tried = 0 self.state = -1 self.state_time = 0 REFUSE_FILTER = mesos_pb2.Filters() REFUSE_FILTER.refuse_seconds = 10 * 60 # 10 mins EXECUTOR_CPUS = 0.01 EXECUTOR_MEMORY = 64 # cache def safe(f): def _(self, *a, **kw): with self.lock: r = f(self, *a, **kw) return r return _ class BaseScheduler(object):