def resourceOffers(self, driver, offers):
'''
Offer resources and launch tasks
'''
for offer in offers:
if all(task.offered for task in self.tasks):
driver.declineOffer(offer.id,
mesos_pb2.Filters(refuse_seconds=FOREVER))
continue
offered_cpus = offered_mem = 0.0
offered_tasks = []
for resource in offer.resources:
if resource.name == "cpus":
offered_cpus = resource.scalar.value
elif resource.name == "mem":
offered_mem = resource.scalar.value
for task in self.tasks:
if task.offered:
continue
if not (task.cpus <= offered_cpus and task.mem <= offered_mem):
continue
offered_cpus -= task.cpus
offered_mem -= task.mem
task.offered = True
offered_tasks.append(task.to_task_info(offer, self.addr))
driver.launchTasks(offer.id, offered_tasks, mesos_pb2.Filters())
def resourceOffers(self, driver, offers):
with self._lock:
log.debug('Got %d resource offers' % len(offers))
self._metrics.resource_offers.add(len(offers))
# Current scheduling algorithm: randomly pick an offer and loop through the list of launchers
# until one decides to use this offer to launch a task.
# It's possible to launch multiple tasks on the same Mesos slave (in different batches of
# offers).
for offer in shuffled(offers):
task_id = None
# 'filters' is set when we need to create a special filter for incompatible roles.
filters = None
# For each offer, launchers are asked to launch a task in the order they are added (FCFS).
for name in self._launchers:
launcher = self._launchers[name]
try:
task_id, _ = launcher.launch(offer)
except MySQLClusterLauncher.IncompatibleRoleError as e:
log.info("Declining offer %s for %s because '%s'" %
(offer.id.value,
INCOMPATIBLE_ROLE_OFFER_REFUSE_DURATION, e))
# This "error" is not severe and we expect this to occur frequently only when Mysos
# first joins the cluster. For a running Mesos cluster this should be somewhat rare
# because we refuse the offer "forever".
filters = mesos_pb2.Filters()
filters.refuse_seconds = INCOMPATIBLE_ROLE_OFFER_REFUSE_DURATION.as_(
Time.SECONDS)
self._metrics.offers_incompatible_role.increment()
break # No need to check with other launchers.
if task_id:
self._metrics.tasks_launched.increment()
self._tasks[task_id] = launcher.cluster_name
# No need to check with other launchers. 'filters' remains unset.
break
if task_id:
break # Some launcher has used this offer. Move on to the next one.
if not filters:
log.debug(
"Declining unused offer %s because no launcher accepted this offer: %s"
% (offer.id.value, offer))
# Mesos scheduler Python binding doesn't deal with filters='None' properly.
# See https://issues.apache.org/jira/browse/MESOS-2567.
filters = mesos_pb2.Filters()
self._metrics.offers_unused.increment()
self._driver.declineOffer(offer.id, filters)
def submit_tasks(self):
prev_q_len = 0
prev_task_len = 0
if (len(self.offers) > 0) and (len(self.taskQueue) > 0):
list = self.taskQueue.__copy__()
for instance in list:
offer = self.select_offer(instance)
if offer:
instance.offer = offer.offer
task = self.new_docker_task(offer.offer, instance)
# TODO(gokrokve) this is not thread safe. Should be properly changed
instance.taskInfo = task
instance.task_id = task.task_id.value
tasks = []
tasks.append(task)
LOG.info("Removing offer for a task")
self.offers.remove(offer)
self.resources.delete_offer(offer.offer)
self.taskQueue.remove(instance)
driver.launchTasks(offer.offer.id, tasks,
mesos_pb2.Filters())
self.tasks[task.task_id.value] = instance
self.persistent_store.storeByTask(task.task_id.value,
instance)
if len(self.offers) == 0:
#No more offers available. Lets wait for new offers
break
def resourceOffers(self, driver, offers):
self.printStatistics()
#TODO(gokrokve) In Kolla-Mesos approach slaves are tagged with special tag (compute, or controller)
#Here we need to filter out offers from computes and never touch controller offers
for offer in offers:
if len(self.offers) > OFFERS_MAX:
driver.declineOffer(offer.id, mesos_pb2.Filters())
else:
print "Got resource offer [%s]" % offer.id.value
for rsc in offer.resources:
LOG.info("Resource %s value: %s" %
(rsc.name, rsc.scalar.value))
cpus = next(rsc.scalar.value for rsc in offer.resources
if rsc.name == "cpus")
mem = next(rsc.scalar.value for rsc in offer.resources
if rsc.name == "mem")
disk = next(rsc.scalar.value for rsc in offer.resources
if rsc.name == "disk") / 1024
print "Offer info: %d cpu %d mem %d disk" % (cpus, mem, disk)
self.resources.add_offer(cpus, mem, disk)
offerInfo = OfferInfo(cpus, mem, offer)
self.offers.append(offerInfo)
if self.shuttingDown:
print "Shutting down: declining offer on [%s]" % offer.hostname
driver.declineOffer(offer.id)
continue
self.submit_tasks()
def __init__(self,
implicitAcknowledgements,
executor,
cpu_alloc,
mem_alloc,
refuse_seconds,
interval=1):
self.implicitAcknowledgements = implicitAcknowledgements
self.executor = executor
self.tasksLaunched = 0
self.tasksFinished = 0
self.messagesSent = 0
self.messagesReceived = 0
self.appQueue = Queue.Queue()
self.appDatadict = {}
# resource requirements
self._cpu_alloc = cpu_alloc
self._mem_alloc = mem_alloc
self._offer_timeout = mesos_pb2.Filters()
self._offer_timeout.refuse_seconds = refuse_seconds
self.clientQueue = Queue.Queue()
self.clientDict = {}
self.interval = interval
thread = threading.Thread(target=self.run, args=())
thread.daemon = True # Daemonize thread
thread.start() # Start the execution
def resourceOffers(self, driver, offers):
logging.debug("resourceOffers (%s, %s)", driver, offers)
for offer in offers:
logging.info("Received offer: %s", offer)
task = mesos_pb2.TaskInfo()
task.task_id.value = "job-{}".format(self.number)
task.slave_id.value = offer.slave_id.value
task.name = "Job {}".format(self.number)
task.command.shell = False
task.command.value = "36000"
task.container.type = mesos_pb2.ContainerInfo.DOCKER
task.container.docker.image = "python:2.7-alpine"
task.container.docker.network = task.container.docker.BRIDGE
task.container.docker.force_pull_image = False
task.container.hostname = task.task_id.value
entrypoint = task.container.docker.parameters.add()
entrypoint.key = "entrypoint"
entrypoint.value = "/bin/sleep"
rmparam = task.container.docker.parameters.add()
rmparam.key = "rm"
rmparam.value = "true"
cpus = task.resources.add()
cpus.name = "cpus"
cpus.type = mesos_pb2.Value.SCALAR
cpus.scalar.value = 2
cpus.role = "*"
mem = task.resources.add()
mem.name = "mem"
mem.type = mesos_pb2.Value.SCALAR
mem.scalar.value = 2927
mem.role = "*"
logging.info("Created a Mesos task for %s", self.number)
self.number += 1
driver.launchTasks(offer.id, [task], filters=mesos_pb2.Filters())
def resourceOffers(self, driver, offers):
'''
Offer resources and launch tasks
'''
for offer in offers:
if all(task.offered for task in self.tasks):
driver.declineOffer(offer.id,
mesos_pb2.Filters(refuse_seconds=FOREVER))
continue
offered_cpus = offered_mem = 0.0
offered_gpus = []
offered_tasks = []
for resource in offer.resources:
if resource.name == "cpus":
offered_cpus = resource.scalar.value
elif resource.name == "mem":
offered_mem = resource.scalar.value
elif resource.name == "gpus":
offered_gpus = resource.set.item
for task in self.tasks:
if task.offered:
continue
if not (task.cpus <= offered_cpus and
task.mem <= offered_mem and
task.gpus <= len(offered_gpus)):
continue
offered_cpus -= task.cpus
offered_mem -= task.mem
gpus = int(math.ceil(task.gpus))
gpu_uuids = offered_gpus[:gpus]
offered_gpus = offered_gpus[gpus:]
task.offered = True
offered_tasks.append(
task.to_task_info(
offer, self.addr,
gpu_uuids=gpu_uuids))
driver.launchTasks(offer.id, offered_tasks, mesos_pb2.Filters())
def resourceOffers(self, driver, offers):
rf = mesos_pb2.Filters()
rf.refuse_seconds = 60
if self.tasks_total_len() == 0:
for o in offers:
driver.launchTasks(o.id, [], rf)
return
random.shuffle(offers)
self.last_offer_time = time.time()
for offer in offers:
attrs = self.getAttributes(offer)
if self.options.group and attrs.get(
'group', 'None') not in self.options.group:
driver.launchTasks(offer.id, [], rf)
continue
cpus, mem, gpus = self.getResources(offer)
logger.debug(
"got resource offer %s: cpus:%s, mem:%s, gpus:%s at %s",
offer.id.value, cpus, mem, gpus, offer.hostname)
sid = offer.slave_id.value
tasks = []
for priority in range(1, self.priority_size + 1):
for t in self.waiting_tasks(priority):
if priority > 1 and cpus <= 1:
break
if cpus >= t.resources.get(
"cpus", 1) and mem >= t.resources.get(
"mem", 100) and (
t.resources.get("gpus", 0) == 0
or attrs.get('gpus', None) is not None):
self.remove_task(t)
t.state = mesos_pb2.TASK_STARTING
t.state_time = time.time()
task = self.create_task(offer, t)
tasks.append(task)
self.task_launched[t.id] = t
self.slaveTasks.setdefault(sid, set()).add(t.id)
cpus -= t.resources.get("cpus", 1)
mem -= t.resources.get("mem", 100)
operation = mesos_pb2.Offer.Operation()
operation.type = mesos_pb2.Offer.Operation.LAUNCH
operation.launch.task_infos.extend(tasks)
driver.acceptOffers([offer.id], [operation])
if len(tasks) > 0:
logger.debug("dispatch %d tasks to slave %s", len(tasks),
offer.hostname)
def resourceOffers(self, driver, offers):
rf = mesos_pb2.Filters()
if not self.task_waiting:
rf.refuse_seconds = 5
for o in offers:
driver.launchTasks(o.id, [], rf)
return
random.shuffle(offers)
self.last_offer_time = time.time()
for offer in offers:
if self.shuttingdown:
print "Shutting down: declining offer on [%s]" % offer.hostname
driver.declineOffer(offer.id)
continue
attrs = self.getAttributes(offer)
if self.options.group and attrs.get(
'group', 'None') not in self.options.group:
driver.launchTasks(offer.id, [], rf)
continue
cpus, mem, gpus = self.getResources(offer)
logger.debug(
"got resource offer %s: cpus:%s, mem:%s, gpus:%s at %s",
offer.id.value, cpus, mem, gpus, offer.hostname)
logger.debug("attributes,gpus:%s", attrs.get('gpus', None))
sid = offer.slave_id.value
tasks = []
while (len(self.task_waiting) > 0 and cpus >= self.cpus
and mem >= self.mem and
(self.gpus == 0 or attrs.get('gpus', None) is not None)):
logger.debug("Accepting resource on slave %s (%s)",
offer.slave_id.value, offer.hostname)
t = self.task_waiting.pop()
t.state = mesos_pb2.TASK_STARTING
t.state_time = time.time()
task = self.create_task(offer, t, cpus)
tasks.append(task)
self.task_launched[t.id] = t
self.slaveTasks.setdefault(sid, set()).add(t.id)
cpus -= self.cpus
mem -= self.mem
# gpus -= self.gpus
operation = mesos_pb2.Offer.Operation()
operation.type = mesos_pb2.Offer.Operation.LAUNCH
operation.launch.task_infos.extend(tasks)
driver.acceptOffers([offer.id], [operation])
def launch_tasks_for_offers(self, driver, offers):
"""For each offer tries to launch all tasks that can fit in there.
Declines offer if no fitting tasks found."""
launched_tasks = []
for offer in offers:
with self.constraint_state_lock:
try:
tasks, new_state = self.tasks_and_state_for_offer(
driver, offer, self.constraint_state)
if tasks is not None and len(tasks) > 0:
operation = mesos_pb2.Offer.Operation()
operation.type = mesos_pb2.Offer.Operation.LAUNCH
operation.launch.task_infos.extend(tasks)
driver.acceptOffers([offer.id], [operation])
for task in tasks:
staging_timer = self.staging_timer_for_task(
self.staging_timeout, driver,
task.task_id.value)
self.tasks_with_flags.setdefault(
task.task_id.value,
MesosTaskParameters(
health=None,
mesos_task_state=TASK_STAGING,
offer=offer,
staging_timer=staging_timer,
))
staging_timer.start()
launched_tasks.extend(tasks)
self.constraint_state = new_state
else:
driver.declineOffer(offer.id)
except ConstraintFailAllTasksError:
paasta_print(
"Offer failed constraints for every task, rejecting 60s"
)
filters = mesos_pb2.Filters()
filters.refuse_seconds = 60
driver.declineOffer(offer.id, filters)
return launched_tasks
def resourceOffers(self, driver, offers):
if self.frozen:
return
if self.within_reconcile_backoff():
paasta_print(
"Declining all offers since we started reconciliation too recently"
)
for offer in offers:
driver.declineOffer(offer.id)
else:
for idx, offer in enumerate(offers):
if offer.slave_id.value in self.blacklisted_slaves:
log.critical(
"Ignoring offer %s from blacklisted slave %s" %
(offer.id.value, offer.slave_id.value))
filters = mesos_pb2.Filters()
filters.refuse_seconds = self.blacklist_timeout
driver.declineOffer(offer.id, filters)
del offers[idx]
self.launch_tasks_for_offers(driver, offers)
def resourceOffers(self, driver, offers):
rf = mesos_pb2.Filters()
if not self.activeJobs:
rf.refuse_seconds = 60 * 5
for o in offers:
driver.launchTasks(o.id, [], rf)
return
start = time.time()
random.shuffle(offers)
cpus = [self.getResource(o.resources, 'cpus') for o in offers]
mems = [
self.getResource(o.resources, 'mem') -
(o.slave_id.value not in self.slaveTasks and EXECUTOR_MEMORY or 0)
for o in offers
]
logger.debug("get %d offers (%s cpus, %s mem), %d jobs", len(offers),
sum(cpus), sum(mems), len(self.activeJobs))
tasks = {}
for job in self.activeJobsQueue:
while True:
launchedTask = False
for i, o in enumerate(offers):
sid = o.slave_id.value
if self.group and (self.getAttribute(
o.attributes, 'group')
or 'none') not in self.group:
continue
if self.slaveFailed.get(sid, 0) >= MAX_FAILED:
continue
if self.slaveTasks.get(sid, 0) >= self.task_per_node:
continue
if mems[i] < self.mem or cpus[i] + 1e-4 < self.cpus:
continue
t = job.slaveOffer(str(o.hostname), cpus[i], mems[i])
if not t:
continue
task = self.createTask(o, job, t, cpus[i])
tasks.setdefault(o.id.value, []).append(task)
logger.debug("dispatch %s into %s", t, o.hostname)
tid = task.task_id.value
self.jobTasks[job.id].add(tid)
self.taskIdToJobId[tid] = job.id
self.taskIdToSlaveId[tid] = sid
self.slaveTasks[sid] = self.slaveTasks.get(sid, 0) + 1
cpus[i] -= min(cpus[i], t.cpus)
mems[i] -= t.mem
launchedTask = True
if not launchedTask:
break
used = time.time() - start
if used > 10:
logger.error("use too much time in slaveOffer: %.2fs", used)
rf.refuse_seconds = 5
for o in offers:
driver.launchTasks(o.id, tasks.get(o.id.value, []), rf)
logger.debug("reply with %d tasks, %s cpus %s mem left",
sum(len(ts) for ts in tasks.values()), sum(cpus),
sum(mems))
def resourceOffers(self, driver, offers):
'''
Offer resources and launch tasks
'''
for offer in offers:
if all(task.offered for _, task in self.jobs.tasks.iteritems()):
driver.declineOffer(offer.id,
mesos_pb2.Filters(refuse_seconds=FOREVER))
continue
offer_ip = offer.url.address.ip
offered_cpus = offered_mem = offered_gpus = 0.0
offered_port_begin = 0
offered_port_end = 0
offered_tasks = []
for resource in offer.resources:
if resource.name == 'cpus':
offered_cpus = resource.scalar.value
elif resource.name == 'mem':
offered_mem = resource.scalar.value
elif resource.name == 'gpus':
offered_gpus = int(resource.scalar.value)
elif resource.name == 'ports':
offered_port_begin = int(resource.ranges.range[0].begin)
offered_port_end = int(resource.ranges.range[0].end)
logger.info(
'Offered cpus: {0} offered mem: {1} offered gpus: {2} offerd port: {3}~{4} on {5}'
.format(offered_cpus, offered_mem, offered_gpus,
offered_port_begin, offered_port_end,
offer.url.address.hostname))
for _, task in self.jobs.tasks.iteritems():
if task.offered:
continue
if not (task.cpus <= offered_cpus and task.mem <= offered_mem
and int(math.ceil(task.gpus)) <= offered_gpus
and offered_port_begin <= offered_port_end
and offered_port_begin != 0):
logger.info(
'/job:{0}/task{1} does not get all needed resources on offer: {2}'
.format(task.job_name, task.task_index,
offer.id.value))
continue
offered_cpus -= task.cpus
offered_mem -= task.mem
offered_gpus -= math.ceil(task.gpus)
offered_tasks.append(
task.to_task_info(offer.slave_id.value, offered_port_begin,
True))
node = '/job:{0}/task:{1}'.format(task.job_name,
task.task_index).replace(
'/',
'_').replace(':', '_')
addr = str(offer_ip) + ':' + str(offered_port_begin)
offered_port_begin += 1
logger.info('Registering in zookeeper {0}, value: {1}'.format(
self.job_zk_dir + '/member/' + node, addr))
self.zk.create(self.job_zk_dir + '/member/' + node, addr)
if task.job_name == 'ps':
self.zk.create(self.job_zk_dir + '/status/' + node,
'false')
task.offered = True
logger.info(
'Allocating resource for /job:{0}/task:{1} successful'.
format(task.job_name, task.task_index))
driver.launchTasks(offer.id, offered_tasks, mesos_pb2.Filters())
def _filters(self, seconds):
f = mesos_pb2.Filters()
f.refuse_seconds = seconds
return f
import dpark.conf as conf
from dpark.util import getuser, memory_str_to_mb
from dpark import moosefs
logger = logging.getLogger('dpark.scheduler')
class Task:
def __init__(self, id):
self.id = id
self.tried = 0
self.state = -1
self.state_time = 0
REFUSE_FILTER = mesos_pb2.Filters()
REFUSE_FILTER.refuse_seconds = 10 * 60 # 10 mins
EXECUTOR_CPUS = 0.01
EXECUTOR_MEMORY = 64 # cache
def safe(f):
def _(self, *a, **kw):
with self.lock:
r = f(self, *a, **kw)
return r
return _
class BaseScheduler(object):
