def resourceOffers(self, driver, offers):
rf = mesos_pb2.Filters()
if not self.activeJobs:
rf.refuse_seconds = 60 * 5
for o in offers:
driver.launchTasks(o.id, [], rf)
return
random.shuffle(offers)
cpus = [self.getResource(o.resources, 'cpus') for o in offers]
mems = [
self.getResource(o.resources, 'mem') -
(o.slave_id.value not in self.slaveTasks and EXECUTOR_MEMORY or 0)
for o in offers
]
logger.debug("get %d offers (%s cpus, %s mem), %d jobs", len(offers),
sum(cpus), sum(mems), len(self.activeJobs))
tasks = {}
launchedTask = False
for job in self.activeJobsQueue:
while True:
launchedTask = False
for i, o in enumerate(offers):
sid = o.slave_id.value
if self.group and (self.getAttribute(
o.attributes, 'group')
or 'none') not in self.group:
continue
if self.slaveFailed.get(sid, 0) >= MAX_FAILED:
continue
if self.slaveTasks.get(sid, 0) >= self.task_per_node:
continue
if mems[i] < self.mem or cpus[i] < self.cpus:
continue
t = job.slaveOffer(str(o.hostname), cpus[i], mems[i])
if not t:
continue
task = self.createTask(o, job, t)
tasks.setdefault(o.id.value, []).append(task)
logger.debug("dispatch %s into %s", t, o.hostname)
tid = task.task_id.value
self.jobTasks[job.id].add(tid)
self.taskIdToJobId[tid] = job.id
self.taskIdToSlaveId[tid] = sid
self.slaveTasks[sid] = self.slaveTasks.get(sid, 0) + 1
cpus[i] -= t.cpus
mems[i] -= t.mem
launchedTask = True
if not launchedTask:
break
rf.refuse_seconds = 5
for o in offers:
driver.launchTasks(o.id, tasks.get(o.id.value, []), rf)
logger.debug("reply with %d tasks, %s cpus %s mem left", len(tasks),
sum(cpus), sum(mems))
import mesos
import mesos_pb2
ctx = zmq.Context()
class Task:
def __init__(self, id):
self.id = id
self.tried = 0
self.state = -1
self.state_time = 0
REFUSE_FILTER = mesos_pb2.Filters()
REFUSE_FILTER.refuse_seconds = -1
def parse_mem(m):
try:
return float(m)
except ValueError:
number, unit = float(m[:-1]), m[-1].lower()
if unit == 'g':
number *= 1024
elif unit == 'k':
number /= 1024
return number