def tick(self):
for framework_name in self.filters.keys():
new_duration = self.filters[framework_name] - 1
self.filters[framework_name] = new_duration
if new_duration <= 0:
event_data = { "framework_name": framework_name }
publish_event(self, "clear_filter", event_data)
print 'Clearing filter for framework %s' % framework_name
del self.filters[framework_name]
for framework_name in self.frameworks.keys():
for task_name in self.frameworks[framework_name].keys():
if self.frameworks[framework_name][task_name].duration is None:
# Task never terminates.
continue
new_duration = self.frameworks[framework_name][task_name].duration - 1
self.frameworks[framework_name][task_name].duration = new_duration
if new_duration <= 0:
print('Task %s completed' % task_name)
del self.frameworks[framework_name][task_name]
update = Update(self.name, framework_name, task_name, "FINISHED")
self.allocator.status_update(update)
def tick(self):
for framework_name in self.filters.keys():
new_duration = self.filters[framework_name] - 1
self.filters[framework_name] = new_duration
if new_duration <= 0:
event_data = {"framework_name": framework_name}
publish_event(self, "clear_filter", event_data)
print 'Clearing filter for framework %s' % framework_name
del self.filters[framework_name]
for framework_name in self.frameworks.keys():
for task_name in self.frameworks[framework_name].keys():
if self.frameworks[framework_name][task_name].duration is None:
# Task never terminates.
continue
new_duration = self.frameworks[framework_name][
task_name].duration - 1
self.frameworks[framework_name][
task_name].duration = new_duration
if new_duration <= 0:
print('Task %s completed' % task_name)
del self.frameworks[framework_name][task_name]
update = Update(self.name, framework_name, task_name,
"FINISHED")
self.allocator.status_update(update)
def recover(self, agent_name, resources, user_name):
event_data = {
"agent_name": agent_name,
"resources": resources,
"user_name": user_name,
}
publish_event(self, "recover_resources", event_data)
self.agents[agent_name].drf.recover(user_name, resources)
def recover(self, agent_name, resources, user_name):
event_data = {
"agent_name": agent_name,
"resources": resources,
"user_name": user_name,
}
publish_event(self, "recover_resources", event_data)
self.agents[agent_name].drf.recover(user_name, resources)
def launch(self, task):
publish_event(self, "launch_task", task)
if task.framework_name not in self.frameworks:
self.frameworks[task.framework_name] = {}
if task.task_name in self.frameworks[task.framework_name]:
print("WARNING: Overriding task %s on agent %s" % (task.task_name, self.name))
self.frameworks[task.framework_name][task.task_name] = task
def decline(self, framework_name, offer, refuse_steps=5):
event_data = {
"framework_name": framework_name,
"agent_name": offer.agent.name,
"resources": offer.resources,
"filter_duration": refuse_steps,
}
publish_event(self, "decline_offer", event_data)
offer.agent.add_filter(framework_name, refuse_steps)
self.recover(offer.agent.name, offer.resources, framework_name)
def launch(self, task):
publish_event(self, "launch_task", task)
if task.framework_name not in self.frameworks:
self.frameworks[task.framework_name] = {}
if task.task_name in self.frameworks[task.framework_name]:
print("WARNING: Overriding task %s on agent %s" %
(task.task_name, self.name))
self.frameworks[task.framework_name][task.task_name] = task
def decline(self, framework_name, offer, refuse_steps=5):
event_data = {
"framework_name": framework_name,
"agent_name": offer.agent.name,
"resources": offer.resources,
"filter_duration": refuse_steps,
}
publish_event(self, "decline_offer", event_data)
offer.agent.add_filter(framework_name, refuse_steps)
self.recover(offer.agent.name, offer.resources, framework_name)
def launch(self, task, offer):
for i in range(len(task.resources.vector)):
if task.resources.vector[i] > offer.resources.vector[i]:
print("Framework tried to launch larger task than offer resources")
return
recover = offer.resources.subtract(task.resources)
self.recover(offer.agent.name, recover, task.framework_name)
# Schedule task on agent
self.agents[offer.agent.name].launch(task)
# Save reference to task to keep track of resources to free up.
if task.framework_name not in self.tasks:
self.tasks[task.framework_name] = {}
publish_event(self, "launch_task", task)
self.tasks[task.framework_name][task.task_name] = task
def launch(self, task, offer):
for i in range(len(task.resources.vector)):
if task.resources.vector[i] > offer.resources.vector[i]:
print(
"Framework tried to launch larger task than offer resources"
)
return
recover = offer.resources.subtract(task.resources)
self.recover(offer.agent.name, recover, task.framework_name)
# Schedule task on agent
self.agents[offer.agent.name].launch(task)
# Save reference to task to keep track of resources to free up.
if task.framework_name not in self.tasks:
self.tasks[task.framework_name] = {}
publish_event(self, "launch_task", task)
self.tasks[task.framework_name][task.task_name] = task
def allocate(self):
for agent_name, agent in self.agents.iteritems():
order = agent.drf.order()
if len(order) == 0:
print("No frameworks to serve")
return
resources = agent.drf.available()
# Ensure we haven't exhausted resources on the agent.
out_of_capacity = False
for resource in resources.vector:
if resource <= 0:
print("Agent %s is out of capacity" % agent_name)
out_of_capacity = True
if out_of_capacity:
continue
# Offer all available resources to first framework.
for framework_name, _ in order:
print 'Allocator considering framework %s' % framework_name
if agent.get_filter(framework_name):
print 'Framework %s is filtered: continue' % framework_name
continue
agent.drf.allocate(framework_name, resources)
# Form offer to scheduler
offer = Offer(agent, resources)
event_data = {
"agent_name": agent.name,
"framework_name": framework_name,
"resources": resources,
}
publish_event(self, "resource_offer", event_data)
self.frameworks[framework_name].offer([offer])
break
def allocate(self):
for agent_name, agent in self.agents.iteritems():
order = agent.drf.order()
if len(order) == 0:
print("No frameworks to serve")
return
resources = agent.drf.available()
# Ensure we haven't exhausted resources on the agent.
out_of_capacity = False
for resource in resources.vector:
if resource <= 0:
print("Agent %s is out of capacity" % agent_name)
out_of_capacity = True
if out_of_capacity:
continue
# Offer all available resources to first framework.
for framework_name, _ in order:
print 'Allocator considering framework %s' % framework_name
if agent.get_filter(framework_name):
print 'Framework %s is filtered: continue' % framework_name
continue
agent.drf.allocate(framework_name, resources)
# Form offer to scheduler
offer = Offer(agent, resources)
event_data = {
"agent_name": agent.name,
"framework_name": framework_name,
"resources": resources,
}
publish_event(self, "resource_offer", event_data)
self.frameworks[framework_name].offer([offer])
break
def add_framework(self, framework):
publish_event(self, "add_framework", framework.name)
self.frameworks[framework.name] = framework
for agent_name, agent in self.agents.iteritems():
agent.add_framework(framework.name)
def remove_agent(self, name):
event_data = {"agent_name": name}
publish_event(self, "remove_agent", event_data)
pass
def add_filter(self, framework_name, duration):
event_data = { "framework_name": framework_name, "duration": duration }
publish_event(self, "add_filter", event_data)
print 'Adding filter with duration %s for framework %s' % (duration,
framework_name)
self.filters[framework_name] = duration
def add_framework(self, name):
publish_event(self, "add_framework", name)
self.drf.add_user(name)
def add_framework(self, name):
publish_event(self, "add_framework", name)
self.drf.add_user(name)
def remove_agent(self, name):
event_data = { "agent_name": name }
publish_event(self, "remove_agent", event_data)
pass
def add_filter(self, framework_name, duration):
event_data = {"framework_name": framework_name, "duration": duration}
publish_event(self, "add_filter", event_data)
print 'Adding filter with duration %s for framework %s' % (
duration, framework_name)
self.filters[framework_name] = duration
def add_framework(self, framework):
publish_event(self, "add_framework", framework.name)
self.frameworks[framework.name] = framework
for agent_name, agent in self.agents.iteritems():
agent.add_framework(framework.name)
def add_agent(self, name, resources):
event_data = {"agent_name": name, "resources": resources}
publish_event(self, "add_agent", event_data)
self.agents[name] = Agent(name, resources, self)
def add_agent(self, name, resources):
event_data = { "agent_name": name, "resources": resources }
publish_event(self, "add_agent", event_data)
self.agents[name] = Agent(name, resources, self)
