def schedule(self):
"""
create inter-priority
"""
def byPriority(wf):
return 0 if wf.priority is None else wf.priority
##simple inter priority sorting
sorted_wfs = sorted(self.workflows, key=byPriority)
wf_jobs = {wf: [] for wf in sorted_wfs}
resources = self.resource_manager.get_resources()
##print("common nodes count:" + str(len(toNodes(resources))))
nodes = HeftHelper.to_nodes(resources)
wf_jobs = {wf: self.make_ranking(wf, nodes) for wf in sorted_wfs}
##new_schedule = self.get_unchanged_schedule(self.old_schedule, time)
new_schedule = Schedule({node: [] for node in nodes})
new_plan = new_schedule.mapping
for (wf, jobs) in wf_jobs.items():
new_schedule = self.mapping([(wf, jobs)],
new_plan,
nodes,
self.commcost,
self.compcost)
new_plan = new_schedule.mapping
return new_schedule
def run(self, current_cleaned_schedule):
## current_cleaned_schedule - this schedule contains only
## finished and executed tasks, all unfinished and failed have been removed already
## current_cleaned_schedule also have down nodes and new added
## ALGORITHM DOESN'T CHECK ADDING OF NEW NODES BY ITSELF
## nodes contain only available now
## 1. get all unscheduled tasks
## 2. sort them by rank
## 3. map on the existed nodes according to current_cleaned_schedule
nodes = self.get_nodes()
live_nodes = [node for node in nodes if node.state != Node.Down]
for_planning = HeftHelper.get_tasks_for_planning(self.workflow, current_cleaned_schedule)
## TODO: check if it sorted properly
for_planning = set([task.id for task in for_planning])
self.wf_jobs = self.make_ranking(self.workflow, live_nodes) if self.ranking is None else self.ranking
sorted_tasks = [task for task in self.wf_jobs if task.id in for_planning]
# print("P: " + str(sorted_tasks))
new_sched = self.mapping([(self.workflow, sorted_tasks)], current_cleaned_schedule.mapping, nodes, self.commcost, self.compcost)
return new_sched
def make_ranking(self, wf, nodes):
##resources = self.resource_manager.get_resources()
##print("common nodes count:" + str(len(toNodes(resources))))
##nodes = HeftHelper.to_nodes(resources)
ranking_func = HeftHelper.build_ranking_func(nodes, self.compcost, self.commcost)
wf_jobs = ranking_func(wf)
return wf_jobs
def mapping(self, sorted_jobs, existing_plan, live_nodes, commcost, compcost):
"""def allocate(job, orders, jobson, prec, compcost, commcost):"""
""" Allocate job to the machine with earliest finish time
Operates in place
"""
## TODO: add finished tasks
jobson = dict()
for (node, items) in existing_plan.items():
for item in items:
if item.state == ScheduleItem.FINISHED or item.state == ScheduleItem.EXECUTING:
jobson[item.job] = node
new_plan = existing_plan
def ft(machine):
#cost = st(machine)
runtime = compcost(task, machine)
cost = st(machine, runtime) + runtime
##print("machine: %s job:%s cost: %s" % (machine.name, task.id, cost))
##print("machine: " + str(machine.name) + " cost: " + str(cost))
return cost
if len(live_nodes) != 0:
## in case if there is not any live nodes we just return the same cleaned schedule
for wf, tasks in sorted_jobs:
##wf_dag = self.convert_to_parent_children_map(wf)
wf_dag = HeftHelper.convert_to_parent_children_map(wf)
prec = reverse_dict(wf_dag)
for task in tasks:
st = partial(self.start_time, wf, task, new_plan, jobson, prec, commcost)
# ress = [(key, ft(key)) for key in new_plan.keys()]
# agent_pair = min(ress, key=lambda x: x[1][0])
# agent = agent_pair[0]
# start = agent_pair[1][0]
# end = agent_pair[1][1]
# agent = min(new_plan.keys(), key=ft)
agent = min(live_nodes, key=ft)
runtime = compcost(task, agent)
start = st(agent, runtime)
end = ft(agent)
# new_plan[agent].append(ScheduleItem(task, start, end))
Schedule.insert_item(new_plan, agent, ScheduleItem(task, start, end))
jobson[task] = agent
new_sched = Schedule(new_plan)
return new_sched
def checkDown(self, node_name, is_down):
nodes = HeftHelper.to_nodes(self.public_resources)
for nd in nodes:
if nd.name == node_name:
if is_down:
nd.state = Node.Down
else:
nd.state = Node.Unknown
pass
def run(self, current_cleaned_schedule):
nodes = self.get_nodes()
live_nodes = [node for node in nodes if node.state != Node.Down]
for_planning = HeftHelper.get_tasks_for_planning(self.workflow, current_cleaned_schedule)
for_planning = set([task.id for task in for_planning])
self.wf_jobs = self.make_ranking(self.workflow, live_nodes) if self.ranking is None else self.ranking
sorted_tasks = [task for task in self.wf_jobs if task.id in for_planning]
# print("P: " + str(sorted_tasks))
new_sched = self.mapping([(self.workflow, sorted_tasks)], current_cleaned_schedule.mapping, nodes, self.commcost, self.compcost)
return new_sched
def __init__(self,
workflow,
resource_manager,
estimator,
task_map,
node_map,
# fixed part of schedule. It need to be accounted when new schedule is built, but it's not possible to cahnge something inside it
fixed_schedule_part):
self.workflow = workflow
self.nodes = HeftHelper.to_nodes(resource_manager.get_resources())
self.estimator = estimator
##TODO: Build it
self.task_map = task_map
##TODO: Build it
self.node_map = node_map
self.fixed_schedule_part = fixed_schedule_part
# construct initial mapping
# eliminate all already scheduled tasks
pass
def isCloudNode(self, node):
result = node.name in [nd.name for nd in HeftHelper.to_nodes(self.public_resources)]
return result
def get_by_softreq(self, soft_reqs):
nodes = HeftHelper.to_nodes(self.public_resources)
def check_reqs(node):
return (soft_reqs in node.soft) or (SoftItem.ANY_SOFT in node.soft)
gotcha = [node for node in nodes if node.state != Node.Down and check_reqs(node)]
return gotcha
def get_nodes(self):
resources = self.resource_manager.get_resources()
nodes = HeftHelper.to_nodes(resources)
return nodes
def generate_public_resources(self):
## TODO: remake it later
#(public_resources, generate_reliability, generate_probability_law_for_(task,node)_pair) = generate public_resource
resCount = 3
resources = list()
for i in range(0, resCount):
res = Resource("public_res_" + str(i))
resources.append(res)
nodeCount = None
if i == 0:
nodeCount = 15
elif i == 1:
nodeCount = 12
elif i == 2:
nodeCount = 9
for j in range(0, nodeCount):
node = Node(res.name + "_node_" + str(j), res, [SoftItem.ANY_SOFT])
# if j == 0:
# node.flops = 10 + 5
# if j == 1:
# node.flops = 15 + 10#10*3
# if j == 2:
# node.flops = 25 + 10#25*3
# if j == 3:
# node.flops = 25 + 10#25*3
# if j == 4:
# node.flops = 30 + 10#30*3
# if j == 5:
# node.flops = 10 + 5
# if j == 6:
# node.flops = 15 + 10#10*3
# if j == 7:
# node.flops = 25 + 10#25*3
# if j == 8:
# node.flops = 25 + 10#25*3
# if j == 9:
# node.flops = 30 + 10#30*3
# if j == 10:
# node.flops = 10 + 5
# if j == 11:
# node.flops = 15 + 10#10*3
# if j == 12:
# node.flops = 25 + 10#25*3
# if j == 13:
# node.flops = 25 + 10#25*3
# if j == 14:
# node.flops = 30 + 10#30*3
if j == 0:
node.flops = 10
if j == 1:
node.flops = 15#10*3
if j == 2:
node.flops = 25#25*3
if j == 3:
node.flops = 25#25*3
if j == 4:
node.flops = 30#30*3
if j == 5:
node.flops = 10
if j == 6:
node.flops = 15#10*3
if j == 7:
node.flops = 25#25*3
if j == 8:
node.flops = 25#25*3
if j == 9:
node.flops = 30#30*3
if j == 10:
node.flops = 10
if j == 11:
node.flops = 15#10*3
if j == 12:
node.flops = 25#25*3
if j == 13:
node.flops = 25#25*3
if j == 14:
node.flops = 30#30*3
res.nodes.add(node)
nodes = HeftHelper.to_nodes(resources)
reliability_map = {node.name: 0.9 for node in nodes}
def probability_estimator(dt, comp_estimation, transfer_estimation):
M = comp_estimation + transfer_estimation
sigma = 0.1 * M
result = 0.5 *(1 + math.erf((dt - M)/sigma))
return result
return (resources, reliability_map, probability_estimator)
