def run(self, current_cleaned_schedule):
## current_cleaned_schedule - this schedule contains only
## finished and executed tasks, all unfinished and failed have been removed already
## current_cleaned_schedule also have down nodes and new added
## ALGORITHM DOESN'T CHECK ADDING OF NEW NODES BY ITSELF
## nodes contain only available now
## 1. get all unscheduled tasks
## 2. sort them by rank
## 3. map on the existed nodes according to current_cleaned_schedule
nodes = self.get_nodes()
live_nodes = [node for node in nodes if node.state != Node.Down]
for_planning = HeftHelper.get_tasks_for_planning(
self.workflow, current_cleaned_schedule)
## TODO: check if it sorted properly
for_planning = set([task.id for task in for_planning])
self.wf_jobs = self.make_ranking(
self.workflow,
live_nodes) if self.ranking is None else self.ranking
sorted_tasks = [
task for task in self.wf_jobs if task.id in for_planning
]
# print("P: " + str(sorted_tasks))
new_sched = self.mapping([(self.workflow, sorted_tasks)],
current_cleaned_schedule.mapping, nodes,
self.commcost, self.compcost)
return new_sched
def __init__(self, workflow, resource_manager, estimator):
self.counter = 0
self.workflow = workflow
##interface Estimator
self.estimator = estimator
self.resource_manager = resource_manager
nodes = resource_manager.get_nodes(
) #list(HeftHelper.to_nodes(resource_manager.get_resources()))
ranking = HeftHelper.build_ranking_func(
nodes, lambda job, agent: estimator.estimate_runtime(job, agent),
lambda ni, nj, A, B: estimator.estimate_transfer_time(
A, B, ni, nj))
sorted_tasks = ranking(self.workflow)
self.nodes = nodes
self.sorted_tasks = sorted_tasks
self.workflow_size = len(sorted_tasks)
self.task_map = {task.id: task for task in sorted_tasks}
self.node_map = {node.name: node for node in nodes}
self.initializing_alg = SimpleRandomizedHeuristic(
self.workflow, self.nodes, self.estimator)
self.initial_chromosome = None ##GAFunctions.schedule_to_chromosome(initial_schedule)
pass
def default_fixed_schedule_part(resource_manager):
fix_schedule_part = Schedule({
node: []
for node in HeftHelper.to_nodes(
resource_manager.get_resources())
})
return fix_schedule_part
def schedule(self):
"""
create inter-priority
"""
def byPriority(wf):
return 0 if wf.priority is None else wf.priority
##simple inter priority sorting
sorted_wfs = sorted(self.workflows, key=byPriority)
wf_jobs = {wf: [] for wf in sorted_wfs}
resources = self.resource_manager.get_resources()
##print("common nodes count:" + str(len(toNodes(resources))))
nodes = HeftHelper.to_nodes(resources)
wf_jobs = {wf: self.make_ranking(wf, nodes) for wf in sorted_wfs}
##new_schedule = self.get_unchanged_schedule(self.old_schedule, time)
new_schedule = Schedule({node: [] for node in nodes})
new_plan = new_schedule.mapping
for (wf, jobs) in wf_jobs.items():
new_schedule = self.mapping([(wf, jobs)], new_plan, nodes,
self.commcost, self.compcost)
new_plan = new_schedule.mapping
return new_schedule
def __call__(self):
stats, logbook = self.stats(), self.logbook()
_wf, rm, estimator = self.env()
heft_schedule = self.heft_schedule()
wf_dag = HeftHelper.convert_to_parent_children_map(_wf)
jobs = set(wf_dag.keys()) | set(x for xx in wf_dag.values() for x in xx)
nodes = rm.get_nodes()
mapMatrix = rd.initMapMatrix(jobs, nodes, estimator)
rankList = rd.initRankList(wf_dag, nodes, estimator)
ordFilter = rd.filterList(_wf)
toolbox = self.toolbox(mapMatrix, rankList, ordFilter)
pop, log, best = rd.run_pso(
toolbox=toolbox,
logbook=logbook,
stats=stats,
gen_curr=0, gen_step=self.GEN, invalidate_fitness=True, initial_pop=None,
w=self.W, c1=self.C1, c2=self.C2, n=self.N, rm=rm, wf=_wf, estimator=estimator, mapMatrix=mapMatrix, rankList=rankList, ordFilter=ordFilter,
)
schedule = rd_order.build_schedule(_wf, rm, estimator, best, mapMatrix, rankList, ordFilter)
Utility.validate_static_schedule(_wf, schedule)
makespan = Utility.makespan(schedule)
#print("Final makespan: {0}".format(makespan))
#print("Heft makespan: {0}".format(Utility.makespan(heft_schedule)))
return makespan
def make_ranking(self, wf, nodes):
##resources = self.resource_manager.get_resources()
##print("common nodes count:" + str(len(toNodes(resources))))
##nodes = HeftHelper.to_nodes(resources)
ranking_func = HeftHelper.build_ranking_func(nodes, self.compcost,
self.commcost)
wf_jobs = ranking_func(wf)
return wf_jobs
def checkDown(self, node_name, is_down):
nodes = HeftHelper.to_nodes(self.public_resources)
for nd in nodes:
if nd.name == node_name:
if is_down:
nd.state = Node.Down
else:
nd.state = Node.Unknown
pass
def mapping(self, sorted_jobs, existing_plan, live_nodes, commcost,
compcost):
"""def allocate(job, orders, jobson, prec, compcost, commcost):"""
""" Allocate job to the machine with earliest finish time
Operates in place
"""
## TODO: add finished tasks
jobson = dict()
for (node, items) in existing_plan.items():
for item in items:
if item.state == ScheduleItem.FINISHED or item.state == ScheduleItem.EXECUTING:
jobson[item.job] = node
new_plan = existing_plan
def ft(machine):
#cost = st(machine)
runtime = compcost(task, machine)
cost = st(machine, runtime) + runtime
##print("machine: %s job:%s cost: %s" % (machine.name, task.id, cost))
##print("machine: " + str(machine.name) + " cost: " + str(cost))
return cost
if len(live_nodes) != 0:
## in case if there is not any live nodes we just return the same cleaned schedule
for wf, tasks in sorted_jobs:
##wf_dag = self.convert_to_parent_children_map(wf)
wf_dag = HeftHelper.convert_to_parent_children_map(wf)
prec = reverse_dict(wf_dag)
for task in tasks:
st = partial(self.start_time, wf, task, new_plan, jobson,
prec, commcost)
# ress = [(key, ft(key)) for key in new_plan.keys()]
# agent_pair = min(ress, key=lambda x: x[1][0])
# agent = agent_pair[0]
# start = agent_pair[1][0]
# end = agent_pair[1][1]
# agent = min(new_plan.keys(), key=ft)
agent = min(live_nodes, key=ft)
runtime = compcost(task, agent)
start = st(agent, runtime)
end = ft(agent)
# new_plan[agent].append(ScheduleItem(task, start, end))
Schedule.insert_item(new_plan, agent,
ScheduleItem(task, start, end))
jobson[task] = agent
new_sched = Schedule(new_plan)
return new_sched
def get_by_softreq(self, soft_reqs):
nodes = HeftHelper.to_nodes(self.public_resources)
def check_reqs(node):
return (soft_reqs in node.soft) or (SoftItem.ANY_SOFT in node.soft)
gotcha = [
node for node in nodes
if node.state != Node.Down and check_reqs(node)
]
return gotcha
def filterList(wf):
wf_dag = HeftHelper.convert_to_parent_children_map(wf)
jobs = set(wf_dag.keys()) | set(x for xx in wf_dag.values() for x in xx)
resultFilter = {job.id:{j.id for j in jobs} for job in jobs}
headTask = wf.head_task
ready = dict()
readyParents = dict()
for task in headTask.children:
getTaskChildFilter(task, resultFilter, ready)
for task in jobs:
getTaskParentFilter(task, resultFilter, readyParents)
return resultFilter
def _run_heft():
dynamic_planner = DynamicHeft(wf, resource_manager, estimator)
nodes = HeftHelper.to_nodes(resource_manager.resources)
current_cleaned_schedule = Schedule({node: [] for node in nodes})
schedule_dynamic_heft = dynamic_planner.run(
current_cleaned_schedule)
self._validate(wf, estimator, schedule_dynamic_heft)
if is_visualized:
viz.visualize_task_node_mapping(wf, schedule_dynamic_heft)
# Utility.create_jedule_visualization(schedule_dynamic_heft, wf_name+'_heft')
pass
return schedule_dynamic_heft
def __init__(self,
workflow,
nodes,
estimator):
self.workflow = workflow
self.nodes = nodes
self.workflow_size = workflow.get_task_count()
self.estimator = estimator
self.task_map = {task.id: task for task in HeftHelper.get_all_tasks(self.workflow)}
self.node_map = {node.name: node for node in nodes}
self.initial_chromosome = None
pass
def __init__(
self,
workflow,
resource_manager,
estimator,
task_map,
node_map,
# fixed part of schedule. It need to be accounted when new schedule is built, but it's not possible to cahnge something inside it
fixed_schedule_part):
self.workflow = workflow
self.nodes = HeftHelper.to_nodes(resource_manager.get_resources())
self.estimator = estimator
##TODO: Build it
self.task_map = task_map
##TODO: Build it
self.node_map = node_map
self.fixed_schedule_part = fixed_schedule_part
# construct initial mapping
# eliminate all already scheduled tasks
pass
def run(self, current_cleaned_schedule):
nodes = self.get_nodes()
live_nodes = [node for node in nodes if node.state != Node.Down]
for_planning = HeftHelper.get_tasks_for_planning(
self.workflow, current_cleaned_schedule)
for_planning = set([task.id for task in for_planning])
self.wf_jobs = self.make_ranking(
self.workflow,
live_nodes) if self.ranking is None else self.ranking
sorted_tasks = [
task for task in self.wf_jobs if task.id in for_planning
]
# print("P: " + str(sorted_tasks))
new_sched = self.mapping([(self.workflow, sorted_tasks)],
current_cleaned_schedule.mapping, nodes,
self.commcost, self.compcost)
return new_sched
def get_nodes(self):
resources = self.resource_manager.get_resources()
nodes = HeftHelper.to_nodes(resources)
return nodes
def isCloudNode(self, node):
result = node.name in [
nd.name for nd in HeftHelper.to_nodes(self.public_resources)
]
return result
from src.core.CommonComponents.ExperimentalManagers import ExperimentResourceManager
from src.core.environment.Utility import Utility, wf
from src.algs.common.MapOrdSchedule import build_schedule, MAPPING_SPECIE, ORDERING_SPECIE, ordering_from_schedule, \
mapping_from_schedule
from src.experiments.cga.mobjective.utility import SimpleTimeCostEstimator
from src.core.environment.ResourceGenerator import ResourceGenerator as rg
from src.algs.common.MapOrdSchedule import fitness as basefitness
_wf = wf("Montage_50")
rm = ExperimentResourceManager(rg.r([10, 15, 25, 30]))
estimator = SimpleTimeCostEstimator(comp_time_cost=0,
transf_time_cost=0,
transferMx=None,
ideal_flops=20,
transfer_time=100)
sorted_tasks = HeftHelper.heft_rank(_wf, rm, estimator)
heft_schedule = run_heft(_wf, rm, estimator)
print(Utility.makespan(heft_schedule))
stats = tools.Statistics(lambda ind: ind.fitness.values[0])
stats.register("avg", numpy.mean)
stats.register("std", numpy.std)
stats.register("min", numpy.min)
stats.register("max", numpy.max)
logbook = tools.Logbook()
logbook.header = ["gen", "evals"] + stats.fields
heft_mapping = mapping_from_schedule(heft_schedule)