def _run_ga(initial_schedule, saveIt=True):
def default_fixed_schedule_part(resource_manager):
fix_schedule_part = Schedule({
node: []
for node in HeftHelper.to_nodes(
resource_manager.get_resources())
})
return fix_schedule_part
fix_schedule_part = default_fixed_schedule_part(resource_manager)
((the_best_individual, pop, schedule, iter_stopped),
logbook) = alg_func(fix_schedule_part, initial_schedule)
self._validate(wf, estimator, schedule)
name = wf_name + "_bundle"
path = '../../resources/saved_schedules/' + name + '.json'
if saveIt:
Utility.save_schedule(path, wf_name,
resource_manager.get_resources(),
estimator.transfer_matrix, ideal_flops,
schedule)
if is_visualized:
viz.visualize_task_node_mapping(wf, schedule)
# Utility.create_jedule_visualization(schedule, wf_name+'_ga')
pass
return schedule, logbook
def __call__(self):
toolbox, stats, logbook = self.toolbox(), self.stats(), self.logbook()
_wf, rm, estimator = self.env()
heft_schedule = self.heft_schedule()
pop, log, best = om.run_pso(
toolbox=toolbox,
logbook=logbook,
stats=stats,
gen_curr=0,
gen_step=self.GEN,
invalidate_fitness=True,
initial_pop=None,
w=self.W,
c1=self.C1,
c2=self.C2,
n=self.N,
)
schedule = om_order.build_schedule(_wf, rm, estimator, best)
Utility.validate_static_schedule(_wf, schedule)
makespan = Utility.makespan(schedule)
#print("Final makespan: {0}".format(makespan))
#print("Heft makespan: {0}".format(Utility.makespan(heft_schedule)))
return makespan
def do_exp():
pop, _logbook, best = run_gsa(toolbox, stats, logbook, pop_size, 0, iter_number, None, kbest, ginit, **{"w":W, "c":C})
schedule = build_schedule(_wf, rm, estimator, best)
Utility.validate_static_schedule(_wf, schedule)
makespan = Utility.makespan(schedule)
print("Final makespan: {0}".format(makespan))
print("Heft makespan: {0}".format(Utility.makespan(heft_schedule)))
return makespan
def do_exp_PEFT(wf_name, trans):
_wf = wf(wf_name)
estimator.transfer_time = trans
#estim = SimpleTimeCostEstimator(comp_time_cost=0, transf_time_cost=0, transferMx=None,
# ideal_flops=20, transfer_time=trans)
peft_schedule = run_peft(_wf, rm, estimator)
#return peft_schedule
Utility.validate_static_schedule(_wf, peft_schedule)
makespan = Utility.makespan(peft_schedule)
return makespan
def do_exp():
best, log, current = run_sa(
toolbox=toolbox,
logbook=logbook,
stats=stats,
initial_solution=initial_state, T=T, N=N
)
solution = {MAPPING_SPECIE: [item for item in best.mapping.items()], ORDERING_SPECIE: best.ordering}
schedule = build_schedule(_wf, estimator, rm, solution)
Utility.validate_static_schedule(_wf, schedule)
makespan = Utility.makespan(schedule)
heft_makespan = Utility.makespan(heft_schedule)
print("Final makespan: {0}".format(makespan))
print("Heft makespan: {0}".format(heft_makespan))
return makespan
def __call__(self):
stats, logbook = self.stats(), self.logbook()
_wf, rm, estimator = self.env()
estimator.transfer_time = 500
heft_schedule = self.heft_schedule()
wf_dag = HeftHelper.convert_to_parent_children_map(_wf)
jobs = set(wf_dag.keys()) | set(x for xx in wf_dag.values()
for x in xx)
nodes = rm.get_nodes()
mapMatrix = initMapMatrix(jobs, nodes, estimator)
rankList = initRankList(wf_dag, nodes, estimator)
ordFilter = filterList(_wf)
toolbox = self.toolbox(mapMatrix, rankList, ordFilter)
pop, log, best = run_pso(
toolbox=toolbox,
logbook=logbook,
stats=stats,
gen_curr=0,
gen_step=self.GEN,
invalidate_fitness=True,
initial_pop=None,
w=self.W,
c1=self.C1,
c2=self.C2,
n=self.N,
rm=rm,
wf=_wf,
estimator=estimator,
mapMatrix=mapMatrix,
rankList=rankList,
ordFilter=ordFilter,
)
#print(str(best.fitness))
schedule = build_schedule(_wf, rm, estimator, best, mapMatrix,
rankList, ordFilter)
Utility.validate_static_schedule(_wf, schedule)
makespan = Utility.makespan(schedule)
if makespan > best.fitness.values[0]:
print("DANGER!!!!!!!!!!!!!!!!!")
print("Final makespan: {0}".format(makespan))
print("Heft makespan: {0}".format(Utility.makespan(heft_schedule)))
return makespan
def recover_ordering(ordering):
corrected_ordering = []
for it in range(len(ordering)):
t = ordering[it]
if Utility.is_enough_to_be_executed(wf, t, corrected_ordering + fixed_tasks_ids):
corrected_ordering.append(t)
else:
#print("INCORRECT")
return False
def __call__(self):
stats, logbook = self.stats(), self.logbook()
_wf, rm, estimator = self.env()
wf_dag = HeftHelper.convert_to_parent_children_map(_wf)
heft_schedule = self.heft_schedule()
nodes = rm.get_nodes()
rankList = initRankList(wf_dag, nodes, estimator)
toolbox = self.toolbox(rankList)
pop, log, best = run_pso(
toolbox=toolbox,
logbook=logbook,
stats=stats,
gen_curr=0,
gen_step=self.GEN,
invalidate_fitness=True,
initial_pop=None,
w=self.W,
c1=self.C1,
c2=self.C2,
n=self.N,
rm=rm,
wf=_wf,
estimator=estimator,
rankList=rankList,
)
#schedule = build_schedule(_wf, rm, estimator, best, mapMatrix, rankList, ordFilter)
schedule = build_schedule(_wf, rm, estimator, best, rankList)
Utility.validate_static_schedule(_wf, schedule)
makespan = Utility.makespan(schedule)
if (makespan > best.fitness.values[0]):
print(
"DANGER!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
)
#print("Final makespan: {0}".format(makespan))
#print("Heft makespan: {0}".format(Utility.makespan(heft_schedule)))
return makespan
def do_exp():
pop, log, best = run_pso(
toolbox=toolbox,
logbook=logbook,
stats=stats,
gen_curr=0, gen_step=GEN, invalidate_fitness=True, initial_pop=None,
w=W, c1=C1, c2=C2, n=N,
)
solution = construct_solution(best, sorted_tasks)
schedule = build_schedule(_wf, estimator, rm, solution)
Utility.validate_static_schedule(_wf, schedule)
makespan = Utility.makespan(schedule)
print("Final makespan: {0}".format(makespan))
print("Heft makespan: {0}".format(Utility.makespan(heft_schedule)))
return makespan
def __call__(self):
toolbox, stats, logbook = self.toolbox(), self.stats(), self.logbook()
_wf, rm, estimator = self.env()
heft_schedule = self.heft_schedule()
pop, log, best = run_gsa(toolbox=toolbox,
logbook=logbook,
statistics=stats,
n=self.N,
iter_number=self.GEN,
kbest=self.KBEST,
ginit=self.G)
schedule = build_schedule(_wf, rm, estimator, best)
Utility.validate_static_schedule(_wf, schedule)
makespan = Utility.makespan(schedule)
print("Final makespan: {0}".format(makespan))
print("Heft makespan: {0}".format(Utility.makespan(heft_schedule)))
return makespan
def do_exp():
pop, log, best = run_gapso(
toolbox=toolbox,
logbook=logbook,
stats=stats,
gen=GEN, n=N, ga=ga, pso=pso
)
best_position = best.entity
solution = construct_solution(best_position, sorted_tasks)
schedule = build_schedule(_wf, estimator, rm, solution)
makespan = Utility.makespan(schedule)
print("Final makespan: {0}".format(makespan))
pass
def fitness(particleind):
position = particleind.entity
ordering = particleind.ordering
solution = construct_solution(position, ordering)
sched = build_schedule(_wf, estimator, rm, solution)
makespan = Utility.makespan(sched)
## TODO: make a real estimation later
fit = FitnessStd(values=(makespan, 0.0))
## TODO: make a normal multi-objective fitness estimation
fit.mofit = makespan
return fit
return basefitness(_wf, rm, estimator, solution)
def recover_ordering(ordering):
corrected_ordering = []
while len(ordering) > 0:
ord_iter = iter(ordering)
while True:
t = next(ord_iter)
if Utility.is_enough_to_be_executed(wf, t, corrected_ordering + fixed_tasks_ids):
ordering.remove((t))
corrected_ordering.append(t)
break
else:
#print("incorrect " + str([task[0] for task in ordering_particle.entity]))
pass
pass
return corrected_ordering
def __call__(self):
_wf = wf(self.wf_name)
rm = ExperimentResourceManager(rg.r([10, 15, 25, 30]))
estimator = ModelTimeEstimator(bandwidth=10)
empty_fixed_schedule_part = Schedule({node: [] for node in rm.get_nodes()})
heft_schedule = run_heft(_wf, rm, estimator)
fixed_schedule = empty_fixed_schedule_part
ga_functions = GAFunctions2(_wf, rm, estimator)
generate = partial(ga_generate, ga_functions=ga_functions,
fixed_schedule_part=fixed_schedule,
current_time=0.0, init_sched_percent=0.05,
initial_schedule=heft_schedule)
stats = tools.Statistics(lambda ind: ind.fitness.values[0])
stats.register("avg", numpy.mean)
stats.register("std", numpy.std)
stats.register("min", numpy.min)
stats.register("max", numpy.max)
logbook = tools.Logbook()
logbook.header = ["gen", "evals"] + stats.fields
toolbox = Toolbox()
toolbox.register("generate", generate)
toolbox.register("evaluate", fit_converter(ga_functions.build_fitness(empty_fixed_schedule_part, 0.0)))
toolbox.register("clone", deepcopy)
toolbox.register("mate", ga_functions.crossover)
toolbox.register("sweep_mutation", ga_functions.sweep_mutation)
toolbox.register("mutate", ga_functions.mutation)
# toolbox.register("select_parents", )
# toolbox.register("select", tools.selTournament, tournsize=4)
toolbox.register("select", tools.selRoulette)
pop, logbook, best = run_ga(toolbox=toolbox,
logbook=logbook,
stats=stats,
**self.GA_PARAMS)
resulted_schedule = ga_functions.build_schedule(best, empty_fixed_schedule_part, 0.0)
ga_makespan = Utility.makespan(resulted_schedule)
return (ga_makespan, resulted_schedule, logbook)
def fitness(wf, rm, estimator, position):
if isinstance(position, Schedule):
sched = position
else:
sched = build_schedule(wf, estimator, rm, position)
# isvalid = Utility.is_static_schedule_valid(wf,sched)
# if not isvalid:
# print("NOT VALID SCHEDULE!")
makespan = Utility.makespan(sched)
## TODO: make a real estimation later
cost = 0.0
Fitness.weights = [-1.0, -1.0]
fit = Fitness(values=(makespan, cost))
## TODO: make a normal multi-objective fitness estimation
fit.mofit = makespan
return fit
def fitness(chromo):
## TODO: remove it later.
# t_ident = str(threading.current_thread().ident)
# t_name = str(threading.current_thread().name)
# print("Time: " + str(current_time) + " Running ga in isolated thread " + t_name + " " + t_ident)
## value of fitness function is the last time point in the schedule
## built from the chromo
## chromo is {Task:Node},{Task:Node},... - fixed length
schedule = builder(chromo, current_time)
time = Utility.makespan(schedule)
# time = 1
# ## TODO: remove it later.
# k = 0
# for i in range(100000):
# k += i
return (1 / time, )
def _construct_environment(self, *args, **kwargs):
wf_name = kwargs["wf_name"]
nodes_conf = kwargs.get("nodes_conf", None)
ideal_flops = kwargs.get("ideal_flops", 20)
transfer_time = kwargs.get("transfer_time", 100)
dax1 = '../../resources/' + wf_name + '.xml'
# wf = Utility.readWorkflow(dax1, wf_name)
_wf = wf(wf_name)
rgen = ResourceGenerator(min_res_count=1,
max_res_count=1,
min_node_count=4,
max_node_count=4,
min_flops=5,
max_flops=10)
resources = rgen.generate()
transferMx = rgen.generateTransferMatrix(resources)
if nodes_conf is None:
bundle = Utility.get_default_bundle()
resources = bundle.dedicated_resources
transferMx = bundle.transfer_mx
ideal_flops = bundle.ideal_flops
##TODO: end
else:
## TODO: refactor it later.
resources = ResourceGenerator.r(nodes_conf)
transferMx = rgen.generateTransferMatrix(resources)
##
estimator = ExperimentEstimator(transferMx, ideal_flops, dict(),
transfer_time)
resource_manager = ExperimentResourceManager(resources)
return (_wf, resource_manager, estimator)
def __call__(self,
wf_name,
ideal_flops,
is_silent=False,
is_visualized=True,
ga_params=DEFAULT_GA_PARAMS,
nodes_conf=None,
transfer_time=100,
heft_initial=True,
**kwargs):
wf = None
## TODO: I know This is a dirty hack
if isinstance(wf_name, Workflow):
wf = wf_name
wf_name = wf.name
print("Proccessing " + str(wf_name))
(_wf, resource_manager,
estimator) = self._construct_environment(wf_name=wf_name,
nodes_conf=nodes_conf,
ideal_flops=ideal_flops,
transfer_time=transfer_time)
wf = wf if wf is not None else _wf
alg_func = GAFactory.default().create_ga(
silent=is_silent,
wf=wf,
resource_manager=resource_manager,
estimator=estimator,
ga_params=ga_params)
def _run_heft():
dynamic_planner = DynamicHeft(wf, resource_manager, estimator)
nodes = HeftHelper.to_nodes(resource_manager.resources)
current_cleaned_schedule = Schedule({node: [] for node in nodes})
schedule_dynamic_heft = dynamic_planner.run(
current_cleaned_schedule)
self._validate(wf, estimator, schedule_dynamic_heft)
if is_visualized:
viz.visualize_task_node_mapping(wf, schedule_dynamic_heft)
# Utility.create_jedule_visualization(schedule_dynamic_heft, wf_name+'_heft')
pass
return schedule_dynamic_heft
# @profile_decorator
def _run_ga(initial_schedule, saveIt=True):
def default_fixed_schedule_part(resource_manager):
fix_schedule_part = Schedule({
node: []
for node in HeftHelper.to_nodes(
resource_manager.get_resources())
})
return fix_schedule_part
fix_schedule_part = default_fixed_schedule_part(resource_manager)
((the_best_individual, pop, schedule, iter_stopped),
logbook) = alg_func(fix_schedule_part, initial_schedule)
self._validate(wf, estimator, schedule)
name = wf_name + "_bundle"
path = '../../resources/saved_schedules/' + name + '.json'
if saveIt:
Utility.save_schedule(path, wf_name,
resource_manager.get_resources(),
estimator.transfer_matrix, ideal_flops,
schedule)
if is_visualized:
viz.visualize_task_node_mapping(wf, schedule)
# Utility.create_jedule_visualization(schedule, wf_name+'_ga')
pass
return schedule, logbook
def _run_sa(initial_schedule):
return None
##================================
##Dynamic Heft Run
##================================
heft_schedule = _run_heft()
##===============================================
##Simulated Annealing
##===============================================
_run_sa(heft_schedule)
##================================
##ga Run
##================================
## TODO: remove time measure
tstart = datetime.now()
# ga_schedule = heft_schedule
if heft_initial:
ga_schedule, logbook = _run_ga(heft_schedule, False)
else:
ga_schedule, logbook = _run_ga(None, False)
# ga_schedule = _run_ga(None)
tend = datetime.now()
tres = tend - tstart
print("Time Result: " + str(tres.total_seconds()))
#print("Count of nodes: " + str(sum(1 if len(items) > 0 else 0 for n, items in ga_schedule.mapping.items())))
print("===========================================")
heft_makespan = Utility.makespan(heft_schedule)
ga_makespan = Utility.makespan(ga_schedule)
print("Profit: " + str((1 - ga_makespan / heft_makespan) * 100))
print("===========================================")
return (ga_makespan, heft_makespan, ga_schedule, heft_schedule,
logbook)
def do_exp(wf_name):
_wf = wf(wf_name)
heft_schedule = run_heft(_wf, rm, estimator)
makespan = Utility.makespan(heft_schedule)
return makespan
from scheduling.src.core.environment.Utility import Utility, wf
from scheduling.src.algs.common.MapOrdSchedule import build_schedule, MAPPING_SPECIE, ORDERING_SPECIE, ordering_from_schedule, \
mapping_from_schedule
from scheduling.src.experiments.cga.mobjective.utility import SimpleTimeCostEstimator
from scheduling.src.core.environment.ResourceGenerator import ResourceGenerator as rg
from scheduling.src.algs.common.MapOrdSchedule import fitness as basefitness
_wf = wf("Montage_50")
rm = ExperimentResourceManager(rg.r([10, 15, 25, 30]))
estimator = SimpleTimeCostEstimator(comp_time_cost=0, transf_time_cost=0, transferMx=None,
ideal_flops=20, transfer_time=100)
sorted_tasks = HeftHelper.heft_rank(_wf, rm, estimator)
heft_schedule = run_heft(_wf, rm, estimator)
print(Utility.makespan(heft_schedule))
stats = tools.Statistics(lambda ind: ind.fitness.values[0])
stats.register("avg", numpy.mean)
stats.register("std", numpy.std)
stats.register("min", numpy.min)
stats.register("max", numpy.max)
logbook = tools.Logbook()
logbook.header = ["gen", "evals"] + stats.fields
heft_mapping = mapping_from_schedule(heft_schedule)
def _validate(self, wf, estimator, schedule):
max_makespan = Utility.makespan(schedule)
seq_time_validaty = Utility.validateNodesSeq(schedule)
sched = deepcopy(schedule)
mark_finished(sched)
Utility.validate_static_schedule(wf, schedule)
return (res[0], res[4])
if __name__ == '__main__':
print("Population size: " + str(PARAMS["ga_params"]["population"]))
_wf = wf(wf_names[0])
rm = ExperimentResourceManager(rg.r(PARAMS["nodes_conf"]))
estimator = SimpleTimeCostEstimator(comp_time_cost=0,
transf_time_cost=0,
transferMx=None,
ideal_flops=PARAMS["ideal_flops"],
transfer_time=PARAMS["transfer_time"])
heft_schedule = run_heft(_wf, rm, estimator)
heft_makespan = Utility.makespan(heft_schedule)
overall_transfer = Utility.overall_transfer_time(heft_schedule, _wf,
estimator)
overall_execution = Utility.overall_execution_time(heft_schedule)
print(
"Heft makespan: {0}, Overall transfer time: {1}, Overall execution time: {2}"
.format(heft_makespan, overall_transfer, overall_execution))
if not only_heft:
exec_count = 100
gen = PARAMS["ga_params"]["generations"]
res_list = [0 for _ in range(gen)]
result = repeat(do_exp_heft_schedule, exec_count)
mean = numpy.mean([makespan for (makespan, list) in result])
for i in range(exec_count):