def heft_exp(saver, wf_name, **params):
_wf = wf(wf_name)
rm = ExperimentResourceManager(rg.r(params["resource_set"]["nodes_conf"]))
estimator = SimpleTimeCostEstimator(**params["estimator_settings"])
dynamic_heft = DynamicHeft(_wf, rm, estimator)
heft_machine = HeftExecutor(rm, heft_planner=dynamic_heft,
**params["executor_params"])
heft_machine.init()
heft_machine.run()
resulted_schedule = heft_machine.current_schedule
Utility.validate_dynamic_schedule(_wf, resulted_schedule)
data = {
"wf_name": wf_name,
"params": params,
"result": {
"makespan": Utility.makespan(resulted_schedule),
## TODO: this function should be remade to adapt under conditions of dynamic env
#"overall_transfer_time": Utility.overall_transfer_time(resulted_schedule, _wf, estimator),
"overall_execution_time": Utility.overall_execution_time(resulted_schedule),
"overall_failed_tasks_count": Utility.overall_failed_tasks_count(resulted_schedule)
}
}
if saver is not None:
saver(data)
return data
def _run_ga(initial_schedule, saveIt=True):
def default_fixed_schedule_part(resource_manager):
fix_schedule_part = Schedule({
node: []
for node in HeftHelper.to_nodes(
resource_manager.get_resources())
})
return fix_schedule_part
fix_schedule_part = default_fixed_schedule_part(resource_manager)
((the_best_individual, pop, schedule, iter_stopped),
logbook) = alg_func(fix_schedule_part, initial_schedule)
self._validate(wf, estimator, schedule)
name = wf_name + "_bundle"
path = '../../resources/saved_schedules/' + name + '.json'
if saveIt:
Utility.save_schedule(path, wf_name,
resource_manager.get_resources(),
estimator.transfer_matrix, ideal_flops,
schedule)
if is_visualized:
viz.visualize_task_node_mapping(wf, schedule)
# Utility.create_jedule_visualization(schedule, wf_name+'_ga')
pass
return schedule
def do_exp(alg_builder, wf_name, **params):
_wf = wf(wf_name)
rm = ExperimentResourceManager(rg.r(params["resource_set"]["nodes_conf"]))
estimator = SimpleTimeCostEstimator(**params["estimator_settings"])
dynamic_heft = DynamicHeft(_wf, rm, estimator)
ga = alg_builder(_wf, rm, estimator,
params["init_sched_percent"],
logbook=None, stats=None,
**params["alg_params"])
machine = GaHeftExecutor(heft_planner=dynamic_heft,
wf=_wf,
resource_manager=rm,
ga_builder=lambda: ga,
**params["executor_params"])
machine.init()
machine.run()
resulted_schedule = machine.current_schedule
Utility.validate_dynamic_schedule(_wf, resulted_schedule)
data = {
"wf_name": wf_name,
"params": params,
"result": {
"makespan": Utility.makespan(resulted_schedule),
## TODO: this function should be remade to adapt under conditions of dynamic env
#"overall_transfer_time": Utility.overall_transfer_time(resulted_schedule, _wf, estimator),
"overall_execution_time": Utility.overall_execution_time(resulted_schedule),
"overall_failed_tasks_count": Utility.overall_failed_tasks_count(resulted_schedule)
}
}
return data
def do_exp():
pop, log, best = run_pso(
toolbox=toolbox,
logbook=logbook,
stats=stats,
gen_curr=0,
gen_step=GEN,
invalidate_fitness=True,
initial_pop=None,
w=W,
c1=C1,
c2=C2,
n=N,
)
solution = construct_solution(best, sorted_tasks)
schedule = build_schedule(_wf, estimator, rm, solution)
Utility.validate_static_schedule(_wf, schedule)
makespan = Utility.makespan(schedule)
print("Final makespan: {0}".format(makespan))
print("Heft makespan: {0}".format(Utility.makespan(heft_schedule)))
return makespan
def do_exp(wf_name, **params):
_wf = wf(wf_name)
ga_makespan, heft_make, ga_schedule, heft_sched = MixRunner()(_wf, **params)
rm = ExperimentResourceManager(rg.r(params["nodes_conf"]))
estimator = SimpleTimeCostEstimator(comp_time_cost=0, transf_time_cost=0, transferMx=None,
ideal_flops=params["ideal_flops"],
transfer_time=params["transfer_time"])
heft_schedule = run_heft(_wf, rm, estimator)
heft_makespan = Utility.makespan(heft_schedule)
data = {
"wf_name": wf_name,
"params": params,
"heft": {
"makespan": heft_makespan,
"overall_transfer_time": Utility.overall_transfer_time(heft_schedule, _wf, estimator),
"overall_execution_time": Utility.overall_execution_time(heft_schedule)
},
"ga": {
"makespan": ga_makespan,
"overall_transfer_time": Utility.overall_transfer_time(ga_schedule, _wf, estimator),
"overall_execution_time": Utility.overall_execution_time(ga_schedule)
},
#"heft_schedule": heft_schedule,
#"ga_schedule": ga_schedule
}
return data
def heft_exp(saver, wf_name, **params):
_wf = wf(wf_name)
rm = ExperimentResourceManager(rg.r(params["resource_set"]["nodes_conf"]))
estimator = SimpleTimeCostEstimator(**params["estimator_settings"])
dynamic_heft = DynamicHeft(_wf, rm, estimator)
heft_machine = HeftExecutor(rm,
heft_planner=dynamic_heft,
**params["executor_params"])
heft_machine.init()
heft_machine.run()
resulted_schedule = heft_machine.current_schedule
Utility.validate_dynamic_schedule(_wf, resulted_schedule)
data = {
"wf_name": wf_name,
"params": params,
"result": {
"makespan":
Utility.makespan(resulted_schedule),
## TODO: this function should be remade to adapt under conditions of dynamic env
#"overall_transfer_time": Utility.overall_transfer_time(resulted_schedule, _wf, estimator),
"overall_execution_time":
Utility.overall_execution_time(resulted_schedule),
"overall_failed_tasks_count":
Utility.overall_failed_tasks_count(resulted_schedule)
}
}
if saver is not None:
saver(data)
return data
def test_fixed_ordering(self):
_wf = wf("Montage_25")
rm = ExperimentResourceManager(rg.r([10, 15, 25, 30]))
estimator = SimpleTimeCostEstimator(comp_time_cost=0,
transf_time_cost=0,
transferMx=None,
ideal_flops=20,
transfer_time=100)
sorted_tasks = HeftHelper.heft_rank(_wf, rm, estimator)
heft_schedule = run_heft(_wf, rm, estimator)
heft_mapping = schedule_to_position(heft_schedule)
heft_gen = lambda: heft_mapping if random.random(
) > 0.95 else generate(_wf, rm, estimator)
toolbox = Toolbox()
# toolbox.register("generate", generate, _wf, rm, estimator)
toolbox.register("generate", heft_gen)
toolbox.register("fitness", fitness, _wf, rm, estimator, sorted_tasks)
toolbox.register("force_vector_matrix", force_vector_matrix, rm)
toolbox.register("velocity_and_position", velocity_and_position, _wf,
rm, estimator)
toolbox.register("G", G)
toolbox.register("kbest", Kbest)
statistics = Statistics()
statistics.register(
"min", lambda pop: numpy.min([p.fitness.mofit for p in pop]))
statistics.register(
"avr", lambda pop: numpy.average([p.fitness.mofit for p in pop]))
statistics.register(
"max", lambda pop: numpy.max([p.fitness.mofit for p in pop]))
statistics.register(
"std", lambda pop: numpy.std([p.fitness.mofit for p in pop]))
logbook = Logbook()
logbook.header = ("gen", "G", "kbest", "min", "avr", "max", "std")
pop_size = 100
iter_number = 100
kbest = pop_size
ginit = 5
final_pop = run_gsa(toolbox, statistics, logbook, pop_size,
iter_number, kbest, ginit)
best = min(final_pop, key=lambda x: toolbox.fitness(x).mofit)
solution = {
MAPPING_SPECIE: list(zip(sorted_tasks, best)),
ORDERING_SPECIE: sorted_tasks
}
schedule = build_schedule(_wf, estimator, rm, solution)
Utility.validate_static_schedule(_wf, schedule)
makespan = Utility.makespan(schedule)
print("Final makespan: {0}".format(makespan))
pass
def do_exp(wf_name):
_wf = wf(wf_name)
rm = ExperimentResourceManager(rg.r([10, 15, 25, 30]))
estimator = SimpleTimeCostEstimator(comp_time_cost=0,
transf_time_cost=0,
transferMx=None,
ideal_flops=20,
transfer_time=100)
empty_fixed_schedule_part = Schedule({node: [] for node in rm.get_nodes()})
heft_schedule = run_heft(_wf, rm, estimator)
ga_functions = GAFunctions2(_wf, rm, estimator)
generate = partial(ga_generate,
ga_functions=ga_functions,
fixed_schedule_part=empty_fixed_schedule_part,
current_time=0.0,
init_sched_percent=0.05,
initial_schedule=heft_schedule)
stats = tools.Statistics(lambda ind: ind.fitness.values[0])
stats.register("avg", numpy.mean)
stats.register("std", numpy.std)
stats.register("min", numpy.min)
stats.register("max", numpy.max)
logbook = tools.Logbook()
logbook.header = ["gen", "evals"] + stats.fields
toolbox = Toolbox()
toolbox.register("generate", generate)
toolbox.register(
"evaluate",
fit_converter(
ga_functions.build_fitness(empty_fixed_schedule_part, 0.0)))
toolbox.register("clone", deepcopy)
toolbox.register("mate", ga_functions.crossover)
toolbox.register("sweep_mutation", ga_functions.sweep_mutation)
toolbox.register("mutate", ga_functions.mutation)
# toolbox.register("select_parents", )
# toolbox.register("select", tools.selTournament, tournsize=4)
toolbox.register("select", tools.selRoulette)
pop, logbook, best = run_ga(toolbox=toolbox,
logbook=logbook,
stats=stats,
**GA_PARAMS)
resulted_schedule = ga_functions.build_schedule(best,
empty_fixed_schedule_part,
0.0)
Utility.validate_static_schedule(_wf, resulted_schedule)
ga_makespan = Utility.makespan(resulted_schedule)
return ga_makespan
def do_exp(wf_name):
_wf = wf(wf_name)
heft_schedule = run_heft(_wf, rm, estimator)
Utility.validate_static_schedule(_wf, heft_schedule)
makespan = Utility.makespan(heft_schedule)
return makespan
def do_exp():
pop, _logbook, best = run_gsa(toolbox, stats, logbook, pop_size, iter_number, kbest, ginit)
solution = {MAPPING_SPECIE: list(best.entity.items()), ORDERING_SPECIE: sorted_tasks}
schedule = build_schedule(_wf, estimator, rm, solution)
Utility.validate_static_schedule(_wf, schedule)
makespan = Utility.makespan(schedule)
print("Final makespan: {0}".format(makespan))
print("Final makespan: {0}".format(makespan))
return makespan
def do_exp():
pop, _logbook, best = run_gsa(toolbox, stats, logbook, pop_size, 0, iter_number, None, kbest, ginit, **{"w":W, "c":C})
schedule = build_schedule(_wf, rm, estimator, best)
Utility.validate_static_schedule(_wf, schedule)
makespan = Utility.makespan(schedule)
print("Final makespan: {0}".format(makespan))
print("Heft makespan: {0}".format(Utility.makespan(heft_schedule)))
return makespan
def _actual_ga_run(self):
##=====================================
##Regular GA
##=====================================
print("GaHeft WITH NEW POP: ")
ga = self.ga_builder()
((best_r, pop_r, schedule_r, stopped_iteration_r), logbook_r) = ga(self.back_cmp.fixed_schedule,
self.back_cmp.initial_schedule,
self.current_time)
##=====================================
##Old pop GA
##=====================================
print("GaHeft WITH OLD POP: ")
cleaned_schedule = self.back_cmp.fixed_schedule
initial_pop = [self._clean_chromosome(ind, self.current_event, cleaned_schedule) for ind in self.past_pop]
## TODO: rethink this hack
result = self.ga_builder()(self.back_cmp.fixed_schedule,
self.back_cmp.initial_schedule,
self.current_time,
initial_population=initial_pop)
((best_op, pop_op, schedule_op, stopped_iteration_op), logbook_op) = result
self.past_pop = pop_op
##=====================================
##Save stat to stat_saver
##=====================================
## TODO: make exception here
task_id = "" if not hasattr(self.current_event, 'task') else str(self.current_event.task.id)
if self.stat_saver is not None:
## TODO: correct pop_agr later
stat_data = {
"wf_name": self.workflow.name,
"event_name": self.current_event.__class__.__name__,
"task_id": task_id,
"with_old_pop": {
"iter": stopped_iteration_op,
"makespan": Utility.makespan(schedule_op),
"pop_aggr": logbook_op
},
"with_random": {
"iter": stopped_iteration_r,
"makespan": Utility.makespan(schedule_r),
"pop_aggr": logbook_r
}
}
self.stat_saver(stat_data)
#TODO: remove this hack later
#self.stop()
self._stop_ga()
return result
def do_triple_island_exp(saver, alg_builder, chromosome_cleaner_builder,
schedule_to_chromosome_converter_builder, wf_name,
**params):
_wf = wf(wf_name)
rm = ExperimentResourceManager(rg.r(params["resource_set"]["nodes_conf"]))
estimator = SimpleTimeCostEstimator(**params["estimator_settings"])
chromosome_cleaner = chromosome_cleaner_builder(_wf, rm, estimator)
dynamic_heft = DynamicHeft(_wf, rm, estimator)
mpga = alg_builder(_wf,
rm,
estimator,
params["init_sched_percent"],
log_book=None,
stats=None,
alg_params=params["alg_params"])
machine = MIGaHeftExecutor(heft_planner=dynamic_heft,
wf=_wf,
resource_manager=rm,
estimator=estimator,
ga_builder=lambda: mpga,
chromosome_cleaner=chromosome_cleaner,
schedule_to_chromosome_converter=
schedule_to_chromosome_converter_builder(
wf, rm, estimator),
**params["executor_params"])
machine.init()
machine.run()
resulted_schedule = machine.current_schedule
Utility.validate_dynamic_schedule(_wf, resulted_schedule)
data = {
"wf_name": wf_name,
"params": params,
"result": {
"makespan":
Utility.makespan(resulted_schedule),
## TODO: this function should be remade to adapt under conditions of dynamic env
#"overall_transfer_time": Utility.overall_transfer_time(resulted_schedule, _wf, estimator),
"overall_execution_time":
Utility.overall_execution_time(resulted_schedule),
"overall_failed_tasks_count":
Utility.overall_failed_tasks_count(resulted_schedule)
}
}
if saver is not None:
saver(data)
return data
def do_exp(wf_name):
_wf = wf(wf_name)
rm = ExperimentResourceManager(rg.r([10, 15, 25, 30]))
estimator = SimpleTimeCostEstimator(comp_time_cost=0, transf_time_cost=0, transferMx=None,
ideal_flops=20, transfer_time=100)
empty_fixed_schedule_part = Schedule({node: [] for node in rm.get_nodes()})
heft_schedule = run_heft(_wf, rm, estimator)
ga_functions = GAFunctions2(_wf, rm, estimator)
generate = partial(ga_generate, ga_functions=ga_functions,
fixed_schedule_part=empty_fixed_schedule_part,
current_time=0.0, init_sched_percent=0.05,
initial_schedule=heft_schedule)
stats = tools.Statistics(lambda ind: ind.fitness.values[0])
stats.register("avg", numpy.mean)
stats.register("std", numpy.std)
stats.register("min", numpy.min)
stats.register("max", numpy.max)
logbook = tools.Logbook()
logbook.header = ["gen", "evals"] + stats.fields
toolbox = Toolbox()
toolbox.register("generate", generate)
toolbox.register("evaluate", fit_converter(ga_functions.build_fitness(empty_fixed_schedule_part, 0.0)))
toolbox.register("clone", deepcopy)
toolbox.register("mate", ga_functions.crossover)
toolbox.register("sweep_mutation", ga_functions.sweep_mutation)
toolbox.register("mutate", ga_functions.mutation)
# toolbox.register("select_parents", )
# toolbox.register("select", tools.selTournament, tournsize=4)
toolbox.register("select", tools.selRoulette)
pop, logbook, best = run_ga(toolbox=toolbox,
logbook=logbook,
stats=stats,
**GA_PARAMS)
resulted_schedule = ga_functions.build_schedule(best, empty_fixed_schedule_part, 0.0)
Utility.validate_static_schedule(_wf, resulted_schedule)
ga_makespan = Utility.makespan(resulted_schedule)
return ga_makespan
def fitness_mapping_and_ordering(ctx,
solution):
env = ctx['env']
schedule = build_schedule(env.wf, env.estimator, env.rm, solution)
result = Utility.makespan(schedule)
#result = ExecutorRunner.extract_result(schedule, True, workflow)
return -result
def do_exp():
best, log, current = run_sa(
toolbox=toolbox,
logbook=logbook,
stats=stats,
initial_solution=initial_state, T=T, N=N
)
solution = {MAPPING_SPECIE: [item for item in best.mapping.items()], ORDERING_SPECIE: best.ordering}
schedule = build_schedule(_wf, estimator, rm, solution)
Utility.validate_static_schedule(_wf, schedule)
makespan = Utility.makespan(schedule)
heft_makespan = Utility.makespan(heft_schedule)
print("Final makespan: {0}".format(makespan))
print("Heft makespan: {0}".format(heft_makespan))
return makespan
def _apply_mh_if_better(self, event, heuristic_resulted_schedule, metaheuristic_resulted_schedule):
t1 = Utility.makespan(metaheuristic_resulted_schedule)
t2 = Utility.makespan(heuristic_resulted_schedule)
print("Replace anyway - {0}".format(self.replace_anyway))
if self.replace_anyway is True or t1 < t2:
## generate new events
self._replace_current_schedule(event, metaheuristic_resulted_schedule)
## if event is TaskStarted event the return value means skip further processing
return True
else:
## TODO: run_ga_yet_another_with_old_genome
# self.ga_computation_manager.run(self.current_schedule, self.current_time)
#self._run_ga_in_background(event)
self.back_cmp = None
return False
pass
def get_bundle(self, the_bundle):
bundle = None
if the_bundle is None:
bundle = Utility.get_default_bundle()
else:
bundle = the_bundle
return bundle
def extract_result(schedule, is_silent, wf):
makespan = Utility.makespan(schedule)
seq_time_validaty = Utility.validateNodesSeq(schedule)
dependency_validaty = Utility.validateParentsAndChildren(schedule, wf)
if not is_silent:
print("=============Res Results====================")
print(" Makespan %s" % str(makespan))
print(" Seq validaty %s" % str(seq_time_validaty))
print(" Dependancy validaty %s" % str(dependency_validaty))
if seq_time_validaty is False:
raise Exception("Sequence validaty check failed")
if dependency_validaty is False:
raise Exception("Dependency validaty check failed")
return (makespan, seq_time_validaty, dependency_validaty)
def extract_result(schedule, is_silent, wf):
makespan = Utility.makespan(schedule)
seq_time_validaty = Utility.validateNodesSeq(schedule)
dependency_validaty = Utility.validateParentsAndChildren(schedule, wf)
if not is_silent:
print("=============Res Results====================")
print(" Makespan %s" % str(makespan))
print(" Seq validaty %s" % str(seq_time_validaty))
print(" Dependancy validaty %s" % str(dependency_validaty))
if seq_time_validaty is False:
raise Exception("Sequence validaty check failed")
if dependency_validaty is False:
raise Exception("Dependency validaty check failed")
return (makespan, seq_time_validaty, dependency_validaty)
def get_bundle(self, the_bundle):
bundle = None
if the_bundle is None:
bundle = Utility.get_default_bundle()
else:
bundle = the_bundle
return bundle
def do_experiment(saver, config, _wf, rm, estimator):
solution, pops, logbook, initial_pops = run_cooperative_ga(**config)
schedule = build_schedule(_wf, estimator, rm, solution)
m = Utility.makespan(schedule)
data = {
"metainfo": {
"interact_individuals_count": config["interact_individuals_count"],
"generations": config["generations"],
"species": [s.name for s in config["species"]],
"pop_sizes": [s.pop_size for s in config["species"]],
"nodes": {n.name: n.flops
for n in config["env"].rm.get_nodes()},
"ideal_inds": {
MAPPING_SPECIE: ms_str_repr,
ORDERING_SPECIE: os_ideal_ind
},
"wf_name": _wf.name
},
"initial_pops": initial_pops,
"final_solution": solution,
"final_makespan": m,
"iterations": logbook
}
saver(data)
return m
def test_fixed_ordering(self):
_wf = wf("Montage_25")
rm = ExperimentResourceManager(rg.r([10, 15, 25, 30]))
estimator = SimpleTimeCostEstimator(comp_time_cost=0, transf_time_cost=0, transferMx=None,
ideal_flops=20, transfer_time=100)
sorted_tasks = HeftHelper.heft_rank(_wf, rm, estimator)
heft_schedule = run_heft(_wf, rm, estimator)
heft_mapping = schedule_to_position(heft_schedule)
heft_gen = lambda: heft_mapping if random.random() > 0.95 else generate(_wf, rm, estimator)
toolbox = Toolbox()
# toolbox.register("generate", generate, _wf, rm, estimator)
toolbox.register("generate", heft_gen)
toolbox.register("fitness", fitness, _wf, rm, estimator, sorted_tasks)
toolbox.register("force_vector_matrix", force_vector_matrix, rm)
toolbox.register("velocity_and_position", velocity_and_position, _wf, rm, estimator)
toolbox.register("G", G)
toolbox.register("kbest", Kbest)
statistics = Statistics()
statistics.register("min", lambda pop: numpy.min([p.fitness.mofit for p in pop]))
statistics.register("avr", lambda pop: numpy.average([p.fitness.mofit for p in pop]))
statistics.register("max", lambda pop: numpy.max([p.fitness.mofit for p in pop]))
statistics.register("std", lambda pop: numpy.std([p.fitness.mofit for p in pop]))
logbook = Logbook()
logbook.header = ("gen", "G", "kbest", "min", "avr", "max", "std")
pop_size = 100
iter_number = 100
kbest = pop_size
ginit = 5
final_pop = run_gsa(toolbox, statistics, logbook, pop_size, iter_number, kbest, ginit)
best = min(final_pop, key=lambda x: toolbox.fitness(x).mofit)
solution = {MAPPING_SPECIE: list(zip(sorted_tasks, best)), ORDERING_SPECIE: sorted_tasks}
schedule = build_schedule(_wf, estimator, rm, solution)
Utility.validate_static_schedule(_wf, schedule)
makespan = Utility.makespan(schedule)
print("Final makespan: {0}".format(makespan))
pass
def wrap():
##Preparing
ut = Utility()
wf_name = 'Montage_25'
dax1 = os.path.join(RESOURCES_PATH, wf_name + '.xml')
#dax2 = '..\\resources\\CyberShake_30.xml'
wf_start_id_1 = "00"
task_postfix_id_1 = "00"
wf_start_id_2 = "10"
task_postfix_id_2 = "10"
deadline_1 = 1000
deadline_2 = 3000
pipeline_1 = ut.generateUrgentPipeline(dax1,
wf_name,
wf_start_id_1,
task_postfix_id_1,
deadline_1)
##pipeline_2 = ut.generateUrgentPipeline(dax2,
## wf_start_id_2,
## task_postfix_id_2,
## deadline_2)
common_pipeline = pipeline_1 ## + pipeline_2
rgen = ResourceGenerator()
resources = rgen.generate()
transferMx = rgen.generateTransferMatrix(resources)
estimator = ExperimentEstimator(transferMx, 20)
planner = StaticHeftPlanner()
planner.estimator = estimator
planner.resource_manager = ExperimentResourceManager(resources)
planner.workflows = common_pipeline
schedule = planner.schedule()
print("planner.global_count: " + str(planner.global_count))
def fitness_ordering_resourceconf(workflow, estimator, solution):
os = solution[ORDERING_SPECIE]
rcs = solution[RESOURCE_CONFIG_SPECIE]
## TODO: refactor this
flops_set = [conf.flops for conf in rcs if conf is not None]
resources = ResourceGenerator.r(flops_set)
resource_manager = ExperimentResourceManager(resources)
heft = DynamicHeft(workflow, resource_manager, estimator, os)
schedule = heft.run({n: [] for n in resource_manager.get_nodes()})
result = Utility.makespan(schedule)
return 1 / result
def _check_event_for_ga_result(self, event):
# check for time to get result from GA running background
if self.back_cmp is None or self.back_cmp.time_to_stop != self.current_time:
return False
else:
result = self._actual_ga_run()
if result is not None:
t1 = Utility.makespan(result[0][2])
t2 = Utility.makespan(self.current_schedule)
if self.replace_anyway is True or t1 < t2:
## generate new events
self._replace_current_schedule(event, result[0][2])
## if event is TaskStarted event the return value means skip further processing
return True
else:
## TODO: run_ga_yet_another_with_old_genome
# self.ga_computation_manager.run(self.current_schedule, self.current_time)
self._run_ga_in_background(event)
return False
def do_exp(wf_name, **params):
_wf = wf(wf_name)
ga_makespan, heft_make, ga_schedule, heft_sched = MixRunner()(_wf,
**params)
rm = ExperimentResourceManager(rg.r(params["nodes_conf"]))
estimator = SimpleTimeCostEstimator(comp_time_cost=0,
transf_time_cost=0,
transferMx=None,
ideal_flops=params["ideal_flops"],
transfer_time=params["transfer_time"])
heft_schedule = run_heft(_wf, rm, estimator)
heft_makespan = Utility.makespan(heft_schedule)
data = {
"wf_name": wf_name,
"params": params,
"heft": {
"makespan":
heft_makespan,
"overall_transfer_time":
Utility.overall_transfer_time(heft_schedule, _wf, estimator),
"overall_execution_time":
Utility.overall_execution_time(heft_schedule)
},
"ga": {
"makespan":
ga_makespan,
"overall_transfer_time":
Utility.overall_transfer_time(ga_schedule, _wf, estimator),
"overall_execution_time":
Utility.overall_execution_time(ga_schedule)
},
#"heft_schedule": heft_schedule,
#"ga_schedule": ga_schedule
}
return data
def __call__(self):
toolbox, stats, logbook = self.toolbox(), self.stats(), self.logbook()
_wf, rm, estimator = self.env()
heft_schedule = self.heft_schedule()
pop, log, best = run_gsa(toolbox=toolbox,
logbook=logbook,
statistics=stats,
n=self.N,
iter_number=self.GEN,
kbest=self.KBEST,
ginit=self.G)
schedule = build_schedule(_wf, rm, estimator, best)
Utility.validate_static_schedule(_wf, schedule)
makespan = Utility.makespan(schedule)
print("Final makespan: {0}".format(makespan))
print("Heft makespan: {0}".format(Utility.makespan(heft_schedule)))
return makespan
def __call__(self):
toolbox, stats, logbook = self.toolbox(), self.stats(), self.logbook()
_wf, rm, estimator = self.env()
heft_schedule = self.heft_schedule()
pop, log, best = run_pso(
toolbox=toolbox,
logbook=logbook,
stats=stats,
gen_curr=0, gen_step=self.GEN, invalidate_fitness=True, initial_pop=None,
w=self.W, c1=self.C1, c2=self.C2, n=self.N,
)
schedule = build_schedule(_wf, rm, estimator, best)
Utility.validate_static_schedule(_wf, schedule)
makespan = Utility.makespan(schedule)
print("Final makespan: {0}".format(makespan))
print("Heft makespan: {0}".format(Utility.makespan(heft_schedule)))
return makespan
def _run_ga(initial_schedule, saveIt=True):
def default_fixed_schedule_part(resource_manager):
fix_schedule_part = Schedule({node: [] for node in HeftHelper.to_nodes(resource_manager.get_resources())})
return fix_schedule_part
fix_schedule_part = default_fixed_schedule_part(resource_manager)
((the_best_individual, pop, schedule, iter_stopped), logbook) = alg_func(fix_schedule_part, initial_schedule)
self._validate(wf, estimator, schedule)
name = wf_name +"_bundle"
path = '../../resources/saved_schedules/' + name + '.json'
if saveIt:
Utility.save_schedule(path, wf_name, resource_manager.get_resources(), estimator.transfer_matrix, ideal_flops, schedule)
if is_visualized:
viz.visualize_task_node_mapping(wf, schedule)
# Utility.create_jedule_visualization(schedule, wf_name+'_ga')
pass
return schedule, logbook
def do_exp():
pop, log, best = run_pso(
toolbox=toolbox,
logbook=logbook,
stats=stats,
gen_curr=0, gen_step=GEN, invalidate_fitness=True, initial_pop=None,
w=W, c1=C1, c2=C2, n=N,
)
best_position = best.entity
solution = construct_solution(best_position, sorted_tasks)
schedule = build_schedule(_wf, estimator, rm, solution)
Utility.validate_static_schedule(_wf, schedule)
makespan = Utility.makespan(schedule)
print("Final makespan: {0}".format(makespan))
print("Heft makespan: {0}".format(Utility.makespan(heft_schedule)))
return makespan
def do_exp(wf_name, **params):
_wf = wf(wf_name)
ga_makespan, heft_make, ga_schedule, heft_sched, logbook = MixRunner()(_wf, **params)
rm = ExperimentResourceManager(rg.r(params["nodes_conf"]))
estimator = TestEstimator(comp_time_cost=0, transf_time_cost=0, bandwidth=float(params["data_intensive_coeff"]), transferMx=None,
ideal_flops=params["ideal_flops"], max_size=float(params["max_size"]),
transfer_time=params["transfer_time"])
heft_schedule = run_heft(_wf, rm, estimator)
heft_makespan = Utility.makespan(heft_schedule)
# print("Heft schedule:")
# print(heft_schedule)
# print("Heft makespan: {0}".format(heft_makespan))
# print("GA schedule:")
# print(ga_schedule)
# print("GA makespan: {0}".format(ga_makespan))
data = {
"wf_name": wf_name,
"params": params,
"heft": {
"makespan": heft_makespan,
"overall_transfer_time": Utility.overall_transfer_time(heft_schedule, _wf, estimator),
"overall_execution_time": Utility.overall_execution_time(heft_schedule)
},
"ga": {
"makespan": ga_makespan,
"overall_transfer_time": Utility.overall_transfer_time(ga_schedule, _wf, estimator),
"overall_execution_time": Utility.overall_execution_time(ga_schedule)
},
"logbook": logbook
#"heft_schedule": heft_schedule,
#"ga_schedule": ga_schedule
}
return data
def fitness_ordering_resourceconf(workflow,
estimator,
solution):
os = solution[ORDERING_SPECIE]
rcs = solution[RESOURCE_CONFIG_SPECIE]
## TODO: refactor this
flops_set = [conf.flops for conf in rcs if conf is not None]
resources = ResourceGenerator.r(flops_set)
resource_manager = ExperimentResourceManager(resources)
heft = DynamicHeft(workflow, resource_manager, estimator, os)
schedule = heft.run({n: [] for n in resource_manager.get_nodes()})
result = Utility.makespan(schedule)
return 1/result
def init(self):
## TODO: replace it with logging
print("Working with initial state of nodes: {0}".format([n.flops for n in self.resource_manager.get_nodes()]))
ga_planner = self.ga_builder()
self.current_schedule = Schedule({node: [] for node in self.resource_manager.get_nodes()})
(result, logbook) = ga_planner(self.current_schedule, None)
self.past_pop = ga_planner.get_pop()
print("Result makespan: " + str(Utility.makespan(result[2])))
self.current_schedule = result[2]
self._post_new_events()
self.failed_once = False
pass
def recover_ordering(ordering):
corrected_ordering = []
while len(ordering) > 0:
ord_iter = iter(ordering)
while True:
t, v = next(ord_iter)
if Utility.is_enough_to_be_executed(wf, t, corrected_ordering + fixed_tasks_ids):
ordering.remove((t, v))
corrected_ordering.append(t)
break
pass
pass
return corrected_ordering
def __call__(self):
toolbox, stats, logbook = self.toolbox(), self.stats(), self.logbook()
_wf, rm, estimator = self.env()
heft_schedule = self.heft_schedule()
pop, log, best = run_gsa(
toolbox=toolbox,
logbook=logbook,
statistics=stats,
pop_size=self.N,
iter_number=self.GEN,
kbest=self.KBEST,
ginit=self.G
)
schedule = build_schedule(_wf, rm, estimator, best)
Utility.validate_static_schedule(_wf, schedule)
makespan = Utility.makespan(schedule)
print("Final makespan: {0}".format(makespan))
print("Heft makespan: {0}".format(Utility.makespan(heft_schedule)))
return makespan
def fitness(particleind):
position = particleind.entity
ordering = particleind.ordering
solution = construct_solution(position, ordering)
sched = build_schedule(_wf, estimator, rm, solution)
makespan = Utility.makespan(sched)
## TODO: make a real estimation later
fit = FitnessStd(values=(makespan, 0.0))
## TODO: make a normal multi-objective fitness estimation
fit.mofit = makespan
return fit
return basefitness(_wf, rm, estimator, solution)
def do_exp():
pop, log, best = run_gapso(
toolbox=toolbox,
logbook=logbook,
stats=stats,
gen=GEN, n=N, ga=ga, pso=pso
)
best_position = best.entity
solution = construct_solution(best_position, sorted_tasks)
schedule = build_schedule(_wf, estimator, rm, solution)
makespan = Utility.makespan(schedule)
print("Final makespan: {0}".format(makespan))
pass
def fitness(particleind):
position = particleind.entity
ordering = particleind.ordering
solution = construct_solution(position, ordering)
sched = build_schedule(_wf, estimator, rm, solution)
makespan = Utility.makespan(sched)
## TODO: make a real estimation later
fit = FitnessStd(values=(makespan, 0.0))
## TODO: make a normal multi-objective fitness estimation
fit.mofit = makespan
return fit
return basefitness(_wf, rm, estimator, solution)
def do_exp():
pop, log, best = run_gapso(toolbox=toolbox,
logbook=logbook,
stats=stats,
gen=GEN,
n=N,
ga=ga,
pso=pso)
best_position = best.entity
solution = construct_solution(best_position, sorted_tasks)
schedule = build_schedule(_wf, estimator, rm, solution)
makespan = Utility.makespan(schedule)
print("Final makespan: {0}".format(makespan))
pass
def recover_ordering(ordering):
corrected_ordering = []
while len(ordering) > 0:
ord_iter = iter(ordering)
while True:
t, v = next(ord_iter)
if Utility.is_enough_to_be_executed(
wf, t, corrected_ordering + fixed_tasks_ids):
ordering.remove((t, v))
corrected_ordering.append(t)
break
pass
pass
return corrected_ordering
def fitness(wf, rm, estimator, position):
if isinstance(position, Schedule):
sched = position
else:
sched = build_schedule(wf, estimator, rm, position)
# isvalid = Utility.is_static_schedule_valid(wf,sched)
# if not isvalid:
# print("NOT VALID SCHEDULE!")
makespan = Utility.makespan(sched)
## TODO: make a real estimation later
cost = 0.0
Fitness.weights = [-1.0, -1.0]
fit = Fitness(values=(makespan, cost))
## TODO: make a normal multi-objective fitness estimation
fit.mofit = makespan
return fit
def fitness(wf, rm, estimator, position):
if isinstance(position, Schedule):
sched = position
else:
sched = build_schedule(wf, estimator, rm, position)
# isvalid = Utility.is_static_schedule_valid(wf,sched)
# if not isvalid:
# print("NOT VALID SCHEDULE!")
makespan = Utility.makespan(sched)
## TODO: make a real estimation later
cost = 0.0
Fitness.weights = [-1.0, -1.0]
fit = Fitness(values=(makespan, cost))
## TODO: make a normal multi-objective fitness estimation
fit.mofit = makespan
return fit
def _actual_ga_run(self):
## this way makes it possible to calculate what time
## ga actually has to find solution
## this value is important when you need account events between
## planned start and stop points
# ga_interval = self.current_time - self.back_cmp.creation_time
## fixed_schedule is actual because
## we can be here only if there haven't been any invalidate events
## such as node failures
## in other case current ga background computation would be dropped
## and we wouldn't get here at all
result = self.ga_builder()(self.back_cmp.fixed_schedule,
# self.back_cmp.initial_schedule,
self.back_cmp.current_schedule,
self.current_time)
print("CURRENT MAKESPAN: {0}".format(Utility.makespan(result[0][2])))
return result
def fitness(chromo):
## TODO: remove it later.
# t_ident = str(threading.current_thread().ident)
# t_name = str(threading.current_thread().name)
# print("Time: " + str(current_time) + " Running ga in isolated thread " + t_name + " " + t_ident)
## value of fitness function is the last time point in the schedule
## built from the chromo
## chromo is {Task:Node},{Task:Node},... - fixed length
schedule = builder(chromo, current_time)
time = Utility.makespan(schedule)
# time = 1
# ## TODO: remove it later.
# k = 0
# for i in range(100000):
# k += i
return (1 / time, )
def fitness(chromo):
## TODO: remove it later.
# t_ident = str(threading.current_thread().ident)
# t_name = str(threading.current_thread().name)
# print("Time: " + str(current_time) + " Running ga in isolated thread " + t_name + " " + t_ident)
## value of fitness function is the last time point in the schedule
## built from the chromo
## chromo is {Task:Node},{Task:Node},... - fixed length
schedule = builder(chromo, current_time)
time = Utility.makespan(schedule)
# time = 1
# ## TODO: remove it later.
# k = 0
# for i in range(100000):
# k += i
return (1/time,)
def _construct_environment(self, *args, **kwargs):
wf_name = kwargs["wf_name"]
nodes_conf = kwargs.get("nodes_conf", None)
ideal_flops = kwargs.get("ideal_flops", 20)
transfer_time = kwargs.get("transfer_time", 100)
bandwidth = float(kwargs.get("data_intensive_coeff", 100))
max_size = float(kwargs.get("max_size", 100))
# dax1 = '../../resources/' + wf_name + '.xml'
# wf = Utility.readWorkflow(dax1, wf_name)
_wf = wf(wf_name)
rgen = ResourceGenerator(min_res_count=1,
max_res_count=1,
min_node_count=4,
max_node_count=4,
min_flops=5,
max_flops=10)
resources = rgen.generate()
transferMx = rgen.generateTransferMatrix(resources)
if nodes_conf is None:
bundle = Utility.get_default_bundle()
resources = bundle.dedicated_resources
transferMx = bundle.transfer_mx
ideal_flops = bundle.ideal_flops
##TODO: end
else:
## TODO: refactor it later.
resources = ResourceGenerator.r(nodes_conf)
transferMx = rgen.generateTransferMatrix(resources)
##
estimator = TestEstimator(comp_time_cost=0, transf_time_cost=0, bandwidth=bandwidth, transferMx=None,
ideal_flops=ideal_flops,
transfer_time=transfer_time,
max_size=max_size)
resource_manager = ExperimentResourceManager(resources)
return (_wf, resource_manager, estimator)
def _construct_environment(self, *args, **kwargs):
wf_name = kwargs["wf_name"]
nodes_conf = kwargs.get("nodes_conf", None)
ideal_flops = kwargs.get("ideal_flops", 20)
transfer_time = kwargs.get("transfer_time", 100)
# dax1 = '../../resources/' + wf_name + '.xml'
# wf = Utility.readWorkflow(dax1, wf_name)
_wf = wf(wf_name)
rgen = ResourceGenerator(min_res_count=1,
max_res_count=1,
min_node_count=4,
max_node_count=4,
min_flops=5,
max_flops=10)
resources = rgen.generate()
transferMx = rgen.generateTransferMatrix(resources)
if nodes_conf is None:
bundle = Utility.get_default_bundle()
resources = bundle.dedicated_resources
transferMx = bundle.transfer_mx
ideal_flops = bundle.ideal_flops
##TODO: end
else:
## TODO: refactor it later.
resources = ResourceGenerator.r(nodes_conf)
transferMx = rgen.generateTransferMatrix(resources)
##
estimator = ExperimentEstimator(transferMx, ideal_flops, dict(),
transfer_time)
resource_manager = ExperimentResourceManager(resources)
return (_wf, resource_manager, estimator)
def _actual_ga_run(self):
## TODO: correct this.
heft_initial = self.schedule_to_chromosome_converter(self.back_cmp.current_schedule)
heft_initial = self._clean_chromosome(heft_initial, self.back_cmp.event, self.back_cmp.fixed_schedule)
##=====================================
##Regular GA
##=====================================
# print("GaHeft WITH NEW POP: ")
# ga = self._get_ga_alg()
# ((best_r, pop_r, schedule_r, stopped_iteration_r), logbook_r) = ga(ga_calc.fixed_schedule_part, ga_calc.initial_schedule, current_time)
## TODO: make here using of param to decide how to build initial population for regular gaheft
## TODO: self.mixed_init_pop doesn't exist and isn't setted anywhere
## TODO: need to refactor and merge with MPGA.py
initial_pop_gaheft = None
if self.mixed_init_pop is True:
heft_initial = tools.initIterate(creator.Individual, lambda: heft_initial)
ga_functions = GAFunctions2(self.workflow, self.resource_manager, self.estimator)
heft_pop = [ga_functions.mutation(deepcopy(heft_initial)) for i in range(self.params["population"])]
cleaned_schedule = self.back_cmp.fixed_schedule
initial_pop_gaheft = [self._clean_chromosome(deepcopy(p), self.back_cmp.event, cleaned_schedule) for p in self.past_pop] + heft_pop
print("GaHeft WITH NEW POP: ")
if self.mpnewVSmpoldmode is True:
print("using multi population gaheft...")
ga = self.mpga_builder()
((best_r, pop_r, schedule_r, stopped_iteration_r), logbook_r) = ga(self.back_cmp.fixed_schedule, None, self.current_time, initial_population=None, only_new_pops=True)
else:
print("using single population gaheft...")
ga = self.ga_builder()
((best_r, pop_r, schedule_r, stopped_iteration_r), logbook_r) = ga(self.back_cmp.fixed_schedule, None, self.current_time, initial_population=initial_pop_gaheft)
##=====================================
##Old pop GA
##=====================================
print("GaHeft WITH OLD POP: ")
cleaned_schedule = self.back_cmp.fixed_schedule
initial_pop = [self._clean_chromosome(ind, self.back_cmp.event, cleaned_schedule) for ind in self.past_pop]
result = self.mpga_builder()(self.back_cmp.fixed_schedule,
heft_initial,
self.current_time,
initial_population=initial_pop)
((best_op, pop_op, schedule_op, stopped_iteration_op), logbook_op) = result
self.past_pop = pop_op
##=====================================
##Save stat to stat_saver
##=====================================
## TODO: make exception here
task_id = "" if not hasattr(self.current_event, 'task') else str(self.current_event.task.id)
if self.stat_saver is not None:
## TODO: correct pop_agr later
stat_data = {
"wf_name": self.workflow.name,
"event_name": self.current_event.__class__.__name__,
"task_id": task_id,
"with_old_pop": {
"iter": stopped_iteration_op,
"makespan": Utility.makespan(schedule_op),
"pop_aggr": logbook_op
},
"with_random": {
"iter": stopped_iteration_r,
"makespan": Utility.makespan(schedule_r),
"pop_aggr": logbook_r
}
}
self.stat_saver(stat_data)
self._stop_ga()
return result
def __call__(self,
wf_name,
ideal_flops,
is_silent=False,
is_visualized=True,
ga_params=DEFAULT_GA_PARAMS,
nodes_conf=None,
transfer_time=100,
heft_initial=True,
**kwargs):
wf = None
## TODO: I know This is a dirty hack
if isinstance(wf_name, Workflow):
wf = wf_name
wf_name = wf.name
print("Proccessing " + str(wf_name))
(_wf, resource_manager,
estimator) = self._construct_environment(wf_name=wf_name,
nodes_conf=nodes_conf,
ideal_flops=ideal_flops,
transfer_time=transfer_time)
wf = wf if wf is not None else _wf
alg_func = GAFactory.default().create_ga(
silent=is_silent,
wf=wf,
resource_manager=resource_manager,
estimator=estimator,
ga_params=ga_params)
def _run_heft():
dynamic_planner = DynamicHeft(wf, resource_manager, estimator)
nodes = HeftHelper.to_nodes(resource_manager.resources)
current_cleaned_schedule = Schedule({node: [] for node in nodes})
schedule_dynamic_heft = dynamic_planner.run(
current_cleaned_schedule)
self._validate(wf, estimator, schedule_dynamic_heft)
if is_visualized:
viz.visualize_task_node_mapping(wf, schedule_dynamic_heft)
# Utility.create_jedule_visualization(schedule_dynamic_heft, wf_name+'_heft')
pass
return schedule_dynamic_heft
# @profile_decorator
def _run_ga(initial_schedule, saveIt=True):
def default_fixed_schedule_part(resource_manager):
fix_schedule_part = Schedule({
node: []
for node in HeftHelper.to_nodes(
resource_manager.get_resources())
})
return fix_schedule_part
fix_schedule_part = default_fixed_schedule_part(resource_manager)
((the_best_individual, pop, schedule, iter_stopped),
logbook) = alg_func(fix_schedule_part, initial_schedule)
self._validate(wf, estimator, schedule)
name = wf_name + "_bundle"
path = '../../resources/saved_schedules/' + name + '.json'
if saveIt:
Utility.save_schedule(path, wf_name,
resource_manager.get_resources(),
estimator.transfer_matrix, ideal_flops,
schedule)
if is_visualized:
viz.visualize_task_node_mapping(wf, schedule)
# Utility.create_jedule_visualization(schedule, wf_name+'_ga')
pass
return schedule
def _run_sa(initial_schedule):
return None
##================================
##Dynamic Heft Run
##================================
heft_schedule = _run_heft()
##===============================================
##Simulated Annealing
##===============================================
_run_sa(heft_schedule)
##================================
##GA Run
##================================
## TODO: remove time measure
tstart = datetime.now()
# ga_schedule = heft_schedule
if heft_initial:
ga_schedule = _run_ga(heft_schedule, False)
else:
ga_schedule = _run_ga(None, False)
# ga_schedule = _run_ga(None)
tend = datetime.now()
tres = tend - tstart
print("Time Result: " + str(tres.total_seconds()))
#print("Count of nodes: " + str(sum(1 if len(items) > 0 else 0 for n, items in ga_schedule.mapping.items())))
print("===========================================")
heft_makespan = Utility.makespan(heft_schedule)
ga_makespan = Utility.makespan(ga_schedule)
print("Profit: " + str((1 - ga_makespan / heft_makespan) * 100))
print("===========================================")
return (ga_makespan, heft_makespan, ga_schedule, heft_schedule)
from heft.experiments.cga.mobjective.utility import SimpleTimeCostEstimator
from heft.core.environment.ResourceGenerator import ResourceGenerator as rg
from heft.algs.common.mapordschedule import fitness as basefitness
_wf = wf("Montage_50")
rm = ExperimentResourceManager(rg.r([10, 15, 25, 30]))
estimator = SimpleTimeCostEstimator(comp_time_cost=0,
transf_time_cost=0,
transferMx=None,
ideal_flops=20,
transfer_time=100)
sorted_tasks = HeftHelper.heft_rank(_wf, rm, estimator)
heft_schedule = run_heft(_wf, rm, estimator)
print(Utility.makespan(heft_schedule))
stats = tools.Statistics(lambda ind: ind.fitness.values[0])
stats.register("avg", numpy.mean)
stats.register("std", numpy.std)
stats.register("min", numpy.min)
stats.register("max", numpy.max)
logbook = tools.Logbook()
logbook.header = ["gen", "evals"] + stats.fields
heft_mapping = mapping_from_schedule(heft_schedule)
heft_ordering = ordering_from_schedule(heft_schedule)
# common params
GEN, N = 1000, 30
def _validate(self, wf, estimator, schedule):
max_makespan = Utility.makespan(schedule)
seq_time_validaty = Utility.validateNodesSeq(schedule)
sched = deepcopy(schedule)
mark_finished(sched)
Utility.validate_static_schedule(wf, schedule)
stat=lambda pop: {
"hamming_distances": hamming_distances(pop, os_ideal_ind),
"unique_inds_count": unique_individuals(pop),
"pcm": pcm(pop),
"gdm": gdm(pop)
})
],
"solstat":
lambda sols: {
"best_components":
hamming_for_best_components(sols, ms_ideal_ind, os_ideal_ind),
"best_components_itself":
best_components_itself(sols),
"best":
-1 * Utility.makespan(
build_schedule(_wf, estimator, rm,
max(sols, key=lambda x: x.fitness)))
},
"analyzers": [mapping_mut_reg.analyze],
"operators": {
# "choose": default_choose,
# "build_solutions": default_build_solutions,
"build_solutions": one_to_one_build_solutions,
"fitness": fitness_mapping_and_ordering,
# "fitness": overhead_fitness_mapping_and_ordering,
# "assign_credits": default_assign_credits
# "assign_credits": bonus2_assign_credits
"assign_credits": max_assign_credits
}
}
def wrap(*args, **kwargs):
x = func(*args, **kwargs)
m = Utility.makespan(x)
return FitnessStd(values=(m, 0.0))