def do_exp(wf_name, **params):
_wf = wf(wf_name)
ga_makespan, heft_make, ga_schedule, heft_sched = MixRunner()(_wf, **params)
rm = ExperimentResourceManager(rg.r(params["nodes_conf"]))
estimator = SimpleTimeCostEstimator(comp_time_cost=0, transf_time_cost=0, transferMx=None,
ideal_flops=params["ideal_flops"],
transfer_time=params["transfer_time"])
heft_schedule = run_heft(_wf, rm, estimator)
heft_makespan = Utility.makespan(heft_schedule)
data = {
"wf_name": wf_name,
"params": params,
"heft": {
"makespan": heft_makespan,
"overall_transfer_time": Utility.overall_transfer_time(heft_schedule, _wf, estimator),
"overall_execution_time": Utility.overall_execution_time(heft_schedule)
},
"ga": {
"makespan": ga_makespan,
"overall_transfer_time": Utility.overall_transfer_time(ga_schedule, _wf, estimator),
"overall_execution_time": Utility.overall_execution_time(ga_schedule)
},
#"heft_schedule": heft_schedule,
#"ga_schedule": ga_schedule
}
return data
def do_exp(alg_builder, wf_name, **params):
_wf = wf(wf_name)
rm = ExperimentResourceManager(rg.r(params["resource_set"]["nodes_conf"]))
estimator = SimpleTimeCostEstimator(**params["estimator_settings"])
dynamic_heft = DynamicHeft(_wf, rm, estimator)
ga = alg_builder(_wf, rm, estimator,
params["init_sched_percent"],
logbook=None, stats=None,
**params["alg_params"])
machine = GaHeftExecutor(heft_planner=dynamic_heft,
wf=_wf,
resource_manager=rm,
ga_builder=lambda: ga,
**params["executor_params"])
machine.init()
machine.run()
resulted_schedule = machine.current_schedule
Utility.validate_dynamic_schedule(_wf, resulted_schedule)
data = {
"wf_name": wf_name,
"params": params,
"result": {
"makespan": Utility.makespan(resulted_schedule),
## TODO: this function should be remade to adapt under conditions of dynamic env
#"overall_transfer_time": Utility.overall_transfer_time(resulted_schedule, _wf, estimator),
"overall_execution_time": Utility.overall_execution_time(resulted_schedule),
"overall_failed_tasks_count": Utility.overall_failed_tasks_count(resulted_schedule)
}
}
return data
def heft_exp(saver, wf_name, **params):
_wf = wf(wf_name)
rm = ExperimentResourceManager(rg.r(params["resource_set"]["nodes_conf"]))
estimator = SimpleTimeCostEstimator(**params["estimator_settings"])
dynamic_heft = DynamicHeft(_wf, rm, estimator)
heft_machine = HeftExecutor(rm, heft_planner=dynamic_heft,
**params["executor_params"])
heft_machine.init()
heft_machine.run()
resulted_schedule = heft_machine.current_schedule
Utility.validate_dynamic_schedule(_wf, resulted_schedule)
data = {
"wf_name": wf_name,
"params": params,
"result": {
"makespan": Utility.makespan(resulted_schedule),
## TODO: this function should be remade to adapt under conditions of dynamic env
#"overall_transfer_time": Utility.overall_transfer_time(resulted_schedule, _wf, estimator),
"overall_execution_time": Utility.overall_execution_time(resulted_schedule),
"overall_failed_tasks_count": Utility.overall_failed_tasks_count(resulted_schedule)
}
}
if saver is not None:
saver(data)
return data
def heft_exp(saver, wf_name, **params):
_wf = wf(wf_name)
rm = ExperimentResourceManager(rg.r(params["resource_set"]["nodes_conf"]))
estimator = SimpleTimeCostEstimator(**params["estimator_settings"])
dynamic_heft = DynamicHeft(_wf, rm, estimator)
heft_machine = HeftExecutor(rm,
heft_planner=dynamic_heft,
**params["executor_params"])
heft_machine.init()
heft_machine.run()
resulted_schedule = heft_machine.current_schedule
Utility.validate_dynamic_schedule(_wf, resulted_schedule)
data = {
"wf_name": wf_name,
"params": params,
"result": {
"makespan":
Utility.makespan(resulted_schedule),
## TODO: this function should be remade to adapt under conditions of dynamic env
#"overall_transfer_time": Utility.overall_transfer_time(resulted_schedule, _wf, estimator),
"overall_execution_time":
Utility.overall_execution_time(resulted_schedule),
"overall_failed_tasks_count":
Utility.overall_failed_tasks_count(resulted_schedule)
}
}
if saver is not None:
saver(data)
return data
def do_exp(wf_name, **params):
_wf = wf(wf_name)
ga_makespan, heft_make, ga_schedule, heft_sched = MixRunner()(_wf,
**params)
rm = ExperimentResourceManager(rg.r(params["nodes_conf"]))
estimator = SimpleTimeCostEstimator(comp_time_cost=0,
transf_time_cost=0,
transferMx=None,
ideal_flops=params["ideal_flops"],
transfer_time=params["transfer_time"])
heft_schedule = run_heft(_wf, rm, estimator)
heft_makespan = Utility.makespan(heft_schedule)
data = {
"wf_name": wf_name,
"params": params,
"heft": {
"makespan":
heft_makespan,
"overall_transfer_time":
Utility.overall_transfer_time(heft_schedule, _wf, estimator),
"overall_execution_time":
Utility.overall_execution_time(heft_schedule)
},
"ga": {
"makespan":
ga_makespan,
"overall_transfer_time":
Utility.overall_transfer_time(ga_schedule, _wf, estimator),
"overall_execution_time":
Utility.overall_execution_time(ga_schedule)
},
#"heft_schedule": heft_schedule,
#"ga_schedule": ga_schedule
}
return data
def do_triple_island_exp(saver, alg_builder, chromosome_cleaner_builder,
schedule_to_chromosome_converter_builder, wf_name,
**params):
_wf = wf(wf_name)
rm = ExperimentResourceManager(rg.r(params["resource_set"]["nodes_conf"]))
estimator = SimpleTimeCostEstimator(**params["estimator_settings"])
chromosome_cleaner = chromosome_cleaner_builder(_wf, rm, estimator)
dynamic_heft = DynamicHeft(_wf, rm, estimator)
mpga = alg_builder(_wf,
rm,
estimator,
params["init_sched_percent"],
log_book=None,
stats=None,
alg_params=params["alg_params"])
machine = MIGaHeftExecutor(heft_planner=dynamic_heft,
wf=_wf,
resource_manager=rm,
estimator=estimator,
ga_builder=lambda: mpga,
chromosome_cleaner=chromosome_cleaner,
schedule_to_chromosome_converter=
schedule_to_chromosome_converter_builder(
wf, rm, estimator),
**params["executor_params"])
machine.init()
machine.run()
resulted_schedule = machine.current_schedule
Utility.validate_dynamic_schedule(_wf, resulted_schedule)
data = {
"wf_name": wf_name,
"params": params,
"result": {
"makespan":
Utility.makespan(resulted_schedule),
## TODO: this function should be remade to adapt under conditions of dynamic env
#"overall_transfer_time": Utility.overall_transfer_time(resulted_schedule, _wf, estimator),
"overall_execution_time":
Utility.overall_execution_time(resulted_schedule),
"overall_failed_tasks_count":
Utility.overall_failed_tasks_count(resulted_schedule)
}
}
if saver is not None:
saver(data)
return data
def do_exp(wf_name, **params):
_wf = wf(wf_name)
ga_makespan, heft_make, ga_schedule, heft_sched, logbook = MixRunner()(_wf, **params)
rm = ExperimentResourceManager(rg.r(params["nodes_conf"]))
estimator = TestEstimator(comp_time_cost=0, transf_time_cost=0, bandwidth=float(params["data_intensive_coeff"]), transferMx=None,
ideal_flops=params["ideal_flops"], max_size=float(params["max_size"]),
transfer_time=params["transfer_time"])
heft_schedule = run_heft(_wf, rm, estimator)
heft_makespan = Utility.makespan(heft_schedule)
# print("Heft schedule:")
# print(heft_schedule)
# print("Heft makespan: {0}".format(heft_makespan))
# print("GA schedule:")
# print(ga_schedule)
# print("GA makespan: {0}".format(ga_makespan))
data = {
"wf_name": wf_name,
"params": params,
"heft": {
"makespan": heft_makespan,
"overall_transfer_time": Utility.overall_transfer_time(heft_schedule, _wf, estimator),
"overall_execution_time": Utility.overall_execution_time(heft_schedule)
},
"ga": {
"makespan": ga_makespan,
"overall_transfer_time": Utility.overall_transfer_time(ga_schedule, _wf, estimator),
"overall_execution_time": Utility.overall_execution_time(ga_schedule)
},
"logbook": logbook
#"heft_schedule": heft_schedule,
#"ga_schedule": ga_schedule
}
return data
res = do_exp_schedule(False)
return (res[0], res[4])
if __name__ == '__main__':
print("Population size: " + str(PARAMS["ga_params"]["population"]))
_wf = wf(wf_names[0])
rm = ExperimentResourceManager(rg.r(PARAMS["nodes_conf"]))
estimator = SimpleTimeCostEstimator(comp_time_cost=0, transf_time_cost=0, transferMx=None,
ideal_flops=PARAMS["ideal_flops"], transfer_time=PARAMS["transfer_time"])
heft_schedule = run_heft(_wf, rm, estimator)
heft_makespan = Utility.makespan(heft_schedule)
overall_transfer = Utility.overall_transfer_time(heft_schedule, _wf, estimator)
overall_execution = Utility.overall_execution_time(heft_schedule)
print("Heft makespan: {0}, Overall transfer time: {1}, Overall execution time: {2}".format(heft_makespan,
overall_transfer,
overall_execution))
if not only_heft:
result = repeat(do_exp_heft_schedule, 1)
mean = numpy.mean(result)
if mean > heft_makespan:
profit = mean / heft_makespan * 100
else:
profit = -heft_makespan / mean * 100
print(result)
print("Heft makespan: {0}, Overall transfer time: {1}, Overall execution time: {2}".format(heft_makespan,
overall_transfer,
def do_inherited_pop_exp(saver, alg_builder, chromosome_cleaner_builder,
wf_name, **params):
_wf = wf(wf_name)
rm = ExperimentResourceManager(rg.r(params["resource_set"]["nodes_conf"]))
estimator = SimpleTimeCostEstimator(**params["estimator_settings"])
chromosome_cleaner = chromosome_cleaner_builder(_wf, rm, estimator)
dynamic_heft = DynamicHeft(_wf, rm, estimator)
logbook = Logbook()
stats = tools.Statistics(lambda ind: ind.fitness.values[0])
stats.register("avg", numpy.mean)
stats.register("std", numpy.std)
stats.register("min", numpy.min)
stats.register("max", numpy.max)
ga = alg_builder(_wf,
rm,
estimator,
params["init_sched_percent"],
log_book=logbook,
stats=stats,
alg_params=params["alg_params"])
machine = GaHeftOldPopExecutor(heft_planner=dynamic_heft,
wf=_wf,
resource_manager=rm,
estimator=estimator,
stat_saver=None,
ga_builder=lambda: ga,
chromosome_cleaner=chromosome_cleaner,
**params["executor_params"])
machine.init()
print("Executor start")
machine.run()
print("Executor stop")
resulted_schedule = machine.current_schedule
stat_data = machine.executor_stat_data
Utility.validate_dynamic_schedule(_wf, resulted_schedule)
data = {
"wf_name": wf_name,
"params": params,
"result": {
"random_init_logbook":
stat_data["random_init_logbook"],
"inherited_init_logbook":
stat_data["inherited_init_logbook"],
"makespan":
Utility.makespan(resulted_schedule),
## TODO: this function should be remade to adapt under conditions of dynamic env
#"overall_transfer_time": Utility.overall_transfer_time(resulted_schedule, _wf, estimator),
"overall_execution_time":
Utility.overall_execution_time(resulted_schedule),
"overall_failed_tasks_count":
Utility.overall_failed_tasks_count(resulted_schedule)
}
}
if saver is not None:
saver(data)
return data
def do_island_inherited_pop_exp(alg_builder, mp_alg_builder, algorithm_builder,
chromosome_cleaner_builder,
schedule_to_chromosome_converter_builder,
wf_name, **params):
_wf = wf(wf_name)
rm = ExperimentResourceManager(rg.r(params["resource_set"]["nodes_conf"]))
estimator = SimpleTimeCostEstimator(**params["estimator_settings"])
chromosome_cleaner = chromosome_cleaner_builder(_wf, rm, estimator)
dynamic_heft = DynamicHeft(_wf, rm, estimator)
ga = alg_builder(_wf,
rm,
estimator,
params["init_sched_percent"],
log_book=None,
stats=None,
alg_params=params["alg_params"])
## TODO: remake this part later.
# def reverse_interface_adapter(func):
# def wrap(*args, **kwargs):
# (best, pop, resulted_schedule, _), logbook = func(*args, **kwargs)
# return pop, logbook, best
# return wrap
mpga = mp_alg_builder(
_wf,
rm,
estimator,
params["init_sched_percent"],
algorithm=partial(algorithm_builder, **params["alg_params"]),
#algorithm=reverse_interface_adapter(ga),
log_book=None,
stats=None,
alg_params=params["alg_params"])
kwargs = dict(params["executor_params"])
kwargs.update(params["alg_params"])
kwargs["ga_params"] = {"population": params["alg_params"]["n"]}
machine = MPGaHeftOldPopExecutor(heft_planner=dynamic_heft,
wf=_wf,
resource_manager=rm,
estimator=estimator,
stat_saver=None,
ga_builder=lambda: ga,
mpga_builder=lambda: mpga,
chromosome_cleaner=chromosome_cleaner,
mpnewVSmpoldmode=False,
mixed_init_pop=False,
emigrant_selection=None,
check_evolution_for_stopping=False,
schedule_to_chromosome_converter=
schedule_to_chromosome_converter_builder(
wf, rm, estimator),
**kwargs)
machine.init()
machine.run()
resulted_schedule = machine.current_schedule
stat_data = machine.executor_stat_data
Utility.validate_dynamic_schedule(_wf, resulted_schedule)
data = {
"wf_name": wf_name,
"params": params,
"result": {
"makespan":
Utility.makespan(resulted_schedule),
## TODO: this function should be remade to adapt under conditions of dynamic env
#"overall_transfer_time": Utility.overall_transfer_time(resulted_schedule, _wf, estimator),
"overall_execution_time":
Utility.overall_execution_time(resulted_schedule),
"overall_failed_tasks_count":
Utility.overall_failed_tasks_count(resulted_schedule)
}
}
return data
if __name__ == '__main__':
print("Population size: " + str(PARAMS["ga_params"]["population"]))
_wf = wf(wf_names[0])
rm = ExperimentResourceManager(rg.r(PARAMS["nodes_conf"]))
estimator = SimpleTimeCostEstimator(comp_time_cost=0,
transf_time_cost=0,
transferMx=None,
ideal_flops=PARAMS["ideal_flops"],
transfer_time=PARAMS["transfer_time"])
heft_schedule = run_heft(_wf, rm, estimator)
heft_makespan = Utility.makespan(heft_schedule)
overall_transfer = Utility.overall_transfer_time(heft_schedule, _wf,
estimator)
overall_execution = Utility.overall_execution_time(heft_schedule)
print(
"Heft makespan: {0}, Overall transfer time: {1}, Overall execution time: {2}"
.format(heft_makespan, overall_transfer, overall_execution))
if not only_heft:
result = repeat(do_exp_heft_schedule, 10)
mean = numpy.mean(result)
#profit = (1 - mean / heft_makespan) * 100
print(result)
print(
"Heft makespan: {0}, Overall transfer time: {1}, Overall execution time: {2}"
.format(heft_makespan, overall_transfer, overall_execution))
print("Mean: {0}".format(mean))
#print("Profit: {0}".format(profit))