def do_exp(wf_name, **params):
_wf = wf(wf_name)
ga_makespan, heft_make, ga_schedule, heft_sched = MixRunner()(_wf, **params)
rm = ExperimentResourceManager(rg.r(params["nodes_conf"]))
estimator = SimpleTimeCostEstimator(comp_time_cost=0, transf_time_cost=0, transferMx=None,
ideal_flops=params["ideal_flops"],
transfer_time=params["transfer_time"])
heft_schedule = run_heft(_wf, rm, estimator)
heft_makespan = Utility.makespan(heft_schedule)
data = {
"wf_name": wf_name,
"params": params,
"heft": {
"makespan": heft_makespan,
"overall_transfer_time": Utility.overall_transfer_time(heft_schedule, _wf, estimator),
"overall_execution_time": Utility.overall_execution_time(heft_schedule)
},
"ga": {
"makespan": ga_makespan,
"overall_transfer_time": Utility.overall_transfer_time(ga_schedule, _wf, estimator),
"overall_execution_time": Utility.overall_execution_time(ga_schedule)
},
#"heft_schedule": heft_schedule,
#"ga_schedule": ga_schedule
}
return data
def test_fixed_ordering(self):
_wf = wf("Montage_25")
rm = ExperimentResourceManager(rg.r([10, 15, 25, 30]))
estimator = SimpleTimeCostEstimator(comp_time_cost=0,
transf_time_cost=0,
transferMx=None,
ideal_flops=20,
transfer_time=100)
sorted_tasks = HeftHelper.heft_rank(_wf, rm, estimator)
heft_schedule = run_heft(_wf, rm, estimator)
heft_mapping = schedule_to_position(heft_schedule)
heft_gen = lambda: heft_mapping if random.random(
) > 0.95 else generate(_wf, rm, estimator)
toolbox = Toolbox()
# toolbox.register("generate", generate, _wf, rm, estimator)
toolbox.register("generate", heft_gen)
toolbox.register("fitness", fitness, _wf, rm, estimator, sorted_tasks)
toolbox.register("force_vector_matrix", force_vector_matrix, rm)
toolbox.register("velocity_and_position", velocity_and_position, _wf,
rm, estimator)
toolbox.register("G", G)
toolbox.register("kbest", Kbest)
statistics = Statistics()
statistics.register(
"min", lambda pop: numpy.min([p.fitness.mofit for p in pop]))
statistics.register(
"avr", lambda pop: numpy.average([p.fitness.mofit for p in pop]))
statistics.register(
"max", lambda pop: numpy.max([p.fitness.mofit for p in pop]))
statistics.register(
"std", lambda pop: numpy.std([p.fitness.mofit for p in pop]))
logbook = Logbook()
logbook.header = ("gen", "G", "kbest", "min", "avr", "max", "std")
pop_size = 100
iter_number = 100
kbest = pop_size
ginit = 5
final_pop = run_gsa(toolbox, statistics, logbook, pop_size,
iter_number, kbest, ginit)
best = min(final_pop, key=lambda x: toolbox.fitness(x).mofit)
solution = {
MAPPING_SPECIE: list(zip(sorted_tasks, best)),
ORDERING_SPECIE: sorted_tasks
}
schedule = build_schedule(_wf, estimator, rm, solution)
Utility.validate_static_schedule(_wf, schedule)
makespan = Utility.makespan(schedule)
print("Final makespan: {0}".format(makespan))
pass
def do_exp(wf_name):
_wf = wf(wf_name)
heft_schedule = run_heft(_wf, rm, estimator)
Utility.validate_static_schedule(_wf, heft_schedule)
makespan = Utility.makespan(heft_schedule)
return makespan
def do_exp(wf_name):
_wf = wf(wf_name)
rm = ExperimentResourceManager(rg.r([10, 15, 25, 30]))
estimator = SimpleTimeCostEstimator(comp_time_cost=0,
transf_time_cost=0,
transferMx=None,
ideal_flops=20,
transfer_time=100)
empty_fixed_schedule_part = Schedule({node: [] for node in rm.get_nodes()})
heft_schedule = run_heft(_wf, rm, estimator)
ga_functions = GAFunctions2(_wf, rm, estimator)
generate = partial(ga_generate,
ga_functions=ga_functions,
fixed_schedule_part=empty_fixed_schedule_part,
current_time=0.0,
init_sched_percent=0.05,
initial_schedule=heft_schedule)
stats = tools.Statistics(lambda ind: ind.fitness.values[0])
stats.register("avg", numpy.mean)
stats.register("std", numpy.std)
stats.register("min", numpy.min)
stats.register("max", numpy.max)
logbook = tools.Logbook()
logbook.header = ["gen", "evals"] + stats.fields
toolbox = Toolbox()
toolbox.register("generate", generate)
toolbox.register(
"evaluate",
fit_converter(
ga_functions.build_fitness(empty_fixed_schedule_part, 0.0)))
toolbox.register("clone", deepcopy)
toolbox.register("mate", ga_functions.crossover)
toolbox.register("sweep_mutation", ga_functions.sweep_mutation)
toolbox.register("mutate", ga_functions.mutation)
# toolbox.register("select_parents", )
# toolbox.register("select", tools.selTournament, tournsize=4)
toolbox.register("select", tools.selRoulette)
pop, logbook, best = run_ga(toolbox=toolbox,
logbook=logbook,
stats=stats,
**GA_PARAMS)
resulted_schedule = ga_functions.build_schedule(best,
empty_fixed_schedule_part,
0.0)
Utility.validate_static_schedule(_wf, resulted_schedule)
ga_makespan = Utility.makespan(resulted_schedule)
return ga_makespan
def do_exp(wf_name):
_wf = wf(wf_name)
rm = ExperimentResourceManager(rg.r([10, 15, 25, 30]))
estimator = SimpleTimeCostEstimator(comp_time_cost=0, transf_time_cost=0, transferMx=None,
ideal_flops=20, transfer_time=100)
empty_fixed_schedule_part = Schedule({node: [] for node in rm.get_nodes()})
heft_schedule = run_heft(_wf, rm, estimator)
ga_functions = GAFunctions2(_wf, rm, estimator)
generate = partial(ga_generate, ga_functions=ga_functions,
fixed_schedule_part=empty_fixed_schedule_part,
current_time=0.0, init_sched_percent=0.05,
initial_schedule=heft_schedule)
stats = tools.Statistics(lambda ind: ind.fitness.values[0])
stats.register("avg", numpy.mean)
stats.register("std", numpy.std)
stats.register("min", numpy.min)
stats.register("max", numpy.max)
logbook = tools.Logbook()
logbook.header = ["gen", "evals"] + stats.fields
toolbox = Toolbox()
toolbox.register("generate", generate)
toolbox.register("evaluate", fit_converter(ga_functions.build_fitness(empty_fixed_schedule_part, 0.0)))
toolbox.register("clone", deepcopy)
toolbox.register("mate", ga_functions.crossover)
toolbox.register("sweep_mutation", ga_functions.sweep_mutation)
toolbox.register("mutate", ga_functions.mutation)
# toolbox.register("select_parents", )
# toolbox.register("select", tools.selTournament, tournsize=4)
toolbox.register("select", tools.selRoulette)
pop, logbook, best = run_ga(toolbox=toolbox,
logbook=logbook,
stats=stats,
**GA_PARAMS)
resulted_schedule = ga_functions.build_schedule(best, empty_fixed_schedule_part, 0.0)
Utility.validate_static_schedule(_wf, resulted_schedule)
ga_makespan = Utility.makespan(resulted_schedule)
return ga_makespan
def test_fixed_ordering(self):
_wf = wf("Montage_25")
rm = ExperimentResourceManager(rg.r([10, 15, 25, 30]))
estimator = SimpleTimeCostEstimator(comp_time_cost=0, transf_time_cost=0, transferMx=None,
ideal_flops=20, transfer_time=100)
sorted_tasks = HeftHelper.heft_rank(_wf, rm, estimator)
heft_schedule = run_heft(_wf, rm, estimator)
heft_mapping = schedule_to_position(heft_schedule)
heft_gen = lambda: heft_mapping if random.random() > 0.95 else generate(_wf, rm, estimator)
toolbox = Toolbox()
# toolbox.register("generate", generate, _wf, rm, estimator)
toolbox.register("generate", heft_gen)
toolbox.register("fitness", fitness, _wf, rm, estimator, sorted_tasks)
toolbox.register("force_vector_matrix", force_vector_matrix, rm)
toolbox.register("velocity_and_position", velocity_and_position, _wf, rm, estimator)
toolbox.register("G", G)
toolbox.register("kbest", Kbest)
statistics = Statistics()
statistics.register("min", lambda pop: numpy.min([p.fitness.mofit for p in pop]))
statistics.register("avr", lambda pop: numpy.average([p.fitness.mofit for p in pop]))
statistics.register("max", lambda pop: numpy.max([p.fitness.mofit for p in pop]))
statistics.register("std", lambda pop: numpy.std([p.fitness.mofit for p in pop]))
logbook = Logbook()
logbook.header = ("gen", "G", "kbest", "min", "avr", "max", "std")
pop_size = 100
iter_number = 100
kbest = pop_size
ginit = 5
final_pop = run_gsa(toolbox, statistics, logbook, pop_size, iter_number, kbest, ginit)
best = min(final_pop, key=lambda x: toolbox.fitness(x).mofit)
solution = {MAPPING_SPECIE: list(zip(sorted_tasks, best)), ORDERING_SPECIE: sorted_tasks}
schedule = build_schedule(_wf, estimator, rm, solution)
Utility.validate_static_schedule(_wf, schedule)
makespan = Utility.makespan(schedule)
print("Final makespan: {0}".format(makespan))
pass
def do_exp(wf_name, **params):
_wf = wf(wf_name)
ga_makespan, heft_make, ga_schedule, heft_sched = MixRunner()(_wf,
**params)
rm = ExperimentResourceManager(rg.r(params["nodes_conf"]))
estimator = SimpleTimeCostEstimator(comp_time_cost=0,
transf_time_cost=0,
transferMx=None,
ideal_flops=params["ideal_flops"],
transfer_time=params["transfer_time"])
heft_schedule = run_heft(_wf, rm, estimator)
heft_makespan = Utility.makespan(heft_schedule)
data = {
"wf_name": wf_name,
"params": params,
"heft": {
"makespan":
heft_makespan,
"overall_transfer_time":
Utility.overall_transfer_time(heft_schedule, _wf, estimator),
"overall_execution_time":
Utility.overall_execution_time(heft_schedule)
},
"ga": {
"makespan":
ga_makespan,
"overall_transfer_time":
Utility.overall_transfer_time(ga_schedule, _wf, estimator),
"overall_execution_time":
Utility.overall_execution_time(ga_schedule)
},
#"heft_schedule": heft_schedule,
#"ga_schedule": ga_schedule
}
return data
def do_exp(wf_name, **params):
_wf = wf(wf_name)
ga_makespan, heft_make, ga_schedule, heft_sched, logbook = MixRunner()(_wf, **params)
rm = ExperimentResourceManager(rg.r(params["nodes_conf"]))
estimator = TestEstimator(comp_time_cost=0, transf_time_cost=0, bandwidth=float(params["data_intensive_coeff"]), transferMx=None,
ideal_flops=params["ideal_flops"], max_size=float(params["max_size"]),
transfer_time=params["transfer_time"])
heft_schedule = run_heft(_wf, rm, estimator)
heft_makespan = Utility.makespan(heft_schedule)
# print("Heft schedule:")
# print(heft_schedule)
# print("Heft makespan: {0}".format(heft_makespan))
# print("GA schedule:")
# print(ga_schedule)
# print("GA makespan: {0}".format(ga_makespan))
data = {
"wf_name": wf_name,
"params": params,
"heft": {
"makespan": heft_makespan,
"overall_transfer_time": Utility.overall_transfer_time(heft_schedule, _wf, estimator),
"overall_execution_time": Utility.overall_execution_time(heft_schedule)
},
"ga": {
"makespan": ga_makespan,
"overall_transfer_time": Utility.overall_transfer_time(ga_schedule, _wf, estimator),
"overall_execution_time": Utility.overall_execution_time(ga_schedule)
},
"logbook": logbook
#"heft_schedule": heft_schedule,
#"ga_schedule": ga_schedule
}
return data
Velocity, Position
from heft.core.CommonComponents.ExperimentalManagers import ExperimentResourceManager
from heft.core.environment.Utility import Utility, wf
from heft.algs.common.mapordschedule import build_schedule, MAPPING_SPECIE, ORDERING_SPECIE, ordering_from_schedule, \
mapping_from_schedule
from heft.experiments.cga.mobjective.utility import SimpleTimeCostEstimator
from heft.core.environment.ResourceGenerator import ResourceGenerator as rg
from heft.algs.common.mapordschedule import fitness as basefitness
_wf = wf("Montage_50")
rm = ExperimentResourceManager(rg.r([10, 15, 25, 30]))
estimator = SimpleTimeCostEstimator(comp_time_cost=0, transf_time_cost=0, transferMx=None,
ideal_flops=20, transfer_time=100)
sorted_tasks = HeftHelper.heft_rank(_wf, rm, estimator)
heft_schedule = run_heft(_wf, rm, estimator)
print(Utility.makespan(heft_schedule))
stats = tools.Statistics(lambda ind: ind.fitness.values[0])
stats.register("avg", numpy.mean)
stats.register("std", numpy.std)
stats.register("min", numpy.min)
stats.register("max", numpy.max)
logbook = tools.Logbook()
logbook.header = ["gen", "evals"] + stats.fields
from heft.algs.common.mapordschedule import build_schedule, MAPPING_SPECIE, ORDERING_SPECIE
from heft.experiments.aggregate_utilities import interval_statistics, interval_stat_string
from heft.experiments.cga.mobjective.utility import SimpleTimeCostEstimator
from heft.core.environment.ResourceGenerator import ResourceGenerator as rg
from heft.experiments.cga.utilities.common import repeat
_wf = wf("Montage_75")
rm = ExperimentResourceManager(rg.r([10, 15, 25, 30]))
estimator = SimpleTimeCostEstimator(comp_time_cost=0,
transf_time_cost=0,
transferMx=None,
ideal_flops=20,
transfer_time=100)
sorted_tasks = HeftHelper.heft_rank(_wf, rm, estimator)
heft_schedule = run_heft(_wf, rm, estimator)
heft_mapping = schedule_to_position(heft_schedule)
heft_mapping.velocity = MappingParticle.Velocity({})
heft_gen = lambda n: [
deepcopy(heft_mapping)
if random.random() > 1.0 else generate(_wf, rm, estimator, 1)[0]
for _ in range(n)
]
W, C1, C2 = 0.1, 0.6, 0.2
GEN, N = 300, 50
toolbox = Toolbox()
toolbox.register("population", heft_gen)
def heft_schedule(self):
if not self._heft_schedule:
self._heft_schedule = run_heft(self._wf, self._rm, self._estimator)
return self._heft_schedule