def do_exp(alg_builder, wf_name, **params):
_wf = wf(wf_name)
rm = ExperimentResourceManager(rg.r(params["resource_set"]["nodes_conf"]))
estimator = SimpleTimeCostEstimator(**params["estimator_settings"])
dynamic_heft = DynamicHeft(_wf, rm, estimator)
ga = alg_builder(_wf, rm, estimator,
params["init_sched_percent"],
logbook=None, stats=None,
**params["alg_params"])
machine = GaHeftExecutor(heft_planner=dynamic_heft,
wf=_wf,
resource_manager=rm,
ga_builder=lambda: ga,
**params["executor_params"])
machine.init()
machine.run()
resulted_schedule = machine.current_schedule
Utility.validate_dynamic_schedule(_wf, resulted_schedule)
data = {
"wf_name": wf_name,
"params": params,
"result": {
"makespan": Utility.makespan(resulted_schedule),
## TODO: this function should be remade to adapt under conditions of dynamic env
#"overall_transfer_time": Utility.overall_transfer_time(resulted_schedule, _wf, estimator),
"overall_execution_time": Utility.overall_execution_time(resulted_schedule),
"overall_failed_tasks_count": Utility.overall_failed_tasks_count(resulted_schedule)
}
}
return data
def env(self):
if not self._wf or not self._rm or not self._estimator:
self._wf = wf(self.wf_name)
self._rm = ExperimentResourceManager(rg.r(self._resorces_set))
self._estimator = SimpleTimeCostEstimator(comp_time_cost=self._comp_time_cost, transf_time_cost=self._transf_time_cost, transferMx=None,
ideal_flops=self._ideal_flops, transfer_time=self._transfer_time)
return self._wf, self._rm, self._estimator
def do_exp(wf_name, **params):
_wf = wf(wf_name)
ga_makespan, heft_make, ga_schedule, heft_sched = MixRunner()(_wf, **params)
rm = ExperimentResourceManager(rg.r(params["nodes_conf"]))
estimator = SimpleTimeCostEstimator(comp_time_cost=0, transf_time_cost=0, transferMx=None,
ideal_flops=params["ideal_flops"],
transfer_time=params["transfer_time"])
heft_schedule = run_heft(_wf, rm, estimator)
heft_makespan = Utility.makespan(heft_schedule)
data = {
"wf_name": wf_name,
"params": params,
"heft": {
"makespan": heft_makespan,
"overall_transfer_time": Utility.overall_transfer_time(heft_schedule, _wf, estimator),
"overall_execution_time": Utility.overall_execution_time(heft_schedule)
},
"ga": {
"makespan": ga_makespan,
"overall_transfer_time": Utility.overall_transfer_time(ga_schedule, _wf, estimator),
"overall_execution_time": Utility.overall_execution_time(ga_schedule)
},
#"heft_schedule": heft_schedule,
#"ga_schedule": ga_schedule
}
return data
def heft_exp(saver, wf_name, **params):
_wf = wf(wf_name)
rm = ExperimentResourceManager(rg.r(params["resource_set"]["nodes_conf"]))
estimator = SimpleTimeCostEstimator(**params["estimator_settings"])
dynamic_heft = DynamicHeft(_wf, rm, estimator)
heft_machine = HeftExecutor(rm, heft_planner=dynamic_heft,
**params["executor_params"])
heft_machine.init()
heft_machine.run()
resulted_schedule = heft_machine.current_schedule
Utility.validate_dynamic_schedule(_wf, resulted_schedule)
data = {
"wf_name": wf_name,
"params": params,
"result": {
"makespan": Utility.makespan(resulted_schedule),
## TODO: this function should be remade to adapt under conditions of dynamic env
#"overall_transfer_time": Utility.overall_transfer_time(resulted_schedule, _wf, estimator),
"overall_execution_time": Utility.overall_execution_time(resulted_schedule),
"overall_failed_tasks_count": Utility.overall_failed_tasks_count(resulted_schedule)
}
}
if saver is not None:
saver(data)
return data
def heft_exp(saver, wf_name, **params):
_wf = wf(wf_name)
rm = ExperimentResourceManager(rg.r(params["resource_set"]["nodes_conf"]))
estimator = SimpleTimeCostEstimator(**params["estimator_settings"])
dynamic_heft = DynamicHeft(_wf, rm, estimator)
heft_machine = HeftExecutor(rm,
heft_planner=dynamic_heft,
**params["executor_params"])
heft_machine.init()
heft_machine.run()
resulted_schedule = heft_machine.current_schedule
Utility.validate_dynamic_schedule(_wf, resulted_schedule)
data = {
"wf_name": wf_name,
"params": params,
"result": {
"makespan":
Utility.makespan(resulted_schedule),
## TODO: this function should be remade to adapt under conditions of dynamic env
#"overall_transfer_time": Utility.overall_transfer_time(resulted_schedule, _wf, estimator),
"overall_execution_time":
Utility.overall_execution_time(resulted_schedule),
"overall_failed_tasks_count":
Utility.overall_failed_tasks_count(resulted_schedule)
}
}
if saver is not None:
saver(data)
return data
def test_fixed_ordering(self):
_wf = wf("Montage_25")
rm = ExperimentResourceManager(rg.r([10, 15, 25, 30]))
estimator = SimpleTimeCostEstimator(comp_time_cost=0,
transf_time_cost=0,
transferMx=None,
ideal_flops=20,
transfer_time=100)
sorted_tasks = HeftHelper.heft_rank(_wf, rm, estimator)
heft_schedule = run_heft(_wf, rm, estimator)
heft_mapping = schedule_to_position(heft_schedule)
heft_gen = lambda: heft_mapping if random.random(
) > 0.95 else generate(_wf, rm, estimator)
toolbox = Toolbox()
# toolbox.register("generate", generate, _wf, rm, estimator)
toolbox.register("generate", heft_gen)
toolbox.register("fitness", fitness, _wf, rm, estimator, sorted_tasks)
toolbox.register("force_vector_matrix", force_vector_matrix, rm)
toolbox.register("velocity_and_position", velocity_and_position, _wf,
rm, estimator)
toolbox.register("G", G)
toolbox.register("kbest", Kbest)
statistics = Statistics()
statistics.register(
"min", lambda pop: numpy.min([p.fitness.mofit for p in pop]))
statistics.register(
"avr", lambda pop: numpy.average([p.fitness.mofit for p in pop]))
statistics.register(
"max", lambda pop: numpy.max([p.fitness.mofit for p in pop]))
statistics.register(
"std", lambda pop: numpy.std([p.fitness.mofit for p in pop]))
logbook = Logbook()
logbook.header = ("gen", "G", "kbest", "min", "avr", "max", "std")
pop_size = 100
iter_number = 100
kbest = pop_size
ginit = 5
final_pop = run_gsa(toolbox, statistics, logbook, pop_size,
iter_number, kbest, ginit)
best = min(final_pop, key=lambda x: toolbox.fitness(x).mofit)
solution = {
MAPPING_SPECIE: list(zip(sorted_tasks, best)),
ORDERING_SPECIE: sorted_tasks
}
schedule = build_schedule(_wf, estimator, rm, solution)
Utility.validate_static_schedule(_wf, schedule)
makespan = Utility.makespan(schedule)
print("Final makespan: {0}".format(makespan))
pass
def do_exp(wf_name):
_wf = wf(wf_name)
rm = ExperimentResourceManager(rg.r([10, 15, 25, 30]))
estimator = SimpleTimeCostEstimator(comp_time_cost=0,
transf_time_cost=0,
transferMx=None,
ideal_flops=20,
transfer_time=100)
empty_fixed_schedule_part = Schedule({node: [] for node in rm.get_nodes()})
heft_schedule = run_heft(_wf, rm, estimator)
ga_functions = GAFunctions2(_wf, rm, estimator)
generate = partial(ga_generate,
ga_functions=ga_functions,
fixed_schedule_part=empty_fixed_schedule_part,
current_time=0.0,
init_sched_percent=0.05,
initial_schedule=heft_schedule)
stats = tools.Statistics(lambda ind: ind.fitness.values[0])
stats.register("avg", numpy.mean)
stats.register("std", numpy.std)
stats.register("min", numpy.min)
stats.register("max", numpy.max)
logbook = tools.Logbook()
logbook.header = ["gen", "evals"] + stats.fields
toolbox = Toolbox()
toolbox.register("generate", generate)
toolbox.register(
"evaluate",
fit_converter(
ga_functions.build_fitness(empty_fixed_schedule_part, 0.0)))
toolbox.register("clone", deepcopy)
toolbox.register("mate", ga_functions.crossover)
toolbox.register("sweep_mutation", ga_functions.sweep_mutation)
toolbox.register("mutate", ga_functions.mutation)
# toolbox.register("select_parents", )
# toolbox.register("select", tools.selTournament, tournsize=4)
toolbox.register("select", tools.selRoulette)
pop, logbook, best = run_ga(toolbox=toolbox,
logbook=logbook,
stats=stats,
**GA_PARAMS)
resulted_schedule = ga_functions.build_schedule(best,
empty_fixed_schedule_part,
0.0)
Utility.validate_static_schedule(_wf, resulted_schedule)
ga_makespan = Utility.makespan(resulted_schedule)
return ga_makespan
def do_exp(wf_name):
_wf = wf(wf_name)
heft_schedule = run_heft(_wf, rm, estimator)
Utility.validate_static_schedule(_wf, heft_schedule)
makespan = Utility.makespan(heft_schedule)
return makespan
def do_triple_island_exp(saver, alg_builder, chromosome_cleaner_builder,
schedule_to_chromosome_converter_builder, wf_name,
**params):
_wf = wf(wf_name)
rm = ExperimentResourceManager(rg.r(params["resource_set"]["nodes_conf"]))
estimator = SimpleTimeCostEstimator(**params["estimator_settings"])
chromosome_cleaner = chromosome_cleaner_builder(_wf, rm, estimator)
dynamic_heft = DynamicHeft(_wf, rm, estimator)
mpga = alg_builder(_wf,
rm,
estimator,
params["init_sched_percent"],
log_book=None,
stats=None,
alg_params=params["alg_params"])
machine = MIGaHeftExecutor(heft_planner=dynamic_heft,
wf=_wf,
resource_manager=rm,
estimator=estimator,
ga_builder=lambda: mpga,
chromosome_cleaner=chromosome_cleaner,
schedule_to_chromosome_converter=
schedule_to_chromosome_converter_builder(
wf, rm, estimator),
**params["executor_params"])
machine.init()
machine.run()
resulted_schedule = machine.current_schedule
Utility.validate_dynamic_schedule(_wf, resulted_schedule)
data = {
"wf_name": wf_name,
"params": params,
"result": {
"makespan":
Utility.makespan(resulted_schedule),
## TODO: this function should be remade to adapt under conditions of dynamic env
#"overall_transfer_time": Utility.overall_transfer_time(resulted_schedule, _wf, estimator),
"overall_execution_time":
Utility.overall_execution_time(resulted_schedule),
"overall_failed_tasks_count":
Utility.overall_failed_tasks_count(resulted_schedule)
}
}
if saver is not None:
saver(data)
return data
def coeff_run():
"""
coefficient of compute/data intensivity
"""
# wf_names = ['Montage_25', 'CyberShake_30', 'Inspiral_30', 'Sipht_30', 'Epigenomics_24']
#all_coeffs = [1/100, 1/50, 1/10, 1/5, 1/2.766, 15, 20, 25, 30, 35, 40, 45, 75]
all_coeffs = [1 / 100, 1 / 50, 1 / 10, 1 / 5, 1 / 2.766, 1, 2.766] + list(
range(5, 101, 5))
# wf_names = [('Montage_25', [10]),
# ('CyberShake_30', [0.1] + list(range(10, 46, 1))),
# ('Inspiral_30', [10, 1, 1/2.766]),
# ('Sipht_30', [0.02] + list(range(30, 50, 1)) + list(range(50, 101, 5))),
# ('Epigenomics_24', list(range(5, 46, 1)) + list(range(45, 101, 5)))]
# wf_names = [('Montage_25', [2.766])]
wf_names = [ #('Montage_25', [10]),
('CyberShake_30', all_coeffs),
#('Inspiral_30', [10, 1, 1/2.766]),
#('Sipht_30', [0.02] + list(range(30, 50, 1)) + list(range(50, 101, 5))),
('Epigenomics_24', all_coeffs)
]
def transfer_time(max_runtime, c):
transfer = max_runtime * BASE_PARAMS["ideal_flops"] / c
return transfer
to_run = []
for wf_name, coeffs in wf_names:
_wf = wf(wf_name)
max_runtime = max(_wf.get_all_unique_tasks(),
key=lambda x: x.runtime).runtime
param_sets = [
copy_and_set(BASE_PARAMS,
transfer_time=transfer_time(max_runtime, c),
data_intensive_coeff=c) for c in coeffs
]
exps = [partial(do_exp, wf_name, **params) for params in param_sets]
to_run = to_run + exps
# m_repeat = lambda n, funcs: [f() for f in funcs for _ in range(n)]
#results = m_repeat(REPEAT_COUNT, to_run)
results = multi_repeat(REPEAT_COUNT, to_run)
saver = UniqueNameSaver(TEMP_PATH, EXPERIMENT_NAME)
for result in results:
saver(result)
# coeff_aggregate(saver.directory, "coeff.png")
pass
def do_exp(wf_name):
_wf = wf(wf_name)
rm = ExperimentResourceManager(rg.r([10, 15, 25, 30]))
estimator = SimpleTimeCostEstimator(comp_time_cost=0, transf_time_cost=0, transferMx=None,
ideal_flops=20, transfer_time=100)
empty_fixed_schedule_part = Schedule({node: [] for node in rm.get_nodes()})
heft_schedule = run_heft(_wf, rm, estimator)
ga_functions = GAFunctions2(_wf, rm, estimator)
generate = partial(ga_generate, ga_functions=ga_functions,
fixed_schedule_part=empty_fixed_schedule_part,
current_time=0.0, init_sched_percent=0.05,
initial_schedule=heft_schedule)
stats = tools.Statistics(lambda ind: ind.fitness.values[0])
stats.register("avg", numpy.mean)
stats.register("std", numpy.std)
stats.register("min", numpy.min)
stats.register("max", numpy.max)
logbook = tools.Logbook()
logbook.header = ["gen", "evals"] + stats.fields
toolbox = Toolbox()
toolbox.register("generate", generate)
toolbox.register("evaluate", fit_converter(ga_functions.build_fitness(empty_fixed_schedule_part, 0.0)))
toolbox.register("clone", deepcopy)
toolbox.register("mate", ga_functions.crossover)
toolbox.register("sweep_mutation", ga_functions.sweep_mutation)
toolbox.register("mutate", ga_functions.mutation)
# toolbox.register("select_parents", )
# toolbox.register("select", tools.selTournament, tournsize=4)
toolbox.register("select", tools.selRoulette)
pop, logbook, best = run_ga(toolbox=toolbox,
logbook=logbook,
stats=stats,
**GA_PARAMS)
resulted_schedule = ga_functions.build_schedule(best, empty_fixed_schedule_part, 0.0)
Utility.validate_static_schedule(_wf, resulted_schedule)
ga_makespan = Utility.makespan(resulted_schedule)
return ga_makespan
def test_fixed_ordering(self):
_wf = wf("Montage_25")
rm = ExperimentResourceManager(rg.r([10, 15, 25, 30]))
estimator = SimpleTimeCostEstimator(comp_time_cost=0, transf_time_cost=0, transferMx=None,
ideal_flops=20, transfer_time=100)
sorted_tasks = HeftHelper.heft_rank(_wf, rm, estimator)
heft_schedule = run_heft(_wf, rm, estimator)
heft_mapping = schedule_to_position(heft_schedule)
heft_gen = lambda: heft_mapping if random.random() > 0.95 else generate(_wf, rm, estimator)
toolbox = Toolbox()
# toolbox.register("generate", generate, _wf, rm, estimator)
toolbox.register("generate", heft_gen)
toolbox.register("fitness", fitness, _wf, rm, estimator, sorted_tasks)
toolbox.register("force_vector_matrix", force_vector_matrix, rm)
toolbox.register("velocity_and_position", velocity_and_position, _wf, rm, estimator)
toolbox.register("G", G)
toolbox.register("kbest", Kbest)
statistics = Statistics()
statistics.register("min", lambda pop: numpy.min([p.fitness.mofit for p in pop]))
statistics.register("avr", lambda pop: numpy.average([p.fitness.mofit for p in pop]))
statistics.register("max", lambda pop: numpy.max([p.fitness.mofit for p in pop]))
statistics.register("std", lambda pop: numpy.std([p.fitness.mofit for p in pop]))
logbook = Logbook()
logbook.header = ("gen", "G", "kbest", "min", "avr", "max", "std")
pop_size = 100
iter_number = 100
kbest = pop_size
ginit = 5
final_pop = run_gsa(toolbox, statistics, logbook, pop_size, iter_number, kbest, ginit)
best = min(final_pop, key=lambda x: toolbox.fitness(x).mofit)
solution = {MAPPING_SPECIE: list(zip(sorted_tasks, best)), ORDERING_SPECIE: sorted_tasks}
schedule = build_schedule(_wf, estimator, rm, solution)
Utility.validate_static_schedule(_wf, schedule)
makespan = Utility.makespan(schedule)
print("Final makespan: {0}".format(makespan))
pass
def coeff_run():
"""
coefficient of compute/data intensivity
"""
# wf_names = ['Montage_25', 'CyberShake_30', 'Inspiral_30', 'Sipht_30', 'Epigenomics_24']
#all_coeffs = [1/100, 1/50, 1/10, 1/5, 1/2.766, 15, 20, 25, 30, 35, 40, 45, 75]
all_coeffs = [1/100, 1/50, 1/10, 1/5, 1/2.766, 1, 2.766] + list(range(5, 101, 5))
# wf_names = [('Montage_25', [10]),
# ('CyberShake_30', [0.1] + list(range(10, 46, 1))),
# ('Inspiral_30', [10, 1, 1/2.766]),
# ('Sipht_30', [0.02] + list(range(30, 50, 1)) + list(range(50, 101, 5))),
# ('Epigenomics_24', list(range(5, 46, 1)) + list(range(45, 101, 5)))]
# wf_names = [('Montage_25', [2.766])]
wf_names = [#('Montage_25', [10]),
('CyberShake_30', all_coeffs),
#('Inspiral_30', [10, 1, 1/2.766]),
#('Sipht_30', [0.02] + list(range(30, 50, 1)) + list(range(50, 101, 5))),
('Epigenomics_24', all_coeffs)]
def transfer_time(max_runtime, c):
transfer = max_runtime * BASE_PARAMS["ideal_flops"] / c
return transfer
to_run = []
for wf_name, coeffs in wf_names:
_wf = wf(wf_name)
max_runtime = max(_wf.get_all_unique_tasks(), key=lambda x: x.runtime).runtime
param_sets = [copy_and_set(BASE_PARAMS, transfer_time=transfer_time(max_runtime, c), data_intensive_coeff=c) for c in coeffs]
exps = [partial(do_exp, wf_name, **params) for params in param_sets]
to_run = to_run + exps
# m_repeat = lambda n, funcs: [f() for f in funcs for _ in range(n)]
#results = m_repeat(REPEAT_COUNT, to_run)
results = multi_repeat(REPEAT_COUNT, to_run)
saver = UniqueNameSaver(TEMP_PATH, EXPERIMENT_NAME)
for result in results:
saver(result)
# coeff_aggregate(saver.directory, "coeff.png")
pass
def do_exp(wf_name, **params):
_wf = wf(wf_name)
ga_makespan, heft_make, ga_schedule, heft_sched = MixRunner()(_wf,
**params)
rm = ExperimentResourceManager(rg.r(params["nodes_conf"]))
estimator = SimpleTimeCostEstimator(comp_time_cost=0,
transf_time_cost=0,
transferMx=None,
ideal_flops=params["ideal_flops"],
transfer_time=params["transfer_time"])
heft_schedule = run_heft(_wf, rm, estimator)
heft_makespan = Utility.makespan(heft_schedule)
data = {
"wf_name": wf_name,
"params": params,
"heft": {
"makespan":
heft_makespan,
"overall_transfer_time":
Utility.overall_transfer_time(heft_schedule, _wf, estimator),
"overall_execution_time":
Utility.overall_execution_time(heft_schedule)
},
"ga": {
"makespan":
ga_makespan,
"overall_transfer_time":
Utility.overall_transfer_time(ga_schedule, _wf, estimator),
"overall_execution_time":
Utility.overall_execution_time(ga_schedule)
},
#"heft_schedule": heft_schedule,
#"ga_schedule": ga_schedule
}
return data
def _construct_environment(self, *args, **kwargs):
wf_name = kwargs["wf_name"]
nodes_conf = kwargs.get("nodes_conf", None)
ideal_flops = kwargs.get("ideal_flops", 20)
transfer_time = kwargs.get("transfer_time", 100)
bandwidth = float(kwargs.get("data_intensive_coeff", 100))
max_size = float(kwargs.get("max_size", 100))
# dax1 = '../../resources/' + wf_name + '.xml'
# wf = Utility.readWorkflow(dax1, wf_name)
_wf = wf(wf_name)
rgen = ResourceGenerator(min_res_count=1,
max_res_count=1,
min_node_count=4,
max_node_count=4,
min_flops=5,
max_flops=10)
resources = rgen.generate()
transferMx = rgen.generateTransferMatrix(resources)
if nodes_conf is None:
bundle = Utility.get_default_bundle()
resources = bundle.dedicated_resources
transferMx = bundle.transfer_mx
ideal_flops = bundle.ideal_flops
##TODO: end
else:
## TODO: refactor it later.
resources = ResourceGenerator.r(nodes_conf)
transferMx = rgen.generateTransferMatrix(resources)
##
estimator = TestEstimator(comp_time_cost=0, transf_time_cost=0, bandwidth=bandwidth, transferMx=None,
ideal_flops=ideal_flops,
transfer_time=transfer_time,
max_size=max_size)
resource_manager = ExperimentResourceManager(resources)
return (_wf, resource_manager, estimator)
def do_exp(wf_name, **params):
_wf = wf(wf_name)
ga_makespan, heft_make, ga_schedule, heft_sched, logbook = MixRunner()(_wf, **params)
rm = ExperimentResourceManager(rg.r(params["nodes_conf"]))
estimator = TestEstimator(comp_time_cost=0, transf_time_cost=0, bandwidth=float(params["data_intensive_coeff"]), transferMx=None,
ideal_flops=params["ideal_flops"], max_size=float(params["max_size"]),
transfer_time=params["transfer_time"])
heft_schedule = run_heft(_wf, rm, estimator)
heft_makespan = Utility.makespan(heft_schedule)
# print("Heft schedule:")
# print(heft_schedule)
# print("Heft makespan: {0}".format(heft_makespan))
# print("GA schedule:")
# print(ga_schedule)
# print("GA makespan: {0}".format(ga_makespan))
data = {
"wf_name": wf_name,
"params": params,
"heft": {
"makespan": heft_makespan,
"overall_transfer_time": Utility.overall_transfer_time(heft_schedule, _wf, estimator),
"overall_execution_time": Utility.overall_execution_time(heft_schedule)
},
"ga": {
"makespan": ga_makespan,
"overall_transfer_time": Utility.overall_transfer_time(ga_schedule, _wf, estimator),
"overall_execution_time": Utility.overall_execution_time(ga_schedule)
},
"logbook": logbook
#"heft_schedule": heft_schedule,
#"ga_schedule": ga_schedule
}
return data
def _construct_environment(self, *args, **kwargs):
wf_name = kwargs["wf_name"]
nodes_conf = kwargs.get("nodes_conf", None)
ideal_flops = kwargs.get("ideal_flops", 20)
transfer_time = kwargs.get("transfer_time", 100)
# dax1 = '../../resources/' + wf_name + '.xml'
# wf = Utility.readWorkflow(dax1, wf_name)
_wf = wf(wf_name)
rgen = ResourceGenerator(min_res_count=1,
max_res_count=1,
min_node_count=4,
max_node_count=4,
min_flops=5,
max_flops=10)
resources = rgen.generate()
transferMx = rgen.generateTransferMatrix(resources)
if nodes_conf is None:
bundle = Utility.get_default_bundle()
resources = bundle.dedicated_resources
transferMx = bundle.transfer_mx
ideal_flops = bundle.ideal_flops
##TODO: end
else:
## TODO: refactor it later.
resources = ResourceGenerator.r(nodes_conf)
transferMx = rgen.generateTransferMatrix(resources)
##
estimator = ExperimentEstimator(transferMx, ideal_flops, dict(),
transfer_time)
resource_manager = ExperimentResourceManager(resources)
return (_wf, resource_manager, estimator)
def get_wf(wf_name, task_postfix_id="00"):
return wf(wf_name, task_postfix_id)
from heft.algs.common.individuals import FitnessStd
from heft.algs.common.mapordschedule import MAPPING_SPECIE, ORDERING_SPECIE, build_schedule
from heft.algs.gsa.SimpleGsaScheme import run_gsa
from heft.algs.gsa.operators import G, Kbest
from heft.algs.gsa.setbasedoperators import force_vector_matrix, velocity_and_position
from heft.algs.heft.DSimpleHeft import run_heft
from heft.algs.heft.HeftHelper import HeftHelper
from heft.algs.pso.sdpso import schedule_to_position, generate, fitness
from heft.core.CommonComponents.ExperimentalManagers import ExperimentResourceManager
from heft.core.environment.ResourceGenerator import ResourceGenerator as rg
from heft.core.environment.Utility import wf, Utility
from heft.experiments.cga.mobjective.utility import SimpleTimeCostEstimator
from heft.experiments.cga.utilities.common import repeat
_wf = wf("Montage_40")
rm = ExperimentResourceManager(rg.r([10, 15, 25, 30]))
estimator = SimpleTimeCostEstimator(comp_time_cost=0, transf_time_cost=0, transferMx=None,
ideal_flops=20, transfer_time=100)
sorted_tasks = HeftHelper.heft_rank(_wf, rm, estimator)
heft_schedule = run_heft(_wf, rm, estimator)
heft_mapping = schedule_to_position(heft_schedule)
heft_gen = lambda n: [deepcopy(heft_mapping) if random.random() > 1.0 else generate(_wf, rm, estimator, 1)[0] for _ in range(n)]
toolbox = Toolbox()
# toolbox.register("generate", generate, _wf, rm, estimator)
toolbox.register("generate", heft_gen)
toolbox.register("fitness", fitness, _wf, rm, estimator, sorted_tasks)
toolbox.register("force_vector_matrix", force_vector_matrix)
def do_island_inherited_pop_exp(alg_builder, mp_alg_builder, algorithm_builder,
chromosome_cleaner_builder,
schedule_to_chromosome_converter_builder,
wf_name, **params):
_wf = wf(wf_name)
rm = ExperimentResourceManager(rg.r(params["resource_set"]["nodes_conf"]))
estimator = SimpleTimeCostEstimator(**params["estimator_settings"])
chromosome_cleaner = chromosome_cleaner_builder(_wf, rm, estimator)
dynamic_heft = DynamicHeft(_wf, rm, estimator)
ga = alg_builder(_wf,
rm,
estimator,
params["init_sched_percent"],
log_book=None,
stats=None,
alg_params=params["alg_params"])
## TODO: remake this part later.
# def reverse_interface_adapter(func):
# def wrap(*args, **kwargs):
# (best, pop, resulted_schedule, _), logbook = func(*args, **kwargs)
# return pop, logbook, best
# return wrap
mpga = mp_alg_builder(
_wf,
rm,
estimator,
params["init_sched_percent"],
algorithm=partial(algorithm_builder, **params["alg_params"]),
#algorithm=reverse_interface_adapter(ga),
log_book=None,
stats=None,
alg_params=params["alg_params"])
kwargs = dict(params["executor_params"])
kwargs.update(params["alg_params"])
kwargs["ga_params"] = {"population": params["alg_params"]["n"]}
machine = MPGaHeftOldPopExecutor(heft_planner=dynamic_heft,
wf=_wf,
resource_manager=rm,
estimator=estimator,
stat_saver=None,
ga_builder=lambda: ga,
mpga_builder=lambda: mpga,
chromosome_cleaner=chromosome_cleaner,
mpnewVSmpoldmode=False,
mixed_init_pop=False,
emigrant_selection=None,
check_evolution_for_stopping=False,
schedule_to_chromosome_converter=
schedule_to_chromosome_converter_builder(
wf, rm, estimator),
**kwargs)
machine.init()
machine.run()
resulted_schedule = machine.current_schedule
stat_data = machine.executor_stat_data
Utility.validate_dynamic_schedule(_wf, resulted_schedule)
data = {
"wf_name": wf_name,
"params": params,
"result": {
"makespan":
Utility.makespan(resulted_schedule),
## TODO: this function should be remade to adapt under conditions of dynamic env
#"overall_transfer_time": Utility.overall_transfer_time(resulted_schedule, _wf, estimator),
"overall_execution_time":
Utility.overall_execution_time(resulted_schedule),
"overall_failed_tasks_count":
Utility.overall_failed_tasks_count(resulted_schedule)
}
}
return data
def do_exp_heft_schedule():
res = do_exp_schedule(True)
return res[0]
def do_exp_ga_schedule():
res = do_exp_schedule(False)
return (res[0], res[4])
if __name__ == '__main__':
print("Population size: " + str(PARAMS["ga_params"]["population"]))
_wf = wf(wf_names[0])
rm = ExperimentResourceManager(rg.r(PARAMS["nodes_conf"]))
estimator = SimpleTimeCostEstimator(comp_time_cost=0,
transf_time_cost=0,
transferMx=None,
ideal_flops=PARAMS["ideal_flops"],
transfer_time=PARAMS["transfer_time"])
heft_schedule = run_heft(_wf, rm, estimator)
heft_makespan = Utility.makespan(heft_schedule)
overall_transfer = Utility.overall_transfer_time(heft_schedule, _wf,
estimator)
overall_execution = Utility.overall_execution_time(heft_schedule)
print(
"Heft makespan: {0}, Overall transfer time: {1}, Overall execution time: {2}"
name = saver(data)
return ga_makespan, heft_makespan, ga_schedule, heft_schedule, name
def do_exp_heft_schedule():
res = do_exp_schedule(True)
return res[0]
def do_exp_ga_schedule():
res = do_exp_schedule(False)
return (res[0], res[4])
if __name__ == '__main__':
print("Population size: " + str(PARAMS["ga_params"]["population"]))
_wf = wf(wf_names[0])
rm = ExperimentResourceManager(rg.r(PARAMS["nodes_conf"]))
estimator = SimpleTimeCostEstimator(comp_time_cost=0, transf_time_cost=0, transferMx=None,
ideal_flops=PARAMS["ideal_flops"], transfer_time=PARAMS["transfer_time"])
heft_schedule = run_heft(_wf, rm, estimator)
heft_makespan = Utility.makespan(heft_schedule)
overall_transfer = Utility.overall_transfer_time(heft_schedule, _wf, estimator)
overall_execution = Utility.overall_execution_time(heft_schedule)
print("Heft makespan: {0}, Overall transfer time: {1}, Overall execution time: {2}".format(heft_makespan,
overall_transfer,
overall_execution))
if not only_heft:
result = repeat(do_exp_heft_schedule, 1)
ordering_heft_based_initialize
from heft.algs.ga.nsga2 import run_nsga2
from heft.algs.heft.DSimpleHeft import run_heft
from heft.algs.heft.HeftHelper import HeftHelper
from heft.algs.pso.gapso import run_gapso
from heft.algs.pso.sdpso import run_pso, update, schedule_to_position, construct_solution, MappingParticle, \
Velocity, Position
from heft.core.CommonComponents.ExperimentalManagers import ExperimentResourceManager
from heft.core.environment.Utility import Utility, wf
from heft.algs.common.mapordschedule import build_schedule, MAPPING_SPECIE, ORDERING_SPECIE, ordering_from_schedule, \
mapping_from_schedule
from heft.experiments.cga.mobjective.utility import SimpleTimeCostEstimator
from heft.core.environment.ResourceGenerator import ResourceGenerator as rg
from heft.algs.common.mapordschedule import fitness as basefitness
_wf = wf("Montage_50")
rm = ExperimentResourceManager(rg.r([10, 15, 25, 30]))
estimator = SimpleTimeCostEstimator(comp_time_cost=0,
transf_time_cost=0,
transferMx=None,
ideal_flops=20,
transfer_time=100)
sorted_tasks = HeftHelper.heft_rank(_wf, rm, estimator)
heft_schedule = run_heft(_wf, rm, estimator)
print(Utility.makespan(heft_schedule))
stats = tools.Statistics(lambda ind: ind.fitness.values[0])
stats.register("avg", numpy.mean)
stats.register("std", numpy.std)
def do_inherited_pop_exp(saver, alg_builder, chromosome_cleaner_builder,
wf_name, **params):
_wf = wf(wf_name)
rm = ExperimentResourceManager(rg.r(params["resource_set"]["nodes_conf"]))
estimator = SimpleTimeCostEstimator(**params["estimator_settings"])
chromosome_cleaner = chromosome_cleaner_builder(_wf, rm, estimator)
dynamic_heft = DynamicHeft(_wf, rm, estimator)
logbook = Logbook()
stats = tools.Statistics(lambda ind: ind.fitness.values[0])
stats.register("avg", numpy.mean)
stats.register("std", numpy.std)
stats.register("min", numpy.min)
stats.register("max", numpy.max)
ga = alg_builder(_wf,
rm,
estimator,
params["init_sched_percent"],
log_book=logbook,
stats=stats,
alg_params=params["alg_params"])
machine = GaHeftOldPopExecutor(heft_planner=dynamic_heft,
wf=_wf,
resource_manager=rm,
estimator=estimator,
stat_saver=None,
ga_builder=lambda: ga,
chromosome_cleaner=chromosome_cleaner,
**params["executor_params"])
machine.init()
print("Executor start")
machine.run()
print("Executor stop")
resulted_schedule = machine.current_schedule
stat_data = machine.executor_stat_data
Utility.validate_dynamic_schedule(_wf, resulted_schedule)
data = {
"wf_name": wf_name,
"params": params,
"result": {
"random_init_logbook":
stat_data["random_init_logbook"],
"inherited_init_logbook":
stat_data["inherited_init_logbook"],
"makespan":
Utility.makespan(resulted_schedule),
## TODO: this function should be remade to adapt under conditions of dynamic env
#"overall_transfer_time": Utility.overall_transfer_time(resulted_schedule, _wf, estimator),
"overall_execution_time":
Utility.overall_execution_time(resulted_schedule),
"overall_failed_tasks_count":
Utility.overall_failed_tasks_count(resulted_schedule)
}
}
if saver is not None:
saver(data)
return data
import numpy
from heft.algs.common.particle_operations import MappingParticle
from heft.algs.heft.DSimpleHeft import run_heft
from heft.algs.heft.HeftHelper import HeftHelper
from heft.algs.pso.mapping_operators import schedule_to_position, fitness, update, generate, construct_solution
from heft.algs.pso.sdpso import run_pso
from heft.core.CommonComponents.ExperimentalManagers import ExperimentResourceManager
from heft.core.environment.Utility import Utility, wf
from heft.algs.common.mapordschedule import build_schedule, MAPPING_SPECIE, ORDERING_SPECIE
from heft.experiments.aggregate_utilities import interval_statistics, interval_stat_string
from heft.experiments.cga.mobjective.utility import SimpleTimeCostEstimator
from heft.core.environment.ResourceGenerator import ResourceGenerator as rg
from heft.experiments.cga.utilities.common import repeat
_wf = wf("Montage_75")
rm = ExperimentResourceManager(rg.r([10, 15, 25, 30]))
estimator = SimpleTimeCostEstimator(comp_time_cost=0,
transf_time_cost=0,
transferMx=None,
ideal_flops=20,
transfer_time=100)
sorted_tasks = HeftHelper.heft_rank(_wf, rm, estimator)
heft_schedule = run_heft(_wf, rm, estimator)
heft_mapping = schedule_to_position(heft_schedule)
heft_mapping.velocity = MappingParticle.Velocity({})
heft_gen = lambda n: [
deepcopy(heft_mapping)
from heft.algs.ga.GAImplementation.GAImpl import GAFactory
from heft.core.CommonComponents.ExperimentalManagers import ExperimentResourceManager, ExperimentEstimator
from heft.core.environment import ResourceGenerator
from heft.core.environment.ResourceManager import Schedule
from heft.core.environment.Utility import profile_decorator, wf
_wf = wf("Montage_100")
resources = ResourceGenerator.r([10, 15, 25, 30])
resource_manager = ExperimentResourceManager(resources)
estimator = ExperimentEstimator(None,
ideal_flops=20,
reliability=1.0,
transfer_time=100)
ga = GAFactory.default().create_ga(silent=True,
wf=_wf,
resource_manager=resource_manager,
estimator=estimator,
ga_params={
"population": 10,
"crossover_probability": 0.8,
"replacing_mutation_probability": 0.5,
"sweep_mutation_probability": 0.4,
"generations": 20
})
@profile_decorator
def fnc():
empty_schedule = Schedule(
{node: []
from deap import tools
from heft.algs.common.individuals import ListBasedIndividual
from heft.algs.ga.coevolution.cga import Env, Specie
from heft.algs.ga.coevolution.operators import MAPPING_SPECIE, ORDERING_SPECIE, build_schedule, mapping_default_initialize, overhead_fitness_mapping_and_ordering, assign_from_transfer_overhead, mapping_all_mutate, default_build_solutions
from heft.core.CommonComponents.ExperimentalManagers import ExperimentResourceManager, ExperimentEstimator
from heft.core.environment.Utility import wf
from heft.experiments.cga.cga_exp import os_ideal_ind, ms_ideal_ind, do_experiment
from heft.experiments.cga.utilities.common import UniqueNameSaver, repeat, extract_ordering_from_file, ArchivedSelector, roulette, build_ms_ideal_ind, build_os_ideal_ind, hamming_distances, unique_individuals, to_seq, pcm, gdm, hamming_for_best_components, best_components_itself
from heft.settings import __root_path__
from heft.core.environment import Utility
_wf = wf("Montage_25")
rm = ExperimentResourceManager(rg.r([10, 15, 25, 30]))
estimator = ExperimentEstimator(None, ideal_flops=20, transfer_time=100)
# selector = ArchivedSelector(5)(tourn)
selector = ArchivedSelector(5)(roulette)
os_representative = extract_ordering_from_file("{0}/temp/cga_exp_example/6685a2b2-78d6-4637-b099-ed91152464f5.json".format(__root_path__),
_wf, estimator, rm)
ms_ideal_ind = build_ms_ideal_ind(_wf, rm)
os_ideal_ind = build_os_ideal_ind(_wf)
# heft_mapping = extract_mapping_from_file("../../temp/heft_etalon_tr100.json")
saver = UniqueNameSaver("{0}/temp/cga_improving_mutation".format(__root_path__))
def coeff_run():
"""
coefficient of compute/data intensivity
"""
# wf_names = ['Montage_25', 'CyberShake_30', 'Inspiral_30', 'Sipht_30', 'Epigenomics_24']
#all_coeffs = [1/100, 1/50, 1/10, 1/5, 1/2.766, 15, 20, 25, 30, 35, 40, 45, 75]
all_coeffs = [1, 5, 10, 25, 50, 100]
# all_coeffs = [0.0005]
# flops = [1, 10, 20, 30, 50, 100]
transfer_coeffs = [0.1, 0.2, 0.5, 1, 2, 5, 10]
# wf_names = [('Montage_25', [10]),
# ('CyberShake_30', [0.1] + list(range(10, 46, 1))),
# ('Inspiral_30', [10, 1, 1/2.766]),
# ('Sipht_30', [0.02] + list(range(30, 50, 1)) + list(range(50, 101, 5))),
# ('Epigenomics_24', list(range(5, 46, 1)) + list(range(45, 101, 5)))]
# wf_names = [('Montage_25', [2.766])]
wf_names = [('Montage_25', all_coeffs),
('CyberShake_30', all_coeffs),
('Inspiral_30', all_coeffs),
('Sipht_30', all_coeffs),
('Epigenomics_24', all_coeffs)]
def transfer_time(max_runtime, flops, max_transfer_size, bandwidth):
real_bandwidth = bandwidth / 8 * 1000000
transfer = max_runtime * flops / (max_transfer_size / real_bandwidth )
return transfer
def compute_max_transfer_size(head_task):
max_size = 0
if head_task.output_files != None:
for task in head_task.children:
parent_keys = set(head_task.output_files.keys())
children_keys = set(task.input_files.keys())
parent_output_as_child_input = parent_keys.intersection(children_keys)
if parent_output_as_child_input != None:
size = 0
for file_name in parent_output_as_child_input:
size += head_task.output_files[file_name].size
if size > max_size:
max_size = size
for task in head_task.children:
size = compute_max_transfer_size(task)
if size > max_size:
max_size = size
return max_size
def compute_mean_transfer_time(head_task):
sizes = []
if head_task.output_files != None:
for task in head_task.children:
parent_keys = set(head_task.output_files.keys())
children_keys = set(task.input_files.keys())
parent_output_as_child_input = parent_keys.intersection(children_keys)
if parent_output_as_child_input != None:
size = 0
for file_name in parent_output_as_child_input:
size += head_task.output_files[file_name].size
sizes.append(size)
for task in head_task.children:
size = compute_max_transfer_size(task)
sizes.append(size)
percentile = numpy.percentile(sizes, 0.95)
# mean_size = statistics.mean(sizes)
# return mean_size
return percentile
def compute_mean_runtime(tasks):
times = []
for task in tasks:
times.append(task.runtime)
percentile = numpy.percentile(times, 0.9)
# return statistics.mean(times)
return percentile
def compute_flops(max_runtime, max_size, bandwidth, coeff):
# real_bandwidth = bandwidth / 8 * 1000000
# flops = coeff * max_size / (max_runtime * real_bandwidth)
flops = coeff * bandwidth / max_runtime
return flops
to_run = []
experiment_id = uuid.uuid4()
for wf_name, coeffs in wf_names:
_wf = wf(wf_name)
max_size = compute_max_transfer_size(_wf.head_task)
mean_size = compute_mean_transfer_time(_wf.head_task)
max_runtime = max(_wf.get_all_unique_tasks(), key=lambda x: x.runtime).runtime
mean_runtime = compute_mean_runtime(_wf.get_all_unique_tasks())
param_sets = [copy_and_set(BASE_PARAMS, max_size=max_size,
data_intensive_coeff=c)
for c in all_coeffs]
exps = [partial(do_exp_and_save, do_exp, experiment_id, wf_name, **params) for params in param_sets]
to_run = to_run + exps
# m_repeat = lambda n, funcs: [f() for f in funcs for _ in range(n)]
# results = m_repeat(REPEAT_COUNT, to_run)
multi_repeat(REPEAT_COUNT, to_run)
# saver = UniqueNameSaver(TEMP_PATH, EXPERIMENT_NAME)
# saver = MongoSaver(MONGO_SERVER_ADDRESS, EXPERIMENT_NAME, EXPERIMENT_ID)
# for result in results:
# saver(result)
print("Experiment ID: {0}".format(experiment_id))
# coeff_aggregate(saver.directory, "coeff.png")
pass
ordering_heft_based_initialize
from heft.algs.ga.nsga2 import run_nsga2
from heft.algs.heft.DSimpleHeft import run_heft
from heft.algs.heft.HeftHelper import HeftHelper
from heft.algs.pso.gapso import run_gapso
from heft.algs.pso.sdpso import run_pso, update, schedule_to_position, construct_solution, MappingParticle, \
Velocity, Position
from heft.core.CommonComponents.ExperimentalManagers import ExperimentResourceManager
from heft.core.environment.Utility import Utility, wf
from heft.algs.common.mapordschedule import build_schedule, MAPPING_SPECIE, ORDERING_SPECIE, ordering_from_schedule, \
mapping_from_schedule
from heft.experiments.cga.mobjective.utility import SimpleTimeCostEstimator
from heft.core.environment.ResourceGenerator import ResourceGenerator as rg
from heft.algs.common.mapordschedule import fitness as basefitness
_wf = wf("Montage_50")
rm = ExperimentResourceManager(rg.r([10, 15, 25, 30]))
estimator = SimpleTimeCostEstimator(comp_time_cost=0, transf_time_cost=0, transferMx=None,
ideal_flops=20, transfer_time=100)
sorted_tasks = HeftHelper.heft_rank(_wf, rm, estimator)
heft_schedule = run_heft(_wf, rm, estimator)
print(Utility.makespan(heft_schedule))
stats = tools.Statistics(lambda ind: ind.fitness.values[0])
stats.register("avg", numpy.mean)
stats.register("std", numpy.std)
def get_wf(wf_name, task_postfix_id="00"):
return wf(wf_name, task_postfix_id)
from deap import base
from deap import creator
from deap import tools
from heft.algs.common.individuals import ListBasedIndividual
from heft.algs.ga.coevolution.cga import Env
from heft.algs.ga.coevolution.operators import mapping_heft_based_initialize, ordering_heft_based_initialize
from heft.core.CommonComponents.ExperimentalManagers import ExperimentResourceManager
from heft.core.environment import ResourceGenerator as rg
from heft.core.environment.Utility import wf
from heft.experiments.cga.mobjective.utility import fitness_makespan_and_cost_map_ord, SimpleTimeCostEstimator
from heft.experiments.cga.utilities.common import extract_ordering_from_ga_file, extract_mapping_from_ga_file
from heft.experiments.cga.utilities.double_chromosome_ga import _mate, _mutate
_wf = wf("Montage_100")
# rm = ExperimentResourceManager(VMResGen.r([10, 15, 25, 30], 4))
rm = ExperimentResourceManager(rg.r([10, 15, 25, 30]))
# estimator = ExperimentEstimator(None, ideal_flops=20, transfer_time=100)
estimator = SimpleTimeCostEstimator(comp_time_cost=10, transf_time_cost=2, transferMx=None, ideal_flops=20, transfer_time=100)
env = Env(_wf, rm, estimator)
pop_size = 200
NGEN = 300
MU = 40
CXPB = 0.9
heft_mapping = extract_mapping_from_ga_file("../../../temp/heft_etalon_full_tr100_m100.json", rm)
heft_ordering = extract_ordering_from_ga_file("../../../temp/heft_etalon_full_tr100_m100.json")
from heft.algs.heft.DSimpleHeft import run_heft
from heft.algs.heft.HeftHelper import HeftHelper
from heft.algs.pso.sdpso import schedule_to_position, generate
from heft.algs.sa.SimulatedAnnealingScheme import run_sa
from heft.algs.sa.mappingops import energy, update_T, mapping_neighbor, transition_probability, State
from heft.core.CommonComponents.ExperimentalManagers import ExperimentResourceManager
from heft.core.environment.Utility import Utility, wf
from heft.algs.common.mapordschedule import build_schedule, MAPPING_SPECIE, ORDERING_SPECIE
from heft.experiments.cga.mobjective.utility import SimpleTimeCostEstimator
from heft.core.environment.ResourceGenerator import ResourceGenerator as rg
## TODO: need to test all of it
from heft.experiments.cga.utilities.common import repeat
_wf = wf("Montage_25")
rm = ExperimentResourceManager(rg.r([10, 15, 25, 30]))
estimator = SimpleTimeCostEstimator(comp_time_cost=0, transf_time_cost=0, transferMx=None,
ideal_flops=20, transfer_time=100)
sorted_tasks = HeftHelper.heft_rank(_wf, rm, estimator)
heft_schedule = run_heft(_wf, rm, estimator)
heft_mapping = schedule_to_position(heft_schedule).entity
initial_state = State()
initial_state.mapping = heft_mapping
# initial_state.mapping = generate(_wf, rm, estimator, 1)[0].entity
initial_state.ordering = sorted_tasks
