def do_exp(wf_name, **params): _wf = wf(wf_name) ga_makespan, heft_make, ga_schedule, heft_sched = MixRunner()(_wf, **params) rm = ExperimentResourceManager(rg.r(params["nodes_conf"])) estimator = SimpleTimeCostEstimator(comp_time_cost=0, transf_time_cost=0, transferMx=None, ideal_flops=params["ideal_flops"], transfer_time=params["transfer_time"]) heft_schedule = run_heft(_wf, rm, estimator) heft_makespan = Utility.makespan(heft_schedule) data = { "wf_name": wf_name, "params": params, "heft": { "makespan": heft_makespan, "overall_transfer_time": Utility.overall_transfer_time(heft_schedule, _wf, estimator), "overall_execution_time": Utility.overall_execution_time(heft_schedule) }, "ga": { "makespan": ga_makespan, "overall_transfer_time": Utility.overall_transfer_time(ga_schedule, _wf, estimator), "overall_execution_time": Utility.overall_execution_time(ga_schedule) }, #"heft_schedule": heft_schedule, #"ga_schedule": ga_schedule } return data
def test_fixed_ordering(self): _wf = wf("Montage_25") rm = ExperimentResourceManager(rg.r([10, 15, 25, 30])) estimator = SimpleTimeCostEstimator(comp_time_cost=0, transf_time_cost=0, transferMx=None, ideal_flops=20, transfer_time=100) sorted_tasks = HeftHelper.heft_rank(_wf, rm, estimator) heft_schedule = run_heft(_wf, rm, estimator) heft_mapping = schedule_to_position(heft_schedule) heft_gen = lambda: heft_mapping if random.random( ) > 0.95 else generate(_wf, rm, estimator) toolbox = Toolbox() # toolbox.register("generate", generate, _wf, rm, estimator) toolbox.register("generate", heft_gen) toolbox.register("fitness", fitness, _wf, rm, estimator, sorted_tasks) toolbox.register("force_vector_matrix", force_vector_matrix, rm) toolbox.register("velocity_and_position", velocity_and_position, _wf, rm, estimator) toolbox.register("G", G) toolbox.register("kbest", Kbest) statistics = Statistics() statistics.register( "min", lambda pop: numpy.min([p.fitness.mofit for p in pop])) statistics.register( "avr", lambda pop: numpy.average([p.fitness.mofit for p in pop])) statistics.register( "max", lambda pop: numpy.max([p.fitness.mofit for p in pop])) statistics.register( "std", lambda pop: numpy.std([p.fitness.mofit for p in pop])) logbook = Logbook() logbook.header = ("gen", "G", "kbest", "min", "avr", "max", "std") pop_size = 100 iter_number = 100 kbest = pop_size ginit = 5 final_pop = run_gsa(toolbox, statistics, logbook, pop_size, iter_number, kbest, ginit) best = min(final_pop, key=lambda x: toolbox.fitness(x).mofit) solution = { MAPPING_SPECIE: list(zip(sorted_tasks, best)), ORDERING_SPECIE: sorted_tasks } schedule = build_schedule(_wf, estimator, rm, solution) Utility.validate_static_schedule(_wf, schedule) makespan = Utility.makespan(schedule) print("Final makespan: {0}".format(makespan)) pass
def do_exp(wf_name): _wf = wf(wf_name) heft_schedule = run_heft(_wf, rm, estimator) Utility.validate_static_schedule(_wf, heft_schedule) makespan = Utility.makespan(heft_schedule) return makespan
def do_exp(wf_name): _wf = wf(wf_name) rm = ExperimentResourceManager(rg.r([10, 15, 25, 30])) estimator = SimpleTimeCostEstimator(comp_time_cost=0, transf_time_cost=0, transferMx=None, ideal_flops=20, transfer_time=100) empty_fixed_schedule_part = Schedule({node: [] for node in rm.get_nodes()}) heft_schedule = run_heft(_wf, rm, estimator) ga_functions = GAFunctions2(_wf, rm, estimator) generate = partial(ga_generate, ga_functions=ga_functions, fixed_schedule_part=empty_fixed_schedule_part, current_time=0.0, init_sched_percent=0.05, initial_schedule=heft_schedule) stats = tools.Statistics(lambda ind: ind.fitness.values[0]) stats.register("avg", numpy.mean) stats.register("std", numpy.std) stats.register("min", numpy.min) stats.register("max", numpy.max) logbook = tools.Logbook() logbook.header = ["gen", "evals"] + stats.fields toolbox = Toolbox() toolbox.register("generate", generate) toolbox.register( "evaluate", fit_converter( ga_functions.build_fitness(empty_fixed_schedule_part, 0.0))) toolbox.register("clone", deepcopy) toolbox.register("mate", ga_functions.crossover) toolbox.register("sweep_mutation", ga_functions.sweep_mutation) toolbox.register("mutate", ga_functions.mutation) # toolbox.register("select_parents", ) # toolbox.register("select", tools.selTournament, tournsize=4) toolbox.register("select", tools.selRoulette) pop, logbook, best = run_ga(toolbox=toolbox, logbook=logbook, stats=stats, **GA_PARAMS) resulted_schedule = ga_functions.build_schedule(best, empty_fixed_schedule_part, 0.0) Utility.validate_static_schedule(_wf, resulted_schedule) ga_makespan = Utility.makespan(resulted_schedule) return ga_makespan
def do_exp(wf_name): _wf = wf(wf_name) rm = ExperimentResourceManager(rg.r([10, 15, 25, 30])) estimator = SimpleTimeCostEstimator(comp_time_cost=0, transf_time_cost=0, transferMx=None, ideal_flops=20, transfer_time=100) empty_fixed_schedule_part = Schedule({node: [] for node in rm.get_nodes()}) heft_schedule = run_heft(_wf, rm, estimator) ga_functions = GAFunctions2(_wf, rm, estimator) generate = partial(ga_generate, ga_functions=ga_functions, fixed_schedule_part=empty_fixed_schedule_part, current_time=0.0, init_sched_percent=0.05, initial_schedule=heft_schedule) stats = tools.Statistics(lambda ind: ind.fitness.values[0]) stats.register("avg", numpy.mean) stats.register("std", numpy.std) stats.register("min", numpy.min) stats.register("max", numpy.max) logbook = tools.Logbook() logbook.header = ["gen", "evals"] + stats.fields toolbox = Toolbox() toolbox.register("generate", generate) toolbox.register("evaluate", fit_converter(ga_functions.build_fitness(empty_fixed_schedule_part, 0.0))) toolbox.register("clone", deepcopy) toolbox.register("mate", ga_functions.crossover) toolbox.register("sweep_mutation", ga_functions.sweep_mutation) toolbox.register("mutate", ga_functions.mutation) # toolbox.register("select_parents", ) # toolbox.register("select", tools.selTournament, tournsize=4) toolbox.register("select", tools.selRoulette) pop, logbook, best = run_ga(toolbox=toolbox, logbook=logbook, stats=stats, **GA_PARAMS) resulted_schedule = ga_functions.build_schedule(best, empty_fixed_schedule_part, 0.0) Utility.validate_static_schedule(_wf, resulted_schedule) ga_makespan = Utility.makespan(resulted_schedule) return ga_makespan
def test_fixed_ordering(self): _wf = wf("Montage_25") rm = ExperimentResourceManager(rg.r([10, 15, 25, 30])) estimator = SimpleTimeCostEstimator(comp_time_cost=0, transf_time_cost=0, transferMx=None, ideal_flops=20, transfer_time=100) sorted_tasks = HeftHelper.heft_rank(_wf, rm, estimator) heft_schedule = run_heft(_wf, rm, estimator) heft_mapping = schedule_to_position(heft_schedule) heft_gen = lambda: heft_mapping if random.random() > 0.95 else generate(_wf, rm, estimator) toolbox = Toolbox() # toolbox.register("generate", generate, _wf, rm, estimator) toolbox.register("generate", heft_gen) toolbox.register("fitness", fitness, _wf, rm, estimator, sorted_tasks) toolbox.register("force_vector_matrix", force_vector_matrix, rm) toolbox.register("velocity_and_position", velocity_and_position, _wf, rm, estimator) toolbox.register("G", G) toolbox.register("kbest", Kbest) statistics = Statistics() statistics.register("min", lambda pop: numpy.min([p.fitness.mofit for p in pop])) statistics.register("avr", lambda pop: numpy.average([p.fitness.mofit for p in pop])) statistics.register("max", lambda pop: numpy.max([p.fitness.mofit for p in pop])) statistics.register("std", lambda pop: numpy.std([p.fitness.mofit for p in pop])) logbook = Logbook() logbook.header = ("gen", "G", "kbest", "min", "avr", "max", "std") pop_size = 100 iter_number = 100 kbest = pop_size ginit = 5 final_pop = run_gsa(toolbox, statistics, logbook, pop_size, iter_number, kbest, ginit) best = min(final_pop, key=lambda x: toolbox.fitness(x).mofit) solution = {MAPPING_SPECIE: list(zip(sorted_tasks, best)), ORDERING_SPECIE: sorted_tasks} schedule = build_schedule(_wf, estimator, rm, solution) Utility.validate_static_schedule(_wf, schedule) makespan = Utility.makespan(schedule) print("Final makespan: {0}".format(makespan)) pass
def do_exp(wf_name, **params): _wf = wf(wf_name) ga_makespan, heft_make, ga_schedule, heft_sched = MixRunner()(_wf, **params) rm = ExperimentResourceManager(rg.r(params["nodes_conf"])) estimator = SimpleTimeCostEstimator(comp_time_cost=0, transf_time_cost=0, transferMx=None, ideal_flops=params["ideal_flops"], transfer_time=params["transfer_time"]) heft_schedule = run_heft(_wf, rm, estimator) heft_makespan = Utility.makespan(heft_schedule) data = { "wf_name": wf_name, "params": params, "heft": { "makespan": heft_makespan, "overall_transfer_time": Utility.overall_transfer_time(heft_schedule, _wf, estimator), "overall_execution_time": Utility.overall_execution_time(heft_schedule) }, "ga": { "makespan": ga_makespan, "overall_transfer_time": Utility.overall_transfer_time(ga_schedule, _wf, estimator), "overall_execution_time": Utility.overall_execution_time(ga_schedule) }, #"heft_schedule": heft_schedule, #"ga_schedule": ga_schedule } return data
def do_exp(wf_name, **params): _wf = wf(wf_name) ga_makespan, heft_make, ga_schedule, heft_sched, logbook = MixRunner()(_wf, **params) rm = ExperimentResourceManager(rg.r(params["nodes_conf"])) estimator = TestEstimator(comp_time_cost=0, transf_time_cost=0, bandwidth=float(params["data_intensive_coeff"]), transferMx=None, ideal_flops=params["ideal_flops"], max_size=float(params["max_size"]), transfer_time=params["transfer_time"]) heft_schedule = run_heft(_wf, rm, estimator) heft_makespan = Utility.makespan(heft_schedule) # print("Heft schedule:") # print(heft_schedule) # print("Heft makespan: {0}".format(heft_makespan)) # print("GA schedule:") # print(ga_schedule) # print("GA makespan: {0}".format(ga_makespan)) data = { "wf_name": wf_name, "params": params, "heft": { "makespan": heft_makespan, "overall_transfer_time": Utility.overall_transfer_time(heft_schedule, _wf, estimator), "overall_execution_time": Utility.overall_execution_time(heft_schedule) }, "ga": { "makespan": ga_makespan, "overall_transfer_time": Utility.overall_transfer_time(ga_schedule, _wf, estimator), "overall_execution_time": Utility.overall_execution_time(ga_schedule) }, "logbook": logbook #"heft_schedule": heft_schedule, #"ga_schedule": ga_schedule } return data
Velocity, Position from heft.core.CommonComponents.ExperimentalManagers import ExperimentResourceManager from heft.core.environment.Utility import Utility, wf from heft.algs.common.mapordschedule import build_schedule, MAPPING_SPECIE, ORDERING_SPECIE, ordering_from_schedule, \ mapping_from_schedule from heft.experiments.cga.mobjective.utility import SimpleTimeCostEstimator from heft.core.environment.ResourceGenerator import ResourceGenerator as rg from heft.algs.common.mapordschedule import fitness as basefitness _wf = wf("Montage_50") rm = ExperimentResourceManager(rg.r([10, 15, 25, 30])) estimator = SimpleTimeCostEstimator(comp_time_cost=0, transf_time_cost=0, transferMx=None, ideal_flops=20, transfer_time=100) sorted_tasks = HeftHelper.heft_rank(_wf, rm, estimator) heft_schedule = run_heft(_wf, rm, estimator) print(Utility.makespan(heft_schedule)) stats = tools.Statistics(lambda ind: ind.fitness.values[0]) stats.register("avg", numpy.mean) stats.register("std", numpy.std) stats.register("min", numpy.min) stats.register("max", numpy.max) logbook = tools.Logbook() logbook.header = ["gen", "evals"] + stats.fields
from heft.algs.common.mapordschedule import build_schedule, MAPPING_SPECIE, ORDERING_SPECIE from heft.experiments.aggregate_utilities import interval_statistics, interval_stat_string from heft.experiments.cga.mobjective.utility import SimpleTimeCostEstimator from heft.core.environment.ResourceGenerator import ResourceGenerator as rg from heft.experiments.cga.utilities.common import repeat _wf = wf("Montage_75") rm = ExperimentResourceManager(rg.r([10, 15, 25, 30])) estimator = SimpleTimeCostEstimator(comp_time_cost=0, transf_time_cost=0, transferMx=None, ideal_flops=20, transfer_time=100) sorted_tasks = HeftHelper.heft_rank(_wf, rm, estimator) heft_schedule = run_heft(_wf, rm, estimator) heft_mapping = schedule_to_position(heft_schedule) heft_mapping.velocity = MappingParticle.Velocity({}) heft_gen = lambda n: [ deepcopy(heft_mapping) if random.random() > 1.0 else generate(_wf, rm, estimator, 1)[0] for _ in range(n) ] W, C1, C2 = 0.1, 0.6, 0.2 GEN, N = 300, 50 toolbox = Toolbox() toolbox.register("population", heft_gen)
def heft_schedule(self): if not self._heft_schedule: self._heft_schedule = run_heft(self._wf, self._rm, self._estimator) return self._heft_schedule