def do_exp(alg_builder, wf_name, **params): _wf = wf(wf_name) rm = ExperimentResourceManager(rg.r(params["resource_set"]["nodes_conf"])) estimator = SimpleTimeCostEstimator(**params["estimator_settings"]) dynamic_heft = DynamicHeft(_wf, rm, estimator) ga = alg_builder(_wf, rm, estimator, params["init_sched_percent"], logbook=None, stats=None, **params["alg_params"]) machine = GaHeftExecutor(heft_planner=dynamic_heft, wf=_wf, resource_manager=rm, ga_builder=lambda: ga, **params["executor_params"]) machine.init() machine.run() resulted_schedule = machine.current_schedule Utility.validate_dynamic_schedule(_wf, resulted_schedule) data = { "wf_name": wf_name, "params": params, "result": { "makespan": Utility.makespan(resulted_schedule), ## TODO: this function should be remade to adapt under conditions of dynamic env #"overall_transfer_time": Utility.overall_transfer_time(resulted_schedule, _wf, estimator), "overall_execution_time": Utility.overall_execution_time(resulted_schedule), "overall_failed_tasks_count": Utility.overall_failed_tasks_count(resulted_schedule) } } return data
def env(self): if not self._wf or not self._rm or not self._estimator: self._wf = wf(self.wf_name) self._rm = ExperimentResourceManager(rg.r(self._resorces_set)) self._estimator = SimpleTimeCostEstimator(comp_time_cost=self._comp_time_cost, transf_time_cost=self._transf_time_cost, transferMx=None, ideal_flops=self._ideal_flops, transfer_time=self._transfer_time) return self._wf, self._rm, self._estimator
def do_exp(wf_name, **params): _wf = wf(wf_name) ga_makespan, heft_make, ga_schedule, heft_sched = MixRunner()(_wf, **params) rm = ExperimentResourceManager(rg.r(params["nodes_conf"])) estimator = SimpleTimeCostEstimator(comp_time_cost=0, transf_time_cost=0, transferMx=None, ideal_flops=params["ideal_flops"], transfer_time=params["transfer_time"]) heft_schedule = run_heft(_wf, rm, estimator) heft_makespan = Utility.makespan(heft_schedule) data = { "wf_name": wf_name, "params": params, "heft": { "makespan": heft_makespan, "overall_transfer_time": Utility.overall_transfer_time(heft_schedule, _wf, estimator), "overall_execution_time": Utility.overall_execution_time(heft_schedule) }, "ga": { "makespan": ga_makespan, "overall_transfer_time": Utility.overall_transfer_time(ga_schedule, _wf, estimator), "overall_execution_time": Utility.overall_execution_time(ga_schedule) }, #"heft_schedule": heft_schedule, #"ga_schedule": ga_schedule } return data
def heft_exp(saver, wf_name, **params): _wf = wf(wf_name) rm = ExperimentResourceManager(rg.r(params["resource_set"]["nodes_conf"])) estimator = SimpleTimeCostEstimator(**params["estimator_settings"]) dynamic_heft = DynamicHeft(_wf, rm, estimator) heft_machine = HeftExecutor(rm, heft_planner=dynamic_heft, **params["executor_params"]) heft_machine.init() heft_machine.run() resulted_schedule = heft_machine.current_schedule Utility.validate_dynamic_schedule(_wf, resulted_schedule) data = { "wf_name": wf_name, "params": params, "result": { "makespan": Utility.makespan(resulted_schedule), ## TODO: this function should be remade to adapt under conditions of dynamic env #"overall_transfer_time": Utility.overall_transfer_time(resulted_schedule, _wf, estimator), "overall_execution_time": Utility.overall_execution_time(resulted_schedule), "overall_failed_tasks_count": Utility.overall_failed_tasks_count(resulted_schedule) } } if saver is not None: saver(data) return data
def test_fixed_ordering(self): _wf = wf("Montage_25") rm = ExperimentResourceManager(rg.r([10, 15, 25, 30])) estimator = SimpleTimeCostEstimator(comp_time_cost=0, transf_time_cost=0, transferMx=None, ideal_flops=20, transfer_time=100) sorted_tasks = HeftHelper.heft_rank(_wf, rm, estimator) heft_schedule = run_heft(_wf, rm, estimator) heft_mapping = schedule_to_position(heft_schedule) heft_gen = lambda: heft_mapping if random.random( ) > 0.95 else generate(_wf, rm, estimator) toolbox = Toolbox() # toolbox.register("generate", generate, _wf, rm, estimator) toolbox.register("generate", heft_gen) toolbox.register("fitness", fitness, _wf, rm, estimator, sorted_tasks) toolbox.register("force_vector_matrix", force_vector_matrix, rm) toolbox.register("velocity_and_position", velocity_and_position, _wf, rm, estimator) toolbox.register("G", G) toolbox.register("kbest", Kbest) statistics = Statistics() statistics.register( "min", lambda pop: numpy.min([p.fitness.mofit for p in pop])) statistics.register( "avr", lambda pop: numpy.average([p.fitness.mofit for p in pop])) statistics.register( "max", lambda pop: numpy.max([p.fitness.mofit for p in pop])) statistics.register( "std", lambda pop: numpy.std([p.fitness.mofit for p in pop])) logbook = Logbook() logbook.header = ("gen", "G", "kbest", "min", "avr", "max", "std") pop_size = 100 iter_number = 100 kbest = pop_size ginit = 5 final_pop = run_gsa(toolbox, statistics, logbook, pop_size, iter_number, kbest, ginit) best = min(final_pop, key=lambda x: toolbox.fitness(x).mofit) solution = { MAPPING_SPECIE: list(zip(sorted_tasks, best)), ORDERING_SPECIE: sorted_tasks } schedule = build_schedule(_wf, estimator, rm, solution) Utility.validate_static_schedule(_wf, schedule) makespan = Utility.makespan(schedule) print("Final makespan: {0}".format(makespan)) pass
def do_exp(wf_name): _wf = wf(wf_name) rm = ExperimentResourceManager(rg.r([10, 15, 25, 30])) estimator = SimpleTimeCostEstimator(comp_time_cost=0, transf_time_cost=0, transferMx=None, ideal_flops=20, transfer_time=100) empty_fixed_schedule_part = Schedule({node: [] for node in rm.get_nodes()}) heft_schedule = run_heft(_wf, rm, estimator) ga_functions = GAFunctions2(_wf, rm, estimator) generate = partial(ga_generate, ga_functions=ga_functions, fixed_schedule_part=empty_fixed_schedule_part, current_time=0.0, init_sched_percent=0.05, initial_schedule=heft_schedule) stats = tools.Statistics(lambda ind: ind.fitness.values[0]) stats.register("avg", numpy.mean) stats.register("std", numpy.std) stats.register("min", numpy.min) stats.register("max", numpy.max) logbook = tools.Logbook() logbook.header = ["gen", "evals"] + stats.fields toolbox = Toolbox() toolbox.register("generate", generate) toolbox.register( "evaluate", fit_converter( ga_functions.build_fitness(empty_fixed_schedule_part, 0.0))) toolbox.register("clone", deepcopy) toolbox.register("mate", ga_functions.crossover) toolbox.register("sweep_mutation", ga_functions.sweep_mutation) toolbox.register("mutate", ga_functions.mutation) # toolbox.register("select_parents", ) # toolbox.register("select", tools.selTournament, tournsize=4) toolbox.register("select", tools.selRoulette) pop, logbook, best = run_ga(toolbox=toolbox, logbook=logbook, stats=stats, **GA_PARAMS) resulted_schedule = ga_functions.build_schedule(best, empty_fixed_schedule_part, 0.0) Utility.validate_static_schedule(_wf, resulted_schedule) ga_makespan = Utility.makespan(resulted_schedule) return ga_makespan
def do_exp(wf_name): _wf = wf(wf_name) heft_schedule = run_heft(_wf, rm, estimator) Utility.validate_static_schedule(_wf, heft_schedule) makespan = Utility.makespan(heft_schedule) return makespan
def do_triple_island_exp(saver, alg_builder, chromosome_cleaner_builder, schedule_to_chromosome_converter_builder, wf_name, **params): _wf = wf(wf_name) rm = ExperimentResourceManager(rg.r(params["resource_set"]["nodes_conf"])) estimator = SimpleTimeCostEstimator(**params["estimator_settings"]) chromosome_cleaner = chromosome_cleaner_builder(_wf, rm, estimator) dynamic_heft = DynamicHeft(_wf, rm, estimator) mpga = alg_builder(_wf, rm, estimator, params["init_sched_percent"], log_book=None, stats=None, alg_params=params["alg_params"]) machine = MIGaHeftExecutor(heft_planner=dynamic_heft, wf=_wf, resource_manager=rm, estimator=estimator, ga_builder=lambda: mpga, chromosome_cleaner=chromosome_cleaner, schedule_to_chromosome_converter= schedule_to_chromosome_converter_builder( wf, rm, estimator), **params["executor_params"]) machine.init() machine.run() resulted_schedule = machine.current_schedule Utility.validate_dynamic_schedule(_wf, resulted_schedule) data = { "wf_name": wf_name, "params": params, "result": { "makespan": Utility.makespan(resulted_schedule), ## TODO: this function should be remade to adapt under conditions of dynamic env #"overall_transfer_time": Utility.overall_transfer_time(resulted_schedule, _wf, estimator), "overall_execution_time": Utility.overall_execution_time(resulted_schedule), "overall_failed_tasks_count": Utility.overall_failed_tasks_count(resulted_schedule) } } if saver is not None: saver(data) return data
def coeff_run(): """ coefficient of compute/data intensivity """ # wf_names = ['Montage_25', 'CyberShake_30', 'Inspiral_30', 'Sipht_30', 'Epigenomics_24'] #all_coeffs = [1/100, 1/50, 1/10, 1/5, 1/2.766, 15, 20, 25, 30, 35, 40, 45, 75] all_coeffs = [1 / 100, 1 / 50, 1 / 10, 1 / 5, 1 / 2.766, 1, 2.766] + list( range(5, 101, 5)) # wf_names = [('Montage_25', [10]), # ('CyberShake_30', [0.1] + list(range(10, 46, 1))), # ('Inspiral_30', [10, 1, 1/2.766]), # ('Sipht_30', [0.02] + list(range(30, 50, 1)) + list(range(50, 101, 5))), # ('Epigenomics_24', list(range(5, 46, 1)) + list(range(45, 101, 5)))] # wf_names = [('Montage_25', [2.766])] wf_names = [ #('Montage_25', [10]), ('CyberShake_30', all_coeffs), #('Inspiral_30', [10, 1, 1/2.766]), #('Sipht_30', [0.02] + list(range(30, 50, 1)) + list(range(50, 101, 5))), ('Epigenomics_24', all_coeffs) ] def transfer_time(max_runtime, c): transfer = max_runtime * BASE_PARAMS["ideal_flops"] / c return transfer to_run = [] for wf_name, coeffs in wf_names: _wf = wf(wf_name) max_runtime = max(_wf.get_all_unique_tasks(), key=lambda x: x.runtime).runtime param_sets = [ copy_and_set(BASE_PARAMS, transfer_time=transfer_time(max_runtime, c), data_intensive_coeff=c) for c in coeffs ] exps = [partial(do_exp, wf_name, **params) for params in param_sets] to_run = to_run + exps # m_repeat = lambda n, funcs: [f() for f in funcs for _ in range(n)] #results = m_repeat(REPEAT_COUNT, to_run) results = multi_repeat(REPEAT_COUNT, to_run) saver = UniqueNameSaver(TEMP_PATH, EXPERIMENT_NAME) for result in results: saver(result) # coeff_aggregate(saver.directory, "coeff.png") pass
def do_exp(wf_name): _wf = wf(wf_name) rm = ExperimentResourceManager(rg.r([10, 15, 25, 30])) estimator = SimpleTimeCostEstimator(comp_time_cost=0, transf_time_cost=0, transferMx=None, ideal_flops=20, transfer_time=100) empty_fixed_schedule_part = Schedule({node: [] for node in rm.get_nodes()}) heft_schedule = run_heft(_wf, rm, estimator) ga_functions = GAFunctions2(_wf, rm, estimator) generate = partial(ga_generate, ga_functions=ga_functions, fixed_schedule_part=empty_fixed_schedule_part, current_time=0.0, init_sched_percent=0.05, initial_schedule=heft_schedule) stats = tools.Statistics(lambda ind: ind.fitness.values[0]) stats.register("avg", numpy.mean) stats.register("std", numpy.std) stats.register("min", numpy.min) stats.register("max", numpy.max) logbook = tools.Logbook() logbook.header = ["gen", "evals"] + stats.fields toolbox = Toolbox() toolbox.register("generate", generate) toolbox.register("evaluate", fit_converter(ga_functions.build_fitness(empty_fixed_schedule_part, 0.0))) toolbox.register("clone", deepcopy) toolbox.register("mate", ga_functions.crossover) toolbox.register("sweep_mutation", ga_functions.sweep_mutation) toolbox.register("mutate", ga_functions.mutation) # toolbox.register("select_parents", ) # toolbox.register("select", tools.selTournament, tournsize=4) toolbox.register("select", tools.selRoulette) pop, logbook, best = run_ga(toolbox=toolbox, logbook=logbook, stats=stats, **GA_PARAMS) resulted_schedule = ga_functions.build_schedule(best, empty_fixed_schedule_part, 0.0) Utility.validate_static_schedule(_wf, resulted_schedule) ga_makespan = Utility.makespan(resulted_schedule) return ga_makespan
def test_fixed_ordering(self): _wf = wf("Montage_25") rm = ExperimentResourceManager(rg.r([10, 15, 25, 30])) estimator = SimpleTimeCostEstimator(comp_time_cost=0, transf_time_cost=0, transferMx=None, ideal_flops=20, transfer_time=100) sorted_tasks = HeftHelper.heft_rank(_wf, rm, estimator) heft_schedule = run_heft(_wf, rm, estimator) heft_mapping = schedule_to_position(heft_schedule) heft_gen = lambda: heft_mapping if random.random() > 0.95 else generate(_wf, rm, estimator) toolbox = Toolbox() # toolbox.register("generate", generate, _wf, rm, estimator) toolbox.register("generate", heft_gen) toolbox.register("fitness", fitness, _wf, rm, estimator, sorted_tasks) toolbox.register("force_vector_matrix", force_vector_matrix, rm) toolbox.register("velocity_and_position", velocity_and_position, _wf, rm, estimator) toolbox.register("G", G) toolbox.register("kbest", Kbest) statistics = Statistics() statistics.register("min", lambda pop: numpy.min([p.fitness.mofit for p in pop])) statistics.register("avr", lambda pop: numpy.average([p.fitness.mofit for p in pop])) statistics.register("max", lambda pop: numpy.max([p.fitness.mofit for p in pop])) statistics.register("std", lambda pop: numpy.std([p.fitness.mofit for p in pop])) logbook = Logbook() logbook.header = ("gen", "G", "kbest", "min", "avr", "max", "std") pop_size = 100 iter_number = 100 kbest = pop_size ginit = 5 final_pop = run_gsa(toolbox, statistics, logbook, pop_size, iter_number, kbest, ginit) best = min(final_pop, key=lambda x: toolbox.fitness(x).mofit) solution = {MAPPING_SPECIE: list(zip(sorted_tasks, best)), ORDERING_SPECIE: sorted_tasks} schedule = build_schedule(_wf, estimator, rm, solution) Utility.validate_static_schedule(_wf, schedule) makespan = Utility.makespan(schedule) print("Final makespan: {0}".format(makespan)) pass
def coeff_run(): """ coefficient of compute/data intensivity """ # wf_names = ['Montage_25', 'CyberShake_30', 'Inspiral_30', 'Sipht_30', 'Epigenomics_24'] #all_coeffs = [1/100, 1/50, 1/10, 1/5, 1/2.766, 15, 20, 25, 30, 35, 40, 45, 75] all_coeffs = [1/100, 1/50, 1/10, 1/5, 1/2.766, 1, 2.766] + list(range(5, 101, 5)) # wf_names = [('Montage_25', [10]), # ('CyberShake_30', [0.1] + list(range(10, 46, 1))), # ('Inspiral_30', [10, 1, 1/2.766]), # ('Sipht_30', [0.02] + list(range(30, 50, 1)) + list(range(50, 101, 5))), # ('Epigenomics_24', list(range(5, 46, 1)) + list(range(45, 101, 5)))] # wf_names = [('Montage_25', [2.766])] wf_names = [#('Montage_25', [10]), ('CyberShake_30', all_coeffs), #('Inspiral_30', [10, 1, 1/2.766]), #('Sipht_30', [0.02] + list(range(30, 50, 1)) + list(range(50, 101, 5))), ('Epigenomics_24', all_coeffs)] def transfer_time(max_runtime, c): transfer = max_runtime * BASE_PARAMS["ideal_flops"] / c return transfer to_run = [] for wf_name, coeffs in wf_names: _wf = wf(wf_name) max_runtime = max(_wf.get_all_unique_tasks(), key=lambda x: x.runtime).runtime param_sets = [copy_and_set(BASE_PARAMS, transfer_time=transfer_time(max_runtime, c), data_intensive_coeff=c) for c in coeffs] exps = [partial(do_exp, wf_name, **params) for params in param_sets] to_run = to_run + exps # m_repeat = lambda n, funcs: [f() for f in funcs for _ in range(n)] #results = m_repeat(REPEAT_COUNT, to_run) results = multi_repeat(REPEAT_COUNT, to_run) saver = UniqueNameSaver(TEMP_PATH, EXPERIMENT_NAME) for result in results: saver(result) # coeff_aggregate(saver.directory, "coeff.png") pass
def _construct_environment(self, *args, **kwargs): wf_name = kwargs["wf_name"] nodes_conf = kwargs.get("nodes_conf", None) ideal_flops = kwargs.get("ideal_flops", 20) transfer_time = kwargs.get("transfer_time", 100) bandwidth = float(kwargs.get("data_intensive_coeff", 100)) max_size = float(kwargs.get("max_size", 100)) # dax1 = '../../resources/' + wf_name + '.xml' # wf = Utility.readWorkflow(dax1, wf_name) _wf = wf(wf_name) rgen = ResourceGenerator(min_res_count=1, max_res_count=1, min_node_count=4, max_node_count=4, min_flops=5, max_flops=10) resources = rgen.generate() transferMx = rgen.generateTransferMatrix(resources) if nodes_conf is None: bundle = Utility.get_default_bundle() resources = bundle.dedicated_resources transferMx = bundle.transfer_mx ideal_flops = bundle.ideal_flops ##TODO: end else: ## TODO: refactor it later. resources = ResourceGenerator.r(nodes_conf) transferMx = rgen.generateTransferMatrix(resources) ## estimator = TestEstimator(comp_time_cost=0, transf_time_cost=0, bandwidth=bandwidth, transferMx=None, ideal_flops=ideal_flops, transfer_time=transfer_time, max_size=max_size) resource_manager = ExperimentResourceManager(resources) return (_wf, resource_manager, estimator)
def do_exp(wf_name, **params): _wf = wf(wf_name) ga_makespan, heft_make, ga_schedule, heft_sched, logbook = MixRunner()(_wf, **params) rm = ExperimentResourceManager(rg.r(params["nodes_conf"])) estimator = TestEstimator(comp_time_cost=0, transf_time_cost=0, bandwidth=float(params["data_intensive_coeff"]), transferMx=None, ideal_flops=params["ideal_flops"], max_size=float(params["max_size"]), transfer_time=params["transfer_time"]) heft_schedule = run_heft(_wf, rm, estimator) heft_makespan = Utility.makespan(heft_schedule) # print("Heft schedule:") # print(heft_schedule) # print("Heft makespan: {0}".format(heft_makespan)) # print("GA schedule:") # print(ga_schedule) # print("GA makespan: {0}".format(ga_makespan)) data = { "wf_name": wf_name, "params": params, "heft": { "makespan": heft_makespan, "overall_transfer_time": Utility.overall_transfer_time(heft_schedule, _wf, estimator), "overall_execution_time": Utility.overall_execution_time(heft_schedule) }, "ga": { "makespan": ga_makespan, "overall_transfer_time": Utility.overall_transfer_time(ga_schedule, _wf, estimator), "overall_execution_time": Utility.overall_execution_time(ga_schedule) }, "logbook": logbook #"heft_schedule": heft_schedule, #"ga_schedule": ga_schedule } return data
def _construct_environment(self, *args, **kwargs): wf_name = kwargs["wf_name"] nodes_conf = kwargs.get("nodes_conf", None) ideal_flops = kwargs.get("ideal_flops", 20) transfer_time = kwargs.get("transfer_time", 100) # dax1 = '../../resources/' + wf_name + '.xml' # wf = Utility.readWorkflow(dax1, wf_name) _wf = wf(wf_name) rgen = ResourceGenerator(min_res_count=1, max_res_count=1, min_node_count=4, max_node_count=4, min_flops=5, max_flops=10) resources = rgen.generate() transferMx = rgen.generateTransferMatrix(resources) if nodes_conf is None: bundle = Utility.get_default_bundle() resources = bundle.dedicated_resources transferMx = bundle.transfer_mx ideal_flops = bundle.ideal_flops ##TODO: end else: ## TODO: refactor it later. resources = ResourceGenerator.r(nodes_conf) transferMx = rgen.generateTransferMatrix(resources) ## estimator = ExperimentEstimator(transferMx, ideal_flops, dict(), transfer_time) resource_manager = ExperimentResourceManager(resources) return (_wf, resource_manager, estimator)
def get_wf(wf_name, task_postfix_id="00"): return wf(wf_name, task_postfix_id)
from heft.algs.common.individuals import FitnessStd from heft.algs.common.mapordschedule import MAPPING_SPECIE, ORDERING_SPECIE, build_schedule from heft.algs.gsa.SimpleGsaScheme import run_gsa from heft.algs.gsa.operators import G, Kbest from heft.algs.gsa.setbasedoperators import force_vector_matrix, velocity_and_position from heft.algs.heft.DSimpleHeft import run_heft from heft.algs.heft.HeftHelper import HeftHelper from heft.algs.pso.sdpso import schedule_to_position, generate, fitness from heft.core.CommonComponents.ExperimentalManagers import ExperimentResourceManager from heft.core.environment.ResourceGenerator import ResourceGenerator as rg from heft.core.environment.Utility import wf, Utility from heft.experiments.cga.mobjective.utility import SimpleTimeCostEstimator from heft.experiments.cga.utilities.common import repeat _wf = wf("Montage_40") rm = ExperimentResourceManager(rg.r([10, 15, 25, 30])) estimator = SimpleTimeCostEstimator(comp_time_cost=0, transf_time_cost=0, transferMx=None, ideal_flops=20, transfer_time=100) sorted_tasks = HeftHelper.heft_rank(_wf, rm, estimator) heft_schedule = run_heft(_wf, rm, estimator) heft_mapping = schedule_to_position(heft_schedule) heft_gen = lambda n: [deepcopy(heft_mapping) if random.random() > 1.0 else generate(_wf, rm, estimator, 1)[0] for _ in range(n)] toolbox = Toolbox() # toolbox.register("generate", generate, _wf, rm, estimator) toolbox.register("generate", heft_gen) toolbox.register("fitness", fitness, _wf, rm, estimator, sorted_tasks) toolbox.register("force_vector_matrix", force_vector_matrix)
def do_island_inherited_pop_exp(alg_builder, mp_alg_builder, algorithm_builder, chromosome_cleaner_builder, schedule_to_chromosome_converter_builder, wf_name, **params): _wf = wf(wf_name) rm = ExperimentResourceManager(rg.r(params["resource_set"]["nodes_conf"])) estimator = SimpleTimeCostEstimator(**params["estimator_settings"]) chromosome_cleaner = chromosome_cleaner_builder(_wf, rm, estimator) dynamic_heft = DynamicHeft(_wf, rm, estimator) ga = alg_builder(_wf, rm, estimator, params["init_sched_percent"], log_book=None, stats=None, alg_params=params["alg_params"]) ## TODO: remake this part later. # def reverse_interface_adapter(func): # def wrap(*args, **kwargs): # (best, pop, resulted_schedule, _), logbook = func(*args, **kwargs) # return pop, logbook, best # return wrap mpga = mp_alg_builder( _wf, rm, estimator, params["init_sched_percent"], algorithm=partial(algorithm_builder, **params["alg_params"]), #algorithm=reverse_interface_adapter(ga), log_book=None, stats=None, alg_params=params["alg_params"]) kwargs = dict(params["executor_params"]) kwargs.update(params["alg_params"]) kwargs["ga_params"] = {"population": params["alg_params"]["n"]} machine = MPGaHeftOldPopExecutor(heft_planner=dynamic_heft, wf=_wf, resource_manager=rm, estimator=estimator, stat_saver=None, ga_builder=lambda: ga, mpga_builder=lambda: mpga, chromosome_cleaner=chromosome_cleaner, mpnewVSmpoldmode=False, mixed_init_pop=False, emigrant_selection=None, check_evolution_for_stopping=False, schedule_to_chromosome_converter= schedule_to_chromosome_converter_builder( wf, rm, estimator), **kwargs) machine.init() machine.run() resulted_schedule = machine.current_schedule stat_data = machine.executor_stat_data Utility.validate_dynamic_schedule(_wf, resulted_schedule) data = { "wf_name": wf_name, "params": params, "result": { "makespan": Utility.makespan(resulted_schedule), ## TODO: this function should be remade to adapt under conditions of dynamic env #"overall_transfer_time": Utility.overall_transfer_time(resulted_schedule, _wf, estimator), "overall_execution_time": Utility.overall_execution_time(resulted_schedule), "overall_failed_tasks_count": Utility.overall_failed_tasks_count(resulted_schedule) } } return data
def do_exp_heft_schedule(): res = do_exp_schedule(True) return res[0] def do_exp_ga_schedule(): res = do_exp_schedule(False) return (res[0], res[4]) if __name__ == '__main__': print("Population size: " + str(PARAMS["ga_params"]["population"])) _wf = wf(wf_names[0]) rm = ExperimentResourceManager(rg.r(PARAMS["nodes_conf"])) estimator = SimpleTimeCostEstimator(comp_time_cost=0, transf_time_cost=0, transferMx=None, ideal_flops=PARAMS["ideal_flops"], transfer_time=PARAMS["transfer_time"]) heft_schedule = run_heft(_wf, rm, estimator) heft_makespan = Utility.makespan(heft_schedule) overall_transfer = Utility.overall_transfer_time(heft_schedule, _wf, estimator) overall_execution = Utility.overall_execution_time(heft_schedule) print( "Heft makespan: {0}, Overall transfer time: {1}, Overall execution time: {2}"
name = saver(data) return ga_makespan, heft_makespan, ga_schedule, heft_schedule, name def do_exp_heft_schedule(): res = do_exp_schedule(True) return res[0] def do_exp_ga_schedule(): res = do_exp_schedule(False) return (res[0], res[4]) if __name__ == '__main__': print("Population size: " + str(PARAMS["ga_params"]["population"])) _wf = wf(wf_names[0]) rm = ExperimentResourceManager(rg.r(PARAMS["nodes_conf"])) estimator = SimpleTimeCostEstimator(comp_time_cost=0, transf_time_cost=0, transferMx=None, ideal_flops=PARAMS["ideal_flops"], transfer_time=PARAMS["transfer_time"]) heft_schedule = run_heft(_wf, rm, estimator) heft_makespan = Utility.makespan(heft_schedule) overall_transfer = Utility.overall_transfer_time(heft_schedule, _wf, estimator) overall_execution = Utility.overall_execution_time(heft_schedule) print("Heft makespan: {0}, Overall transfer time: {1}, Overall execution time: {2}".format(heft_makespan, overall_transfer, overall_execution)) if not only_heft: result = repeat(do_exp_heft_schedule, 1)
ordering_heft_based_initialize from heft.algs.ga.nsga2 import run_nsga2 from heft.algs.heft.DSimpleHeft import run_heft from heft.algs.heft.HeftHelper import HeftHelper from heft.algs.pso.gapso import run_gapso from heft.algs.pso.sdpso import run_pso, update, schedule_to_position, construct_solution, MappingParticle, \ Velocity, Position from heft.core.CommonComponents.ExperimentalManagers import ExperimentResourceManager from heft.core.environment.Utility import Utility, wf from heft.algs.common.mapordschedule import build_schedule, MAPPING_SPECIE, ORDERING_SPECIE, ordering_from_schedule, \ mapping_from_schedule from heft.experiments.cga.mobjective.utility import SimpleTimeCostEstimator from heft.core.environment.ResourceGenerator import ResourceGenerator as rg from heft.algs.common.mapordschedule import fitness as basefitness _wf = wf("Montage_50") rm = ExperimentResourceManager(rg.r([10, 15, 25, 30])) estimator = SimpleTimeCostEstimator(comp_time_cost=0, transf_time_cost=0, transferMx=None, ideal_flops=20, transfer_time=100) sorted_tasks = HeftHelper.heft_rank(_wf, rm, estimator) heft_schedule = run_heft(_wf, rm, estimator) print(Utility.makespan(heft_schedule)) stats = tools.Statistics(lambda ind: ind.fitness.values[0]) stats.register("avg", numpy.mean) stats.register("std", numpy.std)
def do_inherited_pop_exp(saver, alg_builder, chromosome_cleaner_builder, wf_name, **params): _wf = wf(wf_name) rm = ExperimentResourceManager(rg.r(params["resource_set"]["nodes_conf"])) estimator = SimpleTimeCostEstimator(**params["estimator_settings"]) chromosome_cleaner = chromosome_cleaner_builder(_wf, rm, estimator) dynamic_heft = DynamicHeft(_wf, rm, estimator) logbook = Logbook() stats = tools.Statistics(lambda ind: ind.fitness.values[0]) stats.register("avg", numpy.mean) stats.register("std", numpy.std) stats.register("min", numpy.min) stats.register("max", numpy.max) ga = alg_builder(_wf, rm, estimator, params["init_sched_percent"], log_book=logbook, stats=stats, alg_params=params["alg_params"]) machine = GaHeftOldPopExecutor(heft_planner=dynamic_heft, wf=_wf, resource_manager=rm, estimator=estimator, stat_saver=None, ga_builder=lambda: ga, chromosome_cleaner=chromosome_cleaner, **params["executor_params"]) machine.init() print("Executor start") machine.run() print("Executor stop") resulted_schedule = machine.current_schedule stat_data = machine.executor_stat_data Utility.validate_dynamic_schedule(_wf, resulted_schedule) data = { "wf_name": wf_name, "params": params, "result": { "random_init_logbook": stat_data["random_init_logbook"], "inherited_init_logbook": stat_data["inherited_init_logbook"], "makespan": Utility.makespan(resulted_schedule), ## TODO: this function should be remade to adapt under conditions of dynamic env #"overall_transfer_time": Utility.overall_transfer_time(resulted_schedule, _wf, estimator), "overall_execution_time": Utility.overall_execution_time(resulted_schedule), "overall_failed_tasks_count": Utility.overall_failed_tasks_count(resulted_schedule) } } if saver is not None: saver(data) return data
import numpy from heft.algs.common.particle_operations import MappingParticle from heft.algs.heft.DSimpleHeft import run_heft from heft.algs.heft.HeftHelper import HeftHelper from heft.algs.pso.mapping_operators import schedule_to_position, fitness, update, generate, construct_solution from heft.algs.pso.sdpso import run_pso from heft.core.CommonComponents.ExperimentalManagers import ExperimentResourceManager from heft.core.environment.Utility import Utility, wf from heft.algs.common.mapordschedule import build_schedule, MAPPING_SPECIE, ORDERING_SPECIE from heft.experiments.aggregate_utilities import interval_statistics, interval_stat_string from heft.experiments.cga.mobjective.utility import SimpleTimeCostEstimator from heft.core.environment.ResourceGenerator import ResourceGenerator as rg from heft.experiments.cga.utilities.common import repeat _wf = wf("Montage_75") rm = ExperimentResourceManager(rg.r([10, 15, 25, 30])) estimator = SimpleTimeCostEstimator(comp_time_cost=0, transf_time_cost=0, transferMx=None, ideal_flops=20, transfer_time=100) sorted_tasks = HeftHelper.heft_rank(_wf, rm, estimator) heft_schedule = run_heft(_wf, rm, estimator) heft_mapping = schedule_to_position(heft_schedule) heft_mapping.velocity = MappingParticle.Velocity({}) heft_gen = lambda n: [ deepcopy(heft_mapping)
from heft.algs.ga.GAImplementation.GAImpl import GAFactory from heft.core.CommonComponents.ExperimentalManagers import ExperimentResourceManager, ExperimentEstimator from heft.core.environment import ResourceGenerator from heft.core.environment.ResourceManager import Schedule from heft.core.environment.Utility import profile_decorator, wf _wf = wf("Montage_100") resources = ResourceGenerator.r([10, 15, 25, 30]) resource_manager = ExperimentResourceManager(resources) estimator = ExperimentEstimator(None, ideal_flops=20, reliability=1.0, transfer_time=100) ga = GAFactory.default().create_ga(silent=True, wf=_wf, resource_manager=resource_manager, estimator=estimator, ga_params={ "population": 10, "crossover_probability": 0.8, "replacing_mutation_probability": 0.5, "sweep_mutation_probability": 0.4, "generations": 20 }) @profile_decorator def fnc(): empty_schedule = Schedule( {node: []
from deap import tools from heft.algs.common.individuals import ListBasedIndividual from heft.algs.ga.coevolution.cga import Env, Specie from heft.algs.ga.coevolution.operators import MAPPING_SPECIE, ORDERING_SPECIE, build_schedule, mapping_default_initialize, overhead_fitness_mapping_and_ordering, assign_from_transfer_overhead, mapping_all_mutate, default_build_solutions from heft.core.CommonComponents.ExperimentalManagers import ExperimentResourceManager, ExperimentEstimator from heft.core.environment.Utility import wf from heft.experiments.cga.cga_exp import os_ideal_ind, ms_ideal_ind, do_experiment from heft.experiments.cga.utilities.common import UniqueNameSaver, repeat, extract_ordering_from_file, ArchivedSelector, roulette, build_ms_ideal_ind, build_os_ideal_ind, hamming_distances, unique_individuals, to_seq, pcm, gdm, hamming_for_best_components, best_components_itself from heft.settings import __root_path__ from heft.core.environment import Utility _wf = wf("Montage_25") rm = ExperimentResourceManager(rg.r([10, 15, 25, 30])) estimator = ExperimentEstimator(None, ideal_flops=20, transfer_time=100) # selector = ArchivedSelector(5)(tourn) selector = ArchivedSelector(5)(roulette) os_representative = extract_ordering_from_file("{0}/temp/cga_exp_example/6685a2b2-78d6-4637-b099-ed91152464f5.json".format(__root_path__), _wf, estimator, rm) ms_ideal_ind = build_ms_ideal_ind(_wf, rm) os_ideal_ind = build_os_ideal_ind(_wf) # heft_mapping = extract_mapping_from_file("../../temp/heft_etalon_tr100.json") saver = UniqueNameSaver("{0}/temp/cga_improving_mutation".format(__root_path__))
def coeff_run(): """ coefficient of compute/data intensivity """ # wf_names = ['Montage_25', 'CyberShake_30', 'Inspiral_30', 'Sipht_30', 'Epigenomics_24'] #all_coeffs = [1/100, 1/50, 1/10, 1/5, 1/2.766, 15, 20, 25, 30, 35, 40, 45, 75] all_coeffs = [1, 5, 10, 25, 50, 100] # all_coeffs = [0.0005] # flops = [1, 10, 20, 30, 50, 100] transfer_coeffs = [0.1, 0.2, 0.5, 1, 2, 5, 10] # wf_names = [('Montage_25', [10]), # ('CyberShake_30', [0.1] + list(range(10, 46, 1))), # ('Inspiral_30', [10, 1, 1/2.766]), # ('Sipht_30', [0.02] + list(range(30, 50, 1)) + list(range(50, 101, 5))), # ('Epigenomics_24', list(range(5, 46, 1)) + list(range(45, 101, 5)))] # wf_names = [('Montage_25', [2.766])] wf_names = [('Montage_25', all_coeffs), ('CyberShake_30', all_coeffs), ('Inspiral_30', all_coeffs), ('Sipht_30', all_coeffs), ('Epigenomics_24', all_coeffs)] def transfer_time(max_runtime, flops, max_transfer_size, bandwidth): real_bandwidth = bandwidth / 8 * 1000000 transfer = max_runtime * flops / (max_transfer_size / real_bandwidth ) return transfer def compute_max_transfer_size(head_task): max_size = 0 if head_task.output_files != None: for task in head_task.children: parent_keys = set(head_task.output_files.keys()) children_keys = set(task.input_files.keys()) parent_output_as_child_input = parent_keys.intersection(children_keys) if parent_output_as_child_input != None: size = 0 for file_name in parent_output_as_child_input: size += head_task.output_files[file_name].size if size > max_size: max_size = size for task in head_task.children: size = compute_max_transfer_size(task) if size > max_size: max_size = size return max_size def compute_mean_transfer_time(head_task): sizes = [] if head_task.output_files != None: for task in head_task.children: parent_keys = set(head_task.output_files.keys()) children_keys = set(task.input_files.keys()) parent_output_as_child_input = parent_keys.intersection(children_keys) if parent_output_as_child_input != None: size = 0 for file_name in parent_output_as_child_input: size += head_task.output_files[file_name].size sizes.append(size) for task in head_task.children: size = compute_max_transfer_size(task) sizes.append(size) percentile = numpy.percentile(sizes, 0.95) # mean_size = statistics.mean(sizes) # return mean_size return percentile def compute_mean_runtime(tasks): times = [] for task in tasks: times.append(task.runtime) percentile = numpy.percentile(times, 0.9) # return statistics.mean(times) return percentile def compute_flops(max_runtime, max_size, bandwidth, coeff): # real_bandwidth = bandwidth / 8 * 1000000 # flops = coeff * max_size / (max_runtime * real_bandwidth) flops = coeff * bandwidth / max_runtime return flops to_run = [] experiment_id = uuid.uuid4() for wf_name, coeffs in wf_names: _wf = wf(wf_name) max_size = compute_max_transfer_size(_wf.head_task) mean_size = compute_mean_transfer_time(_wf.head_task) max_runtime = max(_wf.get_all_unique_tasks(), key=lambda x: x.runtime).runtime mean_runtime = compute_mean_runtime(_wf.get_all_unique_tasks()) param_sets = [copy_and_set(BASE_PARAMS, max_size=max_size, data_intensive_coeff=c) for c in all_coeffs] exps = [partial(do_exp_and_save, do_exp, experiment_id, wf_name, **params) for params in param_sets] to_run = to_run + exps # m_repeat = lambda n, funcs: [f() for f in funcs for _ in range(n)] # results = m_repeat(REPEAT_COUNT, to_run) multi_repeat(REPEAT_COUNT, to_run) # saver = UniqueNameSaver(TEMP_PATH, EXPERIMENT_NAME) # saver = MongoSaver(MONGO_SERVER_ADDRESS, EXPERIMENT_NAME, EXPERIMENT_ID) # for result in results: # saver(result) print("Experiment ID: {0}".format(experiment_id)) # coeff_aggregate(saver.directory, "coeff.png") pass
from deap import base from deap import creator from deap import tools from heft.algs.common.individuals import ListBasedIndividual from heft.algs.ga.coevolution.cga import Env from heft.algs.ga.coevolution.operators import mapping_heft_based_initialize, ordering_heft_based_initialize from heft.core.CommonComponents.ExperimentalManagers import ExperimentResourceManager from heft.core.environment import ResourceGenerator as rg from heft.core.environment.Utility import wf from heft.experiments.cga.mobjective.utility import fitness_makespan_and_cost_map_ord, SimpleTimeCostEstimator from heft.experiments.cga.utilities.common import extract_ordering_from_ga_file, extract_mapping_from_ga_file from heft.experiments.cga.utilities.double_chromosome_ga import _mate, _mutate _wf = wf("Montage_100") # rm = ExperimentResourceManager(VMResGen.r([10, 15, 25, 30], 4)) rm = ExperimentResourceManager(rg.r([10, 15, 25, 30])) # estimator = ExperimentEstimator(None, ideal_flops=20, transfer_time=100) estimator = SimpleTimeCostEstimator(comp_time_cost=10, transf_time_cost=2, transferMx=None, ideal_flops=20, transfer_time=100) env = Env(_wf, rm, estimator) pop_size = 200 NGEN = 300 MU = 40 CXPB = 0.9 heft_mapping = extract_mapping_from_ga_file("../../../temp/heft_etalon_full_tr100_m100.json", rm) heft_ordering = extract_ordering_from_ga_file("../../../temp/heft_etalon_full_tr100_m100.json")
from heft.algs.heft.DSimpleHeft import run_heft from heft.algs.heft.HeftHelper import HeftHelper from heft.algs.pso.sdpso import schedule_to_position, generate from heft.algs.sa.SimulatedAnnealingScheme import run_sa from heft.algs.sa.mappingops import energy, update_T, mapping_neighbor, transition_probability, State from heft.core.CommonComponents.ExperimentalManagers import ExperimentResourceManager from heft.core.environment.Utility import Utility, wf from heft.algs.common.mapordschedule import build_schedule, MAPPING_SPECIE, ORDERING_SPECIE from heft.experiments.cga.mobjective.utility import SimpleTimeCostEstimator from heft.core.environment.ResourceGenerator import ResourceGenerator as rg ## TODO: need to test all of it from heft.experiments.cga.utilities.common import repeat _wf = wf("Montage_25") rm = ExperimentResourceManager(rg.r([10, 15, 25, 30])) estimator = SimpleTimeCostEstimator(comp_time_cost=0, transf_time_cost=0, transferMx=None, ideal_flops=20, transfer_time=100) sorted_tasks = HeftHelper.heft_rank(_wf, rm, estimator) heft_schedule = run_heft(_wf, rm, estimator) heft_mapping = schedule_to_position(heft_schedule).entity initial_state = State() initial_state.mapping = heft_mapping # initial_state.mapping = generate(_wf, rm, estimator, 1)[0].entity initial_state.ordering = sorted_tasks