def ga_generate(wf, rm, estimator, schedule=None, fixed_schedule_part=None, current_time=0.0): if schedule is not None: return GAFunctions2.schedule_to_chromosome(schedule, fixed_schedule_part) gafunctions = GAFunctions2(wf, rm, estimator) return gafunctions.build_initial(fixed_schedule_part, current_time)
def generate(n, ga_functions, fixed_schedule_part, current_time, init_sched_percent, initial_schedule): init_ind_count = int(n*init_sched_percent) res = [] if initial_schedule is not None and init_ind_count > 0: ga_functions.initial_chromosome = DictBasedIndividual(GAFunctions2.schedule_to_chromosome(initial_schedule)) init_chromosome = ga_functions.initial_chromosome init_arr = [copy.deepcopy(init_chromosome) for _ in range(init_ind_count)] res = res + init_arr if n - init_ind_count > 0: generated_arr = [DictBasedIndividual(ga_functions.random_chromo(fixed_schedule_part, current_time)) for _ in range(n - init_ind_count)] res = res + generated_arr return res
def generate(n, ga_functions, fixed_schedule_part, current_time, init_sched_percent, initial_schedule): init_ind_count = int(n * init_sched_percent) res = [] if initial_schedule is not None and init_ind_count > 0: ga_functions.initial_chromosome = DictBasedIndividual( GAFunctions2.schedule_to_chromosome(initial_schedule)) init_chromosome = ga_functions.initial_chromosome init_arr = [ copy.deepcopy(init_chromosome) for _ in range(init_ind_count) ] res = res + init_arr if n - init_ind_count > 0: generated_arr = [ DictBasedIndividual( ga_functions.random_chromo(fixed_schedule_part, current_time)) for _ in range(n - init_ind_count) ] res = res + generated_arr return res
def _actual_ga_run(self): heft_initial = GAFunctions2.schedule_to_chromosome(self.back_cmp.current_schedule) heft_initial = self._clean_chromosome(heft_initial, self.back_cmp.event, self.back_cmp.fixed_schedule) ##===================================== ##Regular GA ##===================================== # print("GaHeft WITH NEW POP: ") # ga = self._get_ga_alg() # ((best_r, pop_r, schedule_r, stopped_iteration_r), logbook_r) = ga(ga_calc.fixed_schedule_part, ga_calc.initial_schedule, current_time) ## TODO: make here using of param to decide how to build initial population for regular gaheft ## TODO: self.mixed_init_pop doesn't exist and isn't setted anywhere ## TODO: need to refactor and merge with MPGA.py initial_pop_gaheft = None if self.mixed_init_pop is True: heft_initial = tools.initIterate(creator.Individual, lambda: heft_initial) ga_functions = GAFunctions2(self.workflow, self.resource_manager, self.estimator) heft_pop = [ga_functions.mutation(deepcopy(heft_initial)) for i in range(self.params["population"])] cleaned_schedule = self.back_cmp.fixed_schedule initial_pop_gaheft = [self._clean_chromosome(deepcopy(p), self.back_cmp.event, cleaned_schedule) for p in self.past_pop] + heft_pop print("GaHeft WITH NEW POP: ") if self.mpnewVSmpoldmode is True: print("using multi population gaheft...") ga = self.mpga_builder() ((best_r, pop_r, schedule_r, stopped_iteration_r), logbook_r) = ga(self.back_cmp.fixed_schedule, None, self.current_time, initial_population=None, only_new_pops=True) else: print("using single population gaheft...") ga = self.ga_builder() ((best_r, pop_r, schedule_r, stopped_iteration_r), logbook_r) = ga(self.back_cmp.fixed_schedule, None, self.current_time, initial_population=initial_pop_gaheft) ##===================================== ##Old pop GA ##===================================== print("GaHeft WITH OLD POP: ") cleaned_schedule = self.back_cmp.fixed_schedule initial_pop = [self._clean_chromosome(ind, self.back_cmp.event, cleaned_schedule) for ind in self.past_pop] result = self.mpga_builder()(self.back_cmp.fixed_schedule, heft_initial, self.current_time, initial_population=initial_pop) ((best_op, pop_op, schedule_op, stopped_iteration_op), logbook_op) = result self.past_pop = pop_op ##===================================== ##Save stat to stat_saver ##===================================== ## TODO: make exception here task_id = "" if not hasattr(self.current_event, 'task') else str(self.current_event.task.id) if self.stat_saver is not None: ## TODO: correct pop_agr later stat_data = { "wf_name": self.workflow.name, "event_name": self.current_event.__class__.__name__, "task_id": task_id, "with_old_pop": { "iter": stopped_iteration_op, "makespan": Utility.makespan(schedule_op), "pop_aggr": logbook_op }, "with_random": { "iter": stopped_iteration_r, "makespan": Utility.makespan(schedule_r), "pop_aggr": logbook_r } } self.stat_saver(stat_data) self._stop_ga() return result
def __call__(self, fixed_schedule_part, initial_schedule, current_time=0, initial_population=None, only_new_pops=False): ##========================== ## create populations ##========================== ##TODO: remake it creator.create("FitnessMax", base.Fitness, weights=(1.0, )) creator.create("Individual", dict, fitness=creator.FitnessMax) toolbox = base.Toolbox() toolbox.register( "evaluate", ga_functions.build_fitness(fixed_schedule_part, current_time)) toolbox.register( "attr_bool", ga_functions.build_initial(fixed_schedule_part, current_time)) # Structure initializers toolbox.register("individual", tools.initIterate, creator.Individual, toolbox.attr_bool) toolbox.register("population", tools.initRepeat, list, toolbox.individual) ## TODO: replace it with strategy and specilized builder if only_new_pops is True: ## TODO: replace this magic number with parameter populations = [toolbox.population(n=POPSIZE) for i in range(3)] else: ## create populations old_pop = initial_population newpop = toolbox.population(n=POPSIZE) #heft_initial = GAFunctions2.schedule_to_chromosome(initial_schedule) # TODO: this is a stub. Rearchitect information flows and entities responsibilities as soon as you will get the first positive results. heft_initial = initial_schedule if not isinstance( initial_schedule, Schedule) else GAFunctions2.schedule_to_chromosome( initial_schedule, fixed_schedule_part) heft_initial = tools.initIterate(creator.Individual, lambda: heft_initial) heft_pop = [ ga_functions.mutation(deepcopy(heft_initial)) for i in range(POPSIZE) ] populations = [old_pop, newpop, heft_pop] ## TODO: replace this more effective implementation def quick_save(): whole_pop = reduce(lambda x, y: x + y, populations) ## choose new old pop for the next run sorted_whole_pop = sorted(whole_pop, key=lambda x: x.fitness.values, reverse=True) best = sorted_whole_pop[0] new_oldpop = sorted_whole_pop[0:POPSIZE] ## construct final result ## TODO: implement constructor of final result here result = ((best, new_oldpop, None, None), common_logbook) self._save_result(result) return result ##========================== ## run for several migration periods ##========================== common_logbook = tools.Logbook() result = None for k in range(MIGRATIONS): new_pops = [] iter_map = {} for pop in populations: ((best, npop, schedule, stopped_iteration), logbook) = ga_alg(fixed_schedule_part, None, current_time=current_time, initial_population=pop) new_pops.append(npop) # for rec in logbook: # iter = k*GENERATIONS + rec["iter"] # mp = iter_map.get(iter, []) # mp.append({"worst": rec["worst"], "best": rec["best"], "avr": rec["avr"]}) # iter_map[iter] = mp # pass for iter, items in iter_map.items(): best = max(it["best"] for it in items) avr = sum(it["avr"] for it in items) / len(items) worst = min(it["worst"] for it in items) common_logbook.record(iter=iter, worst=worst, best=best, avr=avr) pass populations = new_pops migRing(populations, migrCount, emigrant_selection) result = quick_save() pass # merge all populations in one and evaluate for some time # TODO: test this changes common_pop = functools.reduce(operator.add, populations, []) for k in range(MERGED_POP): ((best, npop, schedule, stopped_iteration), logbook) = ga_alg(fixed_schedule_part, None, current_time=current_time, initial_population=common_pop) common_pop = npop # for rec in logbook: # iter = (all_iters_count - MERGED_POP) + rec["iter"] # common_logbook.record(iter=iter, worst=rec["worst"], best=rec["best"], avr=rec["avr"]) # pass result = quick_save() pass ((best, new_oldpop, x1, x2), x3) = result result = ((best, new_oldpop, ga_functions.build_schedule(best, fixed_schedule_part, current_time), None), common_logbook) self._save_result(result[0]) return result
def __call__(self, fixed_schedule_part, initial_schedule, current_time=0, initial_population=None, only_new_pops=False): ##========================== ## create populations ##========================== ##TODO: remake it creator.create("FitnessMax", base.Fitness, weights=(1.0,)) creator.create("Individual", dict, fitness=creator.FitnessMax) toolbox = base.Toolbox() toolbox.register("evaluate", ga_functions.build_fitness(fixed_schedule_part, current_time)) toolbox.register("attr_bool", ga_functions.build_initial(fixed_schedule_part, current_time)) # Structure initializers toolbox.register("individual", tools.initIterate, creator.Individual, toolbox.attr_bool) toolbox.register("population", tools.initRepeat, list, toolbox.individual) ## TODO: replace it with strategy and specilized builder if only_new_pops is True: ## TODO: replace this magic number with parameter populations = [toolbox.population(n=POPSIZE) for i in range(3)] else: ## create populations old_pop = initial_population newpop = toolbox.population(n=POPSIZE) #heft_initial = GAFunctions2.schedule_to_chromosome(initial_schedule) # TODO: this is a stub. Rearchitect information flows and entities responsibilities as soon as you will get the first positive results. heft_initial = initial_schedule if not isinstance(initial_schedule, Schedule) else GAFunctions2.schedule_to_chromosome(initial_schedule, fixed_schedule_part) heft_initial = tools.initIterate(creator.Individual, lambda: heft_initial) heft_pop = [ga_functions.mutation(deepcopy(heft_initial)) for i in range(POPSIZE)] populations = [old_pop, newpop, heft_pop] ## TODO: replace this more effective implementation def quick_save(): whole_pop = reduce(lambda x, y: x+y, populations) ## choose new old pop for the next run sorted_whole_pop = sorted(whole_pop, key=lambda x: x.fitness.values, reverse=True) best = sorted_whole_pop[0] new_oldpop = sorted_whole_pop[0:POPSIZE] ## construct final result ## TODO: implement constructor of final result here result = ((best, new_oldpop, None, None), common_logbook) self._save_result(result) return result ##========================== ## run for several migration periods ##========================== common_logbook = tools.Logbook() result = None for k in range(MIGRATIONS): new_pops = [] iter_map = {} for pop in populations: ((best, npop, schedule, stopped_iteration), logbook) = ga_alg(fixed_schedule_part, None, current_time=current_time, initial_population=pop) new_pops.append(npop) # for rec in logbook: # iter = k*GENERATIONS + rec["iter"] # mp = iter_map.get(iter, []) # mp.append({"worst": rec["worst"], "best": rec["best"], "avr": rec["avr"]}) # iter_map[iter] = mp # pass for iter, items in iter_map.items(): best = max(it["best"] for it in items) avr = sum(it["avr"] for it in items)/len(items) worst = min(it["worst"] for it in items) common_logbook.record(iter=iter, worst=worst, best=best, avr=avr) pass populations = new_pops migRing(populations, migrCount, emigrant_selection) result = quick_save() pass # merge all populations in one and evaluate for some time # TODO: test this changes common_pop = functools.reduce(operator.add, populations, []) for k in range(MERGED_POP): ((best, npop, schedule, stopped_iteration), logbook) = ga_alg(fixed_schedule_part, None, current_time=current_time, initial_population=common_pop) common_pop = npop # for rec in logbook: # iter = (all_iters_count - MERGED_POP) + rec["iter"] # common_logbook.record(iter=iter, worst=rec["worst"], best=rec["best"], avr=rec["avr"]) # pass result = quick_save() pass ((best, new_oldpop, x1, x2), x3) = result result = ((best, new_oldpop, ga_functions.build_schedule(best, fixed_schedule_part, current_time), None), common_logbook) self._save_result(result[0]) return result
def __call__(self, fixed_schedule_part, initial_schedule, current_time=0, initial_population=None): print("Evaluating...") toolbox.register("evaluate", ga_functions.build_fitness(fixed_schedule_part, current_time)) toolbox.register("attr_bool", ga_functions.build_initial(fixed_schedule_part, current_time)) # Structure initializers toolbox.register("individual", tools.initIterate, creator.Individual, toolbox.attr_bool) toolbox.register("population", tools.initRepeat, list, toolbox.individual) ga_functions.initial_chromosome = GAFunctions2.schedule_to_chromosome(initial_schedule, fixed_schedule_part) if initial_population is None: initial_population = [] if ga_functions.initial_chromosome is None: print("empty_init_solutions") init_solutions = [] else: init_solutions = [creator.Individual(copy.deepcopy(ga_functions.initial_chromosome)) for _ in range(int(POPSIZE*0.9))] pop = initial_population + toolbox.population(n=POPSIZE - len(initial_population) - len(init_solutions)) + init_solutions ## TODO: experimental change history = History() # Decorate the variation operators #toolbox.decorate("mate", history.decorator) # toolbox.decorate("mutate", history.decorator) # Create the population and populate the history #history.update(pop) #=================================================== hallOfFame = deap.tools.HallOfFame(5) stats = tools.Statistics(key=lambda x: 1/x.fitness.values[0]) stats.register("min", numpy.min) stats.register("max", numpy.max) stats.register("avr", numpy.mean) stats.register("std", numpy.std) logbook = tools.Logbook() logbook.header = ["gen"] + stats.fields # Evaluate the entire population fitnesses = list(map(toolbox.evaluate, pop)) for ind, fit in zip(pop, fitnesses): ind.fitness.values = fit previous_raised_avr_individuals = [] # Begin the evolution for g in range(NGEN): # print("Iteration") if self.is_stopped(): break hallOfFame.update(pop) # logbook.record(pop=copy.deepcopy(pop)) # check if evolution process has stopped # if (check_evolution_for_stopping is True) and len(previous_raised_avr_individuals) == repeated_best_count: # length = len(previous_raised_avr_individuals) # whole_sum = sum(previous_raised_avr_individuals) # mean = whole_sum / length # sum2 = sum(abs(x - mean) for x in previous_raised_avr_individuals) # std = sum2/length # ## TODO: uncomment it later. output # # print("std: " + str(std)) # if std < 0.0001: # print(" Evolution process has stopped at " + str(g) + " iteration") # res = self._get_result() # extended_result = (res[0], res[1], res[2], res[3], g) # self._save_result(extended_result) # break # print("-- Generation %i --" % g) # Select the next generation individuals offspring = pop#toolbox.select(pop, len(pop)) # Clone the selected individuals offspring = list(map(toolbox.clone, offspring)) # Apply crossover and mutation on the offspring for child1, child2 in zip(offspring[::2], offspring[1::2]): if random.random() < CXPB: toolbox.mate(child1, child2) del child1.fitness.values del child2.fitness.values for mutant in offspring: if random.random() < SWEEPMUTPB: ga_functions.sweep_mutation(mutant) del mutant.fitness.values continue if random.random() < MUTPB: toolbox.mutate(mutant) del mutant.fitness.values # Evaluate the individuals with an invalid fitness invalid_ind = [ind for ind in offspring if not ind.fitness.valid] fitnesses = list(map(toolbox.evaluate, invalid_ind)) for ind, fit in zip(invalid_ind, fitnesses): ind.fitness.values = fit #pop[:] = offspring # mix with the best individuals of the time sorted_pop = sorted(pop + list(hallOfFame) + offspring, key=lambda x: x.fitness.values, reverse=True) pop = sorted_pop[:Kbest:] + toolbox.select(sorted_pop[Kbest:], POPSIZE - Kbest) # Gather all the fitnesses in one list and print the stats fits = [ind.fitness.values[0] for ind in pop] length = len(pop) mean = sum(fits) / length sum2 = sum(x*x for x in fits) std = abs(sum2 / length - mean**2)**0.5 worst = 1/min(fits) best = 1/max(fits) avr = 1/mean data = stats.compile(pop) logbook.record(gen=g, **data) if not is_silent: print(logbook.stream) # print("-- Generation %i --" % g) # print(" Worst %s" % str(worst)) # print(" Best %s" % str(best)) # print(" Avg %s" % str(avr)) # print(" Std %s" % str(1/std)) best = self._find_best(pop) # the last component is iteration number when evolution stopped result = (best, pop, fixed_schedule_part, current_time, g) self._save_result(result) self._save_pop(pop) if len(previous_raised_avr_individuals) == repeated_best_count: previous_raised_avr_individuals = previous_raised_avr_individuals[1::] previous_raised_avr_individuals.append(1/mean) pass # # import matplotlib.pyplot as plt # import networkx # # graph = networkx.DiGraph(history.genealogy_tree) # graph = graph.reverse() # Make the grah top-down # colors = [toolbox.evaluate(history.genealogy_history[i])[0] for i in graph] # networkx.draw(graph, node_color=colors) # plt.show() # best = self._find_best(pop) # self._save_result((best, pop, fixed_schedule_part, current_time)) ## return the best fitted individual and resulted population print("Ready") return self.get_result(), logbook
def __call__(self, fixed_schedule_part, initial_schedule, current_time=0, initial_population=None): print("Evaluating...") toolbox.register( "evaluate", ga_functions.build_fitness(fixed_schedule_part, current_time)) toolbox.register( "attr_bool", ga_functions.build_initial(fixed_schedule_part, current_time)) # Structure initializers toolbox.register("individual", tools.initIterate, creator.Individual, toolbox.attr_bool) toolbox.register("population", tools.initRepeat, list, toolbox.individual) ga_functions.initial_chromosome = GAFunctions2.schedule_to_chromosome( initial_schedule, fixed_schedule_part) if initial_population is None: initial_population = [] if ga_functions.initial_chromosome is None: print("empty_init_solutions") init_solutions = [] else: init_solutions = [ creator.Individual( copy.deepcopy(ga_functions.initial_chromosome)) for _ in range(int(POPSIZE * 0.9)) ] pop = initial_population + toolbox.population( n=POPSIZE - len(initial_population) - len(init_solutions)) + init_solutions ## TODO: experimental change history = History() # Decorate the variation operators #toolbox.decorate("mate", history.decorator) # toolbox.decorate("mutate", history.decorator) # Create the population and populate the history #history.update(pop) #=================================================== hallOfFame = deap.tools.HallOfFame(5) stats = tools.Statistics(key=lambda x: 1 / x.fitness.values[0]) stats.register("min", numpy.min) stats.register("max", numpy.max) stats.register("avr", numpy.mean) stats.register("std", numpy.std) logbook = tools.Logbook() logbook.header = ["gen"] + stats.fields # Evaluate the entire population fitnesses = list(map(toolbox.evaluate, pop)) for ind, fit in zip(pop, fitnesses): ind.fitness.values = fit previous_raised_avr_individuals = [] # Begin the evolution for g in range(NGEN): # print("Iteration") if self.is_stopped(): break hallOfFame.update(pop) # logbook.record(pop=copy.deepcopy(pop)) # check if evolution process has stopped # if (check_evolution_for_stopping is True) and len(previous_raised_avr_individuals) == repeated_best_count: # length = len(previous_raised_avr_individuals) # whole_sum = sum(previous_raised_avr_individuals) # mean = whole_sum / length # sum2 = sum(abs(x - mean) for x in previous_raised_avr_individuals) # std = sum2/length # ## TODO: uncomment it later. output # # print("std: " + str(std)) # if std < 0.0001: # print(" Evolution process has stopped at " + str(g) + " iteration") # res = self._get_result() # extended_result = (res[0], res[1], res[2], res[3], g) # self._save_result(extended_result) # break # print("-- Generation %i --" % g) # Select the next generation individuals offspring = pop #toolbox.select(pop, len(pop)) # Clone the selected individuals offspring = list(map(toolbox.clone, offspring)) # Apply crossover and mutation on the offspring for child1, child2 in zip(offspring[::2], offspring[1::2]): if random.random() < CXPB: toolbox.mate(child1, child2) del child1.fitness.values del child2.fitness.values for mutant in offspring: if random.random() < SWEEPMUTPB: ga_functions.sweep_mutation(mutant) del mutant.fitness.values continue if random.random() < MUTPB: toolbox.mutate(mutant) del mutant.fitness.values # Evaluate the individuals with an invalid fitness invalid_ind = [ ind for ind in offspring if not ind.fitness.valid ] fitnesses = list(map(toolbox.evaluate, invalid_ind)) for ind, fit in zip(invalid_ind, fitnesses): ind.fitness.values = fit #pop[:] = offspring # mix with the best individuals of the time sorted_pop = sorted(pop + list(hallOfFame) + offspring, key=lambda x: x.fitness.values, reverse=True) pop = sorted_pop[:Kbest:] + toolbox.select( sorted_pop[Kbest:], POPSIZE - Kbest) # Gather all the fitnesses in one list and print the stats fits = [ind.fitness.values[0] for ind in pop] length = len(pop) mean = sum(fits) / length sum2 = sum(x * x for x in fits) std = abs(sum2 / length - mean**2)**0.5 worst = 1 / min(fits) best = 1 / max(fits) avr = 1 / mean data = stats.compile(pop) logbook.record(gen=g, **data) if not is_silent: print(logbook.stream) # print("-- Generation %i --" % g) # print(" Worst %s" % str(worst)) # print(" Best %s" % str(best)) # print(" Avg %s" % str(avr)) # print(" Std %s" % str(1/std)) best = self._find_best(pop) # the last component is iteration number when evolution stopped result = (best, pop, fixed_schedule_part, current_time, g) self._save_result(result) self._save_pop(pop) if len(previous_raised_avr_individuals ) == repeated_best_count: previous_raised_avr_individuals = previous_raised_avr_individuals[ 1::] previous_raised_avr_individuals.append(1 / mean) pass # # import matplotlib.pyplot as plt # import networkx # # graph = networkx.DiGraph(history.genealogy_tree) # graph = graph.reverse() # Make the grah top-down # colors = [toolbox.evaluate(history.genealogy_history[i])[0] for i in graph] # networkx.draw(graph, node_color=colors) # plt.show() # best = self._find_best(pop) # self._save_result((best, pop, fixed_schedule_part, current_time)) ## return the best fitted individual and resulted population print("Ready") return self.get_result(), logbook