def ga_generate(wf,
rm,
estimator,
schedule=None,
fixed_schedule_part=None,
current_time=0.0):
if schedule is not None:
return GAFunctions2.schedule_to_chromosome(schedule,
fixed_schedule_part)
gafunctions = GAFunctions2(wf, rm, estimator)
return gafunctions.build_initial(fixed_schedule_part, current_time)
def alg(fixed_schedule_part,
initial_schedule,
current_time=0.0,
initial_population=None):
ga_functions = GAFunctions2(wf, rm, estimator)
generate = partial(ga_generate,
ga_functions=ga_functions,
fixed_schedule_part=fixed_schedule_part,
current_time=current_time,
init_sched_percent=init_sched_percent,
initial_schedule=initial_schedule)
toolbox = Toolbox()
toolbox.register("generate", generate)
toolbox.register(
"evaluate",
fit_converter(
ga_functions.build_fitness(fixed_schedule_part, current_time)))
toolbox.register("clone", deepcopy)
toolbox.register("mate", ga_functions.crossover)
toolbox.register("sweep_mutation", ga_functions.sweep_mutation)
toolbox.register("mutate", ga_functions.mutation)
# toolbox.register("select_parents", )
toolbox.register("select", tools.selRoulette)
pop, logbook, best = run_ga(toolbox=toolbox, **alg_params)
resulted_schedule = ga_functions.build_schedule(
best, fixed_schedule_part, current_time)
result = (best, pop, resulted_schedule, None), logbook
return result
def do_exp(wf_name):
_wf = wf(wf_name)
rm = ExperimentResourceManager(rg.r([10, 15, 25, 30]))
estimator = SimpleTimeCostEstimator(comp_time_cost=0,
transf_time_cost=0,
transferMx=None,
ideal_flops=20,
transfer_time=100)
empty_fixed_schedule_part = Schedule({node: [] for node in rm.get_nodes()})
heft_schedule = run_heft(_wf, rm, estimator)
ga_functions = GAFunctions2(_wf, rm, estimator)
generate = partial(ga_generate,
ga_functions=ga_functions,
fixed_schedule_part=empty_fixed_schedule_part,
current_time=0.0,
init_sched_percent=0.05,
initial_schedule=heft_schedule)
stats = tools.Statistics(lambda ind: ind.fitness.values[0])
stats.register("avg", numpy.mean)
stats.register("std", numpy.std)
stats.register("min", numpy.min)
stats.register("max", numpy.max)
logbook = tools.Logbook()
logbook.header = ["gen", "evals"] + stats.fields
toolbox = Toolbox()
toolbox.register("generate", generate)
toolbox.register(
"evaluate",
fit_converter(
ga_functions.build_fitness(empty_fixed_schedule_part, 0.0)))
toolbox.register("clone", deepcopy)
toolbox.register("mate", ga_functions.crossover)
toolbox.register("sweep_mutation", ga_functions.sweep_mutation)
toolbox.register("mutate", ga_functions.mutation)
# toolbox.register("select_parents", )
# toolbox.register("select", tools.selTournament, tournsize=4)
toolbox.register("select", tools.selRoulette)
pop, logbook, best = run_ga(toolbox=toolbox,
logbook=logbook,
stats=stats,
**GA_PARAMS)
resulted_schedule = ga_functions.build_schedule(best,
empty_fixed_schedule_part,
0.0)
Utility.validate_static_schedule(_wf, resulted_schedule)
ga_makespan = Utility.makespan(resulted_schedule)
return ga_makespan
def generate(n, ga_functions, fixed_schedule_part,
current_time, init_sched_percent,
initial_schedule):
init_ind_count = int(n*init_sched_percent)
res = []
if initial_schedule is not None and init_ind_count > 0:
ga_functions.initial_chromosome = DictBasedIndividual(GAFunctions2.schedule_to_chromosome(initial_schedule))
init_chromosome = ga_functions.initial_chromosome
init_arr = [copy.deepcopy(init_chromosome) for _ in range(init_ind_count)]
res = res + init_arr
if n - init_ind_count > 0:
generated_arr = [DictBasedIndividual(ga_functions.random_chromo(fixed_schedule_part, current_time))
for _ in range(n - init_ind_count)]
res = res + generated_arr
return res
def generate(n, ga_functions, fixed_schedule_part, current_time,
init_sched_percent, initial_schedule):
init_ind_count = int(n * init_sched_percent)
res = []
if initial_schedule is not None and init_ind_count > 0:
ga_functions.initial_chromosome = DictBasedIndividual(
GAFunctions2.schedule_to_chromosome(initial_schedule))
init_chromosome = ga_functions.initial_chromosome
init_arr = [
copy.deepcopy(init_chromosome) for _ in range(init_ind_count)
]
res = res + init_arr
if n - init_ind_count > 0:
generated_arr = [
DictBasedIndividual(
ga_functions.random_chromo(fixed_schedule_part, current_time))
for _ in range(n - init_ind_count)
]
res = res + generated_arr
return res
def ga_generate(wf, rm, estimator, schedule=None, fixed_schedule_part=None, current_time=0.0):
if schedule is not None:
return GAFunctions2.schedule_to_chromosome(schedule, fixed_schedule_part)
gafunctions = GAFunctions2(wf, rm, estimator)
return gafunctions.build_initial(fixed_schedule_part, current_time)
def _actual_ga_run(self):
## TODO: correct this.
heft_initial = self.schedule_to_chromosome_converter(self.back_cmp.current_schedule)
heft_initial = self._clean_chromosome(heft_initial, self.back_cmp.event, self.back_cmp.fixed_schedule)
##=====================================
##Regular GA
##=====================================
# print("GaHeft WITH NEW POP: ")
# ga = self._get_ga_alg()
# ((best_r, pop_r, schedule_r, stopped_iteration_r), logbook_r) = ga(ga_calc.fixed_schedule_part, ga_calc.initial_schedule, current_time)
## TODO: make here using of param to decide how to build initial population for regular gaheft
## TODO: self.mixed_init_pop doesn't exist and isn't setted anywhere
## TODO: need to refactor and merge with MPGA.py
initial_pop_gaheft = None
if self.mixed_init_pop is True:
heft_initial = tools.initIterate(creator.Individual, lambda: heft_initial)
ga_functions = GAFunctions2(self.workflow, self.resource_manager, self.estimator)
heft_pop = [ga_functions.mutation(deepcopy(heft_initial)) for i in range(self.params["population"])]
cleaned_schedule = self.back_cmp.fixed_schedule
initial_pop_gaheft = [self._clean_chromosome(deepcopy(p), self.back_cmp.event, cleaned_schedule) for p in self.past_pop] + heft_pop
print("GaHeft WITH NEW POP: ")
if self.mpnewVSmpoldmode is True:
print("using multi population gaheft...")
ga = self.mpga_builder()
((best_r, pop_r, schedule_r, stopped_iteration_r), logbook_r) = ga(self.back_cmp.fixed_schedule, None, self.current_time, initial_population=None, only_new_pops=True)
else:
print("using single population gaheft...")
ga = self.ga_builder()
((best_r, pop_r, schedule_r, stopped_iteration_r), logbook_r) = ga(self.back_cmp.fixed_schedule, None, self.current_time, initial_population=initial_pop_gaheft)
##=====================================
##Old pop GA
##=====================================
print("GaHeft WITH OLD POP: ")
cleaned_schedule = self.back_cmp.fixed_schedule
initial_pop = [self._clean_chromosome(ind, self.back_cmp.event, cleaned_schedule) for ind in self.past_pop]
result = self.mpga_builder()(self.back_cmp.fixed_schedule,
heft_initial,
self.current_time,
initial_population=initial_pop)
((best_op, pop_op, schedule_op, stopped_iteration_op), logbook_op) = result
self.past_pop = pop_op
##=====================================
##Save stat to stat_saver
##=====================================
## TODO: make exception here
task_id = "" if not hasattr(self.current_event, 'task') else str(self.current_event.task.id)
if self.stat_saver is not None:
## TODO: correct pop_agr later
stat_data = {
"wf_name": self.workflow.name,
"event_name": self.current_event.__class__.__name__,
"task_id": task_id,
"with_old_pop": {
"iter": stopped_iteration_op,
"makespan": Utility.makespan(schedule_op),
"pop_aggr": logbook_op
},
"with_random": {
"iter": stopped_iteration_r,
"makespan": Utility.makespan(schedule_r),
"pop_aggr": logbook_r
}
}
self.stat_saver(stat_data)
self._stop_ga()
return result
def _actual_ga_run(self):
heft_initial = GAFunctions2.schedule_to_chromosome(self.back_cmp.current_schedule)
heft_initial = self._clean_chromosome(heft_initial, self.back_cmp.event, self.back_cmp.fixed_schedule)
##=====================================
##Regular GA
##=====================================
# print("GaHeft WITH NEW POP: ")
# ga = self._get_ga_alg()
# ((best_r, pop_r, schedule_r, stopped_iteration_r), logbook_r) = ga(ga_calc.fixed_schedule_part, ga_calc.initial_schedule, current_time)
## TODO: make here using of param to decide how to build initial population for regular gaheft
## TODO: self.mixed_init_pop doesn't exist and isn't setted anywhere
## TODO: need to refactor and merge with MPGA.py
initial_pop_gaheft = None
if self.mixed_init_pop is True:
heft_initial = tools.initIterate(creator.Individual, lambda: heft_initial)
ga_functions = GAFunctions2(self.workflow, self.resource_manager, self.estimator)
heft_pop = [ga_functions.mutation(deepcopy(heft_initial)) for i in range(self.params["population"])]
cleaned_schedule = self.back_cmp.fixed_schedule
initial_pop_gaheft = [self._clean_chromosome(deepcopy(p), self.back_cmp.event, cleaned_schedule) for p in self.past_pop] + heft_pop
print("GaHeft WITH NEW POP: ")
if self.mpnewVSmpoldmode is True:
print("using multi population gaheft...")
ga = self.mpga_builder()
((best_r, pop_r, schedule_r, stopped_iteration_r), logbook_r) = ga(self.back_cmp.fixed_schedule, None, self.current_time, initial_population=None, only_new_pops=True)
else:
print("using single population gaheft...")
ga = self.ga_builder()
((best_r, pop_r, schedule_r, stopped_iteration_r), logbook_r) = ga(self.back_cmp.fixed_schedule, None, self.current_time, initial_population=initial_pop_gaheft)
##=====================================
##Old pop GA
##=====================================
print("GaHeft WITH OLD POP: ")
cleaned_schedule = self.back_cmp.fixed_schedule
initial_pop = [self._clean_chromosome(ind, self.back_cmp.event, cleaned_schedule) for ind in self.past_pop]
result = self.mpga_builder()(self.back_cmp.fixed_schedule,
heft_initial,
self.current_time,
initial_population=initial_pop)
((best_op, pop_op, schedule_op, stopped_iteration_op), logbook_op) = result
self.past_pop = pop_op
##=====================================
##Save stat to stat_saver
##=====================================
## TODO: make exception here
task_id = "" if not hasattr(self.current_event, 'task') else str(self.current_event.task.id)
if self.stat_saver is not None:
## TODO: correct pop_agr later
stat_data = {
"wf_name": self.workflow.name,
"event_name": self.current_event.__class__.__name__,
"task_id": task_id,
"with_old_pop": {
"iter": stopped_iteration_op,
"makespan": Utility.makespan(schedule_op),
"pop_aggr": logbook_op
},
"with_random": {
"iter": stopped_iteration_r,
"makespan": Utility.makespan(schedule_r),
"pop_aggr": logbook_r
}
}
self.stat_saver(stat_data)
self._stop_ga()
return result
def __call__(self,
fixed_schedule_part,
initial_schedule,
current_time=0,
initial_population=None,
only_new_pops=False):
##==========================
## create populations
##==========================
##TODO: remake it
creator.create("FitnessMax", base.Fitness, weights=(1.0, ))
creator.create("Individual", dict, fitness=creator.FitnessMax)
toolbox = base.Toolbox()
toolbox.register(
"evaluate",
ga_functions.build_fitness(fixed_schedule_part, current_time))
toolbox.register(
"attr_bool",
ga_functions.build_initial(fixed_schedule_part, current_time))
# Structure initializers
toolbox.register("individual", tools.initIterate,
creator.Individual, toolbox.attr_bool)
toolbox.register("population", tools.initRepeat, list,
toolbox.individual)
## TODO: replace it with strategy and specilized builder
if only_new_pops is True:
## TODO: replace this magic number with parameter
populations = [toolbox.population(n=POPSIZE) for i in range(3)]
else:
## create populations
old_pop = initial_population
newpop = toolbox.population(n=POPSIZE)
#heft_initial = GAFunctions2.schedule_to_chromosome(initial_schedule)
# TODO: this is a stub. Rearchitect information flows and entities responsibilities as soon as you will get the first positive results.
heft_initial = initial_schedule if not isinstance(
initial_schedule,
Schedule) else GAFunctions2.schedule_to_chromosome(
initial_schedule, fixed_schedule_part)
heft_initial = tools.initIterate(creator.Individual,
lambda: heft_initial)
heft_pop = [
ga_functions.mutation(deepcopy(heft_initial))
for i in range(POPSIZE)
]
populations = [old_pop, newpop, heft_pop]
## TODO: replace this more effective implementation
def quick_save():
whole_pop = reduce(lambda x, y: x + y, populations)
## choose new old pop for the next run
sorted_whole_pop = sorted(whole_pop,
key=lambda x: x.fitness.values,
reverse=True)
best = sorted_whole_pop[0]
new_oldpop = sorted_whole_pop[0:POPSIZE]
## construct final result
## TODO: implement constructor of final result here
result = ((best, new_oldpop, None, None), common_logbook)
self._save_result(result)
return result
##==========================
## run for several migration periods
##==========================
common_logbook = tools.Logbook()
result = None
for k in range(MIGRATIONS):
new_pops = []
iter_map = {}
for pop in populations:
((best, npop, schedule, stopped_iteration),
logbook) = ga_alg(fixed_schedule_part,
None,
current_time=current_time,
initial_population=pop)
new_pops.append(npop)
# for rec in logbook:
# iter = k*GENERATIONS + rec["iter"]
# mp = iter_map.get(iter, [])
# mp.append({"worst": rec["worst"], "best": rec["best"], "avr": rec["avr"]})
# iter_map[iter] = mp
# pass
for iter, items in iter_map.items():
best = max(it["best"] for it in items)
avr = sum(it["avr"] for it in items) / len(items)
worst = min(it["worst"] for it in items)
common_logbook.record(iter=iter,
worst=worst,
best=best,
avr=avr)
pass
populations = new_pops
migRing(populations, migrCount, emigrant_selection)
result = quick_save()
pass
# merge all populations in one and evaluate for some time
# TODO: test this changes
common_pop = functools.reduce(operator.add, populations, [])
for k in range(MERGED_POP):
((best, npop, schedule, stopped_iteration),
logbook) = ga_alg(fixed_schedule_part,
None,
current_time=current_time,
initial_population=common_pop)
common_pop = npop
# for rec in logbook:
# iter = (all_iters_count - MERGED_POP) + rec["iter"]
# common_logbook.record(iter=iter, worst=rec["worst"], best=rec["best"], avr=rec["avr"])
# pass
result = quick_save()
pass
((best, new_oldpop, x1, x2), x3) = result
result = ((best, new_oldpop,
ga_functions.build_schedule(best, fixed_schedule_part,
current_time), None),
common_logbook)
self._save_result(result[0])
return result
def __call__(self, fixed_schedule_part, initial_schedule, current_time=0, initial_population=None, only_new_pops=False):
##==========================
## create populations
##==========================
##TODO: remake it
creator.create("FitnessMax", base.Fitness, weights=(1.0,))
creator.create("Individual", dict, fitness=creator.FitnessMax)
toolbox = base.Toolbox()
toolbox.register("evaluate", ga_functions.build_fitness(fixed_schedule_part, current_time))
toolbox.register("attr_bool", ga_functions.build_initial(fixed_schedule_part, current_time))
# Structure initializers
toolbox.register("individual", tools.initIterate, creator.Individual, toolbox.attr_bool)
toolbox.register("population", tools.initRepeat, list, toolbox.individual)
## TODO: replace it with strategy and specilized builder
if only_new_pops is True:
## TODO: replace this magic number with parameter
populations = [toolbox.population(n=POPSIZE) for i in range(3)]
else:
## create populations
old_pop = initial_population
newpop = toolbox.population(n=POPSIZE)
#heft_initial = GAFunctions2.schedule_to_chromosome(initial_schedule)
# TODO: this is a stub. Rearchitect information flows and entities responsibilities as soon as you will get the first positive results.
heft_initial = initial_schedule if not isinstance(initial_schedule, Schedule) else GAFunctions2.schedule_to_chromosome(initial_schedule, fixed_schedule_part)
heft_initial = tools.initIterate(creator.Individual, lambda: heft_initial)
heft_pop = [ga_functions.mutation(deepcopy(heft_initial)) for i in range(POPSIZE)]
populations = [old_pop, newpop, heft_pop]
## TODO: replace this more effective implementation
def quick_save():
whole_pop = reduce(lambda x, y: x+y, populations)
## choose new old pop for the next run
sorted_whole_pop = sorted(whole_pop, key=lambda x: x.fitness.values, reverse=True)
best = sorted_whole_pop[0]
new_oldpop = sorted_whole_pop[0:POPSIZE]
## construct final result
## TODO: implement constructor of final result here
result = ((best, new_oldpop, None, None), common_logbook)
self._save_result(result)
return result
##==========================
## run for several migration periods
##==========================
common_logbook = tools.Logbook()
result = None
for k in range(MIGRATIONS):
new_pops = []
iter_map = {}
for pop in populations:
((best, npop, schedule, stopped_iteration), logbook) = ga_alg(fixed_schedule_part,
None,
current_time=current_time,
initial_population=pop)
new_pops.append(npop)
# for rec in logbook:
# iter = k*GENERATIONS + rec["iter"]
# mp = iter_map.get(iter, [])
# mp.append({"worst": rec["worst"], "best": rec["best"], "avr": rec["avr"]})
# iter_map[iter] = mp
# pass
for iter, items in iter_map.items():
best = max(it["best"] for it in items)
avr = sum(it["avr"] for it in items)/len(items)
worst = min(it["worst"] for it in items)
common_logbook.record(iter=iter, worst=worst, best=best, avr=avr)
pass
populations = new_pops
migRing(populations, migrCount, emigrant_selection)
result = quick_save()
pass
# merge all populations in one and evaluate for some time
# TODO: test this changes
common_pop = functools.reduce(operator.add, populations, [])
for k in range(MERGED_POP):
((best, npop, schedule, stopped_iteration), logbook) = ga_alg(fixed_schedule_part,
None,
current_time=current_time,
initial_population=common_pop)
common_pop = npop
# for rec in logbook:
# iter = (all_iters_count - MERGED_POP) + rec["iter"]
# common_logbook.record(iter=iter, worst=rec["worst"], best=rec["best"], avr=rec["avr"])
# pass
result = quick_save()
pass
((best, new_oldpop, x1, x2), x3) = result
result = ((best, new_oldpop, ga_functions.build_schedule(best, fixed_schedule_part, current_time), None), common_logbook)
self._save_result(result[0])
return result
def __call__(self, fixed_schedule_part, initial_schedule, current_time=0, initial_population=None):
print("Evaluating...")
toolbox.register("evaluate", ga_functions.build_fitness(fixed_schedule_part, current_time))
toolbox.register("attr_bool", ga_functions.build_initial(fixed_schedule_part, current_time))
# Structure initializers
toolbox.register("individual", tools.initIterate, creator.Individual, toolbox.attr_bool)
toolbox.register("population", tools.initRepeat, list, toolbox.individual)
ga_functions.initial_chromosome = GAFunctions2.schedule_to_chromosome(initial_schedule, fixed_schedule_part)
if initial_population is None:
initial_population = []
if ga_functions.initial_chromosome is None:
print("empty_init_solutions")
init_solutions = []
else:
init_solutions = [creator.Individual(copy.deepcopy(ga_functions.initial_chromosome)) for _ in range(int(POPSIZE*0.9))]
pop = initial_population + toolbox.population(n=POPSIZE - len(initial_population) - len(init_solutions)) + init_solutions
## TODO: experimental change
history = History()
# Decorate the variation operators
#toolbox.decorate("mate", history.decorator)
# toolbox.decorate("mutate", history.decorator)
# Create the population and populate the history
#history.update(pop)
#===================================================
hallOfFame = deap.tools.HallOfFame(5)
stats = tools.Statistics(key=lambda x: 1/x.fitness.values[0])
stats.register("min", numpy.min)
stats.register("max", numpy.max)
stats.register("avr", numpy.mean)
stats.register("std", numpy.std)
logbook = tools.Logbook()
logbook.header = ["gen"] + stats.fields
# Evaluate the entire population
fitnesses = list(map(toolbox.evaluate, pop))
for ind, fit in zip(pop, fitnesses):
ind.fitness.values = fit
previous_raised_avr_individuals = []
# Begin the evolution
for g in range(NGEN):
# print("Iteration")
if self.is_stopped():
break
hallOfFame.update(pop)
# logbook.record(pop=copy.deepcopy(pop))
# check if evolution process has stopped
# if (check_evolution_for_stopping is True) and len(previous_raised_avr_individuals) == repeated_best_count:
# length = len(previous_raised_avr_individuals)
# whole_sum = sum(previous_raised_avr_individuals)
# mean = whole_sum / length
# sum2 = sum(abs(x - mean) for x in previous_raised_avr_individuals)
# std = sum2/length
# ## TODO: uncomment it later. output
# # print("std: " + str(std))
# if std < 0.0001:
# print(" Evolution process has stopped at " + str(g) + " iteration")
# res = self._get_result()
# extended_result = (res[0], res[1], res[2], res[3], g)
# self._save_result(extended_result)
# break
# print("-- Generation %i --" % g)
# Select the next generation individuals
offspring = pop#toolbox.select(pop, len(pop))
# Clone the selected individuals
offspring = list(map(toolbox.clone, offspring))
# Apply crossover and mutation on the offspring
for child1, child2 in zip(offspring[::2], offspring[1::2]):
if random.random() < CXPB:
toolbox.mate(child1, child2)
del child1.fitness.values
del child2.fitness.values
for mutant in offspring:
if random.random() < SWEEPMUTPB:
ga_functions.sweep_mutation(mutant)
del mutant.fitness.values
continue
if random.random() < MUTPB:
toolbox.mutate(mutant)
del mutant.fitness.values
# Evaluate the individuals with an invalid fitness
invalid_ind = [ind for ind in offspring if not ind.fitness.valid]
fitnesses = list(map(toolbox.evaluate, invalid_ind))
for ind, fit in zip(invalid_ind, fitnesses):
ind.fitness.values = fit
#pop[:] = offspring
# mix with the best individuals of the time
sorted_pop = sorted(pop + list(hallOfFame) + offspring, key=lambda x: x.fitness.values, reverse=True)
pop = sorted_pop[:Kbest:] + toolbox.select(sorted_pop[Kbest:], POPSIZE - Kbest)
# Gather all the fitnesses in one list and print the stats
fits = [ind.fitness.values[0] for ind in pop]
length = len(pop)
mean = sum(fits) / length
sum2 = sum(x*x for x in fits)
std = abs(sum2 / length - mean**2)**0.5
worst = 1/min(fits)
best = 1/max(fits)
avr = 1/mean
data = stats.compile(pop)
logbook.record(gen=g, **data)
if not is_silent:
print(logbook.stream)
# print("-- Generation %i --" % g)
# print(" Worst %s" % str(worst))
# print(" Best %s" % str(best))
# print(" Avg %s" % str(avr))
# print(" Std %s" % str(1/std))
best = self._find_best(pop)
# the last component is iteration number when evolution stopped
result = (best, pop, fixed_schedule_part, current_time, g)
self._save_result(result)
self._save_pop(pop)
if len(previous_raised_avr_individuals) == repeated_best_count:
previous_raised_avr_individuals = previous_raised_avr_individuals[1::]
previous_raised_avr_individuals.append(1/mean)
pass
#
# import matplotlib.pyplot as plt
# import networkx
#
# graph = networkx.DiGraph(history.genealogy_tree)
# graph = graph.reverse() # Make the grah top-down
# colors = [toolbox.evaluate(history.genealogy_history[i])[0] for i in graph]
# networkx.draw(graph, node_color=colors)
# plt.show()
# best = self._find_best(pop)
# self._save_result((best, pop, fixed_schedule_part, current_time))
## return the best fitted individual and resulted population
print("Ready")
return self.get_result(), logbook
def create_ga(self, *args, **kwargs):
is_silent = kwargs["silent"]
wf = kwargs["wf"]
rm = kwargs["resource_manager"]
estimator = kwargs["estimator"]
ga_params = kwargs["ga_params"]
POPSIZE = ga_params.get("population", self.DEFAULT_POPULATION)
CXPB = ga_params.get('crossover_probability',
self.DEFAULT_CROSSOVER_PROBABILITY)
MUTPB = ga_params.get('replacing_mutation_probability',
self.DEFAULT_REPLACING_MUTATION_PROBABILITY)
NGEN = ga_params.get('generations', self.DEFAULT_GENERATIONS)
SWEEPMUTPB = ga_params.get('sweep_mutation_probability',
self.DEFAULT_SWEEP_MUTATION_PROBABILITY)
Kbest = ga_params.get('Kbest', POPSIZE)
ga_functions = kwargs.get("ga_functions",
GAFunctions2(wf, rm, estimator))
check_evolution_for_stopping = kwargs.get(
"check_evolution_for_stopping", True)
def default_fixed_schedule_part(resource_manager):
fix_schedule_part = Schedule({
node: []
for node in HeftHelper.to_nodes(
resource_manager.get_resources())
})
return fix_schedule_part
##================================
##Create genetic algorithm here
##================================
creator.create("FitnessMax", base.Fitness, weights=(1.0, ))
creator.create("Individual", dict, fitness=creator.FitnessMax)
toolbox = base.Toolbox()
# Attribute generator
toolbox.register("attr_bool", ga_functions.build_initial(None, 0))
# Structure initializers
toolbox.register("individual", tools.initIterate, creator.Individual,
toolbox.attr_bool)
toolbox.register("population", tools.initRepeat, list,
toolbox.individual)
## default case
fix_schedule_part = default_fixed_schedule_part(rm)
toolbox.register("evaluate",
ga_functions.build_fitness(fix_schedule_part, 0))
toolbox.register("mate", ga_functions.crossover)
toolbox.register("mutate", ga_functions.mutation)
# toolbox.register("select", tools.selTournament, tournsize=4)
toolbox.register("select", tools.selRoulette)
# toolbox.register("select", tools.selBest)
# toolbox.register("select", tools.selTournamentDCD)
# toolbox.register("select", tools.selNSGA2)
repeated_best_count = 10
class GAComputation(SynchronizedCheckpointedGA):
EVOLUTION_STOPPED_ITERATION_NUMBER = "EvoStpdIterNum"
def __init__(self):
super().__init__()
pass
@timing
def __call__(self,
fixed_schedule_part,
initial_schedule,
current_time=0,
initial_population=None):
print("Evaluating...")
toolbox.register(
"evaluate",
ga_functions.build_fitness(fixed_schedule_part,
current_time))
toolbox.register(
"attr_bool",
ga_functions.build_initial(fixed_schedule_part,
current_time))
# Structure initializers
toolbox.register("individual", tools.initIterate,
creator.Individual, toolbox.attr_bool)
toolbox.register("population", tools.initRepeat, list,
toolbox.individual)
ga_functions.initial_chromosome = GAFunctions2.schedule_to_chromosome(
initial_schedule, fixed_schedule_part)
if initial_population is None:
initial_population = []
if ga_functions.initial_chromosome is None:
print("empty_init_solutions")
init_solutions = []
else:
init_solutions = [
creator.Individual(
copy.deepcopy(ga_functions.initial_chromosome))
for _ in range(int(POPSIZE * 0.9))
]
pop = initial_population + toolbox.population(
n=POPSIZE - len(initial_population) -
len(init_solutions)) + init_solutions
## TODO: experimental change
history = History()
# Decorate the variation operators
#toolbox.decorate("mate", history.decorator)
# toolbox.decorate("mutate", history.decorator)
# Create the population and populate the history
#history.update(pop)
#===================================================
hallOfFame = deap.tools.HallOfFame(5)
stats = tools.Statistics(key=lambda x: 1 / x.fitness.values[0])
stats.register("min", numpy.min)
stats.register("max", numpy.max)
stats.register("avr", numpy.mean)
stats.register("std", numpy.std)
logbook = tools.Logbook()
logbook.header = ["gen"] + stats.fields
# Evaluate the entire population
fitnesses = list(map(toolbox.evaluate, pop))
for ind, fit in zip(pop, fitnesses):
ind.fitness.values = fit
previous_raised_avr_individuals = []
# Begin the evolution
for g in range(NGEN):
# print("Iteration")
if self.is_stopped():
break
hallOfFame.update(pop)
# logbook.record(pop=copy.deepcopy(pop))
# check if evolution process has stopped
# if (check_evolution_for_stopping is True) and len(previous_raised_avr_individuals) == repeated_best_count:
# length = len(previous_raised_avr_individuals)
# whole_sum = sum(previous_raised_avr_individuals)
# mean = whole_sum / length
# sum2 = sum(abs(x - mean) for x in previous_raised_avr_individuals)
# std = sum2/length
# ## TODO: uncomment it later. output
# # print("std: " + str(std))
# if std < 0.0001:
# print(" Evolution process has stopped at " + str(g) + " iteration")
# res = self._get_result()
# extended_result = (res[0], res[1], res[2], res[3], g)
# self._save_result(extended_result)
# break
# print("-- Generation %i --" % g)
# Select the next generation individuals
offspring = pop #toolbox.select(pop, len(pop))
# Clone the selected individuals
offspring = list(map(toolbox.clone, offspring))
# Apply crossover and mutation on the offspring
for child1, child2 in zip(offspring[::2], offspring[1::2]):
if random.random() < CXPB:
toolbox.mate(child1, child2)
del child1.fitness.values
del child2.fitness.values
for mutant in offspring:
if random.random() < SWEEPMUTPB:
ga_functions.sweep_mutation(mutant)
del mutant.fitness.values
continue
if random.random() < MUTPB:
toolbox.mutate(mutant)
del mutant.fitness.values
# Evaluate the individuals with an invalid fitness
invalid_ind = [
ind for ind in offspring if not ind.fitness.valid
]
fitnesses = list(map(toolbox.evaluate, invalid_ind))
for ind, fit in zip(invalid_ind, fitnesses):
ind.fitness.values = fit
#pop[:] = offspring
# mix with the best individuals of the time
sorted_pop = sorted(pop + list(hallOfFame) + offspring,
key=lambda x: x.fitness.values,
reverse=True)
pop = sorted_pop[:Kbest:] + toolbox.select(
sorted_pop[Kbest:], POPSIZE - Kbest)
# Gather all the fitnesses in one list and print the stats
fits = [ind.fitness.values[0] for ind in pop]
length = len(pop)
mean = sum(fits) / length
sum2 = sum(x * x for x in fits)
std = abs(sum2 / length - mean**2)**0.5
worst = 1 / min(fits)
best = 1 / max(fits)
avr = 1 / mean
data = stats.compile(pop)
logbook.record(gen=g, **data)
if not is_silent:
print(logbook.stream)
# print("-- Generation %i --" % g)
# print(" Worst %s" % str(worst))
# print(" Best %s" % str(best))
# print(" Avg %s" % str(avr))
# print(" Std %s" % str(1/std))
best = self._find_best(pop)
# the last component is iteration number when evolution stopped
result = (best, pop, fixed_schedule_part, current_time, g)
self._save_result(result)
self._save_pop(pop)
if len(previous_raised_avr_individuals
) == repeated_best_count:
previous_raised_avr_individuals = previous_raised_avr_individuals[
1::]
previous_raised_avr_individuals.append(1 / mean)
pass
#
# import matplotlib.pyplot as plt
# import networkx
#
# graph = networkx.DiGraph(history.genealogy_tree)
# graph = graph.reverse() # Make the grah top-down
# colors = [toolbox.evaluate(history.genealogy_history[i])[0] for i in graph]
# networkx.draw(graph, node_color=colors)
# plt.show()
# best = self._find_best(pop)
# self._save_result((best, pop, fixed_schedule_part, current_time))
## return the best fitted individual and resulted population
print("Ready")
return self.get_result(), logbook
def _find_best(self, pop):
# resulted_pop = [(ind, ind.fitness.values[0]) for ind in pop]
# result = max(resulted_pop, key=lambda x: x[1])
# return result[0]
result = max(pop, key=lambda x: x.fitness.values[0])
return result
def _construct_result(self, result):
(best, pop, fixed_schedule_part, current_time,
stopped_iteration) = result
## TODO: make additional structure to return elements
return best, pop, ga_functions.build_schedule(
best, fixed_schedule_part, current_time), stopped_iteration
pass
return GAComputation()
def __call__(self,
fixed_schedule_part,
initial_schedule,
current_time=0,
initial_population=None):
print("Evaluating...")
toolbox.register(
"evaluate",
ga_functions.build_fitness(fixed_schedule_part,
current_time))
toolbox.register(
"attr_bool",
ga_functions.build_initial(fixed_schedule_part,
current_time))
# Structure initializers
toolbox.register("individual", tools.initIterate,
creator.Individual, toolbox.attr_bool)
toolbox.register("population", tools.initRepeat, list,
toolbox.individual)
ga_functions.initial_chromosome = GAFunctions2.schedule_to_chromosome(
initial_schedule, fixed_schedule_part)
if initial_population is None:
initial_population = []
if ga_functions.initial_chromosome is None:
print("empty_init_solutions")
init_solutions = []
else:
init_solutions = [
creator.Individual(
copy.deepcopy(ga_functions.initial_chromosome))
for _ in range(int(POPSIZE * 0.9))
]
pop = initial_population + toolbox.population(
n=POPSIZE - len(initial_population) -
len(init_solutions)) + init_solutions
## TODO: experimental change
history = History()
# Decorate the variation operators
#toolbox.decorate("mate", history.decorator)
# toolbox.decorate("mutate", history.decorator)
# Create the population and populate the history
#history.update(pop)
#===================================================
hallOfFame = deap.tools.HallOfFame(5)
stats = tools.Statistics(key=lambda x: 1 / x.fitness.values[0])
stats.register("min", numpy.min)
stats.register("max", numpy.max)
stats.register("avr", numpy.mean)
stats.register("std", numpy.std)
logbook = tools.Logbook()
logbook.header = ["gen"] + stats.fields
# Evaluate the entire population
fitnesses = list(map(toolbox.evaluate, pop))
for ind, fit in zip(pop, fitnesses):
ind.fitness.values = fit
previous_raised_avr_individuals = []
# Begin the evolution
for g in range(NGEN):
# print("Iteration")
if self.is_stopped():
break
hallOfFame.update(pop)
# logbook.record(pop=copy.deepcopy(pop))
# check if evolution process has stopped
# if (check_evolution_for_stopping is True) and len(previous_raised_avr_individuals) == repeated_best_count:
# length = len(previous_raised_avr_individuals)
# whole_sum = sum(previous_raised_avr_individuals)
# mean = whole_sum / length
# sum2 = sum(abs(x - mean) for x in previous_raised_avr_individuals)
# std = sum2/length
# ## TODO: uncomment it later. output
# # print("std: " + str(std))
# if std < 0.0001:
# print(" Evolution process has stopped at " + str(g) + " iteration")
# res = self._get_result()
# extended_result = (res[0], res[1], res[2], res[3], g)
# self._save_result(extended_result)
# break
# print("-- Generation %i --" % g)
# Select the next generation individuals
offspring = pop #toolbox.select(pop, len(pop))
# Clone the selected individuals
offspring = list(map(toolbox.clone, offspring))
# Apply crossover and mutation on the offspring
for child1, child2 in zip(offspring[::2], offspring[1::2]):
if random.random() < CXPB:
toolbox.mate(child1, child2)
del child1.fitness.values
del child2.fitness.values
for mutant in offspring:
if random.random() < SWEEPMUTPB:
ga_functions.sweep_mutation(mutant)
del mutant.fitness.values
continue
if random.random() < MUTPB:
toolbox.mutate(mutant)
del mutant.fitness.values
# Evaluate the individuals with an invalid fitness
invalid_ind = [
ind for ind in offspring if not ind.fitness.valid
]
fitnesses = list(map(toolbox.evaluate, invalid_ind))
for ind, fit in zip(invalid_ind, fitnesses):
ind.fitness.values = fit
#pop[:] = offspring
# mix with the best individuals of the time
sorted_pop = sorted(pop + list(hallOfFame) + offspring,
key=lambda x: x.fitness.values,
reverse=True)
pop = sorted_pop[:Kbest:] + toolbox.select(
sorted_pop[Kbest:], POPSIZE - Kbest)
# Gather all the fitnesses in one list and print the stats
fits = [ind.fitness.values[0] for ind in pop]
length = len(pop)
mean = sum(fits) / length
sum2 = sum(x * x for x in fits)
std = abs(sum2 / length - mean**2)**0.5
worst = 1 / min(fits)
best = 1 / max(fits)
avr = 1 / mean
data = stats.compile(pop)
logbook.record(gen=g, **data)
if not is_silent:
print(logbook.stream)
# print("-- Generation %i --" % g)
# print(" Worst %s" % str(worst))
# print(" Best %s" % str(best))
# print(" Avg %s" % str(avr))
# print(" Std %s" % str(1/std))
best = self._find_best(pop)
# the last component is iteration number when evolution stopped
result = (best, pop, fixed_schedule_part, current_time, g)
self._save_result(result)
self._save_pop(pop)
if len(previous_raised_avr_individuals
) == repeated_best_count:
previous_raised_avr_individuals = previous_raised_avr_individuals[
1::]
previous_raised_avr_individuals.append(1 / mean)
pass
#
# import matplotlib.pyplot as plt
# import networkx
#
# graph = networkx.DiGraph(history.genealogy_tree)
# graph = graph.reverse() # Make the grah top-down
# colors = [toolbox.evaluate(history.genealogy_history[i])[0] for i in graph]
# networkx.draw(graph, node_color=colors)
# plt.show()
# best = self._find_best(pop)
# self._save_result((best, pop, fixed_schedule_part, current_time))
## return the best fitted individual and resulted population
print("Ready")
return self.get_result(), logbook