def GA():
"""Starts the optimization with the GA algorithm."""
begin = time.time()
# initialize random generator with system time
rand = random.Random()
rand.seed()
# original start individual of the input data
global start_individual
# generate the original start individual from the input data
# return it including the non static land use indices
start_individual, nonstatic_elements = generate_genom(max_range, file_HRU,cfg.mapConfig.file_ASCII_map,
cfg.mapConfig.file_transformation, cfg.mapConfig.file_ID_map,
cfg.mapConfig.four_neighbours)
if len(start_individual) == 0:
msg = "Error: The generated start individual has no elements."
WriteLogMsg(msg)
raise SystemError("Error: The generated start individual has no elements.")
close_window
# determine that 'Bounder' conditions of candidates are equal to
# the integer values of the non static land use indices
bounder_discrete = nonstatic_elements
# initialize inspyred log files
stats_file,individ_file = fh.init_inspyred_logfiles()
# initialize and run GA
ea = ec.GA(rand)
# public attributes
# GA is predefined with rank_selection
if cfg.ea.selector != 'rank_selection':
exec ("%s%s" % ('ea.selector = ', fh.preparing_attribute('selector',cfg.ea.selector)))
msg = 'Selector of the optimization algorithm changed to: %s' % cfg.ea.selector
WriteLogMsg(msg)
# GA is predefined with generational_replacement
if cfg.ea.replacer != 'generational_replacement':
exec ("%s%s" % ('ea.replacer = ', fh.preparing_attribute('replacer',cfg.ea.replacer)))
msg = 'Replacer of the optimization algorithm changed to: %s' % cfg.ea.replacer
WriteLogMsg(msg)
# specify how the new candidates should be varied
# GA is predefined with n_point_crossover,bit_flip_mutation as variators
if cfg.ea.variator != 'n_point_crossover,bit_flip_mutation' and cfg.ea.variator != 'bit_flip_mutation,n_point_crossover':
exec ("%s%s" % ('ea.variator = ', fh.preparing_attribute('variator',cfg.ea.variator)))
msg = 'Variator of the optimization algorithm changed to: %s' % cfg.ea.variator
WriteLogMsg(msg)
# GA is predefined with num_selected = pop_size
if cfg.ea.num_selected != cfg.ea.pop_size:
msg = 'Num_selected of the optimization algorithm changed to: %s' % cfg.ea.num_selected
WriteLogMsg(msg)
exec ("%s%s" % ('ea.migrator = ', fh.preparing_attribute('migrator',cfg.ea.migrator)))
exec ("%s%s" % ('ea.archiver = ', fh.preparing_attribute('archiver',cfg.ea.archiver)))
if cfg.ea.archiver != 'best_archiver':
msg = 'Archiver of the optimization algorithm changed to: %s' % cfg.ea.archiver
WriteLogMsg(msg)
exec ("%s%s" % ('ea.observer = ', fh.preparing_attribute('observer',cfg.ea.observer)))
# specify when the optimization should terminate
exec ("%s%s" % ('ea.terminator = ', fh.preparing_attribute('terminator',cfg.ea.terminator)))
# run optimization, when finished final_pop holds the results
final_pop = ea.evolve(generator = generate_parameter,
# evaluate is the function to start external models
# return results for the optimization algorithm
evaluator = evaluate,
# define population size
pop_size = cfg.ea.pop_size,
# maximize or minimize the problem
maximize = cfg.ea.maximize,
# bound the parameters to an interval
# choose integer values between 1 and max_range in this case
bounder = ec.DiscreteBounder(bounder_discrete),
# minimum population diversity allowed (when using diversity_termination default 0.001)
min_diversity = cfg.ea.min_diversity,
# maximum number of evaluations (default pop_size)
max_evaluations = cfg.ea.max_evaluations,
# maximum number of generations
max_generations = cfg.ea.max_generations,
# number of elites to consider (default 0)
num_elites = cfg.ea.num_elites,
# number of individuals to be selected (default NSGA2 pop_size)
num_selected = cfg.ea.num_selected,
# tournament size (default NSGA2 2)
tournament_size = cfg.ea.tournament_size,
# the rate at which crossover is performed (default 1.0)
crossover_rate = cfg.ea.crossover_rate,
# mutation rate
mutation_rate = cfg.ea.mutation_rate,
# number of crossover points used (default 1)
num_crossover_points = cfg.ea.num_crossover_points,
# a positive integer representing the number of
# closest solutions to consider as a “crowd” (default 2)
crowding_distance = cfg.ea.crowding_distance,
# statistic file
statistics_file = stats_file,
# individuals file
individuals_file = individ_file)
# read out the best individuals
final_arc = ea.archive
# for constrained_tournament_selection:
# create a copy of final_arc only with feasible individuals (for csv file with best feasible solutions)
if 'constrained_tournament_selection' in cfg.ea.selector:
final_arc_feasible = []
end = time.time()
WriteLogMsg("The optimization process needed %d seconds." %(end-begin))
msg = 'Best Solutions: \n'
WriteLogMsg(msg)
# save the map as ascii file in output folder
f_count=1
for f in final_arc:
# for constrained_tournament_selection: with information if individual is infeasible
# and copy feasible solutions in final_arc_feasible
if 'constrained_tournament_selection' in cfg.ea.selector:
if individual_filter(f.candidate) == False:
WriteLogMsg("(infeasible) %s" % f)
# save the map as ascii file in output folder
if file_HRU == 'None' or (file_HRU != 'None' and cfg.mapConfig.file_ID_map != 'None'):
transform_individual_ascii_map(f.candidate,False,f_count,None,None,None,False)
else:
WriteLogMsg("%s" % f)
# save the map as ascii file in output folder
if file_HRU == 'None' or (file_HRU != 'None' and cfg.mapConfig.file_ID_map != 'None'):
transform_individual_ascii_map(f.candidate,False,f_count)
final_arc_feasible.append(f)
else:
WriteLogMsg("%s" % f)
# save the map as ascii file in output folder
if file_HRU == 'None' or (file_HRU != 'None' and cfg.mapConfig.file_ID_map != 'None'):
transform_individual_ascii_map(f.candidate,False,f_count)
f_count += 1
if cfg.ea.maximize == 'True':
if 'constrained_tournament_selection' in cfg.ea.selector and individual_filter(f.candidate) == False:
WriteLogMsg("\nFinal infeasible individual: %s, [%f]" % (max(final_pop).candidate,max(final_pop).fitness))
else:
WriteLogMsg("\nFinal individual: %s, [%f]" % (max(final_pop).candidate,max(final_pop).fitness))
else:
if 'constrained_tournament_selection' in cfg.ea.selector and individual_filter(f.candidate) == False:
WriteLogMsg("\nFinal infeasible individual: %s, [%f]" % (min(final_pop).candidate,min(final_pop).fitness))
else:
WriteLogMsg("\nFinal individual: %s, [%f]" % (min(final_pop).candidate,min(final_pop).fitness))
# save the map as ascii file in output folder
# log the best solutions in a csv file
fh.save_best_solutions(final_arc,1)
# for constrained_tournament_selection: log the best feasible solutions in a csv file
if 'constrained_tournament_selection' in cfg.ea.selector:
fh.save_best_solutions(final_arc_feasible,1)
def NSGA2():
"""Starts the optimization with the NSGA-II algorithm."""
begin = time.time()
# initialize random generator with system time
rand = random.Random()
rand.seed()
# Generate the original start individual from input data
# return it including the non static land use indices
start_individual, nonstatic_elements = generate_genom(max_range, file_HRU,cfg.mapConfig.file_ASCII_map,
cfg.mapConfig.file_transformation, cfg.mapConfig.file_ID_map,
cfg.mapConfig.four_neighbours)
if len(start_individual) == 0:
msg = "Error: The generated start individual has no elements."
WriteLogMsg(msg)
raise SystemError("Error: The generated start individual has no elements.")
close_window
# determine that 'Bounder' conditions of candidates are equal to
# the integer values of the non static land use indices
bounder_discrete = nonstatic_elements
# initialize inspyred log files
stats_file,individ_file = fh.init_inspyred_logfiles()
# initialize and run NSGA2
ea = ec.emo.NSGA2(rand)
# public attributes
# NSGA2 is predefined with tournament_selection
if cfg.ea.selector != 'tournament_selection':
exec ("%s%s" % ('ea.selector = ', fh.preparing_attribute('selector',cfg.ea.selector)))
msg = 'Selector of the optimization algorithm changed to: %s' % cfg.ea.selector
WriteLogMsg(msg)
# NSGA2 is predefined with nsga_replacement
if cfg.ea.replacer != 'nsga_replacement':
exec ("%s%s" % ('ea.replacer = ', fh.preparing_attribute('replacer',cfg.ea.replacer)))
msg = 'Replacer of the optimization algorithm changed to: %s' % cfg.ea.replacer
WriteLogMsg(msg)
# NSGA2 is predefined with best_archiver
if cfg.ea.archiver != 'best_archiver':
exec ("%s%s" % ('ea.archiver = ', fh.preparing_attribute('archiver',cfg.ea.archiver)))
msg = 'Archiver of the optimization algorithm changed to: %s' % cfg.ea.archiver
WriteLogMsg(msg)
exec ("%s%s" % ('ea.migrator = ', fh.preparing_attribute('migrator',cfg.ea.migrator)))
# file observer prints after each generation the best, worst, mean etc. values into the statistic and individual file
exec ("%s%s" % ('ea.observer = ', fh.preparing_attribute('observer',cfg.ea.observer)))
# specify how the new candidates should be varied
exec ("%s%s" % ('ea.variator = ', fh.preparing_attribute('variator',cfg.ea.variator)))
# specify when the optimization should terminate
exec ("%s%s" % ('ea.terminator = ', fh.preparing_attribute('terminator',cfg.ea.terminator)))
# NSGA2 is predefined with num_selected = pop_size
if cfg.ea.num_selected != cfg.ea.pop_size:
msg = 'Num_selected of the optimization algorithm changed to: %s' % cfg.ea.num_selected
WriteLogMsg(msg)
# NSGA2 is predefined with tournament_size = 2
if cfg.ea.tournament_size != 2:
msg = 'Tournament_size of the optimization algorithm changed to: %s' % cfg.ea.tournament_size
WriteLogMsg(msg)
# run optimization, when finished final_pop holds the results
final_pop = ea.evolve(generator = generate_parameter,
# evaluate is the function to start external models
# return results for the optimization algorithm
evaluator = evaluate,
# define population size
pop_size = cfg.ea.pop_size,
# maximize or Minimize the problem (default True)
maximize = cfg.ea.maximize,
# bound the parameters to an interval
# DiscreteBounder: choose integer values between 1 and max_range
bounder = ec.DiscreteBounder(bounder_discrete),
# minimum population diversity allowed (when using diversity_termination default 0.001)
min_diversity = cfg.ea.min_diversity,
# maximum number of generations
max_generations = cfg.ea.max_generations,
# maximum number of evaluations (default pop_size)
max_evaluations = cfg.ea.max_evaluations,
# number of elites to consider (default 0)
num_elites = cfg.ea.num_elites,
# number of individuals to be selected (default NSGA2 pop_size)
num_selected = cfg.ea.num_selected,
# tournament size (default NSGA2 2)
tournament_size = cfg.ea.tournament_size,
# rate at which crossover is performed (default 1.0)
crossover_rate = cfg.ea.crossover_rate,
# rate at which mutation is performed (default 0.1)
mutation_rate = cfg.ea.mutation_rate,
# number of crossover points used (default 1)
num_crossover_points = cfg.ea.num_crossover_points,
# a positive integer representing the number of
# closest solutions to consider as a “crowd” (default 2)
crowding_distance = cfg.ea.crowding_distance,
# statistic file
statistics_file = stats_file,
# individuals file
individuals_file = individ_file)
final_arc = ea.archive
# for constrained_tournament_selection:
# create a copy of final_arc only with feasible individuals (for csv file with best feasible solutions)
if 'constrained_tournament_selection' in cfg.ea.selector:
final_arc_feasible = []
end = time.time()
msg = "The optimization process needed %d seconds." %(end-begin)
fh.WriteLogMsg(msg)
msg = 'Best Solutions: \n'
WriteLogMsg(msg)
f_count=1
for f in final_arc:
# for constrained_tournament_selection: with information if individual is infeasible
# and copy feasible solutions in final_arc_feasible
if 'constrained_tournament_selection' in cfg.ea.selector:
if individual_filter(f.candidate) == False:
WriteLogMsg("(infeasible) %s" % f)
# save the map as ascii file in output folder
if file_HRU == 'None' or (file_HRU != 'None' and cfg.mapConfig.file_ID_map != 'None'):
transform_individual_ascii_map(f.candidate,False,f_count,None,None,None,False)
else:
WriteLogMsg("%s" % f)
# save the map as ascii file in output folder
if file_HRU == 'None' or (file_HRU != 'None' and cfg.mapConfig.file_ID_map != 'None'):
transform_individual_ascii_map(f.candidate,False,f_count)
final_arc_feasible.append(f)
else:
msg = "%s" % f
#msg = "%f" % f
WriteLogMsg(msg)
# save the map as ascii file in output folder
if file_HRU == 'None' or (file_HRU != 'None' and cfg.mapConfig.file_ID_map != 'None'):
transform_individual_ascii_map(f.candidate,False,f_count)
if f_count == 1:
len_fitness = len(f.fitness)
f_count += 1
# plot the best solution in a 2, 3 or 4 dimensional plot
# 2 dimensional plot
if (cfg.ea.plot_results == True and len_fitness == 2):
# log the best solutions in a csv file
fh.save_best_solutions(final_arc,2)
import pylab
x = []
y = []
for f in final_arc:
x.append(f.fitness[0])
y.append(f.fitness[1])
pylab.scatter(x, y, color='r')
fh.savePlot_png(opt_algorithm)
pylab.show()
# for constrained_tournament_selection: create a second plot with feasible solutions
if 'constrained_tournament_selection' in cfg.ea.selector:
# log the best feasible solutions in a csv file
fh.save_best_solutions(final_arc_feasible,2)
x = []
y = []
for f in final_arc_feasible:
x.append(f.fitness[0])
y.append(f.fitness[1])
pylab.scatter(x, y, color='r')
fh.savePlot_png(opt_algorithm)
pylab.show()
# 3 and 4 dimensional plots
if (cfg.ea.plot_results == True and (len_fitness == 3 or len_fitness == 4)):
import warnings
with warnings.catch_warnings():
warnings.filterwarnings("ignore", category=FutureWarning)
from mpl_toolkits.mplot3d import Axes3D
import matplotlib.pyplot as plt
fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')
x = []
y = []
if len_fitness == 3 or len_fitness == 4:
z = []
if len_fitness == 4:
c = []
for f in final_arc:
x.append(f.fitness[0])
y.append(f.fitness[1])
if len(f.fitness) == 3 or len(f.fitness) == 4:
z.append(f.fitness[2])
if len(f.fitness) == 4:
c.append(f.fitness[3])
if len_fitness == 3:
# log the best solutions in a csv file
fh.save_best_solutions(final_arc,3)
ax.scatter(x, y, z, c='r')
if len_fitness == 4:
# log the best solutions in a csv file
fh.save_best_solutions(final_arc,4)
ax.scatter(x, y, z, c=c, cmap=plt.hot())
fh.savePlot_png(opt_algorithm)
plt.show()
# for constrained_tournament_selection: create a second plot with feasible solutions
if 'constrained_tournament_selection' in cfg.ea.selector:
fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')
x = []
y = []
if len_fitness == 3 or len_fitness == 4:
z = []
if len_fitness == 4:
c = []
for f in final_arc_feasible:
x.append(f.fitness[0])
y.append(f.fitness[1])
if len(f.fitness) == 3 or len(f.fitness) == 4:
z.append(f.fitness[2])
if len(f.fitness) == 4:
c.append(f.fitness[3])
if len_fitness == 3:
# log the best solutions in a csv file
fh.save_best_solutions(final_arc_feasible,3)
ax.scatter(x, y, z, c='r')
if len_fitness == 4:
# log the best solutions in a csv file
fh.save_best_solutions(final_arc_feasible,4)
ax.scatter(x, y, z, c=c, cmap=plt.hot())
fh.savePlot_png(opt_algorithm)
plt.show()
# print results without plotting (if plot_results was set to false)
if cfg.ea.plot_results == False:
if len_fitness == 2:
# log the best solutions in a csv file
fh.save_best_solutions(final_arc,2)
if 'constrained_tournament_selection' in cfg.ea.selector:
# log the best feasible solutions in a csv file
fh.save_best_solutions(final_arc_feasible,2)
if len_fitness == 3:
# log the best solutions in a csv file
fh.save_best_solutions(final_arc,3)
if 'constrained_tournament_selection' in cfg.ea.selector:
# log the best feasible solutions in a csv file
fh.save_best_solutions(final_arc_feasible,3)
if len_fitness == 4:
# log the best solutions in a csv file
fh.save_best_solutions(final_arc,4)
if 'constrained_tournament_selection' in cfg.ea.selector:
# log the best feasible solutions in a csv file
fh.save_best_solutions(final_arc_feasible,4)
def evaluate(candidates, args):
"""Evaluate individuals."""
individuals = candidates
# array for not accepted (infeasible) individuals
not_accepted_ind = []
# increment the generation number
global nmbr_generation
nmbr_generation += 1
if len(individuals[0]) < 101:
msg = "Population for generation %d: " % nmbr_generation
WriteLogMsg(msg)
i = 1
genome_queue = multiprocessing.Queue()
# list with infeasible individuals if constrained_tournament_selection is selected
if 'constrained_tournament_selection' in cfg.ea.selector:
infeasible_ind = []
# log the new population set
for param in individuals:
if len(param) < 101:
msg = "%d, %r" % (i, param)
WriteLogMsg(msg)
# check if individuals are subject to special_termination
# (genome consists of zeros)
if all(item is 0 for item in param):
not_accepted_ind.append(i)
# check if individuals are feasible and constrained_tournament_selection is not selected
# or constrained_tournament_selection is selected -> run models for all individuals
elif (('constrained_tournament_selection' not in cfg.ea.selector) and individual_filter(param) == True) or ('constrained_tournament_selection' in cfg.ea.selector):
# add tasks for the multiprocessing processes to the queue
argument = [i,param]
genome_queue.put(argument)
# mark infeasible individuals for constrained_tournament_selection
if 'constrained_tournament_selection' in cfg.ea.selector and individual_filter(param) == False:
infeasible_ind.append(i)
else:
not_accepted_ind.append(i)
i += 1
# transfer also the variables for map creation to the subprocesses
map_info, patchID_map_info, header_all_info = get_from_maphandler()
queue_arg=[genome_queue, map_info, patchID_map_info, header_all_info]
# check/create helping models folder for multiprocessing
fh.copy_models(i-1)
# a list with results for each individual
fitness = []
# count models
number_models = 1
try:
file_model2
number_models += 1
file_model3
number_models += 1
file_model4
number_models += 1
except:
pass
# define maximum number of processes to run in parallel
if options.nthreads == "max cpu cores":
nthreads = multiprocessing.cpu_count()
else:
nthreads = int(options.nthreads)
# hold the multiprocessing processes
jobs = []
k = 0
# every multiprocess for an individual generates later maximum number_models multiprocesses
# if you have 2 cores and 4 models than you should generate only one multiprocess
# in the first level because you need the 2 cores for the multiprocessing of the models
processes = 1
while (max(nthreads, number_models) >= (processes * number_models)) and processes < i:
processes += 1
# create the multiprocessing processes
while k < (processes-1):
p = multiprocessing.Process(target=genome_process_handling,args=(queue_arg,))
jobs.append(p)
p.start()
k += 1
for j in jobs:
# wait until all multiprocessing processes are finished
j.join()
# Collect the fitness values of all individuals from one generation and return a list of them
output_files = []
external_models = []
try:
output_files.append(cfg.modelConfig.file_output1)
output_files.append(cfg.modelConfig.file_output2)
output_files.append(cfg.modelConfig.file_output3)
output_files.append(cfg.modelConfig.file_output4)
except AttributeError:
pass
try:
external_models.append(cfg.modelConfig.model1_folder)
external_models.append(cfg.modelConfig.model2_folder)
external_models.append(cfg.modelConfig.model3_folder)
external_models.append(cfg.modelConfig.model4_folder)
except AttributeError:
pass
# add the logging informations from the child processes in the optimization_log file
fh.join_ind_number_log()
# add the model outputs of one generation to the special output file
fh.summarize_console_outputs(i-1,nmbr_generation,individuals, external_models, not_accepted_ind)
# collect the fitness values of all individuals and models
fitness = fh.collect_fitness_values(opt_algorithm, i-1, fitness, external_models, output_files, not_accepted_ind, cfg.mapConfig.file_worst_fitness)
# for constrained_tournament_selection: print numbers of infeasible individuals
if 'constrained_tournament_selection' in cfg.ea.selector:
WriteLogMsg("infeasible_ind: %s" % infeasible_ind)
msg = "Fitness values are: %r \n" % fitness
WriteLogMsg(msg)
return fitness