def GA(): """Starts the optimization with the GA algorithm.""" begin = time.time() # initialize random generator with system time rand = random.Random() rand.seed() # original start individual of the input data global start_individual # generate the original start individual from the input data # return it including the non static land use indices start_individual, nonstatic_elements = generate_genom(max_range, file_HRU,cfg.mapConfig.file_ASCII_map, cfg.mapConfig.file_transformation, cfg.mapConfig.file_ID_map, cfg.mapConfig.four_neighbours) if len(start_individual) == 0: msg = "Error: The generated start individual has no elements." WriteLogMsg(msg) raise SystemError("Error: The generated start individual has no elements.") close_window # determine that 'Bounder' conditions of candidates are equal to # the integer values of the non static land use indices bounder_discrete = nonstatic_elements # initialize inspyred log files stats_file,individ_file = fh.init_inspyred_logfiles() # initialize and run GA ea = ec.GA(rand) # public attributes # GA is predefined with rank_selection if cfg.ea.selector != 'rank_selection': exec ("%s%s" % ('ea.selector = ', fh.preparing_attribute('selector',cfg.ea.selector))) msg = 'Selector of the optimization algorithm changed to: %s' % cfg.ea.selector WriteLogMsg(msg) # GA is predefined with generational_replacement if cfg.ea.replacer != 'generational_replacement': exec ("%s%s" % ('ea.replacer = ', fh.preparing_attribute('replacer',cfg.ea.replacer))) msg = 'Replacer of the optimization algorithm changed to: %s' % cfg.ea.replacer WriteLogMsg(msg) # specify how the new candidates should be varied # GA is predefined with n_point_crossover,bit_flip_mutation as variators if cfg.ea.variator != 'n_point_crossover,bit_flip_mutation' and cfg.ea.variator != 'bit_flip_mutation,n_point_crossover': exec ("%s%s" % ('ea.variator = ', fh.preparing_attribute('variator',cfg.ea.variator))) msg = 'Variator of the optimization algorithm changed to: %s' % cfg.ea.variator WriteLogMsg(msg) # GA is predefined with num_selected = pop_size if cfg.ea.num_selected != cfg.ea.pop_size: msg = 'Num_selected of the optimization algorithm changed to: %s' % cfg.ea.num_selected WriteLogMsg(msg) exec ("%s%s" % ('ea.migrator = ', fh.preparing_attribute('migrator',cfg.ea.migrator))) exec ("%s%s" % ('ea.archiver = ', fh.preparing_attribute('archiver',cfg.ea.archiver))) if cfg.ea.archiver != 'best_archiver': msg = 'Archiver of the optimization algorithm changed to: %s' % cfg.ea.archiver WriteLogMsg(msg) exec ("%s%s" % ('ea.observer = ', fh.preparing_attribute('observer',cfg.ea.observer))) # specify when the optimization should terminate exec ("%s%s" % ('ea.terminator = ', fh.preparing_attribute('terminator',cfg.ea.terminator))) # run optimization, when finished final_pop holds the results final_pop = ea.evolve(generator = generate_parameter, # evaluate is the function to start external models # return results for the optimization algorithm evaluator = evaluate, # define population size pop_size = cfg.ea.pop_size, # maximize or minimize the problem maximize = cfg.ea.maximize, # bound the parameters to an interval # choose integer values between 1 and max_range in this case bounder = ec.DiscreteBounder(bounder_discrete), # minimum population diversity allowed (when using diversity_termination default 0.001) min_diversity = cfg.ea.min_diversity, # maximum number of evaluations (default pop_size) max_evaluations = cfg.ea.max_evaluations, # maximum number of generations max_generations = cfg.ea.max_generations, # number of elites to consider (default 0) num_elites = cfg.ea.num_elites, # number of individuals to be selected (default NSGA2 pop_size) num_selected = cfg.ea.num_selected, # tournament size (default NSGA2 2) tournament_size = cfg.ea.tournament_size, # the rate at which crossover is performed (default 1.0) crossover_rate = cfg.ea.crossover_rate, # mutation rate mutation_rate = cfg.ea.mutation_rate, # number of crossover points used (default 1) num_crossover_points = cfg.ea.num_crossover_points, # a positive integer representing the number of # closest solutions to consider as a “crowd” (default 2) crowding_distance = cfg.ea.crowding_distance, # statistic file statistics_file = stats_file, # individuals file individuals_file = individ_file) # read out the best individuals final_arc = ea.archive # for constrained_tournament_selection: # create a copy of final_arc only with feasible individuals (for csv file with best feasible solutions) if 'constrained_tournament_selection' in cfg.ea.selector: final_arc_feasible = [] end = time.time() WriteLogMsg("The optimization process needed %d seconds." %(end-begin)) msg = 'Best Solutions: \n' WriteLogMsg(msg) # save the map as ascii file in output folder f_count=1 for f in final_arc: # for constrained_tournament_selection: with information if individual is infeasible # and copy feasible solutions in final_arc_feasible if 'constrained_tournament_selection' in cfg.ea.selector: if individual_filter(f.candidate) == False: WriteLogMsg("(infeasible) %s" % f) # save the map as ascii file in output folder if file_HRU == 'None' or (file_HRU != 'None' and cfg.mapConfig.file_ID_map != 'None'): transform_individual_ascii_map(f.candidate,False,f_count,None,None,None,False) else: WriteLogMsg("%s" % f) # save the map as ascii file in output folder if file_HRU == 'None' or (file_HRU != 'None' and cfg.mapConfig.file_ID_map != 'None'): transform_individual_ascii_map(f.candidate,False,f_count) final_arc_feasible.append(f) else: WriteLogMsg("%s" % f) # save the map as ascii file in output folder if file_HRU == 'None' or (file_HRU != 'None' and cfg.mapConfig.file_ID_map != 'None'): transform_individual_ascii_map(f.candidate,False,f_count) f_count += 1 if cfg.ea.maximize == 'True': if 'constrained_tournament_selection' in cfg.ea.selector and individual_filter(f.candidate) == False: WriteLogMsg("\nFinal infeasible individual: %s, [%f]" % (max(final_pop).candidate,max(final_pop).fitness)) else: WriteLogMsg("\nFinal individual: %s, [%f]" % (max(final_pop).candidate,max(final_pop).fitness)) else: if 'constrained_tournament_selection' in cfg.ea.selector and individual_filter(f.candidate) == False: WriteLogMsg("\nFinal infeasible individual: %s, [%f]" % (min(final_pop).candidate,min(final_pop).fitness)) else: WriteLogMsg("\nFinal individual: %s, [%f]" % (min(final_pop).candidate,min(final_pop).fitness)) # save the map as ascii file in output folder # log the best solutions in a csv file fh.save_best_solutions(final_arc,1) # for constrained_tournament_selection: log the best feasible solutions in a csv file if 'constrained_tournament_selection' in cfg.ea.selector: fh.save_best_solutions(final_arc_feasible,1)
def NSGA2(): """Starts the optimization with the NSGA-II algorithm.""" begin = time.time() # initialize random generator with system time rand = random.Random() rand.seed() # Generate the original start individual from input data # return it including the non static land use indices start_individual, nonstatic_elements = generate_genom(max_range, file_HRU,cfg.mapConfig.file_ASCII_map, cfg.mapConfig.file_transformation, cfg.mapConfig.file_ID_map, cfg.mapConfig.four_neighbours) if len(start_individual) == 0: msg = "Error: The generated start individual has no elements." WriteLogMsg(msg) raise SystemError("Error: The generated start individual has no elements.") close_window # determine that 'Bounder' conditions of candidates are equal to # the integer values of the non static land use indices bounder_discrete = nonstatic_elements # initialize inspyred log files stats_file,individ_file = fh.init_inspyred_logfiles() # initialize and run NSGA2 ea = ec.emo.NSGA2(rand) # public attributes # NSGA2 is predefined with tournament_selection if cfg.ea.selector != 'tournament_selection': exec ("%s%s" % ('ea.selector = ', fh.preparing_attribute('selector',cfg.ea.selector))) msg = 'Selector of the optimization algorithm changed to: %s' % cfg.ea.selector WriteLogMsg(msg) # NSGA2 is predefined with nsga_replacement if cfg.ea.replacer != 'nsga_replacement': exec ("%s%s" % ('ea.replacer = ', fh.preparing_attribute('replacer',cfg.ea.replacer))) msg = 'Replacer of the optimization algorithm changed to: %s' % cfg.ea.replacer WriteLogMsg(msg) # NSGA2 is predefined with best_archiver if cfg.ea.archiver != 'best_archiver': exec ("%s%s" % ('ea.archiver = ', fh.preparing_attribute('archiver',cfg.ea.archiver))) msg = 'Archiver of the optimization algorithm changed to: %s' % cfg.ea.archiver WriteLogMsg(msg) exec ("%s%s" % ('ea.migrator = ', fh.preparing_attribute('migrator',cfg.ea.migrator))) # file observer prints after each generation the best, worst, mean etc. values into the statistic and individual file exec ("%s%s" % ('ea.observer = ', fh.preparing_attribute('observer',cfg.ea.observer))) # specify how the new candidates should be varied exec ("%s%s" % ('ea.variator = ', fh.preparing_attribute('variator',cfg.ea.variator))) # specify when the optimization should terminate exec ("%s%s" % ('ea.terminator = ', fh.preparing_attribute('terminator',cfg.ea.terminator))) # NSGA2 is predefined with num_selected = pop_size if cfg.ea.num_selected != cfg.ea.pop_size: msg = 'Num_selected of the optimization algorithm changed to: %s' % cfg.ea.num_selected WriteLogMsg(msg) # NSGA2 is predefined with tournament_size = 2 if cfg.ea.tournament_size != 2: msg = 'Tournament_size of the optimization algorithm changed to: %s' % cfg.ea.tournament_size WriteLogMsg(msg) # run optimization, when finished final_pop holds the results final_pop = ea.evolve(generator = generate_parameter, # evaluate is the function to start external models # return results for the optimization algorithm evaluator = evaluate, # define population size pop_size = cfg.ea.pop_size, # maximize or Minimize the problem (default True) maximize = cfg.ea.maximize, # bound the parameters to an interval # DiscreteBounder: choose integer values between 1 and max_range bounder = ec.DiscreteBounder(bounder_discrete), # minimum population diversity allowed (when using diversity_termination default 0.001) min_diversity = cfg.ea.min_diversity, # maximum number of generations max_generations = cfg.ea.max_generations, # maximum number of evaluations (default pop_size) max_evaluations = cfg.ea.max_evaluations, # number of elites to consider (default 0) num_elites = cfg.ea.num_elites, # number of individuals to be selected (default NSGA2 pop_size) num_selected = cfg.ea.num_selected, # tournament size (default NSGA2 2) tournament_size = cfg.ea.tournament_size, # rate at which crossover is performed (default 1.0) crossover_rate = cfg.ea.crossover_rate, # rate at which mutation is performed (default 0.1) mutation_rate = cfg.ea.mutation_rate, # number of crossover points used (default 1) num_crossover_points = cfg.ea.num_crossover_points, # a positive integer representing the number of # closest solutions to consider as a “crowd” (default 2) crowding_distance = cfg.ea.crowding_distance, # statistic file statistics_file = stats_file, # individuals file individuals_file = individ_file) final_arc = ea.archive # for constrained_tournament_selection: # create a copy of final_arc only with feasible individuals (for csv file with best feasible solutions) if 'constrained_tournament_selection' in cfg.ea.selector: final_arc_feasible = [] end = time.time() msg = "The optimization process needed %d seconds." %(end-begin) fh.WriteLogMsg(msg) msg = 'Best Solutions: \n' WriteLogMsg(msg) f_count=1 for f in final_arc: # for constrained_tournament_selection: with information if individual is infeasible # and copy feasible solutions in final_arc_feasible if 'constrained_tournament_selection' in cfg.ea.selector: if individual_filter(f.candidate) == False: WriteLogMsg("(infeasible) %s" % f) # save the map as ascii file in output folder if file_HRU == 'None' or (file_HRU != 'None' and cfg.mapConfig.file_ID_map != 'None'): transform_individual_ascii_map(f.candidate,False,f_count,None,None,None,False) else: WriteLogMsg("%s" % f) # save the map as ascii file in output folder if file_HRU == 'None' or (file_HRU != 'None' and cfg.mapConfig.file_ID_map != 'None'): transform_individual_ascii_map(f.candidate,False,f_count) final_arc_feasible.append(f) else: msg = "%s" % f #msg = "%f" % f WriteLogMsg(msg) # save the map as ascii file in output folder if file_HRU == 'None' or (file_HRU != 'None' and cfg.mapConfig.file_ID_map != 'None'): transform_individual_ascii_map(f.candidate,False,f_count) if f_count == 1: len_fitness = len(f.fitness) f_count += 1 # plot the best solution in a 2, 3 or 4 dimensional plot # 2 dimensional plot if (cfg.ea.plot_results == True and len_fitness == 2): # log the best solutions in a csv file fh.save_best_solutions(final_arc,2) import pylab x = [] y = [] for f in final_arc: x.append(f.fitness[0]) y.append(f.fitness[1]) pylab.scatter(x, y, color='r') fh.savePlot_png(opt_algorithm) pylab.show() # for constrained_tournament_selection: create a second plot with feasible solutions if 'constrained_tournament_selection' in cfg.ea.selector: # log the best feasible solutions in a csv file fh.save_best_solutions(final_arc_feasible,2) x = [] y = [] for f in final_arc_feasible: x.append(f.fitness[0]) y.append(f.fitness[1]) pylab.scatter(x, y, color='r') fh.savePlot_png(opt_algorithm) pylab.show() # 3 and 4 dimensional plots if (cfg.ea.plot_results == True and (len_fitness == 3 or len_fitness == 4)): import warnings with warnings.catch_warnings(): warnings.filterwarnings("ignore", category=FutureWarning) from mpl_toolkits.mplot3d import Axes3D import matplotlib.pyplot as plt fig = plt.figure() ax = fig.add_subplot(111, projection='3d') x = [] y = [] if len_fitness == 3 or len_fitness == 4: z = [] if len_fitness == 4: c = [] for f in final_arc: x.append(f.fitness[0]) y.append(f.fitness[1]) if len(f.fitness) == 3 or len(f.fitness) == 4: z.append(f.fitness[2]) if len(f.fitness) == 4: c.append(f.fitness[3]) if len_fitness == 3: # log the best solutions in a csv file fh.save_best_solutions(final_arc,3) ax.scatter(x, y, z, c='r') if len_fitness == 4: # log the best solutions in a csv file fh.save_best_solutions(final_arc,4) ax.scatter(x, y, z, c=c, cmap=plt.hot()) fh.savePlot_png(opt_algorithm) plt.show() # for constrained_tournament_selection: create a second plot with feasible solutions if 'constrained_tournament_selection' in cfg.ea.selector: fig = plt.figure() ax = fig.add_subplot(111, projection='3d') x = [] y = [] if len_fitness == 3 or len_fitness == 4: z = [] if len_fitness == 4: c = [] for f in final_arc_feasible: x.append(f.fitness[0]) y.append(f.fitness[1]) if len(f.fitness) == 3 or len(f.fitness) == 4: z.append(f.fitness[2]) if len(f.fitness) == 4: c.append(f.fitness[3]) if len_fitness == 3: # log the best solutions in a csv file fh.save_best_solutions(final_arc_feasible,3) ax.scatter(x, y, z, c='r') if len_fitness == 4: # log the best solutions in a csv file fh.save_best_solutions(final_arc_feasible,4) ax.scatter(x, y, z, c=c, cmap=plt.hot()) fh.savePlot_png(opt_algorithm) plt.show() # print results without plotting (if plot_results was set to false) if cfg.ea.plot_results == False: if len_fitness == 2: # log the best solutions in a csv file fh.save_best_solutions(final_arc,2) if 'constrained_tournament_selection' in cfg.ea.selector: # log the best feasible solutions in a csv file fh.save_best_solutions(final_arc_feasible,2) if len_fitness == 3: # log the best solutions in a csv file fh.save_best_solutions(final_arc,3) if 'constrained_tournament_selection' in cfg.ea.selector: # log the best feasible solutions in a csv file fh.save_best_solutions(final_arc_feasible,3) if len_fitness == 4: # log the best solutions in a csv file fh.save_best_solutions(final_arc,4) if 'constrained_tournament_selection' in cfg.ea.selector: # log the best feasible solutions in a csv file fh.save_best_solutions(final_arc_feasible,4)
def evaluate(candidates, args): """Evaluate individuals.""" individuals = candidates # array for not accepted (infeasible) individuals not_accepted_ind = [] # increment the generation number global nmbr_generation nmbr_generation += 1 if len(individuals[0]) < 101: msg = "Population for generation %d: " % nmbr_generation WriteLogMsg(msg) i = 1 genome_queue = multiprocessing.Queue() # list with infeasible individuals if constrained_tournament_selection is selected if 'constrained_tournament_selection' in cfg.ea.selector: infeasible_ind = [] # log the new population set for param in individuals: if len(param) < 101: msg = "%d, %r" % (i, param) WriteLogMsg(msg) # check if individuals are subject to special_termination # (genome consists of zeros) if all(item is 0 for item in param): not_accepted_ind.append(i) # check if individuals are feasible and constrained_tournament_selection is not selected # or constrained_tournament_selection is selected -> run models for all individuals elif (('constrained_tournament_selection' not in cfg.ea.selector) and individual_filter(param) == True) or ('constrained_tournament_selection' in cfg.ea.selector): # add tasks for the multiprocessing processes to the queue argument = [i,param] genome_queue.put(argument) # mark infeasible individuals for constrained_tournament_selection if 'constrained_tournament_selection' in cfg.ea.selector and individual_filter(param) == False: infeasible_ind.append(i) else: not_accepted_ind.append(i) i += 1 # transfer also the variables for map creation to the subprocesses map_info, patchID_map_info, header_all_info = get_from_maphandler() queue_arg=[genome_queue, map_info, patchID_map_info, header_all_info] # check/create helping models folder for multiprocessing fh.copy_models(i-1) # a list with results for each individual fitness = [] # count models number_models = 1 try: file_model2 number_models += 1 file_model3 number_models += 1 file_model4 number_models += 1 except: pass # define maximum number of processes to run in parallel if options.nthreads == "max cpu cores": nthreads = multiprocessing.cpu_count() else: nthreads = int(options.nthreads) # hold the multiprocessing processes jobs = [] k = 0 # every multiprocess for an individual generates later maximum number_models multiprocesses # if you have 2 cores and 4 models than you should generate only one multiprocess # in the first level because you need the 2 cores for the multiprocessing of the models processes = 1 while (max(nthreads, number_models) >= (processes * number_models)) and processes < i: processes += 1 # create the multiprocessing processes while k < (processes-1): p = multiprocessing.Process(target=genome_process_handling,args=(queue_arg,)) jobs.append(p) p.start() k += 1 for j in jobs: # wait until all multiprocessing processes are finished j.join() # Collect the fitness values of all individuals from one generation and return a list of them output_files = [] external_models = [] try: output_files.append(cfg.modelConfig.file_output1) output_files.append(cfg.modelConfig.file_output2) output_files.append(cfg.modelConfig.file_output3) output_files.append(cfg.modelConfig.file_output4) except AttributeError: pass try: external_models.append(cfg.modelConfig.model1_folder) external_models.append(cfg.modelConfig.model2_folder) external_models.append(cfg.modelConfig.model3_folder) external_models.append(cfg.modelConfig.model4_folder) except AttributeError: pass # add the logging informations from the child processes in the optimization_log file fh.join_ind_number_log() # add the model outputs of one generation to the special output file fh.summarize_console_outputs(i-1,nmbr_generation,individuals, external_models, not_accepted_ind) # collect the fitness values of all individuals and models fitness = fh.collect_fitness_values(opt_algorithm, i-1, fitness, external_models, output_files, not_accepted_ind, cfg.mapConfig.file_worst_fitness) # for constrained_tournament_selection: print numbers of infeasible individuals if 'constrained_tournament_selection' in cfg.ea.selector: WriteLogMsg("infeasible_ind: %s" % infeasible_ind) msg = "Fitness values are: %r \n" % fitness WriteLogMsg(msg) return fitness