def test_gd(self): optimizer_parameters = RMSPropParameters(learning_rate=0.01, exploration_step_size=0.01, n_random_steps=1, momentum_decay=0.5, n_iteration=1, stop_criterion=np.Inf, seed=99) optimizer = GradientDescentOptimizer( self.trajectory, optimizee_create_individual=self.optimizee.create_individual, optimizee_fitness_weights=(0.1, ), parameters=optimizer_parameters, optimizee_bounding_func=self.optimizee.bounding_func) self.assertIsNotNone(optimizer.parameters) try: self.experiment.run_experiment( optimizee=self.optimizee, optimizee_parameters=self.optimizee_parameters, optimizer=optimizer, optimizer_parameters=optimizer_parameters) except Exception as e: self.fail(e.__name__) print(self.experiment.optimizer) best = list_to_dict( self.experiment.optimizer.current_individual.tolist(), self.experiment.optimizer.optimizee_individual_dict_spec)['coords'] self.assertEqual(best[0], -4.998856251826551) self.assertEqual(best[1], -1.9766742736816023) self.experiment.end_experiment(optimizer)
def bounding_wrapper(*args, **kwargs): if self.optimizee_bounding_func is None: return func(*args, **kwargs) else: # Deap Functions modify individuals in-place, Hence we must do the same result_individuals_deap = func(*args, **kwargs) result_individuals = [list_to_dict(x, self.optimizee_individual_dict_spec) for x in result_individuals_deap] bounded_individuals = [self.optimizee_bounding_func(x) for x in result_individuals] for i, deap_indiv in enumerate(result_individuals_deap): deap_indiv[:] = dict_to_list(bounded_individuals[i]) # print("Bounded Individual: {}".format(bounded_individuals)) return result_individuals_deap
def end(self, traj): """ See :meth:`~l2l.optimizers.optimizer.Optimizer.end` """ best_last_indiv_dict = list_to_dict( self.best_individual_in_run.tolist(), self.optimizee_individual_dict_spec) traj.f_add_result('final_individual', best_last_indiv_dict) traj.f_add_result('final_fitness', self.best_fitness_in_run) traj.f_add_result('n_iteration', self.g + 1) # ------------ Finished all runs and print result --------------- # logger.info("-- End of (successful) ES optimization --")
def end(self, traj): """ See :meth:`~l2l.optimizers.optimizer.Optimizer.end` """ best_last_indiv_dict = list_to_dict( list(self.current_individual), self.optimizee_individual_dict_spec) traj.f_add_result('final_individual', best_last_indiv_dict) traj.f_add_result('final_fitness', self.current_fitness) traj.f_add_result('n_iteration', self.g + 1) logger.info("The last individual was %s with fitness %s", self.current_individual, self.current_fitness) logger.info("-- End of (successful) gradient descent --")
def end(self, traj): """ See :meth:`~l2l.optimizers.optimizer.Optimizer.end` """ best_last_indiv_dict = list_to_dict(self.best_individual_in_run.tolist(), self.optimizee_individual_dict_spec) traj.f_add_result('final_individual', best_last_indiv_dict) traj.f_add_result('final_fitness', self.best_fitness_in_run) traj.f_add_result('n_iteration', self.g + 1) # ------------ Finished all runs and print result --------------- # logger.info("-- End of (successful) CE optimization --") logger.info("-- Final distribution parameters --") for parameter_key, parameter_value in sorted(self.distribution_results.items()): logger.info(' %s: %s', parameter_key, parameter_value)
def end(self, traj): """ See :meth:`~l2l.optimizers.optimizer.Optimizer.end` """ # ------------ Finished all runs and print result --------------- # best_last_indiv_index = np.argmax(self.current_fitness_value_list) best_last_indiv = self.current_individual_list[best_last_indiv_index] best_last_fitness = self.current_fitness_value_list[best_last_indiv_index] best_last_indiv_dict = list_to_dict(best_last_indiv.tolist(), self.optimizee_individual_dict_spec) traj.f_add_result('final_individual', best_last_indiv_dict) traj.f_add_result('final_fitness', best_last_fitness) traj.f_add_result('n_iteration', self.g + 1) logger.info("The best last individual was %s with fitness %s", best_last_indiv, best_last_fitness) logger.info("-- End of (successful) parallel tempering --")
def post_process(self, traj, fitnesses_results): """ See :meth:`~l2l.optimizers.optimizer.Optimizer.post_process` """ n_iteration, smoothing, temp_decay = \ traj.n_iteration, traj.smoothing, traj.temp_decay stop_criterion, n_elite = traj.stop_criterion, traj.n_elite weighted_fitness_list = [] #************************************************************************************************************** # Storing run-information in the trajectory # Reading fitnesses and performing distribution update #************************************************************************************************************** for run_index, fitness in fitnesses_results: # We need to convert the current run index into an ind_idx # (index of individual within one generation) traj.v_idx = run_index ind_index = traj.par.ind_idx traj.f_add_result('$set.$.individual', self.eval_pop[ind_index]) traj.f_add_result('$set.$.fitness', fitness) weighted_fitness_list.append(np.dot(fitness, self.optimizee_fitness_weights)) traj.v_idx = -1 # set trajectory back to default weighted_fitness_list = np.array(weighted_fitness_list).ravel() # Performs descending arg-sort of weighted fitness fitness_sorting_indices = list(reversed(np.argsort(weighted_fitness_list))) # Sorting the data according to fitness sorted_population = self.eval_pop_asarray[fitness_sorting_indices] sorted_fitness = np.asarray(weighted_fitness_list)[fitness_sorting_indices] # Elite individuals are with performance better than or equal to the (1-rho) quantile. # See original describtion of cross entropy for optimization elite_individuals = sorted_population[:n_elite] self.best_individual_in_run = sorted_population[0] self.best_fitness_in_run = sorted_fitness[0] self.gamma = sorted_fitness[n_elite - 1] logger.info("-- End of generation %d --", self.g) logger.info(" Evaluated %d individuals", len(fitnesses_results)) logger.info(' Best Fitness: %.4f', self.best_fitness_in_run) logger.info(' Average Fitness: %.4f', np.mean(sorted_fitness)) logger.debug(' Calculated gamma: %.4f', self.gamma) #************************************************************************************************************** # Storing Generation Parameters / Results in the trajectory #************************************************************************************************************** # These entries correspond to the generation that has been simulated prior to this post-processing run # Documentation of algorithm parameters for the current generation # # generation - The index of the evaluated generation # gamma - The fitness threshold inferred from the evaluated generation # (This is used in sampling the next generation) # T - Temperature used to select non-elite elements among the individuals # of the evaluated generation # best_fitness_in_run - The highest fitness among the individuals in the # evaluated generation # pop_size - Population size generation_result_dict = { 'generation': self.g, 'gamma': self.gamma, 'T': self.T, 'best_fitness_in_run': self.best_fitness_in_run, 'average_fitness_in_run': np.mean(sorted_fitness), 'pop_size': self.pop_size } generation_name = 'generation_{}'.format(self.g) traj.results.generation_params.f_add_result_group(generation_name) traj.results.generation_params.f_add_result( generation_name + '.algorithm_params', generation_result_dict, comment="These are the parameters that correspond to the algorithm, look at the source code" " for `CrossEntropyOptimizer::post_process()` for comments documenting these" " parameters") # new distribution fit individuals_to_be_fitted = elite_individuals # Temperature dependent sampling of non elite individuals if temp_decay > 0: # Keeping non-elite samples with certain probability dependent on temperature (like Simulated Annealing) non_elite_selection_probs = np.clip(np.exp((weighted_fitness_list[n_elite:] - self.gamma) / self.T), a_min=0.0, a_max=1.0) non_elite_selected_indices = self.random_state.binomial(1, non_elite_selection_probs).astype(bool) non_elite_eval_pop_asarray = sorted_population[n_elite:][non_elite_selected_indices] individuals_to_be_fitted = np.concatenate((elite_individuals, non_elite_eval_pop_asarray)) # Fitting New distribution parameters. self.distribution_results = self.current_distribution.fit(individuals_to_be_fitted, smoothing) #Add the results of the distribution fitting to the trajectory traj.results.generation_params.f_add_result( generation_name + '.distribution_params', self.distribution_results, comment="These are the parameters of the distribution inferred from the currently evaluated" " generation") #************************************************************************************************************** # Create the next generation by sampling the inferred distribution #************************************************************************************************************** # Note that this is only done in case the evaluated run is not the last run fitnesses_results.clear() self.eval_pop.clear() # check if to stop if self.g < n_iteration - 1 and self.best_fitness_in_run < stop_criterion: #Sample from the constructed distribution self.eval_pop_asarray = self.current_distribution.sample(self.pop_size) self.eval_pop = [list_to_dict(ind_asarray, self.optimizee_individual_dict_spec) for ind_asarray in self.eval_pop_asarray] # Clip to boundaries if self.optimizee_bounding_func is not None: self.eval_pop = [self.optimizee_bounding_func(individual) for individual in self.eval_pop] self.eval_pop_asarray = np.array([dict_to_list(x) for x in self.eval_pop]) self.g += 1 # Update generation counter self.T *= temp_decay self._expand_trajectory(traj)
def __init__(self, traj, optimizee_create_individual, optimizee_fitness_weights, parameters, optimizee_bounding_func=None): super().__init__( traj, optimizee_create_individual=optimizee_create_individual, optimizee_fitness_weights=optimizee_fitness_weights, parameters=parameters, optimizee_bounding_func=optimizee_bounding_func) self.optimizee_bounding_func = optimizee_bounding_func __, self.optimizee_individual_dict_spec = dict_to_list( optimizee_create_individual(), get_dict_spec=True) traj.f_add_parameter('seed', parameters.seed, comment='Seed for RNG') traj.f_add_parameter('popsize', parameters.popsize, comment='Population size') # 185 traj.f_add_parameter('CXPB', parameters.CXPB, comment='Crossover term') traj.f_add_parameter('MUTPB', parameters.MUTPB, comment='Mutation probability') traj.f_add_parameter('n_iteration', parameters.NGEN, comment='Number of generations') traj.f_add_parameter('indpb', parameters.indpb, comment='Mutation parameter') traj.f_add_parameter('tournsize', parameters.tournsize, comment='Selection parameter') # ------- Create and register functions with DEAP ------- # # delay_rate, slope, std_err, max_fraction_active creator.create("FitnessMax", base.Fitness, weights=self.optimizee_fitness_weights) creator.create("Individual", list, fitness=creator.FitnessMax) toolbox = base.Toolbox() # Structure initializers toolbox.register("individual", tools.initIterate, creator.Individual, lambda: dict_to_list(optimizee_create_individual())) toolbox.register("population", tools.initRepeat, list, toolbox.individual) # Operator registering # This complex piece of code is only necessary because we're using the # DEAP framework and would like to decorate the DEAP mutation operator def bounding_decorator(func): def bounding_wrapper(*args, **kwargs): if self.optimizee_bounding_func is None: return func(*args, **kwargs) else: # Deap Functions modify individuals in-place, Hence we must do the same result_individuals_deap = func(*args, **kwargs) result_individuals = [ list_to_dict(x, self.optimizee_individual_dict_spec) for x in result_individuals_deap ] bounded_individuals = [ self.optimizee_bounding_func(x) for x in result_individuals ] for i, deap_indiv in enumerate(result_individuals_deap): deap_indiv[:] = dict_to_list(bounded_individuals[i]) print("Bounded Individual: {}".format(bounded_individuals)) return result_individuals_deap return bounding_wrapper toolbox.register("mate", tools.cxBlend, alpha=parameters.matepar) toolbox.decorate("mate", bounding_decorator) toolbox.register("mutate", tools.mutGaussian, mu=0, sigma=parameters.mutpar, indpb=traj.indpb) toolbox.decorate("mutate", bounding_decorator) toolbox.register("select", tools.selTournament, tournsize=traj.tournsize) # ------- Initialize Population and Trajectory -------- # # NOTE: The Individual object implements the list interface. self.pop = toolbox.population(n=traj.popsize) self.eval_pop_inds = [ind for ind in self.pop if not ind.fitness.valid] self.eval_pop = [ list_to_dict(ind, self.optimizee_individual_dict_spec) for ind in self.eval_pop_inds ] self.g = 0 # the current generation self.toolbox = toolbox # the DEAP toolbox self.hall_of_fame = HallOfFame(20) self._expand_trajectory(traj)
def __init__(self, traj, optimizee_create_individual, optimizee_fitness_weights, parameters, optimizee_bounding_func=None): super().__init__( traj, optimizee_create_individual=optimizee_create_individual, optimizee_fitness_weights=optimizee_fitness_weights, parameters=parameters, optimizee_bounding_func=optimizee_bounding_func) self.optimizee_bounding_func = optimizee_bounding_func # If a parameter is set to `None`, use default value as described in Wierstra et al. (2014) if parameters.learning_rate_mu is None: learning_rate_mu = 1. else: learning_rate_mu = parameters.learning_rate_mu if parameters.learning_rate_sigma is None: learning_rate_sigma = (3 + np.log(len(parameters.mu))) / ( 5. * np.sqrt(len(parameters.mu))) else: learning_rate_sigma = parameters.learning_rate_sigma if parameters.pop_size is None: pop_size = 4 + int(np.floor(3 * np.log(len(parameters.mu)))) else: pop_size = parameters.pop_size if pop_size < 1: raise ValueError("pop_size needs to be greater than 0") # The following parameters are recorded traj.f_add_parameter('learning_rate_mu', learning_rate_mu, comment='Learning rate mu') traj.f_add_parameter('learning_rate_sigma', learning_rate_sigma, comment='Learning rate mu') traj.f_add_parameter('mu', parameters.mu, comment='Initial mean of search distribution') traj.f_add_parameter( 'sigma', parameters.sigma, comment='Initial standard deviation of search distribution') traj.f_add_parameter('mirrored_sampling_enabled', parameters.mirrored_sampling_enabled, comment='Flag to enable mirrored sampling') traj.f_add_parameter('fitness_shaping_enabled', parameters.fitness_shaping_enabled, comment='Flag to enable fitness shaping') traj.f_add_parameter( 'pop_size', pop_size, comment='Number of minimal individuals simulated in each run') traj.f_add_parameter('n_iteration', parameters.n_iteration, comment='Number of iterations to run') traj.f_add_parameter( 'stop_criterion', parameters.stop_criterion, comment='Stop if best individual reaches this fitness') traj.f_add_parameter( 'seed', np.uint32(parameters.seed), comment='Seed used for random number generation in optimizer') self.random_state = np.random.RandomState(traj.parameters.seed) self.current_individual_arr, self.optimizee_individual_dict_spec = dict_to_list( self.optimizee_create_individual(), get_dict_spec=True) traj.f_add_derived_parameter( 'dimension', self.current_individual_arr.shape, comment='The dimension of the parameter space of the optimizee') # Added a generation-wise parameter logging traj.results.f_add_result_group( 'generation_params', comment='This contains the optimizer parameters that are' ' common across a generation') # The following parameters are recorded as generation parameters i.e. once per generation self.g = 0 # the current generation self.pop_size = pop_size # Population size is dynamic in FACE self.best_fitness_in_run = -np.inf self.best_individual_in_run = None # Set initial parameters of search distribution self.mu = traj.mu self.sigma = traj.sigma # Generate initial distribution self.current_perturbations = self._get_perturbations(traj) current_eval_pop_arr = ( self.mu + self.sigma * self.current_perturbations).tolist() self.eval_pop = [ list_to_dict(ind, self.optimizee_individual_dict_spec) for ind in current_eval_pop_arr ] # Bounding function has to be applied AFTER the individual has been converted to a dict if optimizee_bounding_func is not None: self.eval_pop = [ self.optimizee_bounding_func(ind) for ind in self.eval_pop ] self.eval_pop_arr = np.array( [dict_to_list(ind) for ind in self.eval_pop]) self._expand_trajectory(traj)
def post_process(self, traj, fitnesses_results): """ See :meth:`~l2l.optimizers.optimizer.Optimizer.post_process` """ n_elite, n_iteration, smoothing, temp_decay, min_pop_size, max_pop_size = \ traj.n_elite, traj.n_iteration, traj.smoothing, traj.temp_decay, traj.min_pop_size, traj.max_pop_size stop_criterion, n_expand = traj.stop_criterion, traj.n_expand weighted_fitness_list = [] # ************************************************************************************************************** # Storing run-information in the trajectory # Reading fitnesses and performing distribution update # ************************************************************************************************************** for run_index, fitness in fitnesses_results: # We need to convert the current run index into an ind_idx # (index of individual within one generation) traj.v_idx = run_index ind_index = traj.par.ind_idx traj.f_add_result('$set.$.individual', self.eval_pop[ind_index]) traj.f_add_result('$set.$.fitness', fitness) weighted_fitness_list.append( np.dot(fitness, self.optimizee_fitness_weights)) traj.v_idx = -1 # set trajectory back to default # Performs descending arg-sort of weighted fitness fitness_sorting_indices = list( reversed(np.argsort(weighted_fitness_list))) generation_name = 'generation_{}'.format(self.g) # Sorting the data according to fitness sorted_population = self.eval_pop_asarray[fitness_sorting_indices] sorted_fitess = np.asarray( weighted_fitness_list)[fitness_sorting_indices] # Elite individuals are with performance better than or equal to the (1-rho) quantile. # See original describtion of cross entropy for optimization elite_individuals = sorted_population[:n_elite] previous_best_fitness = self.best_fitness_in_run self.best_individual_in_run = sorted_population[0] self.best_fitness_in_run = sorted_fitess[0] previous_gamma = self.gamma self.gamma = sorted_fitess[n_elite - 1] logger.info("-- End of generation %d --", self.g) logger.info(" Evaluated %d individuals", len(fitnesses_results)) logger.info(' Best Fitness: %.4f', self.best_fitness_in_run) logger.debug(' Calculated gamma: %.4f', self.gamma) # ************************************************************************************************************** # Storing Generation Parameters / Results in the trajectory # ************************************************************************************************************** # These entries correspond to the generation that has been simulated prior to this post-processing run traj.results.generation_params.f_add_result( generation_name + '.g', self.g, comment='The index of the evaluated generation') traj.results.generation_params.f_add_result( generation_name + '.gamma', self.gamma, comment='The fitness threshold inferred from the evaluated ' 'generation (This is used in sampling the next generation') traj.results.generation_params.f_add_result( generation_name + '.T', self.T, comment='Temperature used to select non-elite elements among the' 'individuals of the evaluated generation') traj.results.generation_params.f_add_result( generation_name + '.best_fitness_in_run', self.best_fitness_in_run, comment='The highest fitness among the individuals in the ' 'evaluated generation') traj.results.generation_params.f_add_result(generation_name + '.pop_size', self.pop_size, comment='Population size') # Check stopping if self.g >= n_iteration or self.best_fitness_in_run >= stop_criterion: return expand = True if self.best_fitness_in_run > previous_best_fitness or self.gamma > previous_gamma: # shrink population size self.pop_size = (self.pop_size + min_pop_size) // 2 # new distribution fit individuals_to_be_fitted = elite_individuals # Temperature dependent sampling of non elite individuals if temp_decay > 0: # Keeping non-elite samples with certain probability dependent on temperature (like Simulated Annealing) non_elite_selection_probs = np.clip(np.exp( (weighted_fitness_list[n_elite:] - self.gamma) / self.T), amin=0.0, a_max=1.0) non_elite_selected_indices = self.random_state.binomial( 1, p=non_elite_selection_probs) non_elite_eval_pop_asarray = sorted_population[n_elite:][ non_elite_selected_indices] individuals_to_be_fitted = np.concatenate( (elite_individuals, non_elite_eval_pop_asarray)) # Fitting New distribution parameters. self.distribution_results = self.current_distribution.fit( individuals_to_be_fitted, smoothing) elif self.pop_size + n_expand <= max_pop_size: # Increase pop size by one, resample, FACE part logger.info(' FACE increase population size by %d', n_expand) self.pop_size += n_expand else: # Stop algorithm expand = False logger.warning(' Possibly diverged') # Add the results of the distribution fitting to the trajectory for parameter_key, parameter_value in self.distribution_results.items( ): traj.results.generation_params.f_add_result( generation_name + '.' + parameter_key, parameter_value) # ************************************************************************************************************** # Create the next generation by sampling the inferred distribution # ************************************************************************************************************** # Note that this is only done in case the evaluated run is not the last run fitnesses_results.clear() self.eval_pop.clear() if expand: # Sample from the constructed distribution self.eval_pop_asarray = self.current_distribution.sample( self.pop_size) self.eval_pop = [ list_to_dict(ind_asarray, self.optimizee_individual_dict_spec) for ind_asarray in self.eval_pop_asarray ] # Clip to boundaries if self.optimizee_bounding_func is not None: self.eval_pop = [ self.optimizee_bounding_func(individual) for individual in self.eval_pop ] self.eval_pop_asarray = np.array( [dict_to_list(x) for x in self.eval_pop]) self.g += 1 # Update generation counter self.T *= temp_decay self._expand_trajectory(traj)
def __init__(self, traj, optimizee_create_individual, optimizee_fitness_weights, parameters, optimizee_bounding_func=None): super().__init__(traj, optimizee_create_individual=optimizee_create_individual, optimizee_fitness_weights=optimizee_fitness_weights, parameters=parameters, optimizee_bounding_func=optimizee_bounding_func) self.optimizee_bounding_func = optimizee_bounding_func # The following parameters are recorded traj.f_add_parameter('n_parallel_runs', parameters.n_parallel_runs, comment='Number of parallel simulated annealing runs / Size of Population') traj.f_add_parameter('noisy_step', parameters.noisy_step, comment='Size of the random step') traj.f_add_parameter('n_iteration', parameters.n_iteration, comment='Number of iteration to perform') traj.f_add_parameter('stop_criterion', parameters.stop_criterion, comment='Stopping criterion parameter') traj.f_add_parameter('seed', parameters.seed, comment='Seed for RNG') cooling_schedules_string = '' bounds_list = [] decay_list = [] schedules_list = [] for i in range(0,traj.n_parallel_runs): bounds_list.append(str(parameters.temperature_bounds[i,:])) bounds_list.append(' ') decay_list.append(str(parameters.decay_parameters[i])) decay_list.append(' ') schedules_list.append(str(parameters.cooling_schedules[i])) schedules_list.append(' ') temperature_bounds_string = ''.join(bounds_list) decay_parameters_string = ''.join(decay_list) cooling_schedules_string = ''.join(schedules_list) traj.f_add_parameter('temperature_bounds', temperature_bounds_string, comment='The max and min temperature of the respective schedule') traj.f_add_parameter('decay_parameters', decay_parameters_string, comment='The one parameter, most schedules need') traj.f_add_parameter('cooling_schedules', cooling_schedules_string, comment='The used cooling schedule') _, self.optimizee_individual_dict_spec = dict_to_list(self.optimizee_create_individual(), get_dict_spec=True) # Note that this array stores individuals as an np.array of floats as opposed to Individual-Dicts # This is because this array is used within the context of the simulated annealing algorithm and # Thus needs to handle the optimizee individuals as vectors self.current_individual_list = [np.array(dict_to_list(self.optimizee_create_individual())) for _ in range(parameters.n_parallel_runs)] traj.f_add_result('fitnesses', [], comment='Fitnesses of all individuals') self.T_all = parameters.temperature_bounds[:,0] # Initialize temperature self.T = 1 self.g = 0 # the current generation self.cooling_schedules = parameters.cooling_schedules self.decay_parameters = parameters.decay_parameters self.temperature_bounds = parameters.temperature_bounds # Keep track of current fitness value to decide whether we want the next individual to be accepted or not self.current_fitness_value_list = [-np.Inf] * parameters.n_parallel_runs new_individual_list = [ list_to_dict(ind_as_list + np.random.normal(0.0, parameters.noisy_step, ind_as_list.size) * traj.noisy_step, self.optimizee_individual_dict_spec) for ind_as_list in self.current_individual_list ] if optimizee_bounding_func is not None: new_individual_list = [self.optimizee_bounding_func(ind) for ind in new_individual_list] self.eval_pop = new_individual_list self._expand_trajectory(traj) #initialize container for the indices of the parallel runs self.parallel_indices = [] for i in range(0,traj.n_parallel_runs): self.parallel_indices.append(i) self.available_cooling_schedules = AvailableCoolingSchedules # assert if all cooling schedules are among the known cooling schedules schedule_known = True # start off as True - if any schdule is unknown gets False for i in range(np.size(self.cooling_schedules)): schedule_known = schedule_known and self.cooling_schedules[i] in AvailableCoolingSchedules assert schedule_known, print("Warning: Unknown cooling schedule")
def post_process(self, traj, fitnesses_results): """ See :meth:`~l2l.optimizers.optimizer.Optimizer.post_process` """ old_eval_pop = self.eval_pop.copy() self.eval_pop.clear() logger.info(" Evaluating %i individuals" % len(fitnesses_results)) assert len(fitnesses_results) - 1 == traj.n_random_steps # We need to collect the directions of the random steps along with # the fitness evaluated there fitnesses = np.zeros(traj.n_random_steps) dx = np.zeros((traj.n_random_steps, len(self.current_individual))) weighted_fitness_list = [] for i, (run_index, fitness) in enumerate(fitnesses_results): # We need to convert the current run index into an ind_idx # (index of individual within one generation traj.v_idx = run_index ind_index = traj.par.ind_idx individual = old_eval_pop[ind_index] traj.f_add_result('$set.$.individual', individual) traj.f_add_result('$set.$.fitness', fitness) weighted_fitness = np.dot(fitness, self.optimizee_fitness_weights) weighted_fitness_list.append(weighted_fitness) # The last element of the list is the evaluation of the individual # obtained via gradient descent if i == len(fitnesses_results) - 1: self.current_fitness = weighted_fitness else: fitnesses[i] = weighted_fitness dx[i] = np.array( dict_to_list(individual)) - self.current_individual traj.v_idx = -1 # set the trajectory back to default # Performs descending arg-sort of weighted fitness fitness_sorting_indices = list( reversed(np.argsort(weighted_fitness_list))) old_eval_pop_as_array = np.array( [dict_to_list(x) for x in old_eval_pop]) # Sorting the data according to fitness sorted_population = old_eval_pop_as_array[fitness_sorting_indices] sorted_fitness = np.asarray( weighted_fitness_list)[fitness_sorting_indices] logger.info("-- End of generation %d --", self.g) logger.info(" Evaluated %d individuals", len(fitnesses_results)) logger.info(' Average Fitness: %.4f', np.mean(sorted_fitness)) logger.info(" Current fitness is %.2f", self.current_fitness) logger.info(' Best Fitness: %.4f', sorted_fitness[0]) logger.info(" Best individual is %s", sorted_population[0]) curr_ind_dict = list_to_dict(self.current_individual, self.optimizee_individual_dict_spec) generation_result_dict = { 'generation': self.g, 'current_fitness': self.current_fitness, 'best_fitness_in_run': sorted_fitness[0], 'average_fitness_in_run': np.mean(sorted_fitness), 'current_individual': curr_ind_dict, } generation_name = 'generation_{}'.format(self.g) traj.results.generation_params.f_add_result_group(generation_name) traj.results.generation_params.f_add_result( generation_name + '.algorithm_params', generation_result_dict) logger.info("-- End of iteration {}, current fitness is {} --".format( self.g, self.current_fitness)) max_g = traj.n_iteration - 1 if self.g < max_g and traj.stop_criterion > self.current_fitness: # Create new individual using the appropriate gradient descent self.update_function( traj, np.dot(np.linalg.pinv(dx), fitnesses - self.current_fitness)) current_individual_dict = list_to_dict( list(self.current_individual), self.optimizee_individual_dict_spec) if self.optimizee_bounding_func is not None: current_individual_dict = self.optimizee_bounding_func( current_individual_dict) self.current_individual = np.array( dict_to_list(current_individual_dict)) # Explore the neighbourhood in the parameter space of the # current individual new_individual_list = [ list_to_dict( self.current_individual + self.random_state.normal(0.0, traj.exploration_step_size, self.current_individual.size), self.optimizee_individual_dict_spec) for _ in range(traj.n_random_steps) ] if self.optimizee_bounding_func is not None: new_individual_list = [ self.optimizee_bounding_func(ind) for ind in new_individual_list ] new_individual_list.append(current_individual_dict) fitnesses_results.clear() self.eval_pop = new_individual_list self.g += 1 # Update generation counter self._expand_trajectory(traj)
def __init__(self, traj, optimizee_create_individual, optimizee_fitness_weights, parameters, optimizee_bounding_func=None): super().__init__( traj, optimizee_create_individual=optimizee_create_individual, optimizee_fitness_weights=optimizee_fitness_weights, parameters=parameters, optimizee_bounding_func=optimizee_bounding_func) self.optimizee_bounding_func = optimizee_bounding_func _, self.optimizee_individual_dict_spec = dict_to_list( self.optimizee_create_individual(), get_dict_spec=True) exploration_step_size = parameters.exploration_step_size if isinstance(exploration_step_size, dict): exploration_step_size = dict_to_list(exploration_step_size) traj.f_add_parameter('learning_rate', parameters.learning_rate, comment='Value of learning rate') traj.f_add_parameter('exploration_step_size', exploration_step_size, comment='Standard deviation of the random steps') traj.f_add_parameter('n_random_steps', parameters.n_random_steps, comment='Amount of random steps taken for ' 'calculating the gradient') traj.f_add_parameter('n_iteration', parameters.n_iteration, comment='Number of iteration to perform') traj.f_add_parameter('stop_criterion', parameters.stop_criterion, comment='Stopping criterion parameter') traj.f_add_parameter('seed', np.uint32(parameters.seed), comment='Optimizer random seed') self.random_state = np.random.RandomState(seed=traj.par.seed) # Note that this array stores individuals as an np.array of floats as # opposed to Individual-Dicts # This is because this array is used within the context of the # gradient descent algorithm and thus needs to handle the optimizee # individuals as vectors self.current_individual = np.array( dict_to_list(self.optimizee_create_individual())) # Depending on the algorithm used, initialize the necessary variables self.update_function = None if type(parameters) is ClassicGDParameters: self.init_classic_gd(parameters, traj) elif type(parameters) is StochasticGDParameters: self.init_stochastic_gd(parameters, traj) elif type(parameters) is AdamParameters: self.so_moment = 0.0 self.delta = 0.0 self.init_adam(parameters, traj) elif type(parameters) is RMSPropParameters: self.so_moment = 0.0 self.fo_moment = 0.0 self.init_rmsprop(parameters, traj) else: raise Exception( 'Class of the provided "parameters" argument is not among ' 'the supported types') # Added a generation-wise parameter logging traj.results.f_add_result_group('generation_params', comment='This contains the optimizer ' 'parameters that are common ' 'across a generation') # Explore the neighbourhood in the parameter space of current # individual new_individual_list = [ list_to_dict((self.current_individual + self.random_state.normal( 0.0, exploration_step_size, self.current_individual.size)).tolist(), self.optimizee_individual_dict_spec) for _ in range(parameters.n_random_steps) ] # Also add the current individual to determine it's fitness new_individual_list.append( list_to_dict(list(self.current_individual), self.optimizee_individual_dict_spec)) if optimizee_bounding_func is not None: new_individual_list = [ self.optimizee_bounding_func(ind) for ind in new_individual_list ] # Storing the fitness of the current individual self.current_fitness = -np.Inf self.g = 0 self.eval_pop = new_individual_list self._expand_trajectory(traj)
def post_process(self, traj, fitnesses_results): """ See :meth:`~l2l.optimizers.optimizer.Optimizer.post_process` """ def to_fit(ind): return np.dot(ind.fitness.values, ind.fitness.weights) def spawn(): x = self.optimizee_create_individual() return dict_to_list(self.optimizee_bounding_func(x)) CXPB, MUTPB, NGEN = traj.CXPB, traj.MUTPB, traj.n_iteration logger.info(" Evaluating %i individuals" % len(fitnesses_results)) print(" Evaluating %i individuals" % len(fitnesses_results)) #****************************************************************** # Storing run-information in the trajectory # Reading fitnesses and performing distribution update #****************************************************************** # print("self.g = {}".format(self.g)) # print("len(self.eval_pop_inds) = {}".format(len(self.eval_pop_inds))) # print("len(self.eval_pop) = {}".format(len(self.eval_pop))) # print("len(fitnesses_results) = {}".format(len(fitnesses_results))) for run_index, fitness in fitnesses_results: # We need to convert the current run index into an ind_idx # (index of individual within one generation) traj.v_idx = run_index ind_index = traj.par.ind_idx traj.f_add_result('$set.$.individual', self.eval_pop[ind_index]) traj.f_add_result('$set.$.fitness', fitness) # Use the ind_idx to update the fitness individual = self.eval_pop_inds[ind_index] individual.fitness.values = fitness traj.v_idx = -1 # set the trajectory back to default logger.info("-- End of generation {} --".format(self.g)) print("-- End of generation {} --".format(self.g)) # best_inds = tools.selBest(self.eval_pop_inds, 2) # for best_ind in best_inds: # print("Best individual is %s, %s" % ( # list_to_dict(best_ind, self.optimizee_individual_dict_spec), # best_ind.fitness.values)) # add the bestest individuals this generation to HoF self.hall_of_fame.update(self.eval_pop_inds) logger.info("-- Hall of fame --") # n_bobs = self.n_bobs # bob_inds = tools.selBest(self.hall_of_fame, n_bobs) n_bobs = self.n_hof bob_inds = tools.selBest(self.hall_of_fame, n_bobs) for hof_ind in self.hall_of_fame: sind = str_ind( list_to_dict(hof_ind, self.optimizee_individual_dict_spec) ) logger.info("BoB individual is %s,\n %s" % ( sind, hof_ind.fitness.values)) # print("HOF individual is %s, %s" % ( # hof_ind, hof_ind.fitness.values)) print("BoB individual is %s,\n %s" % ( sind, hof_ind.fitness.values)) #bob_inds = list(map(self.toolbox.clone, bob_inds)) bob_inds = list(map(self.toolbox.clone, self.hall_of_fame)) # ------- Create the next generation by crossover and mutation -------- # if self.g < NGEN - 1: # not necessary for the last generation # Select the next generation individuals # Tournament of population - a list of "pointers" offspring = self.toolbox.select(self.pop, len(self.pop)) # Clone the selected individuals offspring = list(map(self.toolbox.clone, offspring)) #sorts small to big #switch worst-good with best of best #ascending (worst to best) offsp_ids = np.argsort([to_fit(o) for o in offspring]) #descending (best to worst) bob_ids = np.argsort([to_fit(o) for o in bob_inds])[::-1] max_score = to_fit(bob_inds[bob_ids[0]]) min_score = 0.5 * max_score for i in range(n_bobs): off_i = int(offsp_ids[i]) bob_i = int(bob_ids[i]) off_f = to_fit(offspring[off_i]) bob_f = to_fit(bob_inds[bob_i]) if bob_f > off_f: logger.info("Inserting BoB {} to population".format(i+1)) offspring[off_i][:] = bob_inds[bob_i] # Apply crossover and mutation on the offspring for child1, child2 in zip(offspring[::2], offspring[1::2]): f1, f2 = to_fit(child1), to_fit(child2) if random.random() < CXPB: #if both parents are really unfit, replace them with a new couple if f1 <= min_score and f2 <= min_score: logger.info("Both parents had a low score") child1[:] = spawn() child2[:] = spawn() else: self.toolbox.mate(child1, child2) del child1.fitness.values del child2.fitness.values for mutant in offspring[:]: if random.random() < MUTPB: # f = to_fit(mutant) if mutant.fitness.valid else None # print("f = {}".format(f)) # if this was an unfit individual, replace with a "foreigner" # if f is not None and f <= min_score: # logger.info("Mutant had a really low score") # mutant[:] = spawn() # else: self.toolbox.mutate(mutant) del mutant.fitness.values if len(set(map(tuple, offspring))) < len(offspring): logger.info("Mutating more") for i, o1 in enumerate(offspring[:-1]): for o2 in offspring[i+1:]: if tuple(np.round(o1, decimals=4)) == tuple(np.round(o2, decimals=4)): if random.random() < 0.8: self.toolbox.mutate(o2) #o2[:] = spawn() del o2.fitness.values # off_ids = np.random.choice(len(offspring), size=n_bobs, replace=False) # for i in range(n_bobs): # # off_i = int(offsp_ids[i]) # off_i = int(off_ids[i]) # bob_i = int(bob_ids[i]) # logger.info("Inserting BoB {} to population".format(i+1)) # offspring[off_i][:] = bob_inds[bob_i] # del offspring[off_i].fitness.values # The population is entirely replaced by the offspring self.pop[:] = offspring self.eval_pop_inds[:] = [ind for ind in self.pop if not ind.fitness.valid] self.eval_pop[:] = [list_to_dict(ind, self.optimizee_individual_dict_spec) for ind in self.eval_pop_inds] # print("self.g = {}".format(self.g)) # print("len(self.eval_pop_inds) = {}".format(len(self.eval_pop_inds))) # print("len(self.eval_pop) = {}".format(len(self.eval_pop))) self.g += 1 # Update generation counter if len(self.eval_pop) == 0 and self.g < (NGEN - 1): raise Exception("No more mutants to evaluate where generated. " "Increasing population size may help.") self._expand_trajectory(traj)
def __init__(self, traj, optimizee_create_individual, optimizee_fitness_weights, parameters, optimizee_bounding_func=None): super().__init__( traj, optimizee_create_individual=optimizee_create_individual, optimizee_fitness_weights=optimizee_fitness_weights, parameters=parameters, optimizee_bounding_func=optimizee_bounding_func) self.optimizee_bounding_func = optimizee_bounding_func # The following parameters are recorded traj.f_add_parameter( 'n_parallel_runs', parameters.n_parallel_runs, comment= 'Number of parallel simulated annealing runs / Size of Population') traj.f_add_parameter('noisy_step', parameters.noisy_step, comment='Size of the random step') traj.f_add_parameter( 'temp_decay', parameters.temp_decay, comment='A temperature decay parameter (multiplicative)') traj.f_add_parameter('n_iteration', parameters.n_iteration, comment='Number of iteration to perform') traj.f_add_parameter('stop_criterion', parameters.stop_criterion, comment='Stopping criterion parameter') traj.f_add_parameter('seed', np.uint32(parameters.seed), comment='Seed for RNG') _, self.optimizee_individual_dict_spec = dict_to_list( self.optimizee_create_individual(), get_dict_spec=True) # Note that this array stores individuals as an np.array of floats as opposed to Individual-Dicts # This is because this array is used within the context of the simulated annealing algorithm and # Thus needs to handle the optimizee individuals as vectors self.current_individual_list = [ np.array(dict_to_list(self.optimizee_create_individual())) for _ in range(parameters.n_parallel_runs) ] self.random_state = np.random.RandomState(parameters.seed) # The following parameters are NOT recorded self.T = 1. # Initialize temperature self.g = 0 # the current generation # Keep track of current fitness value to decide whether we want the next individual to be accepted or not self.current_fitness_value_list = [-np.Inf ] * parameters.n_parallel_runs new_individual_list = [ list_to_dict( ind_as_list + self.random_state.normal( 0.0, parameters.noisy_step, ind_as_list.size) * traj.noisy_step * self.T, self.optimizee_individual_dict_spec) for ind_as_list in self.current_individual_list ] if optimizee_bounding_func is not None: new_individual_list = [ self.optimizee_bounding_func(ind) for ind in new_individual_list ] self.eval_pop = new_individual_list self._expand_trajectory(traj) self.cooling_schedule = parameters.cooling_schedule
def post_process(self, traj, fitnesses_results): """ See :meth:`~l2l.optimizers.optimizer.Optimizer.post_process` """ noisy_step, temp_decay, n_iteration, stop_criterion = \ traj.noisy_step, traj.temp_decay, traj.n_iteration, traj.stop_criterion old_eval_pop = self.eval_pop.copy() self.eval_pop.clear() temperature = self.T temperature_end = 0 self.T = self.cooling(temperature, self.cooling_schedule, temp_decay, temperature_end, n_iteration) logger.info(" Evaluating %i individuals" % len(fitnesses_results)) assert len(fitnesses_results) == traj.n_parallel_runs weighted_fitness_list = [] for i, (run_index, fitness) in enumerate(fitnesses_results): weighted_fitness = sum( f * w for f, w in zip(fitness, self.optimizee_fitness_weights)) weighted_fitness_list.append(weighted_fitness) # We need to convert the current run index into an ind_idx # (index of individual within one generation) traj.v_idx = run_index ind_index = traj.par.ind_idx individual = old_eval_pop[ind_index] # Accept or reject the new solution current_fitness_value_i = self.current_fitness_value_list[i] r = self.random_state.rand() p = np.exp((weighted_fitness - current_fitness_value_i) / self.T) # Accept if r < p or weighted_fitness >= current_fitness_value_i: self.current_fitness_value_list[i] = weighted_fitness self.current_individual_list[i] = np.array( dict_to_list(individual)) traj.f_add_result('$set.$.individual', individual) # Watchout! if weighted fitness is a tuple/np array it should be converted to a list first here traj.f_add_result('$set.$.fitness', weighted_fitness) current_individual = self.current_individual_list[i] new_individual = list_to_dict( current_individual + self.random_state.randn(current_individual.size) * noisy_step * self.T, self.optimizee_individual_dict_spec) if self.optimizee_bounding_func is not None: new_individual = self.optimizee_bounding_func(new_individual) logger.debug( "Current best fitness for individual %d is %.2f. New individual is %s", i, self.current_fitness_value_list[i], new_individual) self.eval_pop.append(new_individual) logger.debug("Current best fitness within population is %.2f", max(self.current_fitness_value_list)) traj.v_idx = -1 # set the trajectory back to default logger.info("-- End of generation {} --".format(self.g)) # ------- Create the next generation by crossover and mutation -------- # # not necessary for the last generation if self.g < n_iteration - 1 and stop_criterion > max( self.current_fitness_value_list): fitnesses_results.clear() self.g += 1 # Update generation counter self._expand_trajectory(traj)
def post_process(self, traj, fitnesses_results): """ See :meth:`~l2l.optimizers.optimizer.Optimizer.post_process` """ noisy_step, n_iteration, stop_criterion = \ traj.noisy_step, traj.n_iteration, traj.stop_criterion cooling_schedules = self.cooling_schedules decay_parameters = self.decay_parameters temperature_bounds = self.temperature_bounds old_eval_pop = self.eval_pop.copy() self.eval_pop.clear() temperature = self.T_all for i in range(0,traj.n_parallel_runs): self.T_all[self.parallel_indices[i]] = self.cooling(temperature[self.parallel_indices[i]], cooling_schedules[self.parallel_indices[i]], decay_parameters[self.parallel_indices[i]], temperature_bounds[self.parallel_indices[i],:], n_iteration) logger.info(" Evaluating %i individuals" % len(fitnesses_results)) assert len(fitnesses_results) == traj.n_parallel_runs weighted_fitness_list = [] for i, (run_index, fitness) in enumerate(fitnesses_results): self.T = self.T_all[self.parallel_indices[i]] weighted_fitness = sum(f * w for f, w in zip(fitness, self.optimizee_fitness_weights)) weighted_fitness_list.append(weighted_fitness) # We need to convert the current run index into an ind_idx # (index of individual within one generation) traj.v_idx = run_index ind_index = traj.par.ind_idx individual = old_eval_pop[ind_index] # Accept or reject the new solution current_fitness_value_i = self.current_fitness_value_list[i] r = np.random.rand() p = np.exp((weighted_fitness - current_fitness_value_i) / self.T) # Accept if r < p or weighted_fitness >= current_fitness_value_i: self.current_fitness_value_list[i] = weighted_fitness self.current_individual_list[i] = np.array(dict_to_list(individual)) traj.f_add_result('$set.$.individual', individual) # Watchout! if weighted fitness is a tuple/np array it should be converted to a list first here traj.f_add_result('$set.$.fitness', weighted_fitness) current_individual = self.current_individual_list[i] new_individual = list_to_dict(current_individual + np.random.randn(current_individual.size) * noisy_step * self.T, self.optimizee_individual_dict_spec) if self.optimizee_bounding_func is not None: new_individual = self.optimizee_bounding_func(new_individual) logger.debug("Current best fitness for individual %d is %.2f. New individual is %s", i, self.current_fitness_value_list[i], new_individual) self.eval_pop.append(new_individual) # the parallel tempering swapping starts here for i in range(0,traj.n_parallel_runs): #make a random choice from all the other parallel runs compare_indices = [] for j in range(0,traj.n_parallel_runs): compare_indices.append(i) compare_indices.remove(i) random_choice = random.choice(compare_indices) #random variable with unit distribution betwwen 0 and 1 random_variable = np.random.rand() #swap if criterion is met if (self.metropolis_hasting(self.current_fitness_value_list[i], self.current_fitness_value_list[random_choice], self.T_all[i], self.T_all[random_choice]) > random_variable): temp = self.parallel_indices[i] self.parallel_indices[i] = self.parallel_indices[random_choice] self.parallel_indices[random_choice] = temp logger.debug("Current best fitness within population is %.2f", max(self.current_fitness_value_list)) traj.v_idx = -1 # set the trajectory back to default logger.info("-- End of generation {} --".format(self.g)) # ------- Create the next generation by crossover and mutation -------- # # not necessary for the last generation if self.g < n_iteration - 1 and stop_criterion > max(self.current_fitness_value_list): fitnesses_results.clear() self.g += 1 # Update generation counter self._expand_trajectory(traj)
def post_process(self, traj, fitnesses_results): """ See :meth:`~l2l.optimizers.optimizer.Optimizer.post_process` """ n_iteration, stop_criterion, learning_rate = \ traj.n_iteration, traj.stop_criterion, traj.learning_rate noise_std, fitness_shaping_enabled = \ traj.noise_std, traj.fitness_shaping_enabled weighted_fitness_list = [] # ********************************************************************* # Storing run-information in the trajectory # Reading fitnesses and performing distribution update # ********************************************************************* for run_index, fitness in fitnesses_results: # We need to convert the current run index into an ind_idx # (index of individual within one generation) traj.v_idx = run_index ind_index = traj.par.ind_idx traj.f_add_result('$set.$.individual', self.eval_pop[ind_index]) traj.f_add_result('$set.$.fitness', fitness) weighted_fitness_list.append( np.dot(fitness, self.optimizee_fitness_weights)) traj.v_idx = -1 # set trajectory back to default weighted_fitness_list = np.array(weighted_fitness_list).ravel() # NOTE: It is necessary to clear the finesses_results to clear the data # in the reference, and del # ^ is used to make sure it's not used in the rest of this function fitnesses_results.clear() del fitnesses_results # Last fitness is for the previous `current_individual_arr` weighted_fitness_list = weighted_fitness_list[:-1] current_individual_fitness = weighted_fitness_list[-1] # Performs descending arg-sort of weighted fitness fitness_sorting_indices = list( reversed(np.argsort(weighted_fitness_list))) # Sorting the data according to fitness sorted_population = self.eval_pop_arr[fitness_sorting_indices] sorted_fitness = np.asarray( weighted_fitness_list)[fitness_sorting_indices] sorted_perturbations = self.current_perturbations[ fitness_sorting_indices] self.best_individual_in_run = sorted_population[0] self.best_fitness_in_run = sorted_fitness[0] logger.info("-- End of generation %d --", self.g) logger.info(" Evaluated %d individuals", len(weighted_fitness_list) + 1) logger.info(' Best Fitness: %.4f', self.best_fitness_in_run) logger.info(' Average Fitness: %.4f', np.mean(sorted_fitness)) # ********************************************************************* # Storing Generation Parameters / Results in the trajectory # ********************************************************************* # These entries correspond to the generation that has been simulated # prior to this post-processing run # Documentation of algorithm parameters for the current generation # # generation - The index of the evaluated generation # best_fitness_in_run - The highest fitness among the individuals in the # evaluated generation # pop_size - Population size generation_result_dict = { 'generation': self.g, 'best_fitness_in_run': self.best_fitness_in_run, 'current_individual_fitness': current_individual_fitness, 'average_fitness_in_run': np.mean(sorted_fitness), 'pop_size': self.pop_size } generation_name = 'generation_{}'.format(self.g) traj.results.generation_params.f_add_result_group(generation_name) traj.results.generation_params.f_add_result( generation_name + '.algorithm_params', generation_result_dict, comment="These are the parameters that correspond to the " "algorithm. Look at the source code for " "`EvolutionStrategiesOptimizer::post_process()` " "for comments documenting these parameters") if fitness_shaping_enabled: sorted_utilities = [] n_individuals = len(sorted_fitness) for i in range(n_individuals): u = max(0., np.log((n_individuals / 2) + 1) - np.log(i + 1)) sorted_utilities.append(u) sorted_utilities = np.array(sorted_utilities) sorted_utilities /= np.sum(sorted_utilities) sorted_utilities -= (1. / n_individuals) # assert np.sum(sorted_utilities) == 0., \ # "Sum of utilities is not 0, but %.4f" % np.sum(sorted_utilities) fitnesses_to_fit = sorted_utilities else: fitnesses_to_fit = sorted_fitness assert len(fitnesses_to_fit) == len(sorted_perturbations) sum_fits = np.sum( [f * e for f, e in zip(fitnesses_to_fit, sorted_perturbations)], axis=0) weight = len(fitnesses_to_fit) * np.asarray(noise_std)**2 self.current_individual_arr += learning_rate * (sum_fits / weight) # ********************************************************************* # Create the next generation by sampling the inferred distribution # ********************************************************************* # Note that this is only done in case the evaluated run is not the # last run self.eval_pop.clear() # check if to stop max_g = n_iteration - 1 if self.g < max_g and self.best_fitness_in_run < stop_criterion: self.current_perturbations = self._get_perturbations(traj) perturbated = \ self.current_individual_arr + self.current_perturbations current_eval_pop_arr = perturbated.tolist() self.eval_pop[:] = [ list_to_dict(ind, self.optimizee_individual_dict_spec) for ind in current_eval_pop_arr ] self.eval_pop.append( list_to_dict(self.current_individual_arr, self.optimizee_individual_dict_spec)) # Bounding function has to be applied AFTER the individual has been # converted to a dict if self.optimizee_bounding_func is not None: self.eval_pop[:] = [ self.optimizee_bounding_func(ind) for ind in self.eval_pop ] self.eval_pop_arr[:] = np.asarray( [dict_to_list(ind) for ind in self.eval_pop]) self.g += 1 # Update generation counter self._expand_trajectory(traj)
def post_process(self, traj, fitnesses_results): """ See :meth:`~l2l.optimizers.optimizer.Optimizer.post_process` """ CXPB, MUTPB, NGEN = traj.CXPB, traj.MUTPB, traj.n_iteration logger.info(" Evaluating %i individuals" % len(fitnesses_results)) #************************************************************************************************************** # Storing run-information in the trajectory # Reading fitnesses and performing distribution update #************************************************************************************************************** for run_index, fitness in fitnesses_results: # We need to convert the current run index into an ind_idx # (index of individual within one generation) traj.v_idx = run_index ind_index = traj.par.ind_idx traj.f_add_result('$set.$.individual', self.eval_pop[ind_index]) traj.f_add_result('$set.$.fitness', fitness) # Use the ind_idx to update the fitness individual = self.eval_pop_inds[ind_index] individual.fitness.values = fitness traj.v_idx = -1 # set the trajectory back to default logger.info("-- End of generation {} --".format(self.g)) best_inds = tools.selBest(self.eval_pop_inds, 2) for best_ind in best_inds: print("Best individual is %s, %s" % (list_to_dict(best_ind, self.optimizee_individual_dict_spec), best_ind.fitness.values)) self.hall_of_fame.update(self.eval_pop_inds) logger.info("-- Hall of fame --") for hof_ind in tools.selBest(self.hall_of_fame, 2): logger.info( "HOF individual is %s, %s" % (list_to_dict(hof_ind, self.optimizee_individual_dict_spec), hof_ind.fitness.values)) # ------- Create the next generation by crossover and mutation -------- # if self.g < NGEN - 1: # not necessary for the last generation # Select the next generation individuals offspring = self.toolbox.select(self.pop, len(self.pop)) # Clone the selected individuals offspring = list(map(self.toolbox.clone, offspring)) # Apply crossover and mutation on the offspring for child1, child2 in zip(offspring[::2], offspring[1::2]): if random.random() < CXPB: self.toolbox.mate(child1, child2) del child1.fitness.values del child2.fitness.values for mutant in offspring: if random.random() < MUTPB: self.toolbox.mutate(mutant) del mutant.fitness.values if len(set(map(tuple, offspring))) < len(offspring): logger.info("Mutating more") for i, o1 in enumerate(offspring[:-1]): for o2 in offspring[i + 1:]: if tuple(o1) == tuple(o2): if random.random() < 0.8: self.toolbox.mutate(o2) del o2.fitness.values # The population is entirely replaced by the offspring self.pop[:] = offspring self.eval_pop_inds = [ ind for ind in self.pop if not ind.fitness.valid ] self.eval_pop = [ list_to_dict(ind, self.optimizee_individual_dict_spec) for ind in self.eval_pop_inds ] self.g += 1 # Update generation counter self._expand_trajectory(traj)
def __init__(self, traj, optimizee_create_individual, optimizee_fitness_weights, parameters, optimizee_bounding_func=None): super().__init__( traj, optimizee_create_individual=optimizee_create_individual, optimizee_fitness_weights=optimizee_fitness_weights, parameters=parameters, optimizee_bounding_func=optimizee_bounding_func) self.optimizee_bounding_func = optimizee_bounding_func if parameters.pop_size < 1: raise Exception("pop_size needs to be greater than 0") # The following parameters are recorded traj.f_add_parameter('learning_rate', parameters.learning_rate, comment='Learning rate') traj.f_add_parameter('noise_std', parameters.noise_std, comment='Standard deviation of noise') traj.f_add_parameter('mirrored_sampling_enabled', parameters.mirrored_sampling_enabled, comment='Flag to enable mirrored sampling') traj.f_add_parameter('fitness_shaping_enabled', parameters.fitness_shaping_enabled, comment='Flag to enable fitness shaping') traj.f_add_parameter( 'pop_size', parameters.pop_size, comment='Number of minimal individuals simulated in each run') traj.f_add_parameter('n_iteration', parameters.n_iteration, comment='Number of iterations to run') traj.f_add_parameter( 'stop_criterion', parameters.stop_criterion, comment='Stop if best individual reaches this fitness') traj.f_add_parameter( 'seed', np.uint32(parameters.seed), comment='Seed used for random number generation in optimizer') self.random_state = np.random.RandomState(traj.parameters.seed) self.current_individual_arr, self.optimizee_individual_dict_spec = \ dict_to_list( self.optimizee_create_individual(), get_dict_spec=True) noise_std_shape = np.array(parameters.noise_std).shape ind_shape = self.current_individual_arr.shape assert noise_std_shape == () or noise_std_shape == ind_shape traj.f_add_derived_parameter( 'dimension', self.current_individual_arr.shape, comment='The dimension of the parameter space of the optimizee') # Added a generation-wise parameter logging traj.results.f_add_result_group( 'generation_params', comment='This contains the optimizer parameters that are' ' common across a generation') # The following parameters are recorded as generation parameters # i.e. once per generation self.g = 0 # the current generation # Population size is dynamic in FACE self.pop_size = parameters.pop_size self.best_fitness_in_run = -np.inf self.best_individual_in_run = None # The first iteration does not pick the values out of the Gaussian # distribution. It picks randomly (or at-least # as randomly as optimizee_create_individual creates individuals) # Note that this array stores individuals as an np.array of floats as # opposed to Individual-Dicts. # This is because this array is used within the context of the cross # entropy algorithm and thus needs to handle the optimizee individuals # as vectors self.current_perturbations = self._get_perturbations(traj) current_eval_pop_arr = (self.current_individual_arr + self.current_perturbations).tolist() self.eval_pop = [ list_to_dict(ind, self.optimizee_individual_dict_spec) for ind in current_eval_pop_arr ] self.eval_pop.append( list_to_dict(self.current_individual_arr, self.optimizee_individual_dict_spec)) # Bounding function has to be applied AFTER the individual has been # converted to a dict if optimizee_bounding_func is not None: self.eval_pop[:] = [ self.optimizee_bounding_func(ind) for ind in self.eval_pop ] self.eval_pop_arr = np.array( [dict_to_list(ind) for ind in self.eval_pop]) self._expand_trajectory(traj)
def post_process(self, traj, fitnesses_results): """ See :meth:`~l2l.optimizers.optimizer.Optimizer.post_process` """ n_iteration, stop_criterion, fitness_shaping_enabled = \ traj.n_iteration, traj.stop_criterion, traj.fitness_shaping_enabled weighted_fitness_list = [] # ************************************************************************************************************** # Storing run-information in the trajectory # Reading fitnesses and performing distribution update # ************************************************************************************************************** for run_index, fitness in fitnesses_results: # We need to convert the current run index into an ind_idx # (index of individual within one generation) traj.v_idx = run_index ind_index = traj.par.ind_idx traj.f_add_result('$set.$.individual', self.eval_pop[ind_index]) traj.f_add_result('$set.$.fitness', fitness) weighted_fitness_list.append( np.dot(fitness, self.optimizee_fitness_weights)) traj.v_idx = -1 # set trajectory back to default weighted_fitness_list = np.array(weighted_fitness_list).ravel() # NOTE: It is necessary to clear the finesses_results to clear the data in the reference, and del # is used to make sure it's not used in the rest of this function fitnesses_results.clear() del fitnesses_results # Last fitness is for the previous `current_individual_arr` weighted_fitness_list = weighted_fitness_list[:-1] current_individual_fitness = weighted_fitness_list[-1] # Performs descending arg-sort of weighted fitness fitness_sorting_indices = list( reversed(np.argsort(weighted_fitness_list))) # Sorting the data according to fitness sorted_population = self.eval_pop_arr[fitness_sorting_indices] sorted_fitness = np.asarray( weighted_fitness_list)[fitness_sorting_indices] sorted_perturbations = self.current_perturbations[ fitness_sorting_indices] self.best_individual_in_run = sorted_population[0] self.best_fitness_in_run = sorted_fitness[0] logger.info("-- End of generation %d --", self.g) logger.info(" Evaluated %d individuals", len(weighted_fitness_list) + 1) logger.info(' Best Fitness: %.4f', self.best_fitness_in_run) logger.info(' Average Fitness: %.4f', np.mean(sorted_fitness)) # ************************************************************************************************************** # Storing Generation Parameters / Results in the trajectory # ************************************************************************************************************** # These entries correspond to the generation that has been simulated prior to this post-processing run # Documentation of algorithm parameters for the current generation # # generation - The index of the evaluated generation # best_fitness_in_run - The highest fitness among the individuals in the # evaluated generation # pop_size - Population size generation_result_dict = { 'generation': self.g, 'best_fitness_in_run': self.best_fitness_in_run, 'current_individual_fitness': current_individual_fitness, 'average_fitness_in_run': np.mean(sorted_fitness), 'pop_size': self.pop_size } generation_name = 'generation_{}'.format(self.g) traj.results.generation_params.f_add_result_group(generation_name) traj.results.generation_params.f_add_result( generation_name + '.algorithm_params', generation_result_dict, comment="These are the parameters that correspond to the algorithm. " "Look at the source code for `EvolutionStrategiesOptimizer::post_process()` " "for comments documenting these parameters") traj.results.generation_params.f_add_result( generation_name + '.distribution_params', { 'mu': self.mu.copy(), 'sigma': self.sigma.copy() }, comment= "These are the parameters of the distribution that underlies the" " currently evaluated generation") if fitness_shaping_enabled: fitnesses_to_fit = self._compute_utility(sorted_fitness) else: fitnesses_to_fit = sorted_fitness assert len(fitnesses_to_fit) == len(sorted_perturbations) # ************************************************************************************************************** # Update the parameters of the search distribution using the natural gradient in natural coordinates # ************************************************************************************************************** self.mu += traj.learning_rate_mu * traj.sigma * np.dot( fitnesses_to_fit, sorted_perturbations) self.sigma *= np.exp( traj.learning_rate_sigma / 2. * np.dot(fitnesses_to_fit, sorted_perturbations**2 - 1.)) # ************************************************************************************************************** # Create the next generation by sampling the inferred distribution # ************************************************************************************************************** # Note that this is only done in case the evaluated run is not the last run self.eval_pop.clear() # check if to stop if self.g < n_iteration - 1 and self.best_fitness_in_run < stop_criterion: self.current_perturbations = self._get_perturbations(traj) current_eval_pop_arr = ( self.mu + self.sigma * self.current_perturbations).tolist() self.eval_pop = [ list_to_dict(ind, self.optimizee_individual_dict_spec) for ind in current_eval_pop_arr ] # Bounding function has to be applied AFTER the individual has been converted to a dict if self.optimizee_bounding_func is not None: self.eval_pop = [ self.optimizee_bounding_func(ind) for ind in self.eval_pop ] self.eval_pop_arr = np.array( [dict_to_list(ind) for ind in self.eval_pop]) self.g += 1 # Update generation counter self._expand_trajectory(traj)