def test_gd(self):
optimizer_parameters = RMSPropParameters(learning_rate=0.01,
exploration_step_size=0.01,
n_random_steps=1,
momentum_decay=0.5,
n_iteration=1,
stop_criterion=np.Inf,
seed=99)
optimizer = GradientDescentOptimizer(
self.trajectory,
optimizee_create_individual=self.optimizee.create_individual,
optimizee_fitness_weights=(0.1, ),
parameters=optimizer_parameters,
optimizee_bounding_func=self.optimizee.bounding_func)
self.assertIsNotNone(optimizer.parameters)
try:
self.experiment.run_experiment(
optimizee=self.optimizee,
optimizee_parameters=self.optimizee_parameters,
optimizer=optimizer,
optimizer_parameters=optimizer_parameters)
except Exception as e:
self.fail(e.__name__)
print(self.experiment.optimizer)
best = list_to_dict(
self.experiment.optimizer.current_individual.tolist(),
self.experiment.optimizer.optimizee_individual_dict_spec)['coords']
self.assertEqual(best[0], -4.998856251826551)
self.assertEqual(best[1], -1.9766742736816023)
self.experiment.end_experiment(optimizer)
def bounding_wrapper(*args, **kwargs):
if self.optimizee_bounding_func is None:
return func(*args, **kwargs)
else:
# Deap Functions modify individuals in-place, Hence we must do the same
result_individuals_deap = func(*args, **kwargs)
result_individuals = [list_to_dict(x, self.optimizee_individual_dict_spec)
for x in result_individuals_deap]
bounded_individuals = [self.optimizee_bounding_func(x) for x in result_individuals]
for i, deap_indiv in enumerate(result_individuals_deap):
deap_indiv[:] = dict_to_list(bounded_individuals[i])
# print("Bounded Individual: {}".format(bounded_individuals))
return result_individuals_deap
def end(self, traj):
"""
See :meth:`~l2l.optimizers.optimizer.Optimizer.end`
"""
best_last_indiv_dict = list_to_dict(
self.best_individual_in_run.tolist(),
self.optimizee_individual_dict_spec)
traj.f_add_result('final_individual', best_last_indiv_dict)
traj.f_add_result('final_fitness', self.best_fitness_in_run)
traj.f_add_result('n_iteration', self.g + 1)
# ------------ Finished all runs and print result --------------- #
logger.info("-- End of (successful) ES optimization --")
def end(self, traj):
"""
See :meth:`~l2l.optimizers.optimizer.Optimizer.end`
"""
best_last_indiv_dict = list_to_dict(
list(self.current_individual), self.optimizee_individual_dict_spec)
traj.f_add_result('final_individual', best_last_indiv_dict)
traj.f_add_result('final_fitness', self.current_fitness)
traj.f_add_result('n_iteration', self.g + 1)
logger.info("The last individual was %s with fitness %s",
self.current_individual, self.current_fitness)
logger.info("-- End of (successful) gradient descent --")
def end(self, traj):
"""
See :meth:`~l2l.optimizers.optimizer.Optimizer.end`
"""
best_last_indiv_dict = list_to_dict(self.best_individual_in_run.tolist(),
self.optimizee_individual_dict_spec)
traj.f_add_result('final_individual', best_last_indiv_dict)
traj.f_add_result('final_fitness', self.best_fitness_in_run)
traj.f_add_result('n_iteration', self.g + 1)
# ------------ Finished all runs and print result --------------- #
logger.info("-- End of (successful) CE optimization --")
logger.info("-- Final distribution parameters --")
for parameter_key, parameter_value in sorted(self.distribution_results.items()):
logger.info(' %s: %s', parameter_key, parameter_value)
def end(self, traj):
"""
See :meth:`~l2l.optimizers.optimizer.Optimizer.end`
"""
# ------------ Finished all runs and print result --------------- #
best_last_indiv_index = np.argmax(self.current_fitness_value_list)
best_last_indiv = self.current_individual_list[best_last_indiv_index]
best_last_fitness = self.current_fitness_value_list[best_last_indiv_index]
best_last_indiv_dict = list_to_dict(best_last_indiv.tolist(), self.optimizee_individual_dict_spec)
traj.f_add_result('final_individual', best_last_indiv_dict)
traj.f_add_result('final_fitness', best_last_fitness)
traj.f_add_result('n_iteration', self.g + 1)
logger.info("The best last individual was %s with fitness %s", best_last_indiv, best_last_fitness)
logger.info("-- End of (successful) parallel tempering --")
def post_process(self, traj, fitnesses_results):
"""
See :meth:`~l2l.optimizers.optimizer.Optimizer.post_process`
"""
n_iteration, smoothing, temp_decay = \
traj.n_iteration, traj.smoothing, traj.temp_decay
stop_criterion, n_elite = traj.stop_criterion, traj.n_elite
weighted_fitness_list = []
#**************************************************************************************************************
# Storing run-information in the trajectory
# Reading fitnesses and performing distribution update
#**************************************************************************************************************
for run_index, fitness in fitnesses_results:
# We need to convert the current run index into an ind_idx
# (index of individual within one generation)
traj.v_idx = run_index
ind_index = traj.par.ind_idx
traj.f_add_result('$set.$.individual', self.eval_pop[ind_index])
traj.f_add_result('$set.$.fitness', fitness)
weighted_fitness_list.append(np.dot(fitness, self.optimizee_fitness_weights))
traj.v_idx = -1 # set trajectory back to default
weighted_fitness_list = np.array(weighted_fitness_list).ravel()
# Performs descending arg-sort of weighted fitness
fitness_sorting_indices = list(reversed(np.argsort(weighted_fitness_list)))
# Sorting the data according to fitness
sorted_population = self.eval_pop_asarray[fitness_sorting_indices]
sorted_fitness = np.asarray(weighted_fitness_list)[fitness_sorting_indices]
# Elite individuals are with performance better than or equal to the (1-rho) quantile.
# See original describtion of cross entropy for optimization
elite_individuals = sorted_population[:n_elite]
self.best_individual_in_run = sorted_population[0]
self.best_fitness_in_run = sorted_fitness[0]
self.gamma = sorted_fitness[n_elite - 1]
logger.info("-- End of generation %d --", self.g)
logger.info(" Evaluated %d individuals", len(fitnesses_results))
logger.info(' Best Fitness: %.4f', self.best_fitness_in_run)
logger.info(' Average Fitness: %.4f', np.mean(sorted_fitness))
logger.debug(' Calculated gamma: %.4f', self.gamma)
#**************************************************************************************************************
# Storing Generation Parameters / Results in the trajectory
#**************************************************************************************************************
# These entries correspond to the generation that has been simulated prior to this post-processing run
# Documentation of algorithm parameters for the current generation
#
# generation - The index of the evaluated generation
# gamma - The fitness threshold inferred from the evaluated generation
# (This is used in sampling the next generation)
# T - Temperature used to select non-elite elements among the individuals
# of the evaluated generation
# best_fitness_in_run - The highest fitness among the individuals in the
# evaluated generation
# pop_size - Population size
generation_result_dict = {
'generation': self.g,
'gamma': self.gamma,
'T': self.T,
'best_fitness_in_run': self.best_fitness_in_run,
'average_fitness_in_run': np.mean(sorted_fitness),
'pop_size': self.pop_size
}
generation_name = 'generation_{}'.format(self.g)
traj.results.generation_params.f_add_result_group(generation_name)
traj.results.generation_params.f_add_result(
generation_name + '.algorithm_params', generation_result_dict,
comment="These are the parameters that correspond to the algorithm, look at the source code"
" for `CrossEntropyOptimizer::post_process()` for comments documenting these"
" parameters")
# new distribution fit
individuals_to_be_fitted = elite_individuals
# Temperature dependent sampling of non elite individuals
if temp_decay > 0:
# Keeping non-elite samples with certain probability dependent on temperature (like Simulated Annealing)
non_elite_selection_probs = np.clip(np.exp((weighted_fitness_list[n_elite:] - self.gamma) / self.T),
a_min=0.0, a_max=1.0)
non_elite_selected_indices = self.random_state.binomial(1, non_elite_selection_probs).astype(bool)
non_elite_eval_pop_asarray = sorted_population[n_elite:][non_elite_selected_indices]
individuals_to_be_fitted = np.concatenate((elite_individuals, non_elite_eval_pop_asarray))
# Fitting New distribution parameters.
self.distribution_results = self.current_distribution.fit(individuals_to_be_fitted, smoothing)
#Add the results of the distribution fitting to the trajectory
traj.results.generation_params.f_add_result(
generation_name + '.distribution_params', self.distribution_results,
comment="These are the parameters of the distribution inferred from the currently evaluated"
" generation")
#**************************************************************************************************************
# Create the next generation by sampling the inferred distribution
#**************************************************************************************************************
# Note that this is only done in case the evaluated run is not the last run
fitnesses_results.clear()
self.eval_pop.clear()
# check if to stop
if self.g < n_iteration - 1 and self.best_fitness_in_run < stop_criterion:
#Sample from the constructed distribution
self.eval_pop_asarray = self.current_distribution.sample(self.pop_size)
self.eval_pop = [list_to_dict(ind_asarray, self.optimizee_individual_dict_spec)
for ind_asarray in self.eval_pop_asarray]
# Clip to boundaries
if self.optimizee_bounding_func is not None:
self.eval_pop = [self.optimizee_bounding_func(individual) for individual in self.eval_pop]
self.eval_pop_asarray = np.array([dict_to_list(x) for x in self.eval_pop])
self.g += 1 # Update generation counter
self.T *= temp_decay
self._expand_trajectory(traj)
def __init__(self,
traj,
optimizee_create_individual,
optimizee_fitness_weights,
parameters,
optimizee_bounding_func=None):
super().__init__(
traj,
optimizee_create_individual=optimizee_create_individual,
optimizee_fitness_weights=optimizee_fitness_weights,
parameters=parameters,
optimizee_bounding_func=optimizee_bounding_func)
self.optimizee_bounding_func = optimizee_bounding_func
__, self.optimizee_individual_dict_spec = dict_to_list(
optimizee_create_individual(), get_dict_spec=True)
traj.f_add_parameter('seed', parameters.seed, comment='Seed for RNG')
traj.f_add_parameter('popsize',
parameters.popsize,
comment='Population size') # 185
traj.f_add_parameter('CXPB', parameters.CXPB, comment='Crossover term')
traj.f_add_parameter('MUTPB',
parameters.MUTPB,
comment='Mutation probability')
traj.f_add_parameter('n_iteration',
parameters.NGEN,
comment='Number of generations')
traj.f_add_parameter('indpb',
parameters.indpb,
comment='Mutation parameter')
traj.f_add_parameter('tournsize',
parameters.tournsize,
comment='Selection parameter')
# ------- Create and register functions with DEAP ------- #
# delay_rate, slope, std_err, max_fraction_active
creator.create("FitnessMax",
base.Fitness,
weights=self.optimizee_fitness_weights)
creator.create("Individual", list, fitness=creator.FitnessMax)
toolbox = base.Toolbox()
# Structure initializers
toolbox.register("individual", tools.initIterate, creator.Individual,
lambda: dict_to_list(optimizee_create_individual()))
toolbox.register("population", tools.initRepeat, list,
toolbox.individual)
# Operator registering
# This complex piece of code is only necessary because we're using the
# DEAP framework and would like to decorate the DEAP mutation operator
def bounding_decorator(func):
def bounding_wrapper(*args, **kwargs):
if self.optimizee_bounding_func is None:
return func(*args, **kwargs)
else:
# Deap Functions modify individuals in-place, Hence we must do the same
result_individuals_deap = func(*args, **kwargs)
result_individuals = [
list_to_dict(x, self.optimizee_individual_dict_spec)
for x in result_individuals_deap
]
bounded_individuals = [
self.optimizee_bounding_func(x)
for x in result_individuals
]
for i, deap_indiv in enumerate(result_individuals_deap):
deap_indiv[:] = dict_to_list(bounded_individuals[i])
print("Bounded Individual: {}".format(bounded_individuals))
return result_individuals_deap
return bounding_wrapper
toolbox.register("mate", tools.cxBlend, alpha=parameters.matepar)
toolbox.decorate("mate", bounding_decorator)
toolbox.register("mutate",
tools.mutGaussian,
mu=0,
sigma=parameters.mutpar,
indpb=traj.indpb)
toolbox.decorate("mutate", bounding_decorator)
toolbox.register("select",
tools.selTournament,
tournsize=traj.tournsize)
# ------- Initialize Population and Trajectory -------- #
# NOTE: The Individual object implements the list interface.
self.pop = toolbox.population(n=traj.popsize)
self.eval_pop_inds = [ind for ind in self.pop if not ind.fitness.valid]
self.eval_pop = [
list_to_dict(ind, self.optimizee_individual_dict_spec)
for ind in self.eval_pop_inds
]
self.g = 0 # the current generation
self.toolbox = toolbox # the DEAP toolbox
self.hall_of_fame = HallOfFame(20)
self._expand_trajectory(traj)
def __init__(self,
traj,
optimizee_create_individual,
optimizee_fitness_weights,
parameters,
optimizee_bounding_func=None):
super().__init__(
traj,
optimizee_create_individual=optimizee_create_individual,
optimizee_fitness_weights=optimizee_fitness_weights,
parameters=parameters,
optimizee_bounding_func=optimizee_bounding_func)
self.optimizee_bounding_func = optimizee_bounding_func
# If a parameter is set to `None`, use default value as described in Wierstra et al. (2014)
if parameters.learning_rate_mu is None:
learning_rate_mu = 1.
else:
learning_rate_mu = parameters.learning_rate_mu
if parameters.learning_rate_sigma is None:
learning_rate_sigma = (3 + np.log(len(parameters.mu))) / (
5. * np.sqrt(len(parameters.mu)))
else:
learning_rate_sigma = parameters.learning_rate_sigma
if parameters.pop_size is None:
pop_size = 4 + int(np.floor(3 * np.log(len(parameters.mu))))
else:
pop_size = parameters.pop_size
if pop_size < 1:
raise ValueError("pop_size needs to be greater than 0")
# The following parameters are recorded
traj.f_add_parameter('learning_rate_mu',
learning_rate_mu,
comment='Learning rate mu')
traj.f_add_parameter('learning_rate_sigma',
learning_rate_sigma,
comment='Learning rate mu')
traj.f_add_parameter('mu',
parameters.mu,
comment='Initial mean of search distribution')
traj.f_add_parameter(
'sigma',
parameters.sigma,
comment='Initial standard deviation of search distribution')
traj.f_add_parameter('mirrored_sampling_enabled',
parameters.mirrored_sampling_enabled,
comment='Flag to enable mirrored sampling')
traj.f_add_parameter('fitness_shaping_enabled',
parameters.fitness_shaping_enabled,
comment='Flag to enable fitness shaping')
traj.f_add_parameter(
'pop_size',
pop_size,
comment='Number of minimal individuals simulated in each run')
traj.f_add_parameter('n_iteration',
parameters.n_iteration,
comment='Number of iterations to run')
traj.f_add_parameter(
'stop_criterion',
parameters.stop_criterion,
comment='Stop if best individual reaches this fitness')
traj.f_add_parameter(
'seed',
np.uint32(parameters.seed),
comment='Seed used for random number generation in optimizer')
self.random_state = np.random.RandomState(traj.parameters.seed)
self.current_individual_arr, self.optimizee_individual_dict_spec = dict_to_list(
self.optimizee_create_individual(), get_dict_spec=True)
traj.f_add_derived_parameter(
'dimension',
self.current_individual_arr.shape,
comment='The dimension of the parameter space of the optimizee')
# Added a generation-wise parameter logging
traj.results.f_add_result_group(
'generation_params',
comment='This contains the optimizer parameters that are'
' common across a generation')
# The following parameters are recorded as generation parameters i.e. once per generation
self.g = 0 # the current generation
self.pop_size = pop_size # Population size is dynamic in FACE
self.best_fitness_in_run = -np.inf
self.best_individual_in_run = None
# Set initial parameters of search distribution
self.mu = traj.mu
self.sigma = traj.sigma
# Generate initial distribution
self.current_perturbations = self._get_perturbations(traj)
current_eval_pop_arr = (
self.mu + self.sigma * self.current_perturbations).tolist()
self.eval_pop = [
list_to_dict(ind, self.optimizee_individual_dict_spec)
for ind in current_eval_pop_arr
]
# Bounding function has to be applied AFTER the individual has been converted to a dict
if optimizee_bounding_func is not None:
self.eval_pop = [
self.optimizee_bounding_func(ind) for ind in self.eval_pop
]
self.eval_pop_arr = np.array(
[dict_to_list(ind) for ind in self.eval_pop])
self._expand_trajectory(traj)
def post_process(self, traj, fitnesses_results):
"""
See :meth:`~l2l.optimizers.optimizer.Optimizer.post_process`
"""
n_elite, n_iteration, smoothing, temp_decay, min_pop_size, max_pop_size = \
traj.n_elite, traj.n_iteration, traj.smoothing, traj.temp_decay, traj.min_pop_size, traj.max_pop_size
stop_criterion, n_expand = traj.stop_criterion, traj.n_expand
weighted_fitness_list = []
# **************************************************************************************************************
# Storing run-information in the trajectory
# Reading fitnesses and performing distribution update
# **************************************************************************************************************
for run_index, fitness in fitnesses_results:
# We need to convert the current run index into an ind_idx
# (index of individual within one generation)
traj.v_idx = run_index
ind_index = traj.par.ind_idx
traj.f_add_result('$set.$.individual', self.eval_pop[ind_index])
traj.f_add_result('$set.$.fitness', fitness)
weighted_fitness_list.append(
np.dot(fitness, self.optimizee_fitness_weights))
traj.v_idx = -1 # set trajectory back to default
# Performs descending arg-sort of weighted fitness
fitness_sorting_indices = list(
reversed(np.argsort(weighted_fitness_list)))
generation_name = 'generation_{}'.format(self.g)
# Sorting the data according to fitness
sorted_population = self.eval_pop_asarray[fitness_sorting_indices]
sorted_fitess = np.asarray(
weighted_fitness_list)[fitness_sorting_indices]
# Elite individuals are with performance better than or equal to the (1-rho) quantile.
# See original describtion of cross entropy for optimization
elite_individuals = sorted_population[:n_elite]
previous_best_fitness = self.best_fitness_in_run
self.best_individual_in_run = sorted_population[0]
self.best_fitness_in_run = sorted_fitess[0]
previous_gamma = self.gamma
self.gamma = sorted_fitess[n_elite - 1]
logger.info("-- End of generation %d --", self.g)
logger.info(" Evaluated %d individuals", len(fitnesses_results))
logger.info(' Best Fitness: %.4f', self.best_fitness_in_run)
logger.debug(' Calculated gamma: %.4f', self.gamma)
# **************************************************************************************************************
# Storing Generation Parameters / Results in the trajectory
# **************************************************************************************************************
# These entries correspond to the generation that has been simulated prior to this post-processing run
traj.results.generation_params.f_add_result(
generation_name + '.g',
self.g,
comment='The index of the evaluated generation')
traj.results.generation_params.f_add_result(
generation_name + '.gamma',
self.gamma,
comment='The fitness threshold inferred from the evaluated '
'generation (This is used in sampling the next generation')
traj.results.generation_params.f_add_result(
generation_name + '.T',
self.T,
comment='Temperature used to select non-elite elements among the'
'individuals of the evaluated generation')
traj.results.generation_params.f_add_result(
generation_name + '.best_fitness_in_run',
self.best_fitness_in_run,
comment='The highest fitness among the individuals in the '
'evaluated generation')
traj.results.generation_params.f_add_result(generation_name +
'.pop_size',
self.pop_size,
comment='Population size')
# Check stopping
if self.g >= n_iteration or self.best_fitness_in_run >= stop_criterion:
return
expand = True
if self.best_fitness_in_run > previous_best_fitness or self.gamma > previous_gamma:
# shrink population size
self.pop_size = (self.pop_size + min_pop_size) // 2
# new distribution fit
individuals_to_be_fitted = elite_individuals
# Temperature dependent sampling of non elite individuals
if temp_decay > 0:
# Keeping non-elite samples with certain probability dependent on temperature (like Simulated Annealing)
non_elite_selection_probs = np.clip(np.exp(
(weighted_fitness_list[n_elite:] - self.gamma) / self.T),
amin=0.0,
a_max=1.0)
non_elite_selected_indices = self.random_state.binomial(
1, p=non_elite_selection_probs)
non_elite_eval_pop_asarray = sorted_population[n_elite:][
non_elite_selected_indices]
individuals_to_be_fitted = np.concatenate(
(elite_individuals, non_elite_eval_pop_asarray))
# Fitting New distribution parameters.
self.distribution_results = self.current_distribution.fit(
individuals_to_be_fitted, smoothing)
elif self.pop_size + n_expand <= max_pop_size:
# Increase pop size by one, resample, FACE part
logger.info(' FACE increase population size by %d', n_expand)
self.pop_size += n_expand
else:
# Stop algorithm
expand = False
logger.warning(' Possibly diverged')
# Add the results of the distribution fitting to the trajectory
for parameter_key, parameter_value in self.distribution_results.items(
):
traj.results.generation_params.f_add_result(
generation_name + '.' + parameter_key, parameter_value)
# **************************************************************************************************************
# Create the next generation by sampling the inferred distribution
# **************************************************************************************************************
# Note that this is only done in case the evaluated run is not the last run
fitnesses_results.clear()
self.eval_pop.clear()
if expand:
# Sample from the constructed distribution
self.eval_pop_asarray = self.current_distribution.sample(
self.pop_size)
self.eval_pop = [
list_to_dict(ind_asarray, self.optimizee_individual_dict_spec)
for ind_asarray in self.eval_pop_asarray
]
# Clip to boundaries
if self.optimizee_bounding_func is not None:
self.eval_pop = [
self.optimizee_bounding_func(individual)
for individual in self.eval_pop
]
self.eval_pop_asarray = np.array(
[dict_to_list(x) for x in self.eval_pop])
self.g += 1 # Update generation counter
self.T *= temp_decay
self._expand_trajectory(traj)
def __init__(self, traj,
optimizee_create_individual,
optimizee_fitness_weights,
parameters,
optimizee_bounding_func=None):
super().__init__(traj,
optimizee_create_individual=optimizee_create_individual,
optimizee_fitness_weights=optimizee_fitness_weights,
parameters=parameters, optimizee_bounding_func=optimizee_bounding_func)
self.optimizee_bounding_func = optimizee_bounding_func
# The following parameters are recorded
traj.f_add_parameter('n_parallel_runs', parameters.n_parallel_runs,
comment='Number of parallel simulated annealing runs / Size of Population')
traj.f_add_parameter('noisy_step', parameters.noisy_step, comment='Size of the random step')
traj.f_add_parameter('n_iteration', parameters.n_iteration, comment='Number of iteration to perform')
traj.f_add_parameter('stop_criterion', parameters.stop_criterion, comment='Stopping criterion parameter')
traj.f_add_parameter('seed', parameters.seed, comment='Seed for RNG')
cooling_schedules_string = ''
bounds_list = []
decay_list = []
schedules_list = []
for i in range(0,traj.n_parallel_runs):
bounds_list.append(str(parameters.temperature_bounds[i,:]))
bounds_list.append(' ')
decay_list.append(str(parameters.decay_parameters[i]))
decay_list.append(' ')
schedules_list.append(str(parameters.cooling_schedules[i]))
schedules_list.append(' ')
temperature_bounds_string = ''.join(bounds_list)
decay_parameters_string = ''.join(decay_list)
cooling_schedules_string = ''.join(schedules_list)
traj.f_add_parameter('temperature_bounds', temperature_bounds_string,
comment='The max and min temperature of the respective schedule')
traj.f_add_parameter('decay_parameters', decay_parameters_string,
comment='The one parameter, most schedules need')
traj.f_add_parameter('cooling_schedules', cooling_schedules_string,
comment='The used cooling schedule')
_, self.optimizee_individual_dict_spec = dict_to_list(self.optimizee_create_individual(), get_dict_spec=True)
# Note that this array stores individuals as an np.array of floats as opposed to Individual-Dicts
# This is because this array is used within the context of the simulated annealing algorithm and
# Thus needs to handle the optimizee individuals as vectors
self.current_individual_list = [np.array(dict_to_list(self.optimizee_create_individual()))
for _ in range(parameters.n_parallel_runs)]
traj.f_add_result('fitnesses', [], comment='Fitnesses of all individuals')
self.T_all = parameters.temperature_bounds[:,0] # Initialize temperature
self.T = 1
self.g = 0 # the current generation
self.cooling_schedules = parameters.cooling_schedules
self.decay_parameters = parameters.decay_parameters
self.temperature_bounds = parameters.temperature_bounds
# Keep track of current fitness value to decide whether we want the next individual to be accepted or not
self.current_fitness_value_list = [-np.Inf] * parameters.n_parallel_runs
new_individual_list = [
list_to_dict(ind_as_list + np.random.normal(0.0, parameters.noisy_step, ind_as_list.size) * traj.noisy_step,
self.optimizee_individual_dict_spec)
for ind_as_list in self.current_individual_list
]
if optimizee_bounding_func is not None:
new_individual_list = [self.optimizee_bounding_func(ind) for ind in new_individual_list]
self.eval_pop = new_individual_list
self._expand_trajectory(traj)
#initialize container for the indices of the parallel runs
self.parallel_indices = []
for i in range(0,traj.n_parallel_runs):
self.parallel_indices.append(i)
self.available_cooling_schedules = AvailableCoolingSchedules
# assert if all cooling schedules are among the known cooling schedules
schedule_known = True # start off as True - if any schdule is unknown gets False
for i in range(np.size(self.cooling_schedules)):
schedule_known = schedule_known and self.cooling_schedules[i] in AvailableCoolingSchedules
assert schedule_known, print("Warning: Unknown cooling schedule")
def post_process(self, traj, fitnesses_results):
"""
See :meth:`~l2l.optimizers.optimizer.Optimizer.post_process`
"""
old_eval_pop = self.eval_pop.copy()
self.eval_pop.clear()
logger.info(" Evaluating %i individuals" % len(fitnesses_results))
assert len(fitnesses_results) - 1 == traj.n_random_steps
# We need to collect the directions of the random steps along with
# the fitness evaluated there
fitnesses = np.zeros(traj.n_random_steps)
dx = np.zeros((traj.n_random_steps, len(self.current_individual)))
weighted_fitness_list = []
for i, (run_index, fitness) in enumerate(fitnesses_results):
# We need to convert the current run index into an ind_idx
# (index of individual within one generation
traj.v_idx = run_index
ind_index = traj.par.ind_idx
individual = old_eval_pop[ind_index]
traj.f_add_result('$set.$.individual', individual)
traj.f_add_result('$set.$.fitness', fitness)
weighted_fitness = np.dot(fitness, self.optimizee_fitness_weights)
weighted_fitness_list.append(weighted_fitness)
# The last element of the list is the evaluation of the individual
# obtained via gradient descent
if i == len(fitnesses_results) - 1:
self.current_fitness = weighted_fitness
else:
fitnesses[i] = weighted_fitness
dx[i] = np.array(
dict_to_list(individual)) - self.current_individual
traj.v_idx = -1 # set the trajectory back to default
# Performs descending arg-sort of weighted fitness
fitness_sorting_indices = list(
reversed(np.argsort(weighted_fitness_list)))
old_eval_pop_as_array = np.array(
[dict_to_list(x) for x in old_eval_pop])
# Sorting the data according to fitness
sorted_population = old_eval_pop_as_array[fitness_sorting_indices]
sorted_fitness = np.asarray(
weighted_fitness_list)[fitness_sorting_indices]
logger.info("-- End of generation %d --", self.g)
logger.info(" Evaluated %d individuals", len(fitnesses_results))
logger.info(' Average Fitness: %.4f', np.mean(sorted_fitness))
logger.info(" Current fitness is %.2f", self.current_fitness)
logger.info(' Best Fitness: %.4f', sorted_fitness[0])
logger.info(" Best individual is %s", sorted_population[0])
curr_ind_dict = list_to_dict(self.current_individual,
self.optimizee_individual_dict_spec)
generation_result_dict = {
'generation': self.g,
'current_fitness': self.current_fitness,
'best_fitness_in_run': sorted_fitness[0],
'average_fitness_in_run': np.mean(sorted_fitness),
'current_individual': curr_ind_dict,
}
generation_name = 'generation_{}'.format(self.g)
traj.results.generation_params.f_add_result_group(generation_name)
traj.results.generation_params.f_add_result(
generation_name + '.algorithm_params', generation_result_dict)
logger.info("-- End of iteration {}, current fitness is {} --".format(
self.g, self.current_fitness))
max_g = traj.n_iteration - 1
if self.g < max_g and traj.stop_criterion > self.current_fitness:
# Create new individual using the appropriate gradient descent
self.update_function(
traj,
np.dot(np.linalg.pinv(dx), fitnesses - self.current_fitness))
current_individual_dict = list_to_dict(
list(self.current_individual),
self.optimizee_individual_dict_spec)
if self.optimizee_bounding_func is not None:
current_individual_dict = self.optimizee_bounding_func(
current_individual_dict)
self.current_individual = np.array(
dict_to_list(current_individual_dict))
# Explore the neighbourhood in the parameter space of the
# current individual
new_individual_list = [
list_to_dict(
self.current_individual +
self.random_state.normal(0.0, traj.exploration_step_size,
self.current_individual.size),
self.optimizee_individual_dict_spec)
for _ in range(traj.n_random_steps)
]
if self.optimizee_bounding_func is not None:
new_individual_list = [
self.optimizee_bounding_func(ind)
for ind in new_individual_list
]
new_individual_list.append(current_individual_dict)
fitnesses_results.clear()
self.eval_pop = new_individual_list
self.g += 1 # Update generation counter
self._expand_trajectory(traj)
def __init__(self,
traj,
optimizee_create_individual,
optimizee_fitness_weights,
parameters,
optimizee_bounding_func=None):
super().__init__(
traj,
optimizee_create_individual=optimizee_create_individual,
optimizee_fitness_weights=optimizee_fitness_weights,
parameters=parameters,
optimizee_bounding_func=optimizee_bounding_func)
self.optimizee_bounding_func = optimizee_bounding_func
_, self.optimizee_individual_dict_spec = dict_to_list(
self.optimizee_create_individual(), get_dict_spec=True)
exploration_step_size = parameters.exploration_step_size
if isinstance(exploration_step_size, dict):
exploration_step_size = dict_to_list(exploration_step_size)
traj.f_add_parameter('learning_rate',
parameters.learning_rate,
comment='Value of learning rate')
traj.f_add_parameter('exploration_step_size',
exploration_step_size,
comment='Standard deviation of the random steps')
traj.f_add_parameter('n_random_steps',
parameters.n_random_steps,
comment='Amount of random steps taken for '
'calculating the gradient')
traj.f_add_parameter('n_iteration',
parameters.n_iteration,
comment='Number of iteration to perform')
traj.f_add_parameter('stop_criterion',
parameters.stop_criterion,
comment='Stopping criterion parameter')
traj.f_add_parameter('seed',
np.uint32(parameters.seed),
comment='Optimizer random seed')
self.random_state = np.random.RandomState(seed=traj.par.seed)
# Note that this array stores individuals as an np.array of floats as
# opposed to Individual-Dicts
# This is because this array is used within the context of the
# gradient descent algorithm and thus needs to handle the optimizee
# individuals as vectors
self.current_individual = np.array(
dict_to_list(self.optimizee_create_individual()))
# Depending on the algorithm used, initialize the necessary variables
self.update_function = None
if type(parameters) is ClassicGDParameters:
self.init_classic_gd(parameters, traj)
elif type(parameters) is StochasticGDParameters:
self.init_stochastic_gd(parameters, traj)
elif type(parameters) is AdamParameters:
self.so_moment = 0.0
self.delta = 0.0
self.init_adam(parameters, traj)
elif type(parameters) is RMSPropParameters:
self.so_moment = 0.0
self.fo_moment = 0.0
self.init_rmsprop(parameters, traj)
else:
raise Exception(
'Class of the provided "parameters" argument is not among '
'the supported types')
# Added a generation-wise parameter logging
traj.results.f_add_result_group('generation_params',
comment='This contains the optimizer '
'parameters that are common '
'across a generation')
# Explore the neighbourhood in the parameter space of current
# individual
new_individual_list = [
list_to_dict((self.current_individual + self.random_state.normal(
0.0, exploration_step_size,
self.current_individual.size)).tolist(),
self.optimizee_individual_dict_spec)
for _ in range(parameters.n_random_steps)
]
# Also add the current individual to determine it's fitness
new_individual_list.append(
list_to_dict(list(self.current_individual),
self.optimizee_individual_dict_spec))
if optimizee_bounding_func is not None:
new_individual_list = [
self.optimizee_bounding_func(ind)
for ind in new_individual_list
]
# Storing the fitness of the current individual
self.current_fitness = -np.Inf
self.g = 0
self.eval_pop = new_individual_list
self._expand_trajectory(traj)
def post_process(self, traj, fitnesses_results):
"""
See :meth:`~l2l.optimizers.optimizer.Optimizer.post_process`
"""
def to_fit(ind):
return np.dot(ind.fitness.values, ind.fitness.weights)
def spawn():
x = self.optimizee_create_individual()
return dict_to_list(self.optimizee_bounding_func(x))
CXPB, MUTPB, NGEN = traj.CXPB, traj.MUTPB, traj.n_iteration
logger.info(" Evaluating %i individuals" % len(fitnesses_results))
print(" Evaluating %i individuals" % len(fitnesses_results))
#******************************************************************
# Storing run-information in the trajectory
# Reading fitnesses and performing distribution update
#******************************************************************
# print("self.g = {}".format(self.g))
# print("len(self.eval_pop_inds) = {}".format(len(self.eval_pop_inds)))
# print("len(self.eval_pop) = {}".format(len(self.eval_pop)))
# print("len(fitnesses_results) = {}".format(len(fitnesses_results)))
for run_index, fitness in fitnesses_results:
# We need to convert the current run index into an ind_idx
# (index of individual within one generation)
traj.v_idx = run_index
ind_index = traj.par.ind_idx
traj.f_add_result('$set.$.individual', self.eval_pop[ind_index])
traj.f_add_result('$set.$.fitness', fitness)
# Use the ind_idx to update the fitness
individual = self.eval_pop_inds[ind_index]
individual.fitness.values = fitness
traj.v_idx = -1 # set the trajectory back to default
logger.info("-- End of generation {} --".format(self.g))
print("-- End of generation {} --".format(self.g))
# best_inds = tools.selBest(self.eval_pop_inds, 2)
# for best_ind in best_inds:
# print("Best individual is %s, %s" % (
# list_to_dict(best_ind, self.optimizee_individual_dict_spec),
# best_ind.fitness.values))
# add the bestest individuals this generation to HoF
self.hall_of_fame.update(self.eval_pop_inds)
logger.info("-- Hall of fame --")
# n_bobs = self.n_bobs
# bob_inds = tools.selBest(self.hall_of_fame, n_bobs)
n_bobs = self.n_hof
bob_inds = tools.selBest(self.hall_of_fame, n_bobs)
for hof_ind in self.hall_of_fame:
sind = str_ind(
list_to_dict(hof_ind, self.optimizee_individual_dict_spec)
)
logger.info("BoB individual is %s,\n %s" % (
sind, hof_ind.fitness.values))
# print("HOF individual is %s, %s" % (
# hof_ind, hof_ind.fitness.values))
print("BoB individual is %s,\n %s" % (
sind, hof_ind.fitness.values))
#bob_inds = list(map(self.toolbox.clone, bob_inds))
bob_inds = list(map(self.toolbox.clone, self.hall_of_fame))
# ------- Create the next generation by crossover and mutation -------- #
if self.g < NGEN - 1: # not necessary for the last generation
# Select the next generation individuals
# Tournament of population - a list of "pointers"
offspring = self.toolbox.select(self.pop, len(self.pop))
# Clone the selected individuals
offspring = list(map(self.toolbox.clone, offspring))
#sorts small to big
#switch worst-good with best of best
#ascending (worst to best)
offsp_ids = np.argsort([to_fit(o) for o in offspring])
#descending (best to worst)
bob_ids = np.argsort([to_fit(o) for o in bob_inds])[::-1]
max_score = to_fit(bob_inds[bob_ids[0]])
min_score = 0.5 * max_score
for i in range(n_bobs):
off_i = int(offsp_ids[i])
bob_i = int(bob_ids[i])
off_f = to_fit(offspring[off_i])
bob_f = to_fit(bob_inds[bob_i])
if bob_f > off_f:
logger.info("Inserting BoB {} to population".format(i+1))
offspring[off_i][:] = bob_inds[bob_i]
# Apply crossover and mutation on the offspring
for child1, child2 in zip(offspring[::2], offspring[1::2]):
f1, f2 = to_fit(child1), to_fit(child2)
if random.random() < CXPB:
#if both parents are really unfit, replace them with a new couple
if f1 <= min_score and f2 <= min_score:
logger.info("Both parents had a low score")
child1[:] = spawn()
child2[:] = spawn()
else:
self.toolbox.mate(child1, child2)
del child1.fitness.values
del child2.fitness.values
for mutant in offspring[:]:
if random.random() < MUTPB:
# f = to_fit(mutant) if mutant.fitness.valid else None
# print("f = {}".format(f))
# if this was an unfit individual, replace with a "foreigner"
# if f is not None and f <= min_score:
# logger.info("Mutant had a really low score")
# mutant[:] = spawn()
# else:
self.toolbox.mutate(mutant)
del mutant.fitness.values
if len(set(map(tuple, offspring))) < len(offspring):
logger.info("Mutating more")
for i, o1 in enumerate(offspring[:-1]):
for o2 in offspring[i+1:]:
if tuple(np.round(o1, decimals=4)) == tuple(np.round(o2, decimals=4)):
if random.random() < 0.8:
self.toolbox.mutate(o2)
#o2[:] = spawn()
del o2.fitness.values
# off_ids = np.random.choice(len(offspring), size=n_bobs, replace=False)
# for i in range(n_bobs):
# # off_i = int(offsp_ids[i])
# off_i = int(off_ids[i])
# bob_i = int(bob_ids[i])
# logger.info("Inserting BoB {} to population".format(i+1))
# offspring[off_i][:] = bob_inds[bob_i]
# del offspring[off_i].fitness.values
# The population is entirely replaced by the offspring
self.pop[:] = offspring
self.eval_pop_inds[:] = [ind for ind in self.pop if not ind.fitness.valid]
self.eval_pop[:] = [list_to_dict(ind, self.optimizee_individual_dict_spec)
for ind in self.eval_pop_inds]
# print("self.g = {}".format(self.g))
# print("len(self.eval_pop_inds) = {}".format(len(self.eval_pop_inds)))
# print("len(self.eval_pop) = {}".format(len(self.eval_pop)))
self.g += 1 # Update generation counter
if len(self.eval_pop) == 0 and self.g < (NGEN - 1):
raise Exception("No more mutants to evaluate where generated. "
"Increasing population size may help.")
self._expand_trajectory(traj)
def __init__(self,
traj,
optimizee_create_individual,
optimizee_fitness_weights,
parameters,
optimizee_bounding_func=None):
super().__init__(
traj,
optimizee_create_individual=optimizee_create_individual,
optimizee_fitness_weights=optimizee_fitness_weights,
parameters=parameters,
optimizee_bounding_func=optimizee_bounding_func)
self.optimizee_bounding_func = optimizee_bounding_func
# The following parameters are recorded
traj.f_add_parameter(
'n_parallel_runs',
parameters.n_parallel_runs,
comment=
'Number of parallel simulated annealing runs / Size of Population')
traj.f_add_parameter('noisy_step',
parameters.noisy_step,
comment='Size of the random step')
traj.f_add_parameter(
'temp_decay',
parameters.temp_decay,
comment='A temperature decay parameter (multiplicative)')
traj.f_add_parameter('n_iteration',
parameters.n_iteration,
comment='Number of iteration to perform')
traj.f_add_parameter('stop_criterion',
parameters.stop_criterion,
comment='Stopping criterion parameter')
traj.f_add_parameter('seed',
np.uint32(parameters.seed),
comment='Seed for RNG')
_, self.optimizee_individual_dict_spec = dict_to_list(
self.optimizee_create_individual(), get_dict_spec=True)
# Note that this array stores individuals as an np.array of floats as opposed to Individual-Dicts
# This is because this array is used within the context of the simulated annealing algorithm and
# Thus needs to handle the optimizee individuals as vectors
self.current_individual_list = [
np.array(dict_to_list(self.optimizee_create_individual()))
for _ in range(parameters.n_parallel_runs)
]
self.random_state = np.random.RandomState(parameters.seed)
# The following parameters are NOT recorded
self.T = 1. # Initialize temperature
self.g = 0 # the current generation
# Keep track of current fitness value to decide whether we want the next individual to be accepted or not
self.current_fitness_value_list = [-np.Inf
] * parameters.n_parallel_runs
new_individual_list = [
list_to_dict(
ind_as_list + self.random_state.normal(
0.0, parameters.noisy_step, ind_as_list.size) *
traj.noisy_step * self.T, self.optimizee_individual_dict_spec)
for ind_as_list in self.current_individual_list
]
if optimizee_bounding_func is not None:
new_individual_list = [
self.optimizee_bounding_func(ind)
for ind in new_individual_list
]
self.eval_pop = new_individual_list
self._expand_trajectory(traj)
self.cooling_schedule = parameters.cooling_schedule
def post_process(self, traj, fitnesses_results):
"""
See :meth:`~l2l.optimizers.optimizer.Optimizer.post_process`
"""
noisy_step, temp_decay, n_iteration, stop_criterion = \
traj.noisy_step, traj.temp_decay, traj.n_iteration, traj.stop_criterion
old_eval_pop = self.eval_pop.copy()
self.eval_pop.clear()
temperature = self.T
temperature_end = 0
self.T = self.cooling(temperature, self.cooling_schedule, temp_decay,
temperature_end, n_iteration)
logger.info(" Evaluating %i individuals" % len(fitnesses_results))
assert len(fitnesses_results) == traj.n_parallel_runs
weighted_fitness_list = []
for i, (run_index, fitness) in enumerate(fitnesses_results):
weighted_fitness = sum(
f * w for f, w in zip(fitness, self.optimizee_fitness_weights))
weighted_fitness_list.append(weighted_fitness)
# We need to convert the current run index into an ind_idx
# (index of individual within one generation)
traj.v_idx = run_index
ind_index = traj.par.ind_idx
individual = old_eval_pop[ind_index]
# Accept or reject the new solution
current_fitness_value_i = self.current_fitness_value_list[i]
r = self.random_state.rand()
p = np.exp((weighted_fitness - current_fitness_value_i) / self.T)
# Accept
if r < p or weighted_fitness >= current_fitness_value_i:
self.current_fitness_value_list[i] = weighted_fitness
self.current_individual_list[i] = np.array(
dict_to_list(individual))
traj.f_add_result('$set.$.individual', individual)
# Watchout! if weighted fitness is a tuple/np array it should be converted to a list first here
traj.f_add_result('$set.$.fitness', weighted_fitness)
current_individual = self.current_individual_list[i]
new_individual = list_to_dict(
current_individual +
self.random_state.randn(current_individual.size) * noisy_step *
self.T, self.optimizee_individual_dict_spec)
if self.optimizee_bounding_func is not None:
new_individual = self.optimizee_bounding_func(new_individual)
logger.debug(
"Current best fitness for individual %d is %.2f. New individual is %s",
i, self.current_fitness_value_list[i], new_individual)
self.eval_pop.append(new_individual)
logger.debug("Current best fitness within population is %.2f",
max(self.current_fitness_value_list))
traj.v_idx = -1 # set the trajectory back to default
logger.info("-- End of generation {} --".format(self.g))
# ------- Create the next generation by crossover and mutation -------- #
# not necessary for the last generation
if self.g < n_iteration - 1 and stop_criterion > max(
self.current_fitness_value_list):
fitnesses_results.clear()
self.g += 1 # Update generation counter
self._expand_trajectory(traj)
def post_process(self, traj, fitnesses_results):
"""
See :meth:`~l2l.optimizers.optimizer.Optimizer.post_process`
"""
noisy_step, n_iteration, stop_criterion = \
traj.noisy_step, traj.n_iteration, traj.stop_criterion
cooling_schedules = self.cooling_schedules
decay_parameters = self.decay_parameters
temperature_bounds = self.temperature_bounds
old_eval_pop = self.eval_pop.copy()
self.eval_pop.clear()
temperature = self.T_all
for i in range(0,traj.n_parallel_runs):
self.T_all[self.parallel_indices[i]] = self.cooling(temperature[self.parallel_indices[i]], cooling_schedules[self.parallel_indices[i]], decay_parameters[self.parallel_indices[i]], temperature_bounds[self.parallel_indices[i],:], n_iteration)
logger.info(" Evaluating %i individuals" % len(fitnesses_results))
assert len(fitnesses_results) == traj.n_parallel_runs
weighted_fitness_list = []
for i, (run_index, fitness) in enumerate(fitnesses_results):
self.T = self.T_all[self.parallel_indices[i]]
weighted_fitness = sum(f * w for f, w in zip(fitness, self.optimizee_fitness_weights))
weighted_fitness_list.append(weighted_fitness)
# We need to convert the current run index into an ind_idx
# (index of individual within one generation)
traj.v_idx = run_index
ind_index = traj.par.ind_idx
individual = old_eval_pop[ind_index]
# Accept or reject the new solution
current_fitness_value_i = self.current_fitness_value_list[i]
r = np.random.rand()
p = np.exp((weighted_fitness - current_fitness_value_i) / self.T)
# Accept
if r < p or weighted_fitness >= current_fitness_value_i:
self.current_fitness_value_list[i] = weighted_fitness
self.current_individual_list[i] = np.array(dict_to_list(individual))
traj.f_add_result('$set.$.individual', individual)
# Watchout! if weighted fitness is a tuple/np array it should be converted to a list first here
traj.f_add_result('$set.$.fitness', weighted_fitness)
current_individual = self.current_individual_list[i]
new_individual = list_to_dict(current_individual + np.random.randn(current_individual.size) * noisy_step * self.T,
self.optimizee_individual_dict_spec)
if self.optimizee_bounding_func is not None:
new_individual = self.optimizee_bounding_func(new_individual)
logger.debug("Current best fitness for individual %d is %.2f. New individual is %s",
i, self.current_fitness_value_list[i], new_individual)
self.eval_pop.append(new_individual)
# the parallel tempering swapping starts here
for i in range(0,traj.n_parallel_runs):
#make a random choice from all the other parallel runs
compare_indices = []
for j in range(0,traj.n_parallel_runs):
compare_indices.append(i)
compare_indices.remove(i)
random_choice = random.choice(compare_indices)
#random variable with unit distribution betwwen 0 and 1
random_variable = np.random.rand()
#swap if criterion is met
if (self.metropolis_hasting(self.current_fitness_value_list[i], self.current_fitness_value_list[random_choice], self.T_all[i], self.T_all[random_choice]) > random_variable):
temp = self.parallel_indices[i]
self.parallel_indices[i] = self.parallel_indices[random_choice]
self.parallel_indices[random_choice] = temp
logger.debug("Current best fitness within population is %.2f", max(self.current_fitness_value_list))
traj.v_idx = -1 # set the trajectory back to default
logger.info("-- End of generation {} --".format(self.g))
# ------- Create the next generation by crossover and mutation -------- #
# not necessary for the last generation
if self.g < n_iteration - 1 and stop_criterion > max(self.current_fitness_value_list):
fitnesses_results.clear()
self.g += 1 # Update generation counter
self._expand_trajectory(traj)
def post_process(self, traj, fitnesses_results):
"""
See :meth:`~l2l.optimizers.optimizer.Optimizer.post_process`
"""
n_iteration, stop_criterion, learning_rate = \
traj.n_iteration, traj.stop_criterion, traj.learning_rate
noise_std, fitness_shaping_enabled = \
traj.noise_std, traj.fitness_shaping_enabled
weighted_fitness_list = []
# *********************************************************************
# Storing run-information in the trajectory
# Reading fitnesses and performing distribution update
# *********************************************************************
for run_index, fitness in fitnesses_results:
# We need to convert the current run index into an ind_idx
# (index of individual within one generation)
traj.v_idx = run_index
ind_index = traj.par.ind_idx
traj.f_add_result('$set.$.individual', self.eval_pop[ind_index])
traj.f_add_result('$set.$.fitness', fitness)
weighted_fitness_list.append(
np.dot(fitness, self.optimizee_fitness_weights))
traj.v_idx = -1 # set trajectory back to default
weighted_fitness_list = np.array(weighted_fitness_list).ravel()
# NOTE: It is necessary to clear the finesses_results to clear the data
# in the reference, and del
# ^ is used to make sure it's not used in the rest of this function
fitnesses_results.clear()
del fitnesses_results
# Last fitness is for the previous `current_individual_arr`
weighted_fitness_list = weighted_fitness_list[:-1]
current_individual_fitness = weighted_fitness_list[-1]
# Performs descending arg-sort of weighted fitness
fitness_sorting_indices = list(
reversed(np.argsort(weighted_fitness_list)))
# Sorting the data according to fitness
sorted_population = self.eval_pop_arr[fitness_sorting_indices]
sorted_fitness = np.asarray(
weighted_fitness_list)[fitness_sorting_indices]
sorted_perturbations = self.current_perturbations[
fitness_sorting_indices]
self.best_individual_in_run = sorted_population[0]
self.best_fitness_in_run = sorted_fitness[0]
logger.info("-- End of generation %d --", self.g)
logger.info(" Evaluated %d individuals",
len(weighted_fitness_list) + 1)
logger.info(' Best Fitness: %.4f', self.best_fitness_in_run)
logger.info(' Average Fitness: %.4f', np.mean(sorted_fitness))
# *********************************************************************
# Storing Generation Parameters / Results in the trajectory
# *********************************************************************
# These entries correspond to the generation that has been simulated
# prior to this post-processing run
# Documentation of algorithm parameters for the current generation
#
# generation - The index of the evaluated generation
# best_fitness_in_run - The highest fitness among the individuals in the
# evaluated generation
# pop_size - Population size
generation_result_dict = {
'generation': self.g,
'best_fitness_in_run': self.best_fitness_in_run,
'current_individual_fitness': current_individual_fitness,
'average_fitness_in_run': np.mean(sorted_fitness),
'pop_size': self.pop_size
}
generation_name = 'generation_{}'.format(self.g)
traj.results.generation_params.f_add_result_group(generation_name)
traj.results.generation_params.f_add_result(
generation_name + '.algorithm_params',
generation_result_dict,
comment="These are the parameters that correspond to the "
"algorithm. Look at the source code for "
"`EvolutionStrategiesOptimizer::post_process()` "
"for comments documenting these parameters")
if fitness_shaping_enabled:
sorted_utilities = []
n_individuals = len(sorted_fitness)
for i in range(n_individuals):
u = max(0., np.log((n_individuals / 2) + 1) - np.log(i + 1))
sorted_utilities.append(u)
sorted_utilities = np.array(sorted_utilities)
sorted_utilities /= np.sum(sorted_utilities)
sorted_utilities -= (1. / n_individuals)
# assert np.sum(sorted_utilities) == 0., \
# "Sum of utilities is not 0, but %.4f" % np.sum(sorted_utilities)
fitnesses_to_fit = sorted_utilities
else:
fitnesses_to_fit = sorted_fitness
assert len(fitnesses_to_fit) == len(sorted_perturbations)
sum_fits = np.sum(
[f * e for f, e in zip(fitnesses_to_fit, sorted_perturbations)],
axis=0)
weight = len(fitnesses_to_fit) * np.asarray(noise_std)**2
self.current_individual_arr += learning_rate * (sum_fits / weight)
# *********************************************************************
# Create the next generation by sampling the inferred distribution
# *********************************************************************
# Note that this is only done in case the evaluated run is not the
# last run
self.eval_pop.clear()
# check if to stop
max_g = n_iteration - 1
if self.g < max_g and self.best_fitness_in_run < stop_criterion:
self.current_perturbations = self._get_perturbations(traj)
perturbated = \
self.current_individual_arr + self.current_perturbations
current_eval_pop_arr = perturbated.tolist()
self.eval_pop[:] = [
list_to_dict(ind, self.optimizee_individual_dict_spec)
for ind in current_eval_pop_arr
]
self.eval_pop.append(
list_to_dict(self.current_individual_arr,
self.optimizee_individual_dict_spec))
# Bounding function has to be applied AFTER the individual has been
# converted to a dict
if self.optimizee_bounding_func is not None:
self.eval_pop[:] = [
self.optimizee_bounding_func(ind) for ind in self.eval_pop
]
self.eval_pop_arr[:] = np.asarray(
[dict_to_list(ind) for ind in self.eval_pop])
self.g += 1 # Update generation counter
self._expand_trajectory(traj)
def post_process(self, traj, fitnesses_results):
"""
See :meth:`~l2l.optimizers.optimizer.Optimizer.post_process`
"""
CXPB, MUTPB, NGEN = traj.CXPB, traj.MUTPB, traj.n_iteration
logger.info(" Evaluating %i individuals" % len(fitnesses_results))
#**************************************************************************************************************
# Storing run-information in the trajectory
# Reading fitnesses and performing distribution update
#**************************************************************************************************************
for run_index, fitness in fitnesses_results:
# We need to convert the current run index into an ind_idx
# (index of individual within one generation)
traj.v_idx = run_index
ind_index = traj.par.ind_idx
traj.f_add_result('$set.$.individual', self.eval_pop[ind_index])
traj.f_add_result('$set.$.fitness', fitness)
# Use the ind_idx to update the fitness
individual = self.eval_pop_inds[ind_index]
individual.fitness.values = fitness
traj.v_idx = -1 # set the trajectory back to default
logger.info("-- End of generation {} --".format(self.g))
best_inds = tools.selBest(self.eval_pop_inds, 2)
for best_ind in best_inds:
print("Best individual is %s, %s" %
(list_to_dict(best_ind, self.optimizee_individual_dict_spec),
best_ind.fitness.values))
self.hall_of_fame.update(self.eval_pop_inds)
logger.info("-- Hall of fame --")
for hof_ind in tools.selBest(self.hall_of_fame, 2):
logger.info(
"HOF individual is %s, %s" %
(list_to_dict(hof_ind, self.optimizee_individual_dict_spec),
hof_ind.fitness.values))
# ------- Create the next generation by crossover and mutation -------- #
if self.g < NGEN - 1: # not necessary for the last generation
# Select the next generation individuals
offspring = self.toolbox.select(self.pop, len(self.pop))
# Clone the selected individuals
offspring = list(map(self.toolbox.clone, offspring))
# Apply crossover and mutation on the offspring
for child1, child2 in zip(offspring[::2], offspring[1::2]):
if random.random() < CXPB:
self.toolbox.mate(child1, child2)
del child1.fitness.values
del child2.fitness.values
for mutant in offspring:
if random.random() < MUTPB:
self.toolbox.mutate(mutant)
del mutant.fitness.values
if len(set(map(tuple, offspring))) < len(offspring):
logger.info("Mutating more")
for i, o1 in enumerate(offspring[:-1]):
for o2 in offspring[i + 1:]:
if tuple(o1) == tuple(o2):
if random.random() < 0.8:
self.toolbox.mutate(o2)
del o2.fitness.values
# The population is entirely replaced by the offspring
self.pop[:] = offspring
self.eval_pop_inds = [
ind for ind in self.pop if not ind.fitness.valid
]
self.eval_pop = [
list_to_dict(ind, self.optimizee_individual_dict_spec)
for ind in self.eval_pop_inds
]
self.g += 1 # Update generation counter
self._expand_trajectory(traj)
def __init__(self,
traj,
optimizee_create_individual,
optimizee_fitness_weights,
parameters,
optimizee_bounding_func=None):
super().__init__(
traj,
optimizee_create_individual=optimizee_create_individual,
optimizee_fitness_weights=optimizee_fitness_weights,
parameters=parameters,
optimizee_bounding_func=optimizee_bounding_func)
self.optimizee_bounding_func = optimizee_bounding_func
if parameters.pop_size < 1:
raise Exception("pop_size needs to be greater than 0")
# The following parameters are recorded
traj.f_add_parameter('learning_rate',
parameters.learning_rate,
comment='Learning rate')
traj.f_add_parameter('noise_std',
parameters.noise_std,
comment='Standard deviation of noise')
traj.f_add_parameter('mirrored_sampling_enabled',
parameters.mirrored_sampling_enabled,
comment='Flag to enable mirrored sampling')
traj.f_add_parameter('fitness_shaping_enabled',
parameters.fitness_shaping_enabled,
comment='Flag to enable fitness shaping')
traj.f_add_parameter(
'pop_size',
parameters.pop_size,
comment='Number of minimal individuals simulated in each run')
traj.f_add_parameter('n_iteration',
parameters.n_iteration,
comment='Number of iterations to run')
traj.f_add_parameter(
'stop_criterion',
parameters.stop_criterion,
comment='Stop if best individual reaches this fitness')
traj.f_add_parameter(
'seed',
np.uint32(parameters.seed),
comment='Seed used for random number generation in optimizer')
self.random_state = np.random.RandomState(traj.parameters.seed)
self.current_individual_arr, self.optimizee_individual_dict_spec = \
dict_to_list(
self.optimizee_create_individual(), get_dict_spec=True)
noise_std_shape = np.array(parameters.noise_std).shape
ind_shape = self.current_individual_arr.shape
assert noise_std_shape == () or noise_std_shape == ind_shape
traj.f_add_derived_parameter(
'dimension',
self.current_individual_arr.shape,
comment='The dimension of the parameter space of the optimizee')
# Added a generation-wise parameter logging
traj.results.f_add_result_group(
'generation_params',
comment='This contains the optimizer parameters that are'
' common across a generation')
# The following parameters are recorded as generation parameters
# i.e. once per generation
self.g = 0 # the current generation
# Population size is dynamic in FACE
self.pop_size = parameters.pop_size
self.best_fitness_in_run = -np.inf
self.best_individual_in_run = None
# The first iteration does not pick the values out of the Gaussian
# distribution. It picks randomly (or at-least
# as randomly as optimizee_create_individual creates individuals)
# Note that this array stores individuals as an np.array of floats as
# opposed to Individual-Dicts.
# This is because this array is used within the context of the cross
# entropy algorithm and thus needs to handle the optimizee individuals
# as vectors
self.current_perturbations = self._get_perturbations(traj)
current_eval_pop_arr = (self.current_individual_arr +
self.current_perturbations).tolist()
self.eval_pop = [
list_to_dict(ind, self.optimizee_individual_dict_spec)
for ind in current_eval_pop_arr
]
self.eval_pop.append(
list_to_dict(self.current_individual_arr,
self.optimizee_individual_dict_spec))
# Bounding function has to be applied AFTER the individual has been
# converted to a dict
if optimizee_bounding_func is not None:
self.eval_pop[:] = [
self.optimizee_bounding_func(ind) for ind in self.eval_pop
]
self.eval_pop_arr = np.array(
[dict_to_list(ind) for ind in self.eval_pop])
self._expand_trajectory(traj)
def post_process(self, traj, fitnesses_results):
"""
See :meth:`~l2l.optimizers.optimizer.Optimizer.post_process`
"""
n_iteration, stop_criterion, fitness_shaping_enabled = \
traj.n_iteration, traj.stop_criterion, traj.fitness_shaping_enabled
weighted_fitness_list = []
# **************************************************************************************************************
# Storing run-information in the trajectory
# Reading fitnesses and performing distribution update
# **************************************************************************************************************
for run_index, fitness in fitnesses_results:
# We need to convert the current run index into an ind_idx
# (index of individual within one generation)
traj.v_idx = run_index
ind_index = traj.par.ind_idx
traj.f_add_result('$set.$.individual', self.eval_pop[ind_index])
traj.f_add_result('$set.$.fitness', fitness)
weighted_fitness_list.append(
np.dot(fitness, self.optimizee_fitness_weights))
traj.v_idx = -1 # set trajectory back to default
weighted_fitness_list = np.array(weighted_fitness_list).ravel()
# NOTE: It is necessary to clear the finesses_results to clear the data in the reference, and del
# is used to make sure it's not used in the rest of this function
fitnesses_results.clear()
del fitnesses_results
# Last fitness is for the previous `current_individual_arr`
weighted_fitness_list = weighted_fitness_list[:-1]
current_individual_fitness = weighted_fitness_list[-1]
# Performs descending arg-sort of weighted fitness
fitness_sorting_indices = list(
reversed(np.argsort(weighted_fitness_list)))
# Sorting the data according to fitness
sorted_population = self.eval_pop_arr[fitness_sorting_indices]
sorted_fitness = np.asarray(
weighted_fitness_list)[fitness_sorting_indices]
sorted_perturbations = self.current_perturbations[
fitness_sorting_indices]
self.best_individual_in_run = sorted_population[0]
self.best_fitness_in_run = sorted_fitness[0]
logger.info("-- End of generation %d --", self.g)
logger.info(" Evaluated %d individuals",
len(weighted_fitness_list) + 1)
logger.info(' Best Fitness: %.4f', self.best_fitness_in_run)
logger.info(' Average Fitness: %.4f', np.mean(sorted_fitness))
# **************************************************************************************************************
# Storing Generation Parameters / Results in the trajectory
# **************************************************************************************************************
# These entries correspond to the generation that has been simulated prior to this post-processing run
# Documentation of algorithm parameters for the current generation
#
# generation - The index of the evaluated generation
# best_fitness_in_run - The highest fitness among the individuals in the
# evaluated generation
# pop_size - Population size
generation_result_dict = {
'generation': self.g,
'best_fitness_in_run': self.best_fitness_in_run,
'current_individual_fitness': current_individual_fitness,
'average_fitness_in_run': np.mean(sorted_fitness),
'pop_size': self.pop_size
}
generation_name = 'generation_{}'.format(self.g)
traj.results.generation_params.f_add_result_group(generation_name)
traj.results.generation_params.f_add_result(
generation_name + '.algorithm_params',
generation_result_dict,
comment="These are the parameters that correspond to the algorithm. "
"Look at the source code for `EvolutionStrategiesOptimizer::post_process()` "
"for comments documenting these parameters")
traj.results.generation_params.f_add_result(
generation_name + '.distribution_params', {
'mu': self.mu.copy(),
'sigma': self.sigma.copy()
},
comment=
"These are the parameters of the distribution that underlies the"
" currently evaluated generation")
if fitness_shaping_enabled:
fitnesses_to_fit = self._compute_utility(sorted_fitness)
else:
fitnesses_to_fit = sorted_fitness
assert len(fitnesses_to_fit) == len(sorted_perturbations)
# **************************************************************************************************************
# Update the parameters of the search distribution using the natural gradient in natural coordinates
# **************************************************************************************************************
self.mu += traj.learning_rate_mu * traj.sigma * np.dot(
fitnesses_to_fit, sorted_perturbations)
self.sigma *= np.exp(
traj.learning_rate_sigma / 2. *
np.dot(fitnesses_to_fit, sorted_perturbations**2 - 1.))
# **************************************************************************************************************
# Create the next generation by sampling the inferred distribution
# **************************************************************************************************************
# Note that this is only done in case the evaluated run is not the last run
self.eval_pop.clear()
# check if to stop
if self.g < n_iteration - 1 and self.best_fitness_in_run < stop_criterion:
self.current_perturbations = self._get_perturbations(traj)
current_eval_pop_arr = (
self.mu + self.sigma * self.current_perturbations).tolist()
self.eval_pop = [
list_to_dict(ind, self.optimizee_individual_dict_spec)
for ind in current_eval_pop_arr
]
# Bounding function has to be applied AFTER the individual has been converted to a dict
if self.optimizee_bounding_func is not None:
self.eval_pop = [
self.optimizee_bounding_func(ind) for ind in self.eval_pop
]
self.eval_pop_arr = np.array(
[dict_to_list(ind) for ind in self.eval_pop])
self.g += 1 # Update generation counter
self._expand_trajectory(traj)
