def test_stop_rule_true(self):
"""
Tests that the stop rule is built properly when the stop criteria are not met.
"""
bb_obj = BBOptimizer(
black_box=self.parabola,
heuristic="surrogate_model",
max_iteration=nbr_iteration,
time_out=time_out,
parameter_space=parameter_space,
next_parameter_strategy=expected_improvement,
regression_model=GaussianProcessRegressor,
)
# If the number of iterations is exceeded
bb_obj.nbr_iteration = nbr_iteration - 1
self.assertTrue(
bb_obj.stop_rule,
"Exceeded number of iteration did not stop the " "optimizing loop.",
)
bb_obj = BBOptimizer(
black_box=self.parabola,
heuristic="surrogate_model",
max_iteration=nbr_iteration,
time_out=time_out,
parameter_space=parameter_space,
next_parameter_strategy=expected_improvement,
regression_model=GaussianProcessRegressor,
)
# If the time elapsed is exceeded
bb_obj.elapsed_time = time_out - 1
self.assertTrue(
bb_obj.stop_rule, "Exceeded elapsed time did not stop optimizing loop."
)
bb_obj = BBOptimizer(
black_box=self.parabola,
heuristic="surrogate_model",
max_iteration=nbr_iteration,
time_out=time_out,
parameter_space=parameter_space,
next_parameter_strategy=expected_improvement,
regression_model=GaussianProcessRegressor,
)
# If the heuristic stop criterion is met
bb_obj.heuristic.stop = False
self.assertTrue(
bb_obj.stop_rule,
"Internal heuristic stop did not stop the optimizing " "loop.",
)
def test_size_explored_space(self):
"""
Tests that the computation of the size of the explored space works properly, as well as
the percentage of static moves.
"""
history = {
"fitness": np.array([10, 5, 4]),
"parameters": np.array([[1, 2, 3], [2, 3, 4], [1, 2, 3]]),
"truncated": np.array([True, True, True, True, True, True]),
}
bb_obj = BBOptimizer(
black_box=self.parabola,
heuristic="surrogate_model",
max_iteration=nbr_iteration,
initial_sample_size=2,
parameter_space=parameter_space,
next_parameter_strategy=expected_improvement,
regression_model=GaussianProcessRegressor,
)
bb_obj.history = history
expected_static_moves = 1 / 3 * 100
expected_explored_space = 2 / 1440 * 100
real_explored_space, real_static_moves = bb_obj.size_explored_space
self.assertEqual(
expected_static_moves,
real_static_moves,
"Percentage of static moves " "was not computed properly.",
)
self.assertEqual(
expected_explored_space,
real_explored_space,
"Percentage of explored " "space was not computed " "properly.",
)
def test_infer_type(self):
"""Tests that the static method built to infer the types
of an array.
"""
bb_obj = BBOptimizer(
black_box=self.parabola,
heuristic="surrogate_model",
max_iteration=50,
initial_sample_size=2,
parameter_space=parameter_space,
next_parameter_strategy=expected_improvement,
regression_model=GaussianProcessRegressor,
stop_criterion="improvement_criterion",
stop_window=4,
improvement_estimator=min,
improvement_threshold=0.1
)
int_array = np.array([1, 2, 3])
inferred_int_array = bb_obj.infer_type(int_array)
assert_array_equal(int_array, inferred_int_array)
mixed_type_array = np.array(['1', '2', 'toto'])
expected_mixed_type_array = np.array([1, 2, 'toto'], dtype=object)
inferred_mixed_type_array = bb_obj.infer_type(mixed_type_array)
assert_array_equal(inferred_mixed_type_array,
expected_mixed_type_array)
def test_reevaluation(self):
"""Tests the proper implementation of the reevaluation feature.
This feature is tested by checking in the history that no re-evaluation has been done
(as is the case with this particular optimizer configuration).
With re-evaluate set to True, there are only 3 different values in the history of
parametrizations. With re-evaluate set to False, all parameters of the history are
different.
"""
nbr_iteration = 5
bb_obj = BBOptimizer(
black_box=self.parabola,
heuristic="genetic_algorithm",
max_iteration=nbr_iteration,
initial_sample_size=2,
parameter_space=parameter_space,
selection_method=tournament_pick,
crossover_method=double_point_crossover,
mutation_method=mutate_chromosome_to_neighbor,
mutation_rate=0.6,
reevaluate=False,
max_retry=5,
)
bb_obj.optimize()
self.assertEqual(
np.unique(bb_obj.history["parameters"].astype(
str), axis=0).shape[0],
7,
"Some parameter values have been evaluated several times",
)
def test_reset(self):
"""
Tests that the "reset" method reset the attributes.
"""
bb_obj = BBOptimizer(
black_box=self.parabola,
heuristic="surrogate_model",
max_iteration=nbr_iteration,
initial_sample_size=2,
parameter_space=parameter_space,
next_parameter_strategy=expected_improvement,
regression_model=GaussianProcessRegressor,
)
bb_obj.optimize()
bb_obj.reset()
self.assertEqual(bb_obj.nbr_iteration, 0)
self.assertEqual(bb_obj.elapsed_time, 0)
self.assertEqual(bb_obj.launched, False)
self.assertDictEqual(
bb_obj.history,
{
"fitness": None,
"parameters": None,
"initialization": None,
"resampled": None,
"truncated": None,
},
)
def setup_bb_optimizer(self) -> BBOptimizer:
"""Setups the black-box from the configuration."""
# Create the BBOptimizer object using the different options in the
# configuration
# If pruning is enabled, parse corresponding fields
if self.configuration.pruning:
pruning = True
max_step_cost = (
self.bb_wrapper.default_target_value
if self.configuration.pruning.max_step_duration == "default"
else self.configuration.pruning.max_step_duration
)
else:
max_step_cost = None
pruning = False
self.bb_optimizer = BBOptimizer(
black_box=self.bb_wrapper,
parameter_space=self.configuration.component_parameter_space,
max_iteration=self.nbr_iteration,
async_optim=pruning,
max_step_cost=max_step_cost,
**self.configuration.bbo_parameters,
)
return self.bb_optimizer
def test_optimizer_simple_resampling_noresample(self):
"""Tests that the _select_next_parameters_method works as expected
when using simple resampling strategy and the parameters should not be resampled"""
np.random.seed(1)
bb_obj = BBOptimizer(
black_box=self.parabola,
heuristic="surrogate_model",
max_iteration=nbr_iteration,
initial_sample_size=2,
parameter_space=parameter_space,
next_parameter_strategy=expected_improvement,
regression_model=GaussianProcessRegressor,
resampling_policy="simple_resampling",
nbr_resamples=2,
)
bb_obj.history = {
"fitness": np.array([10, 5, 4, 2, 15, 20]),
"parameters": np.array(
[[1, 2, 3], [2, 3, 3], [1, 3, 3], [1, 5, 3], [1, 1, 3], [1, 5, 3]]
),
"truncated": np.array([True, True, True, True, True, True]),
"resampled": np.array([True, True, True, True, True, True]),
"initialization": np.array([True, True, True, True, True, True]),
}
parameter = bb_obj._select_next_parameters()
np.testing.assert_array_equal(parameter, [1, 3, 2])
def test_simulated_annealing_with_restart(self):
"""
Tests that the simulated annealing algorithm works properly when used through the
optimizer interface when using a restart.
"""
bb_obj = BBOptimizer(
black_box=self.fake_black_box_categorical,
parameter_space=np.array(
[
np.arange(-5, 5, 1),
np.arange(-6, 6, 1),
np.arange(-6, 6, 1),
np.arange(-6, 6, 1),
np.array(["tutu", "toto"])
]
).T,
heuristic="simulated_annealing",
initial_sample_size=2,
max_iteration=10,
initial_temperature=1000,
cooldown_function=multiplicative_schedule,
neighbor_function=hop_to_next_value,
cooling_factor=3,
restart=random_restart,
bernouilli_parameter=0.2,
)
bb_obj.optimize()
print(bb_obj.history["fitness"])
def test_probabilistic_pick_double_crossover(self):
"""
Tests that the optimization works properly when using the tournament pick method for
selection of the fittest parents + double crossover method.
"""
bb_obj = BBOptimizer(
black_box=self.fake_black_box_categorical,
parameter_space=np.array(
[
np.arange(-5, 5, 1),
np.arange(-6, 6, 1),
np.arange(-6, 6, 1),
np.arange(-6, 6, 1),
np.array(["tutu", "toto"])
]
).T,
heuristic="genetic_algorithm",
initial_sample_size=2,
max_iteration=10,
selection_method=tournament_pick,
crossover_method=double_point_crossover,
mutation_method=mutate_chromosome_to_neighbor,
pool_size=5,
mutation_rate=0.1,
elitism=False,
)
bb_obj.optimize()
def test_transform_function(self):
"""
Tests that the transform function works properly by transforming the output of the
compute method of the black-box function.
"""
def perf_function(x):
return x + 2
bb_obj = BBOptimizer(
black_box=self.parabola,
heuristic="surrogate_model",
max_iteration=nbr_iteration,
parameter_space=parameter_space,
perf_function=perf_function,
regression_model=GaussianProcessRegressor,
next_parameter_strategy=expected_improvement,
)
self.assertEqual(
bb_obj.compute_result(np.array([5, 0])),
27,
"Use of a transformation "
"function black-box output "
"did not work properly.",
)
def test_surrogate_model_censored_bayesian_ei(self):
"""
Tests that the optimization with regression trees and expected improvement behaves as
expected.
"""
bb_obj = BBOptimizer(
black_box=self.fake_async_black_box,
parameter_space=np.array(
[
np.arange(-5, 5, 1),
np.arange(-6, 6, 1),
np.arange(-6, 6, 1),
np.arange(-6, 6, 1),
]
).T,
heuristic="surrogate_model",
regression_model=CensoredGaussianProcesses,
next_parameter_strategy=expected_improvement,
initial_sample_size=5,
async_optim=True,
max_step_cost=1,
max_iteration=10,
max_retry=10,
)
bb_obj.optimize()
print(bb_obj.history)
def test_optimization_categorical(self):
"""Tests that the optimization performs correctly whenever using
a categorical variable.
"""
np.random.seed(10)
categorical_parameter_space = np.array(
[np.arange(-5, 5, 1), np.arange(-6, 6, 1),
np.arange(-6, 6, 1), ["toto", "tutu"]]
).T
bb_obj = BBOptimizer(
black_box=self.categorical_parabola,
heuristic="surrogate_model",
max_iteration=10,
initial_sample_size=2,
parameter_space=categorical_parameter_space,
next_parameter_strategy=expected_improvement,
regression_model=GaussianProcessRegressor,
)
bb_obj.optimize()
best_parameters, best_fitness = bb_obj._get_best_performance()
expected_best_parameters = np.array([0, 1, 1, "toto"], dtype=object)
expected_best_fitness = 21
np.testing.assert_array_equal(
best_parameters,
expected_best_parameters
)
self.assertEqual(best_fitness, expected_best_fitness)
def test_global_exploration_cost(self):
"""
Tests that the global exploration cost is properly computed.
"""
history = {
"fitness": np.array([10, 5, 6, 2, 15, 4]),
"parameters": np.array([[1, 2], [2, 3], [1, 3], [4, 3], [2, 1], [1, 5]]),
"truncated": np.array([True, True, True, True, True, True]),
}
bb_obj = BBOptimizer(
black_box=self.parabola,
heuristic="surrogate_model",
max_iteration=nbr_iteration,
initial_sample_size=2,
parameter_space=parameter_space,
next_parameter_strategy=expected_improvement,
regression_model=GaussianProcessRegressor,
)
bb_obj.history = history
expected_number_states = 3
expected_performance_cost = 16
real_number_states, real_performance_cost = bb_obj.global_exploration_cost
self.assertEqual(
expected_number_states,
real_number_states,
"Percentage of static moves " "was not computed properly.",
)
self.assertEqual(
expected_performance_cost,
real_performance_cost,
"Percentage of explored " "space was not computed " "properly.",
)
def test_fitness_gain_per_iteration(self):
"""
Tests that the fitness gain per iteration is properly computed.
"""
history = {
"fitness": np.array([10, 5, 6, 2, 15, 4]),
"parameters": np.array([[1, 2], [2, 3], [1, 3], [4, 3], [2, 1], [1, 5]]),
"truncated": np.array([True, True, True, True, True, True]),
}
bb_obj = BBOptimizer(
black_box=self.parabola,
heuristic="surrogate_model",
max_iteration=nbr_iteration,
initial_sample_size=2,
parameter_space=parameter_space,
next_parameter_strategy=expected_improvement,
regression_model=GaussianProcessRegressor,
)
bb_obj.history = history
bb_obj.launched = True
expected_gain_per_iteration = np.array([-5, 1, -4, 13, -11])
np.testing.assert_array_equal(
expected_gain_per_iteration,
bb_obj.fitness_gain_per_iteration,
"Computation of fitness gain per iteration did not work as " "expected.",
)
def test_pruning_parameters_config(self):
"""Tests that parsing the config is done properly and that the BBOptimizer class can be
properly instanciated.
"""
shaman_config = SHAManConfig.from_yaml(PRUNING_CONFIG, "component_1")
BBOptimizer(black_box=FakeBlackBox,
parameter_space=shaman_config.component_parameter_space,
async_optim=True,
max_step_cost=shaman_config.pruning.max_step_duration,
**shaman_config.bbo_parameters)
def test_no_compute_method(self):
"""
Tests that when there is no compute method, an error is raised.
"""
with self.assertRaises(AttributeError):
BBOptimizer(
black_box=self.no_compute,
heuristic="surrogate_model",
max_iteration=nbr_iteration,
parameter_space=parameter_space,
)
def test_missing_heuristic_argument(self):
"""
Tests that an exception is raised when there's a missing argument for a heuristic.
"""
with self.assertRaises(Exception):
BBOptimizer(
black_box=self.parabola,
heuristic="surrogate_model",
max_iteration=nbr_iteration,
parameter_space=parameter_space,
next_parameter_strategy=expected_improvement,
)
def test_incorrect_heuristic_name(self):
"""
Tests that when an incorrect heuristic name is passed as argument, an error is raised.
"""
with self.assertRaises(ValueError):
BBOptimizer(
black_box=self.parabola,
heuristic="makes no sense !",
max_iteration=nbr_iteration,
parameter_space=parameter_space,
next_parameter_strategy=expected_improvement,
regression_model=GaussianProcessRegressor,
)
def test_stop_criterion_unknown(self):
"""Tests that initializing a BBOptimizer object with an unknown stop criterion returns
a ValueError error.
"""
with self.assertRaises(ValueError):
BBOptimizer(
black_box=self.parabola,
heuristic="surrogate_model",
max_iteration=nbr_iteration,
initial_sample_size=2,
parameter_space=parameter_space,
next_parameter_strategy=expected_improvement,
regression_model=GaussianProcessRegressor,
stop_criterion="unknown_criterion"
)
def test_incorrect_selection_name(self):
"""
Tests that when an incorrect name for the selection method is passed as an argument,
a ValueError is raised.
"""
with self.assertRaises(ValueError):
BBOptimizer(
black_box=self.parabola,
heuristic="surrogate_model",
max_iteration=nbr_iteration,
parameter_space=parameter_space,
next_parameter_strategy=expected_improvement,
initial_draw_method="i do not exist!",
regression_model=GaussianProcessRegressor,
)
def test_select_parameters(self):
"""Test the function _select_next_parameters when there is no retry."""
np.random.seed(10)
bb_obj = BBOptimizer(
black_box=self.parabola,
heuristic="surrogate_model",
max_iteration=1,
initial_sample_size=2,
parameter_space=parameter_space,
next_parameter_strategy=expected_improvement,
regression_model=GaussianProcessRegressor,
)
bb_obj._initialize()
parameter = bb_obj._select_next_parameters()
np.testing.assert_array_equal(parameter, np.array([-5, 2, -6]))
def test_optimizer_resampling_no_exist(self):
"""Tests that an error is raised when the asked for resampling policy does
not exist.
"""
with self.assertRaises(ValueError):
bb_obj = BBOptimizer(
black_box=self.parabola,
heuristic="surrogate_model",
max_iteration=nbr_iteration,
initial_sample_size=2,
parameter_space=parameter_space,
next_parameter_strategy=expected_improvement,
regression_model=GaussianProcessRegressor,
resampling_policy="i do not exist",
)
def test_initialize(self):
"""Tests that the initialization step happens correctly, by appending the proper
number of fitness and parametrization values."""
bb_obj = BBOptimizer(
black_box=self.parabola,
heuristic="surrogate_model",
max_iteration=nbr_iteration,
initial_sample_size=2,
parameter_space=parameter_space,
next_parameter_strategy=expected_improvement,
regression_model=GaussianProcessRegressor,
)
bb_obj._initialize(callbacks=[lambda x: x])
self.assertTrue(len(bb_obj.history["fitness"]), 2)
self.assertTrue(len(bb_obj.history["parameters"]), 2)
def test_summarize_except(self):
"""
Tests that the call to "summarize" raises an exception if the experiment is not yet
launched.
"""
bb_obj = BBOptimizer(
black_box=self.parabola,
heuristic="surrogate_model",
max_iteration=nbr_iteration,
initial_sample_size=2,
parameter_space=parameter_space,
next_parameter_strategy=expected_improvement,
regression_model=GaussianProcessRegressor,
)
self.assertRaises(Exception, bb_obj.summarize)
def test_fitness_gain_per_iteration_not_launched(self):
"""
Tests that the fitness gain per iteration is None if the experiment is not launched.
"""
bb_obj = BBOptimizer(
black_box=self.parabola,
heuristic="surrogate_model",
max_iteration=nbr_iteration,
initial_sample_size=2,
parameter_space=parameter_space,
next_parameter_strategy=expected_improvement,
regression_model=GaussianProcessRegressor,
)
bb_obj.launched = False
self.assertEqual(bb_obj.fitness_gain_per_iteration, None)
def test_summarize(self):
"""
Tests that the call to "summarize" does not raise an error.
"""
bb_obj = BBOptimizer(
black_box=self.parabola,
heuristic="surrogate_model",
max_iteration=nbr_iteration,
initial_sample_size=2,
parameter_space=parameter_space,
next_parameter_strategy=expected_improvement,
regression_model=GaussianProcessRegressor,
)
bb_obj.optimize()
bb_obj.summarize()
def test_noise_reduction_bbo(self):
"""Tests that the noise reduction parameters are properly added to the bbo_kwargs and
the BBOptimizer object can be initialized.
"""
shaman_config = SHAManConfig.from_yaml(NOISE_REDUCTION_CONFIG,
"component_1")
# Check that the dictionary has been properly updated
assert "nbr_resamples" in shaman_config.bbo_parameters
assert "fitness_aggregation" in shaman_config.bbo_parameters
assert "resampling_policy" in shaman_config.bbo_parameters
# Check that the estimator function is properly parsed
self.assertEqual(shaman_config.bbo_parameters["estimator"],
numpy.median)
# Check that the BBOptimizer class can be properly instanciated
BBOptimizer(black_box=FakeBlackBox,
parameter_space=shaman_config.component_parameter_space,
**shaman_config.bbo_parameters)
def test_append_parameters_new_history(self):
"""
Tests that the parameters are correctly added to an empty history.
"""
# Manually create the history of the BBOptimizer
bb_obj = BBOptimizer(
black_box=self.parabola,
heuristic="surrogate_model",
max_iteration=nbr_iteration,
parameter_space=parameter_space,
next_parameter_strategy=expected_improvement,
regression_model=GaussianProcessRegressor,
)
# Test the append method
bb_obj._append_parameters([1, 3])
np.testing.assert_array_equal(
bb_obj.history["parameters"], np.array([1, 3]))
def test_append_performance_new_history(self):
"""
Tests that appending a fitness value on an empty history works properly.
"""
# Manually create the history of the BBOptimizer
bb_obj = BBOptimizer(
black_box=self.parabola,
heuristic="surrogate_model",
max_iteration=nbr_iteration,
parameter_space=parameter_space,
next_parameter_strategy=expected_improvement,
regression_model=GaussianProcessRegressor,
)
# Tests the append method
bb_obj._append_fitness(10)
np.testing.assert_array_equal(
bb_obj.history["fitness"], np.array(np.array([10]))
)
def test_optimizer_process_simple_resampling(self):
"""Tests that the .optimize() method works correctly when using simple resampling"""
np.random.seed(5)
bb_obj = BBOptimizer(
black_box=self.parabola,
heuristic="surrogate_model",
max_iteration=nbr_iteration,
initial_sample_size=2,
parameter_space=parameter_space,
next_parameter_strategy=expected_improvement,
regression_model=GaussianProcessRegressor,
resampling_policy="simple_resampling",
nbr_resamples=2,
)
bb_obj.optimize()
np.testing.assert_array_equal(
bb_obj.history["fitness"], np.array(
[16., 61., 61., 9., 9., 0., 0.])
)
