def test_transform_function(self):
"""
Tests that the transform function works properly by transforming the output of the
compute method of the black-box function.
"""
def perf_function(x):
return x + 2
bb_obj = BBOptimizer(
black_box=self.parabola,
heuristic="surrogate_model",
max_iteration=nbr_iteration,
parameter_space=parameter_space,
perf_function=perf_function,
regression_model=GaussianProcessRegressor,
next_parameter_strategy=expected_improvement,
)
self.assertEqual(
bb_obj.compute_result(np.array([5, 0])),
27,
"Use of a transformation "
"function black-box output "
"did not work properly.",
)
def setup_bb_optimizer(self) -> BBOptimizer:
"""Setups the black-box from the configuration."""
# Create the BBOptimizer object using the different options in the
# configuration
# If pruning is enabled, parse corresponding fields
if self.configuration.pruning:
pruning = True
max_step_cost = (
self.bb_wrapper.default_target_value
if self.configuration.pruning.max_step_duration == "default"
else self.configuration.pruning.max_step_duration
)
else:
max_step_cost = None
pruning = False
self.bb_optimizer = BBOptimizer(
black_box=self.bb_wrapper,
parameter_space=self.configuration.component_parameter_space,
max_iteration=self.nbr_iteration,
async_optim=pruning,
max_step_cost=max_step_cost,
**self.configuration.bbo_parameters,
)
return self.bb_optimizer
def test_surrogate_model_censored_bayesian_ei(self):
"""
Tests that the optimization with regression trees and expected improvement behaves as
expected.
"""
bb_obj = BBOptimizer(
black_box=self.fake_async_black_box,
parameter_space=np.array(
[
np.arange(-5, 5, 1),
np.arange(-6, 6, 1),
np.arange(-6, 6, 1),
np.arange(-6, 6, 1),
]
).T,
heuristic="surrogate_model",
regression_model=CensoredGaussianProcesses,
next_parameter_strategy=expected_improvement,
initial_sample_size=5,
async_optim=True,
max_step_cost=1,
max_iteration=10,
max_retry=10,
)
bb_obj.optimize()
print(bb_obj.history)
def test_optimization_categorical(self):
"""Tests that the optimization performs correctly whenever using
a categorical variable.
"""
np.random.seed(10)
categorical_parameter_space = np.array(
[np.arange(-5, 5, 1), np.arange(-6, 6, 1),
np.arange(-6, 6, 1), ["toto", "tutu"]]
).T
bb_obj = BBOptimizer(
black_box=self.categorical_parabola,
heuristic="surrogate_model",
max_iteration=10,
initial_sample_size=2,
parameter_space=categorical_parameter_space,
next_parameter_strategy=expected_improvement,
regression_model=GaussianProcessRegressor,
)
bb_obj.optimize()
best_parameters, best_fitness = bb_obj._get_best_performance()
expected_best_parameters = np.array([0, 1, 1, "toto"], dtype=object)
expected_best_fitness = 21
np.testing.assert_array_equal(
best_parameters,
expected_best_parameters
)
self.assertEqual(best_fitness, expected_best_fitness)
def test_simulated_annealing_with_restart(self):
"""
Tests that the simulated annealing algorithm works properly when used through the
optimizer interface when using a restart.
"""
bb_obj = BBOptimizer(
black_box=self.fake_black_box_categorical,
parameter_space=np.array(
[
np.arange(-5, 5, 1),
np.arange(-6, 6, 1),
np.arange(-6, 6, 1),
np.arange(-6, 6, 1),
np.array(["tutu", "toto"])
]
).T,
heuristic="simulated_annealing",
initial_sample_size=2,
max_iteration=10,
initial_temperature=1000,
cooldown_function=multiplicative_schedule,
neighbor_function=hop_to_next_value,
cooling_factor=3,
restart=random_restart,
bernouilli_parameter=0.2,
)
bb_obj.optimize()
print(bb_obj.history["fitness"])
def test_size_explored_space(self):
"""
Tests that the computation of the size of the explored space works properly, as well as
the percentage of static moves.
"""
history = {
"fitness": np.array([10, 5, 4]),
"parameters": np.array([[1, 2, 3], [2, 3, 4], [1, 2, 3]]),
"truncated": np.array([True, True, True, True, True, True]),
}
bb_obj = BBOptimizer(
black_box=self.parabola,
heuristic="surrogate_model",
max_iteration=nbr_iteration,
initial_sample_size=2,
parameter_space=parameter_space,
next_parameter_strategy=expected_improvement,
regression_model=GaussianProcessRegressor,
)
bb_obj.history = history
expected_static_moves = 1 / 3 * 100
expected_explored_space = 2 / 1440 * 100
real_explored_space, real_static_moves = bb_obj.size_explored_space
self.assertEqual(
expected_static_moves,
real_static_moves,
"Percentage of static moves " "was not computed properly.",
)
self.assertEqual(
expected_explored_space,
real_explored_space,
"Percentage of explored " "space was not computed " "properly.",
)
def test_infer_type(self):
"""Tests that the static method built to infer the types
of an array.
"""
bb_obj = BBOptimizer(
black_box=self.parabola,
heuristic="surrogate_model",
max_iteration=50,
initial_sample_size=2,
parameter_space=parameter_space,
next_parameter_strategy=expected_improvement,
regression_model=GaussianProcessRegressor,
stop_criterion="improvement_criterion",
stop_window=4,
improvement_estimator=min,
improvement_threshold=0.1
)
int_array = np.array([1, 2, 3])
inferred_int_array = bb_obj.infer_type(int_array)
assert_array_equal(int_array, inferred_int_array)
mixed_type_array = np.array(['1', '2', 'toto'])
expected_mixed_type_array = np.array([1, 2, 'toto'], dtype=object)
inferred_mixed_type_array = bb_obj.infer_type(mixed_type_array)
assert_array_equal(inferred_mixed_type_array,
expected_mixed_type_array)
def test_global_exploration_cost(self):
"""
Tests that the global exploration cost is properly computed.
"""
history = {
"fitness": np.array([10, 5, 6, 2, 15, 4]),
"parameters": np.array([[1, 2], [2, 3], [1, 3], [4, 3], [2, 1], [1, 5]]),
"truncated": np.array([True, True, True, True, True, True]),
}
bb_obj = BBOptimizer(
black_box=self.parabola,
heuristic="surrogate_model",
max_iteration=nbr_iteration,
initial_sample_size=2,
parameter_space=parameter_space,
next_parameter_strategy=expected_improvement,
regression_model=GaussianProcessRegressor,
)
bb_obj.history = history
expected_number_states = 3
expected_performance_cost = 16
real_number_states, real_performance_cost = bb_obj.global_exploration_cost
self.assertEqual(
expected_number_states,
real_number_states,
"Percentage of static moves " "was not computed properly.",
)
self.assertEqual(
expected_performance_cost,
real_performance_cost,
"Percentage of explored " "space was not computed " "properly.",
)
def test_reevaluation(self):
"""Tests the proper implementation of the reevaluation feature.
This feature is tested by checking in the history that no re-evaluation has been done
(as is the case with this particular optimizer configuration).
With re-evaluate set to True, there are only 3 different values in the history of
parametrizations. With re-evaluate set to False, all parameters of the history are
different.
"""
nbr_iteration = 5
bb_obj = BBOptimizer(
black_box=self.parabola,
heuristic="genetic_algorithm",
max_iteration=nbr_iteration,
initial_sample_size=2,
parameter_space=parameter_space,
selection_method=tournament_pick,
crossover_method=double_point_crossover,
mutation_method=mutate_chromosome_to_neighbor,
mutation_rate=0.6,
reevaluate=False,
max_retry=5,
)
bb_obj.optimize()
self.assertEqual(
np.unique(bb_obj.history["parameters"].astype(
str), axis=0).shape[0],
7,
"Some parameter values have been evaluated several times",
)
def test_fitness_gain_per_iteration(self):
"""
Tests that the fitness gain per iteration is properly computed.
"""
history = {
"fitness": np.array([10, 5, 6, 2, 15, 4]),
"parameters": np.array([[1, 2], [2, 3], [1, 3], [4, 3], [2, 1], [1, 5]]),
"truncated": np.array([True, True, True, True, True, True]),
}
bb_obj = BBOptimizer(
black_box=self.parabola,
heuristic="surrogate_model",
max_iteration=nbr_iteration,
initial_sample_size=2,
parameter_space=parameter_space,
next_parameter_strategy=expected_improvement,
regression_model=GaussianProcessRegressor,
)
bb_obj.history = history
bb_obj.launched = True
expected_gain_per_iteration = np.array([-5, 1, -4, 13, -11])
np.testing.assert_array_equal(
expected_gain_per_iteration,
bb_obj.fitness_gain_per_iteration,
"Computation of fitness gain per iteration did not work as " "expected.",
)
def test_optimizer_simple_resampling_noresample(self):
"""Tests that the _select_next_parameters_method works as expected
when using simple resampling strategy and the parameters should not be resampled"""
np.random.seed(1)
bb_obj = BBOptimizer(
black_box=self.parabola,
heuristic="surrogate_model",
max_iteration=nbr_iteration,
initial_sample_size=2,
parameter_space=parameter_space,
next_parameter_strategy=expected_improvement,
regression_model=GaussianProcessRegressor,
resampling_policy="simple_resampling",
nbr_resamples=2,
)
bb_obj.history = {
"fitness": np.array([10, 5, 4, 2, 15, 20]),
"parameters": np.array(
[[1, 2, 3], [2, 3, 3], [1, 3, 3], [1, 5, 3], [1, 1, 3], [1, 5, 3]]
),
"truncated": np.array([True, True, True, True, True, True]),
"resampled": np.array([True, True, True, True, True, True]),
"initialization": np.array([True, True, True, True, True, True]),
}
parameter = bb_obj._select_next_parameters()
np.testing.assert_array_equal(parameter, [1, 3, 2])
def test_probabilistic_pick_double_crossover(self):
"""
Tests that the optimization works properly when using the tournament pick method for
selection of the fittest parents + double crossover method.
"""
bb_obj = BBOptimizer(
black_box=self.fake_black_box_categorical,
parameter_space=np.array(
[
np.arange(-5, 5, 1),
np.arange(-6, 6, 1),
np.arange(-6, 6, 1),
np.arange(-6, 6, 1),
np.array(["tutu", "toto"])
]
).T,
heuristic="genetic_algorithm",
initial_sample_size=2,
max_iteration=10,
selection_method=tournament_pick,
crossover_method=double_point_crossover,
mutation_method=mutate_chromosome_to_neighbor,
pool_size=5,
mutation_rate=0.1,
elitism=False,
)
bb_obj.optimize()
def test_initialize(self):
"""Tests that the initialization step happens correctly, by appending the proper
number of fitness and parametrization values."""
bb_obj = BBOptimizer(
black_box=self.parabola,
heuristic="surrogate_model",
max_iteration=nbr_iteration,
initial_sample_size=2,
parameter_space=parameter_space,
next_parameter_strategy=expected_improvement,
regression_model=GaussianProcessRegressor,
)
bb_obj._initialize(callbacks=[lambda x: x])
self.assertTrue(len(bb_obj.history["fitness"]), 2)
self.assertTrue(len(bb_obj.history["parameters"]), 2)
def test_fitness_gain_per_iteration_not_launched(self):
"""
Tests that the fitness gain per iteration is None if the experiment is not launched.
"""
bb_obj = BBOptimizer(
black_box=self.parabola,
heuristic="surrogate_model",
max_iteration=nbr_iteration,
initial_sample_size=2,
parameter_space=parameter_space,
next_parameter_strategy=expected_improvement,
regression_model=GaussianProcessRegressor,
)
bb_obj.launched = False
self.assertEqual(bb_obj.fitness_gain_per_iteration, None)
def test_select_parameters(self):
"""Test the function _select_next_parameters when there is no retry."""
np.random.seed(10)
bb_obj = BBOptimizer(
black_box=self.parabola,
heuristic="surrogate_model",
max_iteration=1,
initial_sample_size=2,
parameter_space=parameter_space,
next_parameter_strategy=expected_improvement,
regression_model=GaussianProcessRegressor,
)
bb_obj._initialize()
parameter = bb_obj._select_next_parameters()
np.testing.assert_array_equal(parameter, np.array([-5, 2, -6]))
def test_append_parameters_new_history(self):
"""
Tests that the parameters are correctly added to an empty history.
"""
# Manually create the history of the BBOptimizer
bb_obj = BBOptimizer(
black_box=self.parabola,
heuristic="surrogate_model",
max_iteration=nbr_iteration,
parameter_space=parameter_space,
next_parameter_strategy=expected_improvement,
regression_model=GaussianProcessRegressor,
)
# Test the append method
bb_obj._append_parameters([1, 3])
np.testing.assert_array_equal(
bb_obj.history["parameters"], np.array([1, 3]))
def test_append_performance_new_history(self):
"""
Tests that appending a fitness value on an empty history works properly.
"""
# Manually create the history of the BBOptimizer
bb_obj = BBOptimizer(
black_box=self.parabola,
heuristic="surrogate_model",
max_iteration=nbr_iteration,
parameter_space=parameter_space,
next_parameter_strategy=expected_improvement,
regression_model=GaussianProcessRegressor,
)
# Tests the append method
bb_obj._append_fitness(10)
np.testing.assert_array_equal(
bb_obj.history["fitness"], np.array(np.array([10]))
)
def test_optimizer_process_simple_resampling(self):
"""Tests that the .optimize() method works correctly when using simple resampling"""
np.random.seed(5)
bb_obj = BBOptimizer(
black_box=self.parabola,
heuristic="surrogate_model",
max_iteration=nbr_iteration,
initial_sample_size=2,
parameter_space=parameter_space,
next_parameter_strategy=expected_improvement,
regression_model=GaussianProcessRegressor,
resampling_policy="simple_resampling",
nbr_resamples=2,
)
bb_obj.optimize()
np.testing.assert_array_equal(
bb_obj.history["fitness"], np.array(
[16., 61., 61., 9., 9., 0., 0.])
)
def test_pruning_parameters_config(self):
"""Tests that parsing the config is done properly and that the BBOptimizer class can be
properly instanciated.
"""
shaman_config = SHAManConfig.from_yaml(PRUNING_CONFIG, "component_1")
BBOptimizer(black_box=FakeBlackBox,
parameter_space=shaman_config.component_parameter_space,
async_optim=True,
max_step_cost=shaman_config.pruning.max_step_duration,
**shaman_config.bbo_parameters)
def test_no_compute_method(self):
"""
Tests that when there is no compute method, an error is raised.
"""
with self.assertRaises(AttributeError):
BBOptimizer(
black_box=self.no_compute,
heuristic="surrogate_model",
max_iteration=nbr_iteration,
parameter_space=parameter_space,
)
def test_total_iteration(self):
"""
Tests that the total number of iterations is properly computed (and indirectly that the
stop criterion properly stops on the number of iterations).
"""
bb_obj = BBOptimizer(
black_box=self.parabola,
heuristic="surrogate_model",
max_iteration=nbr_iteration,
initial_sample_size=2,
parameter_space=parameter_space,
next_parameter_strategy=expected_improvement,
regression_model=GaussianProcessRegressor,
)
bb_obj.optimize()
self.assertEqual(
nbr_iteration + 2,
bb_obj.total_iteration,
"Total number of iterations " "was not computed properly.",
)
def test_stop_criterion(self):
"""Tests that if the stop criterion is matched, the optimization
is stopped early and does not reach the maximum number of iterations
"""
np.random.seed(5)
bb_obj = BBOptimizer(
black_box=self.parabola,
heuristic="surrogate_model",
max_iteration=50,
initial_sample_size=2,
parameter_space=parameter_space,
next_parameter_strategy=expected_improvement,
regression_model=GaussianProcessRegressor,
stop_criterion="improvement_criterion",
stop_window=4,
improvement_estimator=min,
improvement_threshold=0.1
)
bb_obj.optimize()
self.assertTrue(len(bb_obj.history["fitness"]) < 51)
def test_exhaustive_search(self):
"""
Tests that the exhaustive search heuristic tests all the parametrization when the budget is
equal to the size of the parametric grid.
"""
parametric_grid = np.array(
[np.arange(-5, 5, 1), np.arange(-6, 6, 1), ]).T
bb_obj = BBOptimizer(
black_box=self.fake_black_box,
parameter_space=parametric_grid,
heuristic="exhaustive_search",
initial_sample_size=2,
max_iteration=120,
)
bb_obj.optimize()
exhaustive_grid = np.array(np.meshgrid(*parametric_grid)).T.reshape(
-1, len(parametric_grid)
)
np.testing.assert_array_equal(
bb_obj.history["parameters"][2:], exhaustive_grid)
def test_missing_heuristic_argument(self):
"""
Tests that an exception is raised when there's a missing argument for a heuristic.
"""
with self.assertRaises(Exception):
BBOptimizer(
black_box=self.parabola,
heuristic="surrogate_model",
max_iteration=nbr_iteration,
parameter_space=parameter_space,
next_parameter_strategy=expected_improvement,
)
def test_initialize_callback(self):
"""Tests that the initialization step happens correctly when using a callback function,
which prints the square root of the sum of the fitness values and the sum of the parameters"""
np.random.seed(10)
# Capture the stdout to check if the print happenned correctly
captured_output = io.StringIO()
sys.stdout = captured_output
bb_obj = BBOptimizer(
black_box=self.parabola,
heuristic="surrogate_model",
max_iteration=nbr_iteration,
initial_sample_size=2,
parameter_space=parameter_space,
next_parameter_strategy=expected_improvement,
regression_model=GaussianProcessRegressor,
)
bb_obj._initialize(callbacks=[mock_callback_1])
# Redirect the stdout to not mess with the system
sys.stdout = sys.__stdout__
expected = "Result: 3 + 5\nResult: 4 + 6\n"
self.assertEqual(expected, captured_output.getvalue())
def test_optimization_step(self):
"""Tests that the optimization step runs properly when using the default callback.
It checks that the parameter and the corresponding performance measure is properly added to
the history.
"""
np.random.seed(10)
bb_obj = BBOptimizer(
black_box=self.parabola,
heuristic="surrogate_model",
max_iteration=1,
initial_sample_size=2,
parameter_space=parameter_space,
next_parameter_strategy=expected_improvement,
regression_model=GaussianProcessRegressor,
reevaluate=False,
)
test_parameter = np.array([10, 10, 10])
bb_obj._optimization_step(test_parameter)
np.testing.assert_array_equal(
bb_obj.history["parameters"][-1], test_parameter)
self.assertEqual(bb_obj.history["fitness"][-1], 200)
def test_optimization_callback(self):
"""Tests that the optimization step runs properly when using a custom callback.
It checks that there is the proper output in the stdout.
"""
np.random.seed(10)
captured_output = io.StringIO()
sys.stdout = captured_output
bb_obj = BBOptimizer(
black_box=self.parabola,
heuristic="surrogate_model",
max_iteration=1,
initial_sample_size=2,
parameter_space=parameter_space,
next_parameter_strategy=expected_improvement,
regression_model=GaussianProcessRegressor,
)
bb_obj.optimize(callbacks=[lambda x: print(x["fitness"][0])])
# Redirect the stdout to not mess with the system
sys.stdout = sys.__stdout__
expected = "10.0\n10.0\n10.0\n"
self.assertEqual(expected, captured_output.getvalue())
def test_get_best_performance(self):
"""Tests that the best performance is properly returned when using the method
_get_best_performance"""
bb_obj = BBOptimizer(
black_box=self.parabola,
heuristic="surrogate_model",
max_iteration=nbr_iteration,
initial_sample_size=2,
parameter_space=parameter_space,
next_parameter_strategy=expected_improvement,
regression_model=GaussianProcessRegressor,
)
bb_obj.history["fitness"] = np.array([2, 1, 3])
bb_obj.history["parameters"] = np.array(
[np.array([1, 2]), np.array([3, 4]), np.array([5, 6])]
)
expected_perf = 1
expected_parameters = np.array([3, 4])
best_param, best_fitness = bb_obj._get_best_performance()
np.testing.assert_array_equal(best_param, expected_parameters)
self.assertEqual(best_fitness, 1)
def test_reset(self):
"""
Tests that the "reset" method reset the attributes.
"""
bb_obj = BBOptimizer(
black_box=self.parabola,
heuristic="surrogate_model",
max_iteration=nbr_iteration,
initial_sample_size=2,
parameter_space=parameter_space,
next_parameter_strategy=expected_improvement,
regression_model=GaussianProcessRegressor,
)
bb_obj.optimize()
bb_obj.reset()
self.assertEqual(bb_obj.nbr_iteration, 0)
self.assertEqual(bb_obj.elapsed_time, 0)
self.assertEqual(bb_obj.launched, False)
self.assertDictEqual(
bb_obj.history,
{
"fitness": None,
"parameters": None,
"initialization": None,
"resampled": None,
"truncated": None,
},
)
def test_stop_rule_true(self):
"""
Tests that the stop rule is built properly when the stop criteria are not met.
"""
bb_obj = BBOptimizer(
black_box=self.parabola,
heuristic="surrogate_model",
max_iteration=nbr_iteration,
time_out=time_out,
parameter_space=parameter_space,
next_parameter_strategy=expected_improvement,
regression_model=GaussianProcessRegressor,
)
# If the number of iterations is exceeded
bb_obj.nbr_iteration = nbr_iteration - 1
self.assertTrue(
bb_obj.stop_rule,
"Exceeded number of iteration did not stop the " "optimizing loop.",
)
bb_obj = BBOptimizer(
black_box=self.parabola,
heuristic="surrogate_model",
max_iteration=nbr_iteration,
time_out=time_out,
parameter_space=parameter_space,
next_parameter_strategy=expected_improvement,
regression_model=GaussianProcessRegressor,
)
# If the time elapsed is exceeded
bb_obj.elapsed_time = time_out - 1
self.assertTrue(
bb_obj.stop_rule, "Exceeded elapsed time did not stop optimizing loop."
)
bb_obj = BBOptimizer(
black_box=self.parabola,
heuristic="surrogate_model",
max_iteration=nbr_iteration,
time_out=time_out,
parameter_space=parameter_space,
next_parameter_strategy=expected_improvement,
regression_model=GaussianProcessRegressor,
)
# If the heuristic stop criterion is met
bb_obj.heuristic.stop = False
self.assertTrue(
bb_obj.stop_rule,
"Internal heuristic stop did not stop the optimizing " "loop.",
)