def test_stop_rule_true(self): """ Tests that the stop rule is built properly when the stop criteria are not met. """ bb_obj = BBOptimizer( black_box=self.parabola, heuristic="surrogate_model", max_iteration=nbr_iteration, time_out=time_out, parameter_space=parameter_space, next_parameter_strategy=expected_improvement, regression_model=GaussianProcessRegressor, ) # If the number of iterations is exceeded bb_obj.nbr_iteration = nbr_iteration - 1 self.assertTrue( bb_obj.stop_rule, "Exceeded number of iteration did not stop the " "optimizing loop.", ) bb_obj = BBOptimizer( black_box=self.parabola, heuristic="surrogate_model", max_iteration=nbr_iteration, time_out=time_out, parameter_space=parameter_space, next_parameter_strategy=expected_improvement, regression_model=GaussianProcessRegressor, ) # If the time elapsed is exceeded bb_obj.elapsed_time = time_out - 1 self.assertTrue( bb_obj.stop_rule, "Exceeded elapsed time did not stop optimizing loop." ) bb_obj = BBOptimizer( black_box=self.parabola, heuristic="surrogate_model", max_iteration=nbr_iteration, time_out=time_out, parameter_space=parameter_space, next_parameter_strategy=expected_improvement, regression_model=GaussianProcessRegressor, ) # If the heuristic stop criterion is met bb_obj.heuristic.stop = False self.assertTrue( bb_obj.stop_rule, "Internal heuristic stop did not stop the optimizing " "loop.", )
def test_size_explored_space(self): """ Tests that the computation of the size of the explored space works properly, as well as the percentage of static moves. """ history = { "fitness": np.array([10, 5, 4]), "parameters": np.array([[1, 2, 3], [2, 3, 4], [1, 2, 3]]), "truncated": np.array([True, True, True, True, True, True]), } bb_obj = BBOptimizer( black_box=self.parabola, heuristic="surrogate_model", max_iteration=nbr_iteration, initial_sample_size=2, parameter_space=parameter_space, next_parameter_strategy=expected_improvement, regression_model=GaussianProcessRegressor, ) bb_obj.history = history expected_static_moves = 1 / 3 * 100 expected_explored_space = 2 / 1440 * 100 real_explored_space, real_static_moves = bb_obj.size_explored_space self.assertEqual( expected_static_moves, real_static_moves, "Percentage of static moves " "was not computed properly.", ) self.assertEqual( expected_explored_space, real_explored_space, "Percentage of explored " "space was not computed " "properly.", )
def test_infer_type(self): """Tests that the static method built to infer the types of an array. """ bb_obj = BBOptimizer( black_box=self.parabola, heuristic="surrogate_model", max_iteration=50, initial_sample_size=2, parameter_space=parameter_space, next_parameter_strategy=expected_improvement, regression_model=GaussianProcessRegressor, stop_criterion="improvement_criterion", stop_window=4, improvement_estimator=min, improvement_threshold=0.1 ) int_array = np.array([1, 2, 3]) inferred_int_array = bb_obj.infer_type(int_array) assert_array_equal(int_array, inferred_int_array) mixed_type_array = np.array(['1', '2', 'toto']) expected_mixed_type_array = np.array([1, 2, 'toto'], dtype=object) inferred_mixed_type_array = bb_obj.infer_type(mixed_type_array) assert_array_equal(inferred_mixed_type_array, expected_mixed_type_array)
def test_reevaluation(self): """Tests the proper implementation of the reevaluation feature. This feature is tested by checking in the history that no re-evaluation has been done (as is the case with this particular optimizer configuration). With re-evaluate set to True, there are only 3 different values in the history of parametrizations. With re-evaluate set to False, all parameters of the history are different. """ nbr_iteration = 5 bb_obj = BBOptimizer( black_box=self.parabola, heuristic="genetic_algorithm", max_iteration=nbr_iteration, initial_sample_size=2, parameter_space=parameter_space, selection_method=tournament_pick, crossover_method=double_point_crossover, mutation_method=mutate_chromosome_to_neighbor, mutation_rate=0.6, reevaluate=False, max_retry=5, ) bb_obj.optimize() self.assertEqual( np.unique(bb_obj.history["parameters"].astype( str), axis=0).shape[0], 7, "Some parameter values have been evaluated several times", )
def test_reset(self): """ Tests that the "reset" method reset the attributes. """ bb_obj = BBOptimizer( black_box=self.parabola, heuristic="surrogate_model", max_iteration=nbr_iteration, initial_sample_size=2, parameter_space=parameter_space, next_parameter_strategy=expected_improvement, regression_model=GaussianProcessRegressor, ) bb_obj.optimize() bb_obj.reset() self.assertEqual(bb_obj.nbr_iteration, 0) self.assertEqual(bb_obj.elapsed_time, 0) self.assertEqual(bb_obj.launched, False) self.assertDictEqual( bb_obj.history, { "fitness": None, "parameters": None, "initialization": None, "resampled": None, "truncated": None, }, )
def setup_bb_optimizer(self) -> BBOptimizer: """Setups the black-box from the configuration.""" # Create the BBOptimizer object using the different options in the # configuration # If pruning is enabled, parse corresponding fields if self.configuration.pruning: pruning = True max_step_cost = ( self.bb_wrapper.default_target_value if self.configuration.pruning.max_step_duration == "default" else self.configuration.pruning.max_step_duration ) else: max_step_cost = None pruning = False self.bb_optimizer = BBOptimizer( black_box=self.bb_wrapper, parameter_space=self.configuration.component_parameter_space, max_iteration=self.nbr_iteration, async_optim=pruning, max_step_cost=max_step_cost, **self.configuration.bbo_parameters, ) return self.bb_optimizer
def test_optimizer_simple_resampling_noresample(self): """Tests that the _select_next_parameters_method works as expected when using simple resampling strategy and the parameters should not be resampled""" np.random.seed(1) bb_obj = BBOptimizer( black_box=self.parabola, heuristic="surrogate_model", max_iteration=nbr_iteration, initial_sample_size=2, parameter_space=parameter_space, next_parameter_strategy=expected_improvement, regression_model=GaussianProcessRegressor, resampling_policy="simple_resampling", nbr_resamples=2, ) bb_obj.history = { "fitness": np.array([10, 5, 4, 2, 15, 20]), "parameters": np.array( [[1, 2, 3], [2, 3, 3], [1, 3, 3], [1, 5, 3], [1, 1, 3], [1, 5, 3]] ), "truncated": np.array([True, True, True, True, True, True]), "resampled": np.array([True, True, True, True, True, True]), "initialization": np.array([True, True, True, True, True, True]), } parameter = bb_obj._select_next_parameters() np.testing.assert_array_equal(parameter, [1, 3, 2])
def test_simulated_annealing_with_restart(self): """ Tests that the simulated annealing algorithm works properly when used through the optimizer interface when using a restart. """ bb_obj = BBOptimizer( black_box=self.fake_black_box_categorical, parameter_space=np.array( [ np.arange(-5, 5, 1), np.arange(-6, 6, 1), np.arange(-6, 6, 1), np.arange(-6, 6, 1), np.array(["tutu", "toto"]) ] ).T, heuristic="simulated_annealing", initial_sample_size=2, max_iteration=10, initial_temperature=1000, cooldown_function=multiplicative_schedule, neighbor_function=hop_to_next_value, cooling_factor=3, restart=random_restart, bernouilli_parameter=0.2, ) bb_obj.optimize() print(bb_obj.history["fitness"])
def test_probabilistic_pick_double_crossover(self): """ Tests that the optimization works properly when using the tournament pick method for selection of the fittest parents + double crossover method. """ bb_obj = BBOptimizer( black_box=self.fake_black_box_categorical, parameter_space=np.array( [ np.arange(-5, 5, 1), np.arange(-6, 6, 1), np.arange(-6, 6, 1), np.arange(-6, 6, 1), np.array(["tutu", "toto"]) ] ).T, heuristic="genetic_algorithm", initial_sample_size=2, max_iteration=10, selection_method=tournament_pick, crossover_method=double_point_crossover, mutation_method=mutate_chromosome_to_neighbor, pool_size=5, mutation_rate=0.1, elitism=False, ) bb_obj.optimize()
def test_transform_function(self): """ Tests that the transform function works properly by transforming the output of the compute method of the black-box function. """ def perf_function(x): return x + 2 bb_obj = BBOptimizer( black_box=self.parabola, heuristic="surrogate_model", max_iteration=nbr_iteration, parameter_space=parameter_space, perf_function=perf_function, regression_model=GaussianProcessRegressor, next_parameter_strategy=expected_improvement, ) self.assertEqual( bb_obj.compute_result(np.array([5, 0])), 27, "Use of a transformation " "function black-box output " "did not work properly.", )
def test_surrogate_model_censored_bayesian_ei(self): """ Tests that the optimization with regression trees and expected improvement behaves as expected. """ bb_obj = BBOptimizer( black_box=self.fake_async_black_box, parameter_space=np.array( [ np.arange(-5, 5, 1), np.arange(-6, 6, 1), np.arange(-6, 6, 1), np.arange(-6, 6, 1), ] ).T, heuristic="surrogate_model", regression_model=CensoredGaussianProcesses, next_parameter_strategy=expected_improvement, initial_sample_size=5, async_optim=True, max_step_cost=1, max_iteration=10, max_retry=10, ) bb_obj.optimize() print(bb_obj.history)
def test_optimization_categorical(self): """Tests that the optimization performs correctly whenever using a categorical variable. """ np.random.seed(10) categorical_parameter_space = np.array( [np.arange(-5, 5, 1), np.arange(-6, 6, 1), np.arange(-6, 6, 1), ["toto", "tutu"]] ).T bb_obj = BBOptimizer( black_box=self.categorical_parabola, heuristic="surrogate_model", max_iteration=10, initial_sample_size=2, parameter_space=categorical_parameter_space, next_parameter_strategy=expected_improvement, regression_model=GaussianProcessRegressor, ) bb_obj.optimize() best_parameters, best_fitness = bb_obj._get_best_performance() expected_best_parameters = np.array([0, 1, 1, "toto"], dtype=object) expected_best_fitness = 21 np.testing.assert_array_equal( best_parameters, expected_best_parameters ) self.assertEqual(best_fitness, expected_best_fitness)
def test_global_exploration_cost(self): """ Tests that the global exploration cost is properly computed. """ history = { "fitness": np.array([10, 5, 6, 2, 15, 4]), "parameters": np.array([[1, 2], [2, 3], [1, 3], [4, 3], [2, 1], [1, 5]]), "truncated": np.array([True, True, True, True, True, True]), } bb_obj = BBOptimizer( black_box=self.parabola, heuristic="surrogate_model", max_iteration=nbr_iteration, initial_sample_size=2, parameter_space=parameter_space, next_parameter_strategy=expected_improvement, regression_model=GaussianProcessRegressor, ) bb_obj.history = history expected_number_states = 3 expected_performance_cost = 16 real_number_states, real_performance_cost = bb_obj.global_exploration_cost self.assertEqual( expected_number_states, real_number_states, "Percentage of static moves " "was not computed properly.", ) self.assertEqual( expected_performance_cost, real_performance_cost, "Percentage of explored " "space was not computed " "properly.", )
def test_fitness_gain_per_iteration(self): """ Tests that the fitness gain per iteration is properly computed. """ history = { "fitness": np.array([10, 5, 6, 2, 15, 4]), "parameters": np.array([[1, 2], [2, 3], [1, 3], [4, 3], [2, 1], [1, 5]]), "truncated": np.array([True, True, True, True, True, True]), } bb_obj = BBOptimizer( black_box=self.parabola, heuristic="surrogate_model", max_iteration=nbr_iteration, initial_sample_size=2, parameter_space=parameter_space, next_parameter_strategy=expected_improvement, regression_model=GaussianProcessRegressor, ) bb_obj.history = history bb_obj.launched = True expected_gain_per_iteration = np.array([-5, 1, -4, 13, -11]) np.testing.assert_array_equal( expected_gain_per_iteration, bb_obj.fitness_gain_per_iteration, "Computation of fitness gain per iteration did not work as " "expected.", )
def test_pruning_parameters_config(self): """Tests that parsing the config is done properly and that the BBOptimizer class can be properly instanciated. """ shaman_config = SHAManConfig.from_yaml(PRUNING_CONFIG, "component_1") BBOptimizer(black_box=FakeBlackBox, parameter_space=shaman_config.component_parameter_space, async_optim=True, max_step_cost=shaman_config.pruning.max_step_duration, **shaman_config.bbo_parameters)
def test_no_compute_method(self): """ Tests that when there is no compute method, an error is raised. """ with self.assertRaises(AttributeError): BBOptimizer( black_box=self.no_compute, heuristic="surrogate_model", max_iteration=nbr_iteration, parameter_space=parameter_space, )
def test_missing_heuristic_argument(self): """ Tests that an exception is raised when there's a missing argument for a heuristic. """ with self.assertRaises(Exception): BBOptimizer( black_box=self.parabola, heuristic="surrogate_model", max_iteration=nbr_iteration, parameter_space=parameter_space, next_parameter_strategy=expected_improvement, )
def test_incorrect_heuristic_name(self): """ Tests that when an incorrect heuristic name is passed as argument, an error is raised. """ with self.assertRaises(ValueError): BBOptimizer( black_box=self.parabola, heuristic="makes no sense !", max_iteration=nbr_iteration, parameter_space=parameter_space, next_parameter_strategy=expected_improvement, regression_model=GaussianProcessRegressor, )
def test_stop_criterion_unknown(self): """Tests that initializing a BBOptimizer object with an unknown stop criterion returns a ValueError error. """ with self.assertRaises(ValueError): BBOptimizer( black_box=self.parabola, heuristic="surrogate_model", max_iteration=nbr_iteration, initial_sample_size=2, parameter_space=parameter_space, next_parameter_strategy=expected_improvement, regression_model=GaussianProcessRegressor, stop_criterion="unknown_criterion" )
def test_incorrect_selection_name(self): """ Tests that when an incorrect name for the selection method is passed as an argument, a ValueError is raised. """ with self.assertRaises(ValueError): BBOptimizer( black_box=self.parabola, heuristic="surrogate_model", max_iteration=nbr_iteration, parameter_space=parameter_space, next_parameter_strategy=expected_improvement, initial_draw_method="i do not exist!", regression_model=GaussianProcessRegressor, )
def test_select_parameters(self): """Test the function _select_next_parameters when there is no retry.""" np.random.seed(10) bb_obj = BBOptimizer( black_box=self.parabola, heuristic="surrogate_model", max_iteration=1, initial_sample_size=2, parameter_space=parameter_space, next_parameter_strategy=expected_improvement, regression_model=GaussianProcessRegressor, ) bb_obj._initialize() parameter = bb_obj._select_next_parameters() np.testing.assert_array_equal(parameter, np.array([-5, 2, -6]))
def test_optimizer_resampling_no_exist(self): """Tests that an error is raised when the asked for resampling policy does not exist. """ with self.assertRaises(ValueError): bb_obj = BBOptimizer( black_box=self.parabola, heuristic="surrogate_model", max_iteration=nbr_iteration, initial_sample_size=2, parameter_space=parameter_space, next_parameter_strategy=expected_improvement, regression_model=GaussianProcessRegressor, resampling_policy="i do not exist", )
def test_initialize(self): """Tests that the initialization step happens correctly, by appending the proper number of fitness and parametrization values.""" bb_obj = BBOptimizer( black_box=self.parabola, heuristic="surrogate_model", max_iteration=nbr_iteration, initial_sample_size=2, parameter_space=parameter_space, next_parameter_strategy=expected_improvement, regression_model=GaussianProcessRegressor, ) bb_obj._initialize(callbacks=[lambda x: x]) self.assertTrue(len(bb_obj.history["fitness"]), 2) self.assertTrue(len(bb_obj.history["parameters"]), 2)
def test_summarize_except(self): """ Tests that the call to "summarize" raises an exception if the experiment is not yet launched. """ bb_obj = BBOptimizer( black_box=self.parabola, heuristic="surrogate_model", max_iteration=nbr_iteration, initial_sample_size=2, parameter_space=parameter_space, next_parameter_strategy=expected_improvement, regression_model=GaussianProcessRegressor, ) self.assertRaises(Exception, bb_obj.summarize)
def test_fitness_gain_per_iteration_not_launched(self): """ Tests that the fitness gain per iteration is None if the experiment is not launched. """ bb_obj = BBOptimizer( black_box=self.parabola, heuristic="surrogate_model", max_iteration=nbr_iteration, initial_sample_size=2, parameter_space=parameter_space, next_parameter_strategy=expected_improvement, regression_model=GaussianProcessRegressor, ) bb_obj.launched = False self.assertEqual(bb_obj.fitness_gain_per_iteration, None)
def test_summarize(self): """ Tests that the call to "summarize" does not raise an error. """ bb_obj = BBOptimizer( black_box=self.parabola, heuristic="surrogate_model", max_iteration=nbr_iteration, initial_sample_size=2, parameter_space=parameter_space, next_parameter_strategy=expected_improvement, regression_model=GaussianProcessRegressor, ) bb_obj.optimize() bb_obj.summarize()
def test_noise_reduction_bbo(self): """Tests that the noise reduction parameters are properly added to the bbo_kwargs and the BBOptimizer object can be initialized. """ shaman_config = SHAManConfig.from_yaml(NOISE_REDUCTION_CONFIG, "component_1") # Check that the dictionary has been properly updated assert "nbr_resamples" in shaman_config.bbo_parameters assert "fitness_aggregation" in shaman_config.bbo_parameters assert "resampling_policy" in shaman_config.bbo_parameters # Check that the estimator function is properly parsed self.assertEqual(shaman_config.bbo_parameters["estimator"], numpy.median) # Check that the BBOptimizer class can be properly instanciated BBOptimizer(black_box=FakeBlackBox, parameter_space=shaman_config.component_parameter_space, **shaman_config.bbo_parameters)
def test_append_parameters_new_history(self): """ Tests that the parameters are correctly added to an empty history. """ # Manually create the history of the BBOptimizer bb_obj = BBOptimizer( black_box=self.parabola, heuristic="surrogate_model", max_iteration=nbr_iteration, parameter_space=parameter_space, next_parameter_strategy=expected_improvement, regression_model=GaussianProcessRegressor, ) # Test the append method bb_obj._append_parameters([1, 3]) np.testing.assert_array_equal( bb_obj.history["parameters"], np.array([1, 3]))
def test_append_performance_new_history(self): """ Tests that appending a fitness value on an empty history works properly. """ # Manually create the history of the BBOptimizer bb_obj = BBOptimizer( black_box=self.parabola, heuristic="surrogate_model", max_iteration=nbr_iteration, parameter_space=parameter_space, next_parameter_strategy=expected_improvement, regression_model=GaussianProcessRegressor, ) # Tests the append method bb_obj._append_fitness(10) np.testing.assert_array_equal( bb_obj.history["fitness"], np.array(np.array([10])) )
def test_optimizer_process_simple_resampling(self): """Tests that the .optimize() method works correctly when using simple resampling""" np.random.seed(5) bb_obj = BBOptimizer( black_box=self.parabola, heuristic="surrogate_model", max_iteration=nbr_iteration, initial_sample_size=2, parameter_space=parameter_space, next_parameter_strategy=expected_improvement, regression_model=GaussianProcessRegressor, resampling_policy="simple_resampling", nbr_resamples=2, ) bb_obj.optimize() np.testing.assert_array_equal( bb_obj.history["fitness"], np.array( [16., 61., 61., 9., 9., 0., 0.]) )