def test_create_loop_state_wrong_size_input_fails(): x = np.array([[1], [2], [3], [4]]) y = np.array([[4], [5], [6], [7]]) c = np.array([[4], [5], [6], [7], [8]]) with pytest.raises(ValueError): create_loop_state(x, y, cost=c)
def __init__(self, model: VanillaBayesianQuadrature, acquisition: Acquisition = None, model_updater: ModelUpdater = None): """ The loop for vanilla Bayesian Quadrature :param model: the vanilla Bayesian quadrature method :param acquisition: The acquisition function that is be used to collect new points. default, IntegralVarianceReduction :param model_updater: Defines how and when the quadrature model is updated if new data arrives. Defaults to updating hyper-parameters every iteration. """ if acquisition is None: acquisition = IntegralVarianceReduction(model) if model_updater is None: model_updater = FixedIntervalUpdater(model, 1) space = ParameterSpace(model.integral_bounds.convert_to_list_of_continuous_parameters()) acquisition_optimizer = AcquisitionOptimizer(space) candidate_point_calculator = SequentialPointCalculator(acquisition, acquisition_optimizer) loop_state = create_loop_state(model.X, model.Y) super().__init__(candidate_point_calculator, model_updater, loop_state) self.model = model
def __init__(self, model: VanillaBayesianQuadrature, acquisition: Acquisition = None, model_updater: ModelUpdater = None): """ The loop for vanilla Bayesian Quadrature :param model: the vanilla Bayesian quadrature method :param acquisition: The acquisition function that is be used to collect new points. default, IntegralVarianceReduction :param model_updater: Defines how and when the quadrature model is updated if new data arrives. Defaults to updating hyper-parameters every iteration. """ self.model = model if acquisition is None: acquisition = IntegralVarianceReduction(self.model) if model_updater is None: model_updater = FixedIntervalUpdater(self.model, 1) space = ParameterSpace(self.model.integral_bounds. convert_to_list_of_continuous_parameters()) acquisition_optimizer = AcquisitionOptimizer(space) candidate_point_calculator = SequentialPointCalculator( acquisition, acquisition_optimizer) loop_state = create_loop_state(self.model.X, self.model.Y) super().__init__(candidate_point_calculator, model_updater, loop_state)
def test_iteration_end_event(): space = ParameterSpace([ContinuousParameter('x', 0, 1)]) def user_function(x): return x x_test = np.linspace(0, 1)[:, None] y_test = user_function(x_test) x_init = np.linspace(0, 1, 5)[:, None] y_init = user_function(x_init) gpy_model = GPy.models.GPRegression(x_init, y_init) model = GPyModelWrapper(gpy_model) mse = [] def compute_mse(self, loop_state): mse.append(np.mean(np.square(model.predict(x_test)[0] - y_test))) loop_state = create_loop_state(x_init, y_init) acquisition = ModelVariance(model) acquisition_optimizer = AcquisitionOptimizer(space) candidate_point_calculator = SequentialPointCalculator( acquisition, acquisition_optimizer) model_updater = FixedIntervalUpdater(model) loop = OuterLoop(candidate_point_calculator, model_updater, loop_state) loop.iteration_end_event.append(compute_mse) loop.run_loop(user_function, 5) assert len(mse) == 5
def test_cost_returns_none(): x = np.array([[1], [2], [3], [4]]) y = np.array([[4], [5], [6], [7]]) loop_state = create_loop_state(x[:3, :], y[:3, :]) assert np.array_equiv(loop_state.cost, np.array([None, None, None]))
def test_iteration_end_event(): space = ParameterSpace([ContinuousParameter('x', 0, 1)]) def user_function(x): return x x_test = np.linspace(0, 1)[:, None] y_test = user_function(x_test) x_init = np.linspace(0, 1, 5)[:, None] y_init = user_function(x_init) gpy_model = GPy.models.GPRegression(x_init, y_init) model = GPyModelWrapper(gpy_model) mse = [] def compute_mse(self, loop_state): mse.append(np.mean(np.square(model.predict(x_test)[0] - y_test))) loop_state = create_loop_state(x_init, y_init) acquisition = ModelVariance(model) acquisition_optimizer = AcquisitionOptimizer(space) candidate_point_calculator = SequentialPointCalculator(acquisition, acquisition_optimizer) model_updater = FixedIntervalUpdater(model) loop = OuterLoop(candidate_point_calculator, model_updater, loop_state) loop.iteration_end_event.append(compute_mse) loop.run_loop(user_function, 5) assert len(mse) == 5
def test_create_loop_state(): x_init = np.array([[1], [2], [3]]) y_init = np.array([[4], [5], [6]]) loop_state = create_loop_state(x_init, y_init) assert_array_equal(loop_state.X, x_init) assert_array_equal(loop_state.Y, y_init) assert loop_state.iteration == 0
def test_loop_state_update_error(): x = np.array([[1], [2], [3], [4]]) y = np.array([[4], [5], [6], [7]]) loop_state = create_loop_state(x[:3, :], y[:3, :]) with pytest.raises(ValueError): loop_state.update(None) with pytest.raises(ValueError): loop_state.update([])
def test_loop_state_update(): x = np.array([[1], [2], [3], [4]]) y = np.array([[4], [5], [6], [7]]) loop_state = create_loop_state(x[:3, :], y[:3, :]) step_result = UserFunctionResult(x[3, :], y[3, :]) loop_state.update([step_result]) assert_array_equal(loop_state.X, x) assert_array_equal(loop_state.Y, y) assert loop_state.iteration == 1
def __init__(self, space: ParameterSpace, X_init: np.ndarray, Y_init: np.ndarray, cost_init: np.ndarray, s_min: float, s_max: float, update_interval: int = 1, num_eval_points: int = 2000, marginalize_hypers: bool = True): """ Implements FAst Bayesian Optimization for LArge DataSets as described in: Fast Bayesian hyperparameter optimization on large datasets A. Klein and S. Falkner and S. Bartels and P. Hennig and F. Hutter Electronic Journal of Statistics (2017) :param space: input space where the optimization is carried out. :param X_init: initial data points :param Y_init: initial function values :param cost_init: initial costs :param s_min: smallest possible dataset size :param s_max: highest possible dataset size :param update_interval: number of iterations between optimization of model hyper-parameters. Defaults to 1. :param num_eval_points: number of points to evaluate the acquisition function :param marginalize_hypers: if true, marginalize over the GP hyperparameters """ l = space.parameters l.extend([ContinuousParameter("s", s_min, s_max)]) extended_space = ParameterSpace(l) model_objective = FabolasModel(X_init=X_init, Y_init=Y_init, s_min=s_min, s_max=s_max) model_cost = FabolasModel(X_init=X_init, Y_init=cost_init[:, None], s_min=s_min, s_max=s_max) if marginalize_hypers: acquisition_generator = lambda model: ContinuousFidelityEntropySearch(model_objective, space=extended_space, target_fidelity_index=len( extended_space.parameters) - 1) entropy_search = IntegratedHyperParameterAcquisition(model_objective, acquisition_generator) else: entropy_search = ContinuousFidelityEntropySearch(model_objective, space=extended_space, target_fidelity_index=len(extended_space.parameters) - 1) acquisition = acquisition_per_expected_cost(entropy_search, model_cost) model_updater_objective = FixedIntervalUpdater(model_objective, update_interval) model_updater_cost = FixedIntervalUpdater(model_cost, update_interval, lambda state: state.cost) acquisition_optimizer = RandomSearchAcquisitionOptimizer(extended_space, num_eval_points=num_eval_points) candidate_point_calculator = SequentialPointCalculator(acquisition, acquisition_optimizer) loop_state = create_loop_state(model_objective.X, model_objective.Y, model_cost.Y) super(CostSensitiveBayesianOptimizationLoop, self).__init__(candidate_point_calculator, [model_updater_objective, model_updater_cost], loop_state)
def test_batch_point_calculator(mock_model): acquisition = mock.create_autospec(Acquisition) acquisition_optimizer = mock.create_autospec(GradientAcquisitionOptimizer) acquisition_optimizer.optimize.return_value = (np.zeros((1, 1)), 0) batch_size = 10 calculator = GreedyBatchPointCalculator(mock_model, acquisition, acquisition_optimizer, batch_size) loop_state = create_loop_state(np.zeros((1, 1)), np.zeros((1, 1))) next_points = calculator.compute_next_points(loop_state) assert next_points.shape[0] == batch_size
def test_local_penalization(): parameter_space = ParameterSpace([ContinuousParameter('x', 0, 1)]) acquisition_optimizer = AcquisitionOptimizer(parameter_space) x_init = np.random.rand(5, 1) y_init = np.random.rand(5, 1) gpy_model = GPy.models.GPRegression(x_init, y_init) model = GPyModelWrapper(gpy_model) acquisition = ExpectedImprovement(model) batch_size = 5 lp_calc = LocalPenalizationPointCalculator(acquisition, acquisition_optimizer, model, parameter_space, batch_size) loop_state = create_loop_state(x_init, y_init) new_points = lp_calc.compute_next_points(loop_state) assert new_points.shape == (batch_size, 1)
def test_local_penalization(): parameter_space = ParameterSpace([ContinuousParameter("x", 0, 1)]) acquisition_optimizer = GradientAcquisitionOptimizer(parameter_space) x_init = np.random.rand(5, 1) y_init = np.random.rand(5, 1) gpy_model = GPy.models.GPRegression(x_init, y_init) model = GPyModelWrapper(gpy_model) acquisition = ExpectedImprovement(model) batch_size = 5 lp_calc = LocalPenalizationPointCalculator(acquisition, acquisition_optimizer, model, parameter_space, batch_size) loop_state = create_loop_state(x_init, y_init) new_points = lp_calc.compute_next_points(loop_state) assert new_points.shape == (batch_size, 1)
def test_minimum_observed_value_metric(): x_observations = np.random.rand(50, 2) y_observations = np.random.rand(50, 2) mock_model = mock.create_autospec(IModel) model_updater_mock = mock.create_autospec(ModelUpdater) model_updater_mock.model = mock_model mock_loop = mock.create_autospec(OuterLoop) mock_loop.model_updaters = [model_updater_mock] loop_state = create_loop_state(x_observations, y_observations) loop_state.metrics = dict() metric = MinimumObservedValueMetric() metric_value = metric.evaluate(mock_loop, loop_state) assert metric_value.shape == (2, )
def test_time_metric(): x_observations = np.random.rand(50, 2) y_observations = np.random.rand(50, 2) mock_model = mock.create_autospec(IModel) model_updater_mock = mock.create_autospec(ModelUpdater) model_updater_mock.model = mock_model mock_loop = mock.create_autospec(OuterLoop) mock_loop.model_updater = model_updater_mock loop_state = create_loop_state(x_observations, y_observations) loop_state.metrics = dict() name = 'time' metric = TimeMetric(name) metric.reset() metric_value = metric.evaluate(mock_loop, loop_state) assert metric_value.shape == (1, )
def test_cumulative_costs(): x_observations = np.random.rand(50, 2) y_observations = np.random.rand(50, 2) c_observations = np.random.rand(50, 1) mock_model = mock.create_autospec(IModel) model_updater_mock = mock.create_autospec(ModelUpdater) model_updater_mock.model = mock_model mock_loop = mock.create_autospec(OuterLoop) mock_loop.model_updater = model_updater_mock loop_state = create_loop_state(x_observations, y_observations, cost=c_observations) loop_state.metrics = dict() name = 'cost' metric = CumulativeCostMetric(name) metric.reset() metric_value = metric.evaluate(mock_loop, loop_state) assert metric_value == np.cumsum(c_observations)[-1] assert metric_value.shape == (1, )
def test_create_loop_error(): x_init = np.array([[1], [2], [3]]) y_init = np.array([[4], [5]]) with pytest.raises(ValueError): create_loop_state(x_init, y_init)