def test_bayesian_optimizer_optimize_raises_for_invalid_keys( datasets: dict[str, Dataset], models: dict[str, TrainableProbabilisticModel]) -> None: search_space = Box([-1], [1]) optimizer = BayesianOptimizer(lambda x: {"foo": Dataset(x, x)}, search_space) rule = FixedAcquisitionRule([[0.0]]) with pytest.raises(ValueError): optimizer.optimize(10, datasets, models, rule)
def test_box_contains_raises_on_point_of_different_shape( bound_shape: ShapeLike, point_shape: ShapeLike, ) -> None: box = Box(tf.zeros(bound_shape), tf.ones(bound_shape)) point = tf.zeros(point_shape) with pytest.raises(ValueError): _ = point in box
def test_gumbel_samples_are_minima() -> None: search_space = Box([0, 0], [1, 1]) x_range = tf.linspace(0.0, 1.0, 5) x_range = tf.cast(x_range, dtype=tf.float64) xs = tf.reshape(tf.stack(tf.meshgrid(x_range, x_range, indexing="ij"), axis=-1), (-1, 2)) ys = quadratic(xs) dataset = Dataset(xs, ys) model = QuadraticMeanAndRBFKernel() gumbel_sampler = GumbelSampler(5, model) query_points = search_space.sample(100) query_points = tf.concat([dataset.query_points, query_points], 0) gumbel_samples = gumbel_sampler.sample(query_points) fmean, _ = model.predict(dataset.query_points) assert max(gumbel_samples) < min(fmean)
def one_dimensional_range(lower: float, upper: float) -> Box: """ :param lower: The box lower bound. :param upper: The box upper bound. :return: A one-dimensional box with range given by ``lower`` and ``upper``, and bound dtype `tf.float32`. :raise ValueError: If ``lower`` is not less than ``upper``. """ return Box(tf.constant([lower], dtype=tf.float32), tf.constant([upper], tf.float32))
def test_box_contains_raises_on_point_of_different_shape( bound_shape: ShapeLike, point_shape: ShapeLike, ) -> None: box = Box(tf.zeros(bound_shape), tf.ones(bound_shape)) point = tf.zeros(point_shape) with pytest.raises(TF_DEBUGGING_ERROR_TYPES): _ = point in box
def test_locally_penalized_expected_improvement_raises_when_called_with_invalid_base( ) -> None: search_space = Box([0, 0], [1, 1]) base_builder = NegativeLowerConfidenceBound() with pytest.raises(ValueError): LocalPenalizationAcquisitionFunction( search_space, base_acquisition_function_builder=base_builder # type: ignore )
def test_trust_region_state_deepcopy() -> None: tr_state = TrustRegion.State( Box(tf.constant([1.2]), tf.constant([3.4])), tf.constant(5.6), tf.constant(7.8), False ) tr_state_copy = copy.deepcopy(tr_state) npt.assert_allclose(tr_state_copy.acquisition_space.lower, tr_state.acquisition_space.lower) npt.assert_allclose(tr_state_copy.acquisition_space.upper, tr_state.acquisition_space.upper) npt.assert_allclose(tr_state_copy.eps, tr_state.eps) npt.assert_allclose(tr_state_copy.y_min, tr_state.y_min) assert tr_state_copy.is_global == tr_state.is_global
def test_product_space_discretize_raises_if_sample_larger_than_discretization( num_samples: int, ) -> None: space_A = Box([-1], [2]) space_B = DiscreteSearchSpace(tf.ones([100, 2], dtype=tf.float64)) product_space = TaggedProductSearchSpace(spaces=[space_A, space_B]) dss = product_space.discretize(num_samples) with pytest.raises(tf.errors.InvalidArgumentError): dss.sample(num_samples + 1)
def test_bayesian_optimizer_optimize_raises_for_invalid_rule_keys_and_default_acquisition( ) -> None: optimizer = BayesianOptimizer(lambda x: x[:1], Box([-1], [1])) data, models = { "foo": empty_dataset([1], [1]) }, { "foo": _PseudoTrainableQuadratic() } with pytest.raises(ValueError): optimizer.optimize(3, data, models)
def test_locally_penalized_expected_improvement_raises_when_called_before_initialization( ) -> None: data = Dataset(tf.zeros([3, 2], dtype=tf.float64), tf.ones([3, 2], dtype=tf.float64)) search_space = Box([0, 0], [1, 1]) pending_points = tf.zeros([1, 2]) with pytest.raises(ValueError): LocalPenalizationAcquisitionFunction( search_space).prepare_acquisition_function( data, QuadraticMeanAndRBFKernel(), pending_points)
def test_multi_objective_optimizer_finds_pareto_front_of_the_VLMOP2_function( num_steps: int, acquisition_rule: AcquisitionRule[TensorType, Box], convergence_threshold: float) -> None: search_space = Box([-2, -2], [2, 2]) def build_stacked_independent_objectives_model( data: Dataset) -> ModelStack: gprs = [] for idx in range(2): single_obj_data = Dataset( data.query_points, tf.gather(data.observations, [idx], axis=1)) variance = tf.math.reduce_variance(single_obj_data.observations) kernel = gpflow.kernels.Matern52( variance, tf.constant([0.2, 0.2], tf.float64)) gpr = gpflow.models.GPR(single_obj_data.astuple(), kernel, noise_variance=1e-5) gpflow.utilities.set_trainable(gpr.likelihood, False) gprs.append((GaussianProcessRegression(gpr), 1)) return ModelStack(*gprs) observer = mk_observer(VLMOP2().objective(), OBJECTIVE) initial_query_points = search_space.sample(10) initial_data = observer(initial_query_points) model = build_stacked_independent_objectives_model(initial_data[OBJECTIVE]) dataset = (BayesianOptimizer(observer, search_space).optimize( num_steps, initial_data, { OBJECTIVE: model }, acquisition_rule).try_get_final_datasets()[OBJECTIVE]) # A small log hypervolume difference corresponds to a succesful optimization. ref_point = get_reference_point(dataset.observations) obs_hv = Pareto(dataset.observations).hypervolume_indicator(ref_point) ideal_pf = tf.cast(VLMOP2().gen_pareto_optimal_points(100), dtype=tf.float64) ideal_hv = Pareto(ideal_pf).hypervolume_indicator(ref_point) assert tf.math.log(ideal_hv - obs_hv) < convergence_threshold
def test_bayesian_optimizer_optimize_tracked_state() -> None: class _CountingRule(AcquisitionRule[int, Box]): def acquire( self, search_space: Box, datasets: Mapping[str, Dataset], models: Mapping[str, ProbabilisticModel], state: int | None = None, ) -> tuple[TensorType, int]: new_state = 0 if state is None else state + 1 return tf.constant([[10.0]], tf.float64) + new_state, new_state class _DecreasingVarianceModel(QuadraticMeanAndRBFKernel, TrainableProbabilisticModel): def __init__(self, data: Dataset): super().__init__() self._data = data def predict(self, query_points: TensorType) -> tuple[TensorType, TensorType]: mean, var = super().predict(query_points) return mean, var / len(self._data) def update(self, dataset: Dataset) -> None: self._data = dataset def optimize(self, dataset: Dataset) -> None: pass initial_data = mk_dataset([[0.0]], [[0.0]]) model = _DecreasingVarianceModel(initial_data) _, history = (BayesianOptimizer(_quadratic_observer, Box([0], [1])).optimize(3, { "": initial_data }, { "": model }, _CountingRule()).astuple()) assert [record.acquisition_state for record in history] == [None, 0, 1] assert_datasets_allclose(history[0].datasets[""], initial_data) assert_datasets_allclose(history[1].datasets[""], mk_dataset([[0.0], [10.0]], [[0.0], [100.0]])) assert_datasets_allclose( history[2].datasets[""], mk_dataset([[0.0], [10.0], [11.0]], [[0.0], [100.0], [121.0]])) for step in range(3): assert history[step].model == history[step].models[""] assert history[step].dataset == history[step].datasets[""] _, variance_from_saved_model = (history[step].models[""].predict( tf.constant([[0.0]], tf.float64))) npt.assert_allclose(variance_from_saved_model, 1.0 / (step + 1))
def test_optimizer_finds_minima_of_the_branin_function( num_steps: int, acquisition_rule: AcquisitionRule) -> None: search_space = Box(tf.constant([0.0, 0.0], tf.float64), tf.constant([1.0, 1.0], tf.float64)) def build_model(data: Dataset) -> GaussianProcessRegression: variance = tf.math.reduce_variance(data.observations) kernel = gpflow.kernels.Matern52(variance, tf.constant([0.2, 0.2], tf.float64)) gpr = gpflow.models.GPR((data.query_points, data.observations), kernel, noise_variance=1e-5) gpflow.utilities.set_trainable(gpr.likelihood, False) return GaussianProcessRegression(gpr) initial_query_points = search_space.sample(5) observer = mk_observer(branin, OBJECTIVE) initial_data = observer(initial_query_points) model = build_model(initial_data[OBJECTIVE]) res = BayesianOptimizer(observer, search_space).optimize(num_steps, initial_data, {OBJECTIVE: model}, acquisition_rule) if res.error is not None: raise res.error dataset = res.datasets[OBJECTIVE] arg_min_idx = tf.squeeze(tf.argmin(dataset.observations, axis=0)) best_y = dataset.observations[arg_min_idx] best_x = dataset.query_points[arg_min_idx] argmin = tf.cast(BRANIN_GLOBAL_ARGMIN, tf.float64) relative_argmin_err = tf.abs((best_x - argmin) / argmin) # these accuracies are the current best for the given number of optimization steps, which makes # this is a regression test assert tf.reduce_any(tf.reduce_all(relative_argmin_err < 0.03, axis=-1), axis=0) npt.assert_allclose(best_y, BRANIN_GLOBAL_MINIMUM, rtol=0.03)
def test_product_space_discretize_returns_search_space_with_correct_number_of_points( num_samples: int, ) -> None: space_A = Box([-1], [2]) space_B = DiscreteSearchSpace(tf.ones([100, 2], dtype=tf.float64)) product_space = TaggedProductSearchSpace(spaces=[space_A, space_B]) dss = product_space.discretize(num_samples) samples = dss.sample(num_samples) assert len(samples) == num_samples
def test_product_space_discretize_returns_search_space_with_only_points_contained_within_box( num_samples: int, ) -> None: space_A = Box([-1], [2]) space_B = DiscreteSearchSpace(tf.ones([100, 2], dtype=tf.float64)) product_space = TaggedProductSearchSpace(spaces=[space_A, space_B]) dss = product_space.discretize(num_samples) samples = dss.sample(num_samples) assert all(sample in product_space for sample in samples)
def test_locally_penalized_expected_improvement_builder_raises_for_invalid_pending_points_shape( pending_points, ) -> None: data = Dataset(tf.zeros([3, 2], dtype=tf.float64), tf.ones([3, 2], dtype=tf.float64)) space = Box([0, 0], [1, 1]) builder = LocalPenalizationAcquisitionFunction(search_space=space) builder.prepare_acquisition_function(data, QuadraticMeanAndRBFKernel(), None) # first initialize with pytest.raises(TF_DEBUGGING_ERROR_TYPES): builder.prepare_acquisition_function(data, QuadraticMeanAndRBFKernel(), pending_points)
def test_bayesian_optimizer_optimize_raises_for_negative_steps( num_steps: int) -> None: optimizer = BayesianOptimizer(_quadratic_observer, Box([-1], [1])) data, models = { "": empty_dataset([1], [1]) }, { "": _PseudoTrainableQuadratic() } with pytest.raises(ValueError, match="num_steps"): optimizer.optimize(num_steps, data, models)
def test_joint_batch_acquisition_rule_acquire() -> None: search_space = Box(tf.constant([-2.2, -1.0]), tf.constant([1.3, 3.3])) num_query_points = 4 acq = _JointBatchModelMinusMeanMaximumSingleBuilder() ego: EfficientGlobalOptimization[Box] = EfficientGlobalOptimization( acq, num_query_points=num_query_points ) dataset = Dataset(tf.zeros([0, 2]), tf.zeros([0, 1])) query_point = ego.acquire_single(search_space, dataset, QuadraticMeanAndRBFKernel()) npt.assert_allclose(query_point, [[0.0, 0.0]] * num_query_points, atol=1e-3)
def test_bayesian_optimizer_optimize_for_failed_step( observer: Observer, model: TrainableProbabilisticModel, rule: AcquisitionRule) -> None: optimizer = BayesianOptimizer(observer, Box([0], [1])) data, models = {"": mk_dataset([[0.0]], [[0.0]])}, {"": model} result, history = optimizer.optimize(3, data, models, rule).astuple() with pytest.raises(_Whoops): result.unwrap() assert len(history) == 1
def test_batch_acquisition_rule_acquire() -> None: search_space = Box(tf.constant([-2.2, -1.0]), tf.constant([1.3, 3.3])) num_query_points = 4 ego = BatchAcquisitionRule(num_query_points, _BatchModelMinusMeanMaximumSingleBuilder()) dataset = Dataset(tf.zeros([0, 2]), tf.zeros([0, 1])) query_point, _ = ego.acquire(search_space, {OBJECTIVE: dataset}, {OBJECTIVE: QuadraticMeanAndRBFKernel()}) npt.assert_allclose(query_point, [[0.0, 0.0]] * num_query_points, atol=1e-3)
def test_optimize_continuous_raises_for_impossible_optimization( num_optimization_runs: int, num_recovery_runs: int) -> None: search_space = Box([-1, -1], [1, 2]) optimizer = generate_continuous_optimizer( num_optimization_runs=num_optimization_runs, num_recovery_runs=num_recovery_runs) with pytest.raises(FailedOptimizationError) as e: optimizer(search_space, _delta_function(10)) assert (str(e.value) == f""" Acquisition function optimization failed, even after {num_recovery_runs + num_optimization_runs} restarts. """)
def test_product_space_get_subspace() -> None: space_A = Box([-1, -2], [2, 3]) space_B = DiscreteSearchSpace(tf.constant([[-0.5, 0.5]])) space_C = Box([-1], [2]) product_space = TaggedProductSearchSpace( spaces=[space_A, space_B, space_C], tags=["A", "B", "C"] ) subspace_A = product_space.get_subspace("A") assert isinstance(subspace_A, Box) npt.assert_array_equal(subspace_A.lower, [-1, -2]) npt.assert_array_equal(subspace_A.upper, [2, 3]) subspace_B = product_space.get_subspace("B") assert isinstance(subspace_B, DiscreteSearchSpace) npt.assert_array_equal(subspace_B.points, tf.constant([[-0.5, 0.5]])) subspace_C = product_space.get_subspace("C") assert isinstance(subspace_C, Box) npt.assert_array_equal(subspace_C.lower, [-1]) npt.assert_array_equal(subspace_C.upper, [2])
def test_bayesian_optimizer_optimize_doesnt_track_state_if_told_not_to() -> None: class _UncopyableModel(_PseudoTrainableQuadratic): def __deepcopy__(self, memo: dict[int, object]) -> NoReturn: assert False data, models = {OBJECTIVE: empty_dataset([1], [1])}, {OBJECTIVE: _UncopyableModel()} history = ( BayesianOptimizer(_quadratic_observer, Box([-1], [1])) .optimize(5, data, models, track_state=False) .history ) assert len(history) == 0
def test_discrete_thompson_sampling_acquire_returns_correct_shape( num_fourier_features: bool, num_query_points: int ) -> None: search_space = Box(tf.constant([-2.2, -1.0]), tf.constant([1.3, 3.3])) ts = DiscreteThompsonSampling(100, num_query_points, num_fourier_features=num_fourier_features) dataset = Dataset(tf.zeros([1, 2], dtype=tf.float64), tf.zeros([1, 1], dtype=tf.float64)) model = QuadraticMeanAndRBFKernel(noise_variance=tf.constant(1.0, dtype=tf.float64)) model.kernel = ( gpflow.kernels.RBF() ) # need a gpflow kernel object for random feature decompositions query_points = ts.acquire_single(search_space, model, dataset=dataset) npt.assert_array_equal(query_points.shape, tf.constant([num_query_points, 2]))
def test_rff_thompson_samples_are_minima() -> None: search_space = Box([0.0, 0.0], [1.0, 1.0]) model = QuadraticMeanAndRBFKernel(noise_variance=tf.constant(1e-5, dtype=tf.float64)) model.kernel = ( gpflow.kernels.RBF() ) # need a gpflow kernel object for random feature decompositions x_range = tf.linspace(0.0, 1.0, 5) x_range = tf.cast(x_range, dtype=tf.float64) xs = tf.reshape(tf.stack(tf.meshgrid(x_range, x_range, indexing="ij"), axis=-1), (-1, 2)) ys = quadratic(xs) dataset = Dataset(xs, ys) sampler = RandomFourierFeatureThompsonSampler( 1, model, dataset, num_features=100, sample_min_value=True ) query_points = search_space.sample(100) query_points = tf.concat([dataset.query_points, query_points], 0) thompson_samples = sampler.sample(query_points) fmean, _ = model.predict(dataset.query_points) assert max(thompson_samples) < min(fmean)
def test_min_value_entropy_search_builder_gumbel_samples(mocked_mves) -> None: dataset = Dataset(tf.zeros([3, 2], dtype=tf.float64), tf.ones([3, 2], dtype=tf.float64)) search_space = Box([0, 0], [1, 1]) builder = MinValueEntropySearch(search_space) model = QuadraticMeanAndRBFKernel() builder.prepare_acquisition_function(dataset, model) mocked_mves.assert_called_once() # check that the Gumbel samples look sensible gumbel_samples = mocked_mves.call_args[0][1] query_points = builder._search_space.sample(num_samples=builder._grid_size) query_points = tf.concat([dataset.query_points, query_points], 0) fmean, _ = model.predict(query_points) assert max(gumbel_samples) < min(fmean)
def test_trust_region_for_unsuccessful_local_to_global_trust_region_reduced( rule: AcquisitionRule[TensorType, Box] ) -> None: tr = TrustRegion(rule) dataset = Dataset(tf.constant([[0.1, 0.2], [-0.1, -0.2]]), tf.constant([[0.4], [0.5]])) lower_bound = tf.constant([-2.2, -1.0]) upper_bound = tf.constant([1.3, 3.3]) search_space = Box(lower_bound, upper_bound) eps = 0.5 * (search_space.upper - search_space.lower) / 10 previous_y_min = dataset.observations[0] is_global = False acquisition_space = Box(dataset.query_points[0] - eps, dataset.query_points[0] + eps) previous_state = TrustRegion.State(acquisition_space, eps, previous_y_min, is_global) current_state, _ = tr.acquire( search_space, {OBJECTIVE: dataset}, {OBJECTIVE: QuadraticMeanAndRBFKernel()} )(previous_state) assert current_state is not None npt.assert_array_less(current_state.eps, previous_state.eps) # current TR smaller than previous assert current_state.is_global npt.assert_array_almost_equal(current_state.acquisition_space.lower, lower_bound)
def test_efficient_global_optimization( optimizer: AcquisitionOptimizer[Box]) -> None: class NegQuadratic(AcquisitionFunctionBuilder): def prepare_acquisition_function( self, datasets: Mapping[str, Dataset], models: Mapping[str, ProbabilisticModel]) -> AcquisitionFunction: return lambda x: -quadratic(tf.squeeze(x, -2) - 1) search_space = Box([-10], [10]) ego = EfficientGlobalOptimization(NegQuadratic(), optimizer) data, model = empty_dataset([1], [1]), QuadraticMeanAndRBFKernel(x_shift=1) query_point, _ = ego.acquire(search_space, {"": data}, {"": model}) npt.assert_allclose(query_point, [[1]], rtol=1e-4)
def test_trust_region_for_default_state() -> None: tr = TrustRegion(NegativeLowerConfidenceBound(0)) dataset = Dataset(tf.constant([[0.1, 0.2]]), tf.constant([[0.012]])) lower_bound = tf.constant([-2.2, -1.0]) upper_bound = tf.constant([1.3, 3.3]) search_space = Box(lower_bound, upper_bound) query_point, state = tr.acquire_single(search_space, dataset, QuadraticMeanAndRBFKernel(), None) npt.assert_array_almost_equal(query_point, tf.constant([[0.0, 0.0]]), 5) npt.assert_array_almost_equal(state.acquisition_space.lower, lower_bound) npt.assert_array_almost_equal(state.acquisition_space.upper, upper_bound) npt.assert_array_almost_equal(state.y_min, [0.012]) assert state.is_global
def test_bayesian_optimizer_calls_observer_once_per_iteration(steps: int) -> None: class _CountingObserver: call_count = 0 def __call__(self, x: tf.Tensor) -> Dataset: self.call_count += 1 return Dataset(x, tf.reduce_sum(x ** 2, axis=-1, keepdims=True)) observer = _CountingObserver() optimizer = BayesianOptimizer(observer, Box([-1], [1])) data = mk_dataset([[0.5]], [[0.25]]) optimizer.optimize(steps, data, _PseudoTrainableQuadratic()).final_result.unwrap() assert observer.call_count == steps