def test_bayesian_optimizer_optimize_raises_for_invalid_keys(
datasets: dict[str, Dataset],
models: dict[str, TrainableProbabilisticModel]) -> None:
search_space = Box([-1], [1])
optimizer = BayesianOptimizer(lambda x: {"foo": Dataset(x, x)},
search_space)
rule = FixedAcquisitionRule([[0.0]])
with pytest.raises(ValueError):
optimizer.optimize(10, datasets, models, rule)
def test_box_contains_raises_on_point_of_different_shape(
bound_shape: ShapeLike,
point_shape: ShapeLike,
) -> None:
box = Box(tf.zeros(bound_shape), tf.ones(bound_shape))
point = tf.zeros(point_shape)
with pytest.raises(ValueError):
_ = point in box
def test_gumbel_samples_are_minima() -> None:
search_space = Box([0, 0], [1, 1])
x_range = tf.linspace(0.0, 1.0, 5)
x_range = tf.cast(x_range, dtype=tf.float64)
xs = tf.reshape(tf.stack(tf.meshgrid(x_range, x_range, indexing="ij"), axis=-1), (-1, 2))
ys = quadratic(xs)
dataset = Dataset(xs, ys)
model = QuadraticMeanAndRBFKernel()
gumbel_sampler = GumbelSampler(5, model)
query_points = search_space.sample(100)
query_points = tf.concat([dataset.query_points, query_points], 0)
gumbel_samples = gumbel_sampler.sample(query_points)
fmean, _ = model.predict(dataset.query_points)
assert max(gumbel_samples) < min(fmean)
def one_dimensional_range(lower: float, upper: float) -> Box:
"""
:param lower: The box lower bound.
:param upper: The box upper bound.
:return: A one-dimensional box with range given by ``lower`` and ``upper``, and bound dtype
`tf.float32`.
:raise ValueError: If ``lower`` is not less than ``upper``.
"""
return Box(tf.constant([lower], dtype=tf.float32), tf.constant([upper], tf.float32))
def test_box_contains_raises_on_point_of_different_shape(
bound_shape: ShapeLike,
point_shape: ShapeLike,
) -> None:
box = Box(tf.zeros(bound_shape), tf.ones(bound_shape))
point = tf.zeros(point_shape)
with pytest.raises(TF_DEBUGGING_ERROR_TYPES):
_ = point in box
def test_locally_penalized_expected_improvement_raises_when_called_with_invalid_base(
) -> None:
search_space = Box([0, 0], [1, 1])
base_builder = NegativeLowerConfidenceBound()
with pytest.raises(ValueError):
LocalPenalizationAcquisitionFunction(
search_space,
base_acquisition_function_builder=base_builder # type: ignore
)
def test_trust_region_state_deepcopy() -> None:
tr_state = TrustRegion.State(
Box(tf.constant([1.2]), tf.constant([3.4])), tf.constant(5.6), tf.constant(7.8), False
)
tr_state_copy = copy.deepcopy(tr_state)
npt.assert_allclose(tr_state_copy.acquisition_space.lower, tr_state.acquisition_space.lower)
npt.assert_allclose(tr_state_copy.acquisition_space.upper, tr_state.acquisition_space.upper)
npt.assert_allclose(tr_state_copy.eps, tr_state.eps)
npt.assert_allclose(tr_state_copy.y_min, tr_state.y_min)
assert tr_state_copy.is_global == tr_state.is_global
def test_product_space_discretize_raises_if_sample_larger_than_discretization(
num_samples: int,
) -> None:
space_A = Box([-1], [2])
space_B = DiscreteSearchSpace(tf.ones([100, 2], dtype=tf.float64))
product_space = TaggedProductSearchSpace(spaces=[space_A, space_B])
dss = product_space.discretize(num_samples)
with pytest.raises(tf.errors.InvalidArgumentError):
dss.sample(num_samples + 1)
def test_bayesian_optimizer_optimize_raises_for_invalid_rule_keys_and_default_acquisition(
) -> None:
optimizer = BayesianOptimizer(lambda x: x[:1], Box([-1], [1]))
data, models = {
"foo": empty_dataset([1], [1])
}, {
"foo": _PseudoTrainableQuadratic()
}
with pytest.raises(ValueError):
optimizer.optimize(3, data, models)
def test_locally_penalized_expected_improvement_raises_when_called_before_initialization(
) -> None:
data = Dataset(tf.zeros([3, 2], dtype=tf.float64),
tf.ones([3, 2], dtype=tf.float64))
search_space = Box([0, 0], [1, 1])
pending_points = tf.zeros([1, 2])
with pytest.raises(ValueError):
LocalPenalizationAcquisitionFunction(
search_space).prepare_acquisition_function(
data, QuadraticMeanAndRBFKernel(), pending_points)
def test_multi_objective_optimizer_finds_pareto_front_of_the_VLMOP2_function(
num_steps: int, acquisition_rule: AcquisitionRule[TensorType, Box],
convergence_threshold: float) -> None:
search_space = Box([-2, -2], [2, 2])
def build_stacked_independent_objectives_model(
data: Dataset) -> ModelStack:
gprs = []
for idx in range(2):
single_obj_data = Dataset(
data.query_points, tf.gather(data.observations, [idx], axis=1))
variance = tf.math.reduce_variance(single_obj_data.observations)
kernel = gpflow.kernels.Matern52(
variance, tf.constant([0.2, 0.2], tf.float64))
gpr = gpflow.models.GPR(single_obj_data.astuple(),
kernel,
noise_variance=1e-5)
gpflow.utilities.set_trainable(gpr.likelihood, False)
gprs.append((GaussianProcessRegression(gpr), 1))
return ModelStack(*gprs)
observer = mk_observer(VLMOP2().objective(), OBJECTIVE)
initial_query_points = search_space.sample(10)
initial_data = observer(initial_query_points)
model = build_stacked_independent_objectives_model(initial_data[OBJECTIVE])
dataset = (BayesianOptimizer(observer, search_space).optimize(
num_steps, initial_data, {
OBJECTIVE: model
}, acquisition_rule).try_get_final_datasets()[OBJECTIVE])
# A small log hypervolume difference corresponds to a succesful optimization.
ref_point = get_reference_point(dataset.observations)
obs_hv = Pareto(dataset.observations).hypervolume_indicator(ref_point)
ideal_pf = tf.cast(VLMOP2().gen_pareto_optimal_points(100),
dtype=tf.float64)
ideal_hv = Pareto(ideal_pf).hypervolume_indicator(ref_point)
assert tf.math.log(ideal_hv - obs_hv) < convergence_threshold
def test_bayesian_optimizer_optimize_tracked_state() -> None:
class _CountingRule(AcquisitionRule[int, Box]):
def acquire(
self,
search_space: Box,
datasets: Mapping[str, Dataset],
models: Mapping[str, ProbabilisticModel],
state: int | None = None,
) -> tuple[TensorType, int]:
new_state = 0 if state is None else state + 1
return tf.constant([[10.0]], tf.float64) + new_state, new_state
class _DecreasingVarianceModel(QuadraticMeanAndRBFKernel,
TrainableProbabilisticModel):
def __init__(self, data: Dataset):
super().__init__()
self._data = data
def predict(self,
query_points: TensorType) -> tuple[TensorType, TensorType]:
mean, var = super().predict(query_points)
return mean, var / len(self._data)
def update(self, dataset: Dataset) -> None:
self._data = dataset
def optimize(self, dataset: Dataset) -> None:
pass
initial_data = mk_dataset([[0.0]], [[0.0]])
model = _DecreasingVarianceModel(initial_data)
_, history = (BayesianOptimizer(_quadratic_observer,
Box([0],
[1])).optimize(3, {
"": initial_data
}, {
"": model
}, _CountingRule()).astuple())
assert [record.acquisition_state for record in history] == [None, 0, 1]
assert_datasets_allclose(history[0].datasets[""], initial_data)
assert_datasets_allclose(history[1].datasets[""],
mk_dataset([[0.0], [10.0]], [[0.0], [100.0]]))
assert_datasets_allclose(
history[2].datasets[""],
mk_dataset([[0.0], [10.0], [11.0]], [[0.0], [100.0], [121.0]]))
for step in range(3):
assert history[step].model == history[step].models[""]
assert history[step].dataset == history[step].datasets[""]
_, variance_from_saved_model = (history[step].models[""].predict(
tf.constant([[0.0]], tf.float64)))
npt.assert_allclose(variance_from_saved_model, 1.0 / (step + 1))
def test_optimizer_finds_minima_of_the_branin_function(
num_steps: int, acquisition_rule: AcquisitionRule) -> None:
search_space = Box(tf.constant([0.0, 0.0], tf.float64),
tf.constant([1.0, 1.0], tf.float64))
def build_model(data: Dataset) -> GaussianProcessRegression:
variance = tf.math.reduce_variance(data.observations)
kernel = gpflow.kernels.Matern52(variance,
tf.constant([0.2, 0.2], tf.float64))
gpr = gpflow.models.GPR((data.query_points, data.observations),
kernel,
noise_variance=1e-5)
gpflow.utilities.set_trainable(gpr.likelihood, False)
return GaussianProcessRegression(gpr)
initial_query_points = search_space.sample(5)
observer = mk_observer(branin, OBJECTIVE)
initial_data = observer(initial_query_points)
model = build_model(initial_data[OBJECTIVE])
res = BayesianOptimizer(observer,
search_space).optimize(num_steps, initial_data,
{OBJECTIVE: model},
acquisition_rule)
if res.error is not None:
raise res.error
dataset = res.datasets[OBJECTIVE]
arg_min_idx = tf.squeeze(tf.argmin(dataset.observations, axis=0))
best_y = dataset.observations[arg_min_idx]
best_x = dataset.query_points[arg_min_idx]
argmin = tf.cast(BRANIN_GLOBAL_ARGMIN, tf.float64)
relative_argmin_err = tf.abs((best_x - argmin) / argmin)
# these accuracies are the current best for the given number of optimization steps, which makes
# this is a regression test
assert tf.reduce_any(tf.reduce_all(relative_argmin_err < 0.03, axis=-1),
axis=0)
npt.assert_allclose(best_y, BRANIN_GLOBAL_MINIMUM, rtol=0.03)
def test_product_space_discretize_returns_search_space_with_correct_number_of_points(
num_samples: int,
) -> None:
space_A = Box([-1], [2])
space_B = DiscreteSearchSpace(tf.ones([100, 2], dtype=tf.float64))
product_space = TaggedProductSearchSpace(spaces=[space_A, space_B])
dss = product_space.discretize(num_samples)
samples = dss.sample(num_samples)
assert len(samples) == num_samples
def test_product_space_discretize_returns_search_space_with_only_points_contained_within_box(
num_samples: int,
) -> None:
space_A = Box([-1], [2])
space_B = DiscreteSearchSpace(tf.ones([100, 2], dtype=tf.float64))
product_space = TaggedProductSearchSpace(spaces=[space_A, space_B])
dss = product_space.discretize(num_samples)
samples = dss.sample(num_samples)
assert all(sample in product_space for sample in samples)
def test_locally_penalized_expected_improvement_builder_raises_for_invalid_pending_points_shape(
pending_points, ) -> None:
data = Dataset(tf.zeros([3, 2], dtype=tf.float64),
tf.ones([3, 2], dtype=tf.float64))
space = Box([0, 0], [1, 1])
builder = LocalPenalizationAcquisitionFunction(search_space=space)
builder.prepare_acquisition_function(data, QuadraticMeanAndRBFKernel(),
None) # first initialize
with pytest.raises(TF_DEBUGGING_ERROR_TYPES):
builder.prepare_acquisition_function(data, QuadraticMeanAndRBFKernel(),
pending_points)
def test_bayesian_optimizer_optimize_raises_for_negative_steps(
num_steps: int) -> None:
optimizer = BayesianOptimizer(_quadratic_observer, Box([-1], [1]))
data, models = {
"": empty_dataset([1], [1])
}, {
"": _PseudoTrainableQuadratic()
}
with pytest.raises(ValueError, match="num_steps"):
optimizer.optimize(num_steps, data, models)
def test_joint_batch_acquisition_rule_acquire() -> None:
search_space = Box(tf.constant([-2.2, -1.0]), tf.constant([1.3, 3.3]))
num_query_points = 4
acq = _JointBatchModelMinusMeanMaximumSingleBuilder()
ego: EfficientGlobalOptimization[Box] = EfficientGlobalOptimization(
acq, num_query_points=num_query_points
)
dataset = Dataset(tf.zeros([0, 2]), tf.zeros([0, 1]))
query_point = ego.acquire_single(search_space, dataset, QuadraticMeanAndRBFKernel())
npt.assert_allclose(query_point, [[0.0, 0.0]] * num_query_points, atol=1e-3)
def test_bayesian_optimizer_optimize_for_failed_step(
observer: Observer, model: TrainableProbabilisticModel,
rule: AcquisitionRule) -> None:
optimizer = BayesianOptimizer(observer, Box([0], [1]))
data, models = {"": mk_dataset([[0.0]], [[0.0]])}, {"": model}
result, history = optimizer.optimize(3, data, models, rule).astuple()
with pytest.raises(_Whoops):
result.unwrap()
assert len(history) == 1
def test_batch_acquisition_rule_acquire() -> None:
search_space = Box(tf.constant([-2.2, -1.0]), tf.constant([1.3, 3.3]))
num_query_points = 4
ego = BatchAcquisitionRule(num_query_points,
_BatchModelMinusMeanMaximumSingleBuilder())
dataset = Dataset(tf.zeros([0, 2]), tf.zeros([0, 1]))
query_point, _ = ego.acquire(search_space, {OBJECTIVE: dataset},
{OBJECTIVE: QuadraticMeanAndRBFKernel()})
npt.assert_allclose(query_point, [[0.0, 0.0]] * num_query_points,
atol=1e-3)
def test_optimize_continuous_raises_for_impossible_optimization(
num_optimization_runs: int, num_recovery_runs: int) -> None:
search_space = Box([-1, -1], [1, 2])
optimizer = generate_continuous_optimizer(
num_optimization_runs=num_optimization_runs,
num_recovery_runs=num_recovery_runs)
with pytest.raises(FailedOptimizationError) as e:
optimizer(search_space, _delta_function(10))
assert (str(e.value) == f"""
Acquisition function optimization failed,
even after {num_recovery_runs + num_optimization_runs} restarts.
""")
def test_product_space_get_subspace() -> None:
space_A = Box([-1, -2], [2, 3])
space_B = DiscreteSearchSpace(tf.constant([[-0.5, 0.5]]))
space_C = Box([-1], [2])
product_space = TaggedProductSearchSpace(
spaces=[space_A, space_B, space_C], tags=["A", "B", "C"]
)
subspace_A = product_space.get_subspace("A")
assert isinstance(subspace_A, Box)
npt.assert_array_equal(subspace_A.lower, [-1, -2])
npt.assert_array_equal(subspace_A.upper, [2, 3])
subspace_B = product_space.get_subspace("B")
assert isinstance(subspace_B, DiscreteSearchSpace)
npt.assert_array_equal(subspace_B.points, tf.constant([[-0.5, 0.5]]))
subspace_C = product_space.get_subspace("C")
assert isinstance(subspace_C, Box)
npt.assert_array_equal(subspace_C.lower, [-1])
npt.assert_array_equal(subspace_C.upper, [2])
def test_bayesian_optimizer_optimize_doesnt_track_state_if_told_not_to() -> None:
class _UncopyableModel(_PseudoTrainableQuadratic):
def __deepcopy__(self, memo: dict[int, object]) -> NoReturn:
assert False
data, models = {OBJECTIVE: empty_dataset([1], [1])}, {OBJECTIVE: _UncopyableModel()}
history = (
BayesianOptimizer(_quadratic_observer, Box([-1], [1]))
.optimize(5, data, models, track_state=False)
.history
)
assert len(history) == 0
def test_discrete_thompson_sampling_acquire_returns_correct_shape(
num_fourier_features: bool, num_query_points: int
) -> None:
search_space = Box(tf.constant([-2.2, -1.0]), tf.constant([1.3, 3.3]))
ts = DiscreteThompsonSampling(100, num_query_points, num_fourier_features=num_fourier_features)
dataset = Dataset(tf.zeros([1, 2], dtype=tf.float64), tf.zeros([1, 1], dtype=tf.float64))
model = QuadraticMeanAndRBFKernel(noise_variance=tf.constant(1.0, dtype=tf.float64))
model.kernel = (
gpflow.kernels.RBF()
) # need a gpflow kernel object for random feature decompositions
query_points = ts.acquire_single(search_space, model, dataset=dataset)
npt.assert_array_equal(query_points.shape, tf.constant([num_query_points, 2]))
def test_rff_thompson_samples_are_minima() -> None:
search_space = Box([0.0, 0.0], [1.0, 1.0])
model = QuadraticMeanAndRBFKernel(noise_variance=tf.constant(1e-5, dtype=tf.float64))
model.kernel = (
gpflow.kernels.RBF()
) # need a gpflow kernel object for random feature decompositions
x_range = tf.linspace(0.0, 1.0, 5)
x_range = tf.cast(x_range, dtype=tf.float64)
xs = tf.reshape(tf.stack(tf.meshgrid(x_range, x_range, indexing="ij"), axis=-1), (-1, 2))
ys = quadratic(xs)
dataset = Dataset(xs, ys)
sampler = RandomFourierFeatureThompsonSampler(
1, model, dataset, num_features=100, sample_min_value=True
)
query_points = search_space.sample(100)
query_points = tf.concat([dataset.query_points, query_points], 0)
thompson_samples = sampler.sample(query_points)
fmean, _ = model.predict(dataset.query_points)
assert max(thompson_samples) < min(fmean)
def test_min_value_entropy_search_builder_gumbel_samples(mocked_mves) -> None:
dataset = Dataset(tf.zeros([3, 2], dtype=tf.float64), tf.ones([3, 2], dtype=tf.float64))
search_space = Box([0, 0], [1, 1])
builder = MinValueEntropySearch(search_space)
model = QuadraticMeanAndRBFKernel()
builder.prepare_acquisition_function(dataset, model)
mocked_mves.assert_called_once()
# check that the Gumbel samples look sensible
gumbel_samples = mocked_mves.call_args[0][1]
query_points = builder._search_space.sample(num_samples=builder._grid_size)
query_points = tf.concat([dataset.query_points, query_points], 0)
fmean, _ = model.predict(query_points)
assert max(gumbel_samples) < min(fmean)
def test_trust_region_for_unsuccessful_local_to_global_trust_region_reduced(
rule: AcquisitionRule[TensorType, Box]
) -> None:
tr = TrustRegion(rule)
dataset = Dataset(tf.constant([[0.1, 0.2], [-0.1, -0.2]]), tf.constant([[0.4], [0.5]]))
lower_bound = tf.constant([-2.2, -1.0])
upper_bound = tf.constant([1.3, 3.3])
search_space = Box(lower_bound, upper_bound)
eps = 0.5 * (search_space.upper - search_space.lower) / 10
previous_y_min = dataset.observations[0]
is_global = False
acquisition_space = Box(dataset.query_points[0] - eps, dataset.query_points[0] + eps)
previous_state = TrustRegion.State(acquisition_space, eps, previous_y_min, is_global)
current_state, _ = tr.acquire(
search_space, {OBJECTIVE: dataset}, {OBJECTIVE: QuadraticMeanAndRBFKernel()}
)(previous_state)
assert current_state is not None
npt.assert_array_less(current_state.eps, previous_state.eps) # current TR smaller than previous
assert current_state.is_global
npt.assert_array_almost_equal(current_state.acquisition_space.lower, lower_bound)
def test_efficient_global_optimization(
optimizer: AcquisitionOptimizer[Box]) -> None:
class NegQuadratic(AcquisitionFunctionBuilder):
def prepare_acquisition_function(
self, datasets: Mapping[str, Dataset],
models: Mapping[str,
ProbabilisticModel]) -> AcquisitionFunction:
return lambda x: -quadratic(tf.squeeze(x, -2) - 1)
search_space = Box([-10], [10])
ego = EfficientGlobalOptimization(NegQuadratic(), optimizer)
data, model = empty_dataset([1], [1]), QuadraticMeanAndRBFKernel(x_shift=1)
query_point, _ = ego.acquire(search_space, {"": data}, {"": model})
npt.assert_allclose(query_point, [[1]], rtol=1e-4)
def test_trust_region_for_default_state() -> None:
tr = TrustRegion(NegativeLowerConfidenceBound(0))
dataset = Dataset(tf.constant([[0.1, 0.2]]), tf.constant([[0.012]]))
lower_bound = tf.constant([-2.2, -1.0])
upper_bound = tf.constant([1.3, 3.3])
search_space = Box(lower_bound, upper_bound)
query_point, state = tr.acquire_single(search_space, dataset,
QuadraticMeanAndRBFKernel(), None)
npt.assert_array_almost_equal(query_point, tf.constant([[0.0, 0.0]]), 5)
npt.assert_array_almost_equal(state.acquisition_space.lower, lower_bound)
npt.assert_array_almost_equal(state.acquisition_space.upper, upper_bound)
npt.assert_array_almost_equal(state.y_min, [0.012])
assert state.is_global
def test_bayesian_optimizer_calls_observer_once_per_iteration(steps: int) -> None:
class _CountingObserver:
call_count = 0
def __call__(self, x: tf.Tensor) -> Dataset:
self.call_count += 1
return Dataset(x, tf.reduce_sum(x ** 2, axis=-1, keepdims=True))
observer = _CountingObserver()
optimizer = BayesianOptimizer(observer, Box([-1], [1]))
data = mk_dataset([[0.5]], [[0.25]])
optimizer.optimize(steps, data, _PseudoTrainableQuadratic()).final_result.unwrap()
assert observer.call_count == steps
