def test_numerical_gradient():
debug_output = False
random = np.random.RandomState(42)
eps = 1e-6
for model in default_models():
ei = EIAcquisitionFunction(model)
for iter in range(10):
high = 1.0 if iter < 5 else 0.02
x = random.uniform(low=0.0, high=high, size=(2, ))
f0, analytical_gradient = ei.compute_acq_with_gradients(x)
analytical_gradient = analytical_gradient.flatten()
f0 = f0[0]
if debug_output:
print('x0 = {}, f(x_0) = {}, grad(x_0) = {}'.format(
x, f0, analytical_gradient))
for i in range(2):
h = np.zeros_like(x)
h[i] = eps
fpeps = ei.compute_acq(x + h)
fmeps = ei.compute_acq(x - h)
numerical_derivative = (fpeps - fmeps) / (2 * eps)
if debug_output:
print(
'f(x0+eps) = {}, f(x0-eps) = {}, findiff = {}, deriv = {}'
.format(fpeps[0], fmeps[0], numerical_derivative[0],
analytical_gradient[i]))
np.testing.assert_almost_equal(numerical_derivative.item(),
analytical_gradient[i],
decimal=4)
def test_sanity_check():
# - test that values are negative as we should be returning *minus* expected improvement
# - test that values that are further from evaluated candidates have higher expected improvement
# given similar mean
# - test that points closer to better points have higher expected improvement
for model in default_models(do_mcmc=False):
ei = EIAcquisitionFunction(model)
X = np.array([
(0.0, 0.0), # 0
(1.0, 0.0), # 1
(0.0, 1.0), # 2
(1.0, 1.0), # 3
(0.2, 0.0), # 4
(0.0, 0.2), # 5
(0.1, 0.0), # 6
(0.0, 0.1), # 7
(0.1, 0.1), # 8
(0.9, 0.9), # 9
])
_acq = ei.compute_acq(X).flatten()
#print('Negative EI values:')
#print(_acq)
acq = list(_acq)
assert all(a <= 0 for a in acq), acq
# lower evaluations should correspond to better acquisition
# second inequality is less equal because last two values are likely zero
assert acq[0] < acq[1] <= acq[3], acq
# Note: The improvement here is tiny, just 0.01%:
assert acq[8] < acq[9], acq
# further from an evaluated point should correspond to better acquisition
assert acq[6] < acq[4] < acq[1], acq
assert acq[7] < acq[5] < acq[2], acq
def test_value_same_as_with_gradient():
# test that compute_acq and compute_acq_with_gradients return the same acquisition values
for model in default_models():
ei = EIAcquisitionFunction(model)
random = np.random.RandomState(42)
X = random.uniform(low=0.0, high=1.0, size=(10, 2))
# assert same as computation with gradients
np.testing.assert_almost_equal(
ei.compute_acq(X).flatten(),
ei.compute_acq_with_gradients(X)[0].flatten())
def test_value_same_in_batch():
for model in default_models():
ei = EIAcquisitionFunction(model)
random = np.random.RandomState(42)
X = random.uniform(low=0.0, high=1.0, size=(10, 2))
# assert same as computation with gradients
batch_result, batch_gradient = ei.compute_acq_with_gradients(X)
for i, xi in enumerate(X):
assert xi.shape == (2, )
assert ei.compute_acq(xi) == batch_result[i]
acq, dacq = ei.compute_acq_with_gradients(xi)
assert acq <= 0
assert acq == batch_result[i]
np.testing.assert_array_equal(dacq.flatten(), batch_gradient[i, :])
def test_optimization_improves():
debug_output = False
# Pick a random point, optimize and the expected improvement should be better:
# But only if the starting point is not too far from the origin
random = np.random.RandomState(42)
for model in default_models():
ei = EIAcquisitionFunction(model)
opt = LBFGSOptimizeAcquisition(model.state, model,
EIAcquisitionFunction)
if debug_output:
print('\n\nGP MCMC' if model.does_mcmc() else 'GP Opt')
fzero = ei.compute_acq(np.zeros((2, )))[0]
print('f(0) = {}'.format(fzero))
if debug_output and not model.does_mcmc():
print('Hyperpars: {}'.format(model.get_params()))
# Plot the thing!
plot_ei_mean_std(model, ei, max_grid=0.001)
plot_ei_mean_std(model, ei, max_grid=0.01)
plot_ei_mean_std(model, ei, max_grid=0.1)
plot_ei_mean_std(model, ei, max_grid=1.0)
non_zero_acq_at_least_once = False
for iter in range(10):
#initial_point = random.uniform(low=0.0, high=1.0, size=(2,))
initial_point = random.uniform(low=0.0, high=0.1, size=(2, ))
acq0, df0 = ei.compute_acq_with_gradients(initial_point)
if debug_output:
print('\nInitial point: f(x0) = {}, x0 = {}'.format(
acq0, initial_point))
print('grad0 = {}'.format(df0[0]))
if acq0 != 0:
non_zero_acq_at_least_once = True
optimized = np.array(opt.optimize(tuple(initial_point)))
acq_opt = ei.compute_acq(optimized)
if debug_output:
print('Final point: f(x1) = {}, x1 = {}'.format(
acq_opt, optimized))
assert acq_opt < 0
assert acq_opt < acq0
assert non_zero_acq_at_least_once
def test_best_value():
# test that the best value affects expected improvement
for model in default_models():
ei = EIAcquisitionFunction(model)
random = np.random.RandomState(42)
test_X = random.uniform(low=0.0, high=1.0, size=(10, 2))
acq_best0 = list(ei.compute_acq(test_X).flatten())
acq0_best0 = ei.compute_acq(np.array((0, 0)))
# override current best
def new_current_best():
return np.array([10])
model.current_best = new_current_best
acq_best2 = list(ei.compute_acq(test_X).flatten())
acq0_best2 = ei.compute_acq(np.array((0, 0)))
# if the best is only 2 the acquisition function should be better (lower value)
assert all(a2 < a0 for a2, a0 in zip(acq_best2, acq_best0))
# there should be a considerable gap at the point of the best evaluation
assert acq0_best2 < acq0_best0 - 1.0
def test_gp_fantasizing():
"""
Compare whether acquisition function evaluations (values, gradients) with
fantasizing are the same as averaging them by hand.
"""
random_seed = 4567
_set_seeds(random_seed)
num_fantasy_samples = 10
num_pending = 5
hp_ranges = HyperparameterRanges_Impl(
HyperparameterRangeContinuous('x', 0.0, 1.0, LinearScaling()),
HyperparameterRangeContinuous('y', 0.0, 1.0, LinearScaling()))
X = [
(0.0, 0.0),
(1.0, 0.0),
(0.0, 1.0),
(1.0, 1.0),
]
num_data = len(X)
Y = [
dictionarize_objective(np.random.randn(1, 1)) for _ in range(num_data)
]
# Draw fantasies. This is done for a number of fixed pending candidates
# The model parameters are fit in the first iteration, when there are
# no pending candidates
# Note: It is important to not normalize targets, because this would be
# done on the observed targets only, not the fantasized ones, so it
# would be hard to compare below.
pending_evaluations = []
for _ in range(num_pending):
pending_cand = tuple(np.random.rand(2, ))
pending_evaluations.append(PendingEvaluation(pending_cand))
state = TuningJobState(hp_ranges,
[CandidateEvaluation(x, y) for x, y in zip(X, Y)],
failed_candidates=[],
pending_evaluations=pending_evaluations)
gpmodel = default_gpmodel(state,
random_seed,
optimization_config=DEFAULT_OPTIMIZATION_CONFIG)
model = GPMXNetModel(state,
DEFAULT_METRIC,
random_seed,
gpmodel,
fit_parameters=True,
num_fantasy_samples=num_fantasy_samples,
normalize_targets=False)
fantasy_samples = model.fantasy_samples
# Evaluate acquisition function and gradients with fantasizing
num_test = 50
X_test = np.vstack([
hp_ranges.to_ndarray(tuple(np.random.rand(2, )))
for _ in range(num_test)
])
acq_func = EIAcquisitionFunction(model)
fvals, grads = acq_func.compute_acq_with_gradients(X_test)
# Do the same computation by averaging by hand
fvals_cmp = np.empty((num_fantasy_samples, ) + fvals.shape)
grads_cmp = np.empty((num_fantasy_samples, ) + grads.shape)
X_full = X + state.pending_candidates
for it in range(num_fantasy_samples):
Y_full = Y + [
dictionarize_objective(eval.fantasies[DEFAULT_METRIC][:, it])
for eval in fantasy_samples
]
state2 = TuningJobState(
hp_ranges,
[CandidateEvaluation(x, y) for x, y in zip(X_full, Y_full)],
failed_candidates=[],
pending_evaluations=[])
# We have to skip parameter optimization here
model2 = GPMXNetModel(state2,
DEFAULT_METRIC,
random_seed,
gpmodel,
fit_parameters=False,
num_fantasy_samples=num_fantasy_samples,
normalize_targets=False)
acq_func2 = EIAcquisitionFunction(model2)
fvals_, grads_ = acq_func2.compute_acq_with_gradients(X_test)
fvals_cmp[it, :] = fvals_
grads_cmp[it, :] = grads_
# Comparison
fvals2 = np.mean(fvals_cmp, axis=0)
grads2 = np.mean(grads_cmp, axis=0)
assert np.allclose(fvals, fvals2)
assert np.allclose(grads, grads2)