def __compute_alternative_params(self):
# Copied directly from skopt
transformed_bounds = np.array(self.__opt.space.transformed_bounds)
est = clone(self.__opt.base_estimator)
with warnings.catch_warnings():
warnings.simplefilter("ignore")
est.fit(self.__opt.space.transform(self.__opt.Xi), self.__opt.yi)
X = self.__opt.space.transform(self.__opt.space.rvs(
n_samples=self.__opt.n_points, random_state=self.__opt.rng))
values = _gaussian_acquisition(X=X, model=est, y_opt=np.min(self.__opt.yi),
acq_func='EI',
acq_func_kwargs=dict(n_points=10000))
print('original point ei: %s' % np.min(values))
discount_width = .5
values = self.__discount_leased_params(X, values, discount_width)
while np.min(values) > -1e-5 and discount_width > 1e-2:
discount_width *= .9
values = _gaussian_acquisition(X=X, model=est, y_opt=np.min(self.__opt.yi),
acq_func='EI',
acq_func_kwargs=dict(n_points=10000))
values = self.__discount_leased_params(X, values, discount_width)
next_x = X[np.argmin(values)]
print('new point ei: %s' % np.min(values))
if not self.__opt.space.is_categorical:
next_x = np.clip(next_x, transformed_bounds[:, 0], transformed_bounds[:, 1])
return self.__opt.space.inverse_transform(next_x.reshape((1, -1)))[0]
def evaluate(X, surrogate_model, y_opt, acq_func_kwargs=None):
return _gaussian_acquisition(
X=X,
model=surrogate_model,
y_opt=y_opt,
acq_func="PI",
acq_func_kwargs=acq_func_kwargs,
)
def test_acquisition_per_second(acq_func):
X = np.reshape(np.linspace(4.0, 8.0, 10), (-1, 1))
y = np.vstack((np.ones(10), np.ravel(np.log(X)))).T
cgpr = ConstantGPRSurrogate(Space(((1.0, 9.0),)))
cgpr.fit(X, y)
X_pred = np.reshape(np.linspace(1.0, 11.0, 20), (-1, 1))
indices = np.arange(6)
vals = _gaussian_acquisition(X_pred, cgpr, y_opt=1.0, acq_func=acq_func)
for fast, slow in zip(indices[:-1], indices[1:]):
assert vals[slow] > vals[fast]
acq_wo_time = _gaussian_acquisition(
X, cgpr.estimators_[0], y_opt=1.2, acq_func=acq_func[:2])
acq_with_time = _gaussian_acquisition(
X, cgpr, y_opt=1.2, acq_func=acq_func)
assert_array_almost_equal(acq_wo_time / acq_with_time, np.ravel(X), 2)
def test_acquisition_per_second(acq_func):
X = np.reshape(np.linspace(4.0, 8.0, 10), (-1, 1))
y = np.vstack((np.ones(10), np.ravel(np.log(X)))).T
cgpr = ConstantGPRSurrogate(Space(((1.0, 9.0),)))
cgpr.fit(X, y)
X_pred = np.reshape(np.linspace(1.0, 11.0, 20), (-1, 1))
indices = np.arange(6)
vals = _gaussian_acquisition(X_pred, cgpr, y_opt=1.0, acq_func=acq_func)
for fast, slow in zip(indices[:-1], indices[1:]):
assert_greater(vals[slow], vals[fast])
acq_wo_time = _gaussian_acquisition(
X, cgpr.estimators_[0], y_opt=1.2, acq_func=acq_func[:2])
acq_with_time = _gaussian_acquisition(
X, cgpr, y_opt=1.2, acq_func=acq_func)
assert_array_almost_equal(acq_wo_time / acq_with_time, np.ravel(X), 2)
def _tell(self, x, y, fit=True):
"""Perform the actual work of incorporating one or more new points.
See `tell()` for the full description.
This method exists to give access to the internals of adding points
by side stepping all input validation and transformation."""
if "ps" in self.acq_func:
if is_2Dlistlike(x):
self.Xi.extend(x)
self.yi.extend(y)
self._n_initial_points -= len(y)
elif is_listlike(x):
self.Xi.append(x)
self.yi.append(y)
self._n_initial_points -= 1
# if y isn't a scalar it means we have been handed a batch of points
elif is_listlike(y) and is_2Dlistlike(x):
self.Xi.extend(x)
self.yi.extend(y)
self._n_initial_points -= len(y)
elif is_listlike(x):
self.Xi.append(x)
self.yi.append(y)
self._n_initial_points -= 1
else:
raise ValueError("Type of arguments `x` (%s) and `y` (%s) "
"not compatible." % (type(x), type(y)))
# optimizer learned something new - discard cache
self.cache_ = {}
# after being "told" n_initial_points we switch from sampling
# random points to using a surrogate model
#mprint("starting fit gpr......")
if (fit and self._n_initial_points <= 0
and self.base_estimator_ is not None):
#print('start finding next x.....')
transformed_bounds = np.array(self.space.transformed_bounds)
est = clone(self.base_estimator_)
start_clock = time.time()
with warnings.catch_warnings():
warnings.simplefilter("ignore")
est.fit(self.space.transform(self.Xi), self.yi)
self.regressiontime = time.time() - start_clock
self.tt_regressiontime.append(self.regressiontime)
if hasattr(self, "next_xs_") and self.acq_func == "gp_hedge":
self.gains_ -= est.predict(np.vstack(self.next_xs_))
self.models.append(est)
#mprint("finished fit gpr......")
# even with BFGS as optimizer we want to sample a large number
# of points and then pick the best ones as starting points
start_clock_ac = time.time()
X = self.space.transform(
self.space.rvs(n_samples=self.n_points, random_state=self.rng))
self.next_xs_ = []
for cand_acq_func in self.cand_acq_funcs_:
values = _gaussian_acquisition(
X=X,
model=est,
y_opt=np.min(self.yi),
acq_func=cand_acq_func,
acq_func_kwargs=self.acq_func_kwargs)
self.values = values
# Find the minimum of the acquisition function by randomly
# sampling points from the space
if self.acq_optimizer == "sampling":
next_x = X[np.argmin(values)]
# Use BFGS to find the mimimum of the acquisition function, the
# minimization starts from `n_restarts_optimizer` different
# points and the best minimum is used
elif self.acq_optimizer == "lbfgs":
x0 = X[np.argsort(values)[:self.n_restarts_optimizer]]
with warnings.catch_warnings():
warnings.simplefilter("ignore")
results = Parallel(n_jobs=self.n_jobs)(
delayed(fmin_l_bfgs_b)(
gaussian_acquisition_1D,
x,
args=(est, np.min(self.yi), cand_acq_func,
self.acq_func_kwargs),
bounds=self.space.transformed_bounds,
approx_grad=False,
maxiter=20) for x in x0)
cand_xs = np.array([r[0] for r in results])
cand_acqs = np.array([r[1] for r in results])
next_x = cand_xs[np.argmin(cand_acqs)]
self.actime = start_clock_ac - time.time()
self.tt_actime.append(self.actime)
# lbfgs should handle this but just in case there are
# precision errors.
if not self.space.is_categorical:
next_x = np.clip(next_x, transformed_bounds[:, 0],
transformed_bounds[:, 1])
self.next_xs_.append(next_x)
if self.acq_func == "gp_hedge":
logits = np.array(self.gains_)
logits -= np.max(logits)
exp_logits = np.exp(self.eta * logits)
probs = exp_logits / np.sum(exp_logits)
next_x = self.next_xs_[np.argmax(self.rng.multinomial(
1, probs))]
else:
next_x = self.next_xs_[0]
# note the need for [0] at the end
self._next_x = self.space.inverse_transform(next_x.reshape(
(1, -1)))[0]
# Pack results
return create_result(self.Xi,
self.yi,
self.space,
self.rng,
models=self.models)
def _tell(self, x, y, fit=True):
"""Perform the actual work of incorporating one or more new points. See :meth:`tell` for
the full description. This method exists to give access to the internals of adding points
by side-stepping all input validation and transformation"""
#################### Collect Search Points and Evaluations ####################
# TODO: Clean up below - Looks like the 4 extend/append blocks may be duplicated
if "ps" in self.acq_func:
if is_2d_list_like(x):
self.Xi.extend(x)
self.yi.extend(y)
self._n_initial_points -= len(y)
elif is_list_like(x):
self.Xi.append(x)
self.yi.append(y)
self._n_initial_points -= 1
# If `y` isn't a scalar, we have been handed a batch of points
elif is_list_like(y) and is_2d_list_like(x):
self.Xi.extend(x)
self.yi.extend(y)
self._n_initial_points -= len(y)
elif is_list_like(x):
self.Xi.append(x)
self.yi.append(y)
self._n_initial_points -= 1
else:
raise ValueError(
f"Incompatible argument types: `x` ({type(x)}) and `y` ({type(y)})"
)
# Optimizer learned something new. Discard `cache_`
self.cache_ = {}
#################### Fit Surrogate Model ####################
# After being `tell`-ed `n_initial_points`, use surrogate model instead of random sampling
# TODO: Clean up and separate below. Pretty hard to follow the whole thing
if fit and self._n_initial_points <= 0 and self.base_estimator is not None:
transformed_bounds = np.array(self.space.transformed_bounds)
est = clone(self.base_estimator)
with warnings.catch_warnings():
warnings.simplefilter("ignore")
est.fit(self.space.transform(self.Xi), self.yi)
if hasattr(self, "next_xs_") and self.acq_func == "gp_hedge":
self.gains_ -= est.predict(np.vstack(self.next_xs_))
self.models.append(est)
# Even with BFGS optimizer, we want to sample a large number of points, and
# pick the best ones as starting points
X = self.space.transform(
self.space.rvs(n_samples=self.n_points, random_state=self.rng))
self.next_xs_ = []
for cand_acq_func in self.cand_acq_funcs_:
# TODO: Rename `values` - Maybe `utilities`?
values = _gaussian_acquisition(
X=X,
model=est,
y_opt=np.min(self.yi),
acq_func=cand_acq_func,
acq_func_kwargs=self.acq_func_kwargs,
)
#################### Find Acquisition Function Minimum ####################
# Find acquisition function minimum by randomly sampling points from the space
if self.acq_optimizer == "sampling":
next_x = X[np.argmin(values)]
# Use BFGS to find the minimum of the acquisition function, the minimization starts
# from `n_restarts_optimizer` different points and the best minimum is used
elif self.acq_optimizer == "lbfgs":
x0 = X[np.argsort(values)[:self.n_restarts_optimizer]]
with warnings.catch_warnings():
warnings.simplefilter("ignore")
results = Parallel(n_jobs=self.n_jobs)(
delayed(fmin_l_bfgs_b)(
gaussian_acquisition_1D,
x,
args=(est, np.min(self.yi), cand_acq_func,
self.acq_func_kwargs),
bounds=self.space.transformed_bounds,
approx_grad=False,
maxiter=20,
) for x in x0)
cand_xs = np.array([r[0] for r in results])
cand_acqs = np.array([r[1] for r in results])
next_x = cand_xs[np.argmin(cand_acqs)]
else:
# `acq_optimizer` should have already been checked, so this shouldn't be hit,
# but, it's here anyways to prevent complaints about `next_x` not existing in
# the absence of this `else` clause
raise RuntimeError(
f"Invalid `acq_optimizer` value: {self.acq_optimizer}")
# L-BFGS-B should handle this, but just in case of precision errors...
if not self.space.is_categorical:
next_x = np.clip(next_x, transformed_bounds[:, 0],
transformed_bounds[:, 1])
self.next_xs_.append(next_x)
if self.acq_func == "gp_hedge":
logits = np.array(self.gains_)
logits -= np.max(logits)
exp_logits = np.exp(self.eta * logits)
probs = exp_logits / np.sum(exp_logits)
next_x = self.next_xs_[np.argmax(self.rng.multinomial(
1, probs))]
else:
next_x = self.next_xs_[0]
# Note the need for [0] at the end
self._next_x = self.space.inverse_transform(next_x.reshape(
(1, -1)))[0]
# Pack results
return create_result(self.Xi,
self.yi,
self.space,
self.rng,
models=self.models)
def test_gaussian_acquisition_check_inputs():
model = ConstantGPRSurrogate(Space(((1.0, 9.0), )))
with pytest.raises(ValueError) as err:
_gaussian_acquisition(np.arange(1, 5), model)
assert ("it must be 2-dimensional" in err.value.args[0])
def _tell(self, x, y, fit=True):
# Copied from skopt
"""Perform the actual work of incorporating one or more new points.
See `tell()` for the full description.
This method exists to give access to the internals of adding points
by side stepping all input validation and transformation."""
if "ps" in self.acq_func:
if is_2Dlistlike(x):
self.Xi.extend(x)
self.yi.extend(y)
self._n_initial_points -= len(y)
elif is_listlike(x):
self.Xi.append(x)
self.yi.append(y)
self._n_initial_points -= 1
# if y isn't a scalar it means we have been handed a batch of points
elif is_listlike(y) and is_2Dlistlike(x):
self.Xi.extend(x)
self.yi.extend(y)
self._n_initial_points -= len(y)
elif is_listlike(x):
self.Xi.append(x)
self.yi.append(y)
self._n_initial_points -= 1
else:
raise ValueError(
"Type of arguments `x` (%s) and `y` (%s) not compatible." %
(type(x), type(y)))
# optimizer learned something new - discard cache
self.cache_ = {}
# after being "told" n_initial_points we switch from sampling
# random points to using a surrogate model
if fit and self._n_initial_points <= 0 and self.base_estimator_ is not None:
transformed_bounds = np.array(self.space.transformed_bounds)
est = clone(self.base_estimator_)
with warnings.catch_warnings():
warnings.simplefilter("ignore")
est.fit(self.space.transform(self.Xi), self.yi)
if hasattr(self, "next_xs_") and self.acq_func == "gp_hedge":
self.gains_ -= est.predict(np.vstack(self.next_xs_))
self.models.append(est)
# We're gonna lie to the estimator by telling it a loss for the points that are still being evaluated,
# similar to what we do when we ask for multiple points in ask().
points_running = self.watch_list.keys()
num_points_running = len(points_running)
points_to_lie_about = [
self.all_dim_values[run_ind] for run_ind in points_running
]
strategy = "cl_mean"
if strategy == "cl_min":
y_lie = np.min(self.yi) if self.yi else 0.0 # CL-min lie
elif strategy == "cl_mean":
y_lie = np.mean(self.yi) if self.yi else 0.0 # CL-mean lie
else:
y_lie = np.max(self.yi) if self.yi else 0.0 # CL-max lie
# Lie to the fake optimizer.
fake_est = copy.deepcopy(est)
X_to_tell = self.Xi + points_to_lie_about
X_to_tell = self.space.transform(X_to_tell)
y_to_tell = self.yi + list(np.ones(num_points_running) * y_lie)
fake_est.fit(X_to_tell, y_to_tell)
# even with BFGS as optimizer we want to sample a large number
# of points and then pick the best ones as starting points
# X = self.space.transform(self.space.rvs(
# n_samples=self.n_points, random_state=self.rng))
Xspace = self.space.rvs(n_samples=self.n_points,
random_state=self.rng)
param_thr = self.adapt_param['param_thr']
par_cnt_scheme = self.adapt_param['par_cnt_scheme']
suitable_X, _ = check_parameter_count_for_sample(
Xspace, self.hyper_param_names, param_thr, par_cnt_scheme)
# for x in Xspace:
# vals_suitable, _ = check_parameter_count_for_sample(
# x, self.hyper_param_names, param_thr, par_cnt_scheme)
# suitable_X.append(vals_suitable)
Xspace = [
Xspace[ind] for ind, suit in enumerate(suitable_X) if suit
]
X = self.space.transform(Xspace)
self.next_xs_ = []
for cand_acq_func in self.cand_acq_funcs_:
values = _gaussian_acquisition(
X=X,
model=fake_est,
y_opt=np.min(self.yi),
acq_func=cand_acq_func,
acq_func_kwargs=self.acq_func_kwargs)
# Find the minimum of the acquisition function by randomly
# sampling points from the space
if self.acq_optimizer == "sampling":
next_x = X[np.argmin(values)]
# Use BFGS to find the mimimum of the acquisition function, the
# minimization starts from `n_restarts_optimizer` different
# points and the best minimum is used
elif self.acq_optimizer == "lbfgs":
x0 = X[np.argsort(values)[:self.n_restarts_optimizer]]
with warnings.catch_warnings():
warnings.simplefilter("ignore")
results = Parallel(n_jobs=self.n_jobs)(
delayed(fmin_l_bfgs_b)(
gaussian_acquisition_1D,
x,
args=(fake_est, np.min(self.yi), cand_acq_func,
self.acq_func_kwargs),
bounds=self.space.transformed_bounds,
approx_grad=False,
maxiter=20) for x in x0)
cand_xs = np.array([r[0] for r in results])
cand_acqs = np.array([r[1] for r in results])
next_x = cand_xs[np.argmin(cand_acqs)]
# lbfgs should handle this but just in case there are
# precision errors.
if not self.space.is_categorical:
next_x = np.clip(next_x, transformed_bounds[:, 0],
transformed_bounds[:, 1])
self.next_xs_.append(next_x)
if self.acq_func == "gp_hedge":
logits = np.array(self.gains_)
logits -= np.max(logits)
exp_logits = np.exp(self.eta * logits)
probs = exp_logits / np.sum(exp_logits)
next_x = self.next_xs_[np.argmax(self.rng.multinomial(
1, probs))]
else:
next_x = self.next_xs_[0]
# note the need for [0] at the end
self._next_x = self.space.inverse_transform(next_x.reshape(
(1, -1)))[0]
# Pack results
return create_result(self.Xi,
self.yi,
self.space,
self.rng,
models=self.models)
x_obs = np.array([-4, -1.5, 0, 1.5, 2.5, 2.7])
y_obs = f(x_obs)
x_s = np.linspace(-5, 3, 80)
i = 0
cov_amplitude = ConstantKernel(1.0)
other_kernel = Matern()
print(x_obs, y_obs)
est = GaussianProcessRegressor(kernel=cov_amplitude*other_kernel)
b_s = x_s
s = []
while i < len(x_s):
x_obs = x_obs.reshape(len(x_obs), 1)
x_s = x_s.reshape(len(x_s), 1)
est.fit(x_obs, y_obs)
values = _gaussian_acquisition(X=x_s, model=est)
next_x = b_s[np.random.randint(0, 80)]
s.append(next_x)
x_obs = np.append(x_obs, np.array(next_x))
y_obs = np.append(y_obs, f(next_x))
i = i + 1
import matplotlib.pyplot as plt
y_mean, y_std = est.predict(np.array(s).reshape(len(s), 1), return_std=True)
y_upper = y_mean + y_std
y_lower = y_mean - y_std
l1, l2, l3 = plt.plot(xs, ys, 'r-', s, y_mean, 'b-', s, y_upper, 'g-')
plt.legend(handles=[l1, l2, l3], labels=['True Function', 'Mean', 'UPPER'])
plt.show()
def _tell(self, x, y, fit=True):
if 'ps' in self.acq_func:
# TODO: ps
pass
elif is_listlike(y) and is_2d_listlike(x):
self.xi.extend(x)
self.yi.extend(y)
self.n_initial_points -= len(y)
elif is_listlike(x):
self.xi.append(x)
self.yi.append(y)
self.n_initial_points -= 1
else:
raise ValueError('x {} and y {} are not compatible'.format(x, y))
self.cache = {}
if (fit and self.n_initial_points <= 0
and self.base_estimator is not None):
transformed_bounds = np.array(self.space.transformed_bounds)
estimator = self.base_estimator
estimator.fit(self.space.transform(self.xi), self.yi)
if hasattr(self, 'next_xs_') and self.acq_func == 'gp_hedge':
self.gains -= estimator.predict(np.vstack(self.next_xs_))
if self.max_model_queue_size is None:
self.models.append(estimator)
elif len(self.models) < self.max_model_queue_size:
self.models.append(estimator)
else:
self.models.pop(0)
self.models.append(estimator)
x = self.space.transform(
self.space.rvs(n_samples=self.n_points, random_state=self.rs))
self.next_xs_ = []
for cand_acq_func in self.cand_acq_funcs:
values = _gaussian_acquisition(
X=x,
model=estimator,
y_opt=np.min(self.yi),
acq_func=cand_acq_func,
acq_func_kwargs=self.acq_func_kwargs)
if self.acq_optimizer == 'sampling':
next_x = x[np.argmin(values)]
elif self.acq_optimizer == 'lbfgs':
x0 = np.asarray(x)[np.argsort(values)
[:self.n_restarts_optimizer]]
res = [
fmin_l_bfgs_b(gaussian_acquisition_1D,
xx,
args=(estimator, np.min(self.yi),
cand_acq_func,
self.acq_func_kwargs),
bounds=self.space.transformed_bounds,
approx_grad=False,
maxiter=20) for xx in x0
]
cand_xs = np.array([r[0] for r in res])
cand_acqs = np.array([r[1] for r in res])
next_x = cand_xs[np.argmin(cand_acqs)]
if not self.space.is_categorical:
next_x = np.clip(next_x, transformed_bounds[:, 0],
transformed_bounds[:, 1])
self.next_xs_.append(next_x)
if self.acq_func == 'gp_hedge':
logits = np.array(self.gains)
logits -= np.max(logits)
exp_logits = np.exp(self.eta * logits)
probs = exp_logits / np.sum(exp_logits)
next_x = self.next_xs_[np.argmax(self.rs.multinomial(1,
probs))]
else:
next_x = self.next_xs_[0]
self._next_x = self.space.inverse_transform(next_x.reshape(
(1, -1)))[0]
return create_result(self.xi,
self.yi,
self.space,
self.rs,
models=self.models)
