def get_incumbent(self):
"""
Returns the best observed point and its function value
Returns
----------
incumbent: ndarray (D,)
current incumbent
incumbent_value: ndarray (N,)
the observed value of the incumbent
"""
if self.normalize_input:
X = zero_mean_unit_var_unnormalization(self.X, self.x_mean,
self.x_std)
m = self.predict(X)[0]
else:
m = self.predict(self.X)[0]
best_idx = np.argmin(self.y)
inc = self.X[best_idx]
inc_value = m[best_idx]
if self.normalize_input:
inc = zero_mean_unit_var_unnormalization(inc, self.x_mean,
self.x_std)
if self.normalize_output:
inc_value = zero_mean_unit_var_unnormalization(
inc_value, self.y_mean, self.y_std)
return inc, inc_value
def predict(self, X_test, return_individual_predictions=False, *args, **kwargs):
"""
Returns the predictive mean and variance of the objective function at
the given test points.
Parameters
----------
X_test: np.ndarray (N, D)
Input test points
return_individual_predictions: bool
If set to true than the individual predictions of all samples are returned.
Returns
----------
np.array(N,)
predictive mean
np.array(N,)
predictive variance
"""
if not self.is_trained:
logging.error("Model is not trained!")
return
# Normalize input
if self.normalize_input:
X_, _, _ = zero_mean_unit_var_normalization(X_test, self.x_mean, self.x_std)
else:
X_ = X_test
f_out = []
theta_noise = []
for sample in self.samples:
lasagne.layers.set_all_param_values(self.net, sample)
out = self.single_predict(X_)
f_out.append(out[:, 0])
theta_noise.append(np.exp(out[:, 1]))
f_out = np.asarray(f_out)
theta_noise = np.asarray(theta_noise)
if return_individual_predictions:
if self.normalize_output:
f_out = zero_mean_unit_var_unnormalization(f_out, self.y_mean, self.y_std)
theta_noise *= self.y_std**2
return f_out, theta_noise
m = np.mean(f_out, axis=0)
# Total variance
# v = np.mean(f_out ** 2 + theta_noise, axis=0) - m ** 2
v = np.mean((f_out - m) ** 2, axis=0)
if self.normalize_output:
m = zero_mean_unit_var_unnormalization(m, self.y_mean, self.y_std)
v *= self.y_std ** 2
return m, v
def sample_functions(self, X_test, n_funcs=1):
"""
Samples F function values from the current posterior at the N
specified test point.
Parameters
----------
X_test: np.ndarray (N, D)
Input test points
n_funcs: int
Number of function values that are drawn at each test point.
Returns
----------
np.array(F, N)
The F function values drawn at the N test points.
"""
if self.normalize_input:
X_test_norm, _, _ = zero_mean_unit_var_normalization(
X_test, self.x_mean, self.x_std)
else:
X_test_norm = X_test
f = np.zeros([n_funcs, X_test_norm.shape[0]])
for i in range(n_funcs):
lasagne.layers.set_all_param_values(self.net, self.samples[i])
out = self.single_predict(X_test_norm)[:, 0]
if self.normalize_output:
f[i, :] = zero_mean_unit_var_unnormalization(
out, self.y_mean, self.y_std)
else:
f[i, :] = out
return f
def sample_functions(self, X_test, n_funcs=1):
"""
Samples F function values from the current posterior at the N
specified test points.
Parameters
----------
X_test: np.ndarray (N, D)
Input test points
n_funcs: int
Number of function values that are drawn at each test point.
Returns
----------
function_samples: np.array(F, N)
The F function values drawn at the N test points.
"""
if self.normalize_input:
X_test_norm, _, _ = normalization.zero_one_normalization(X_test, self.lower, self.upper)
else:
X_test_norm = X_test
if not self.is_trained:
raise Exception('Model has to be trained first!')
funcs = self.gp.sample_conditional(self.y, X_test_norm, n_funcs)
if self.normalize_output:
funcs = normalization.zero_mean_unit_var_unnormalization(funcs, self.y_mean, self.y_std)
if len(funcs.shape) == 1:
return funcs[None, :]
else:
return funcs
def sample_functions(self, X_test, n_funcs=1):
"""
Samples F function values from the current posterior at the N
specified test points.
Parameters
----------
X_test: np.ndarray (N, D)
Input test points
n_funcs: int
Number of function values that are drawn at each test point.
Returns
----------
function_samples: np.array(F, N)
The F function values drawn at the N test points.
"""
if self.normalize_input:
X_test_norm, _, _ = normalization.zero_one_normalization(
X_test, self.lower, self.upper)
else:
X_test_norm = X_test
if not self.is_trained:
raise Exception('Model has to be trained first!')
funcs = self.gp.sample_conditional(self.y, X_test_norm, n_funcs)
if self.normalize_output:
funcs = normalization.zero_mean_unit_var_unnormalization(
funcs, self.y_mean, self.y_std)
if len(funcs.shape) == 1:
return funcs[None, :]
else:
return funcs
def predict(self, X_test, **kwargs):
r"""
Returns the predictive mean and variance of the objective function
at X average over all hyperparameter samples.
The mean is computed by:
:math \mu(x) = \frac{1}{M}\sum_{i=1}^{M}\mu_m(x)
And the variance by:
:math \sigma^2(x) = (\frac{1}{M}\sum_{i=1}^{M}(\sigma^2_m(x) + \mu_m(x)^2) - \mu^2
Parameters
----------
X_test: np.ndarray (N, D)
Input test points
Returns
----------
np.array(N,)
predictive mean
np.array(N,)
predictive variance
"""
if not self.is_trained:
raise Exception('Model has to be trained first!')
if self.normalize_input:
X_test_norm, _, _ = normalization.zero_one_normalization(
X_test, self.lower, self.upper)
else:
X_test_norm = X_test
mu = np.zeros([len(self.models), X_test_norm.shape[0]])
var = np.zeros([len(self.models), X_test_norm.shape[0]])
for i, model in enumerate(self.models):
mu[i], var[i] = model.predict(X_test_norm)
# See the Algorithm Runtime Prediction paper by Hutter et al.
# for the derivation of the total variance
m = mu.mean(axis=0)
#v = np.mean(mu ** 2 + var) - m ** 2
v = var.mean(axis=0)
if self.normalize_output:
m = normalization.zero_mean_unit_var_unnormalization(
m, self.y_mean, self.y_std)
# Clip negative variances and set them to the smallest
# positive float value
if v.shape[0] == 1:
v = np.clip(v, np.finfo(v.dtype).eps, np.inf)
else:
v = np.clip(v, np.finfo(v.dtype).eps, np.inf)
v[np.where((v < np.finfo(v.dtype).eps)
& (v > -np.finfo(v.dtype).eps))] = 0
return m, v
def predict(self, X_test):
r"""
Returns the predictive mean and variance of the objective function at
the given test points.
Parameters
----------
X_test: np.ndarray (N, D)
N input test points
Returns
----------
np.array(N,)
predictive mean
np.array(N,)
predictive variance
"""
# Normalize inputs
if self.normalize_input:
X_, _, _ = zero_mean_unit_var_normalization(
X_test, self.X_mean, self.X_std)
else:
X_ = X_test
# Get features from the net
layers = lasagne.layers.get_all_layers(self.network)
theta = lasagne.layers.get_output(layers[:-1], X_)[-1].eval()
# Marginalise predictions over hyperparameters of the BLR
mu = np.zeros([len(self.models), X_test.shape[0]])
var = np.zeros([len(self.models), X_test.shape[0]])
for i, m in enumerate(self.models):
mu[i], var[i] = m.predict(theta)
# See the algorithm runtime prediction paper by Hutter et al
# for the derivation of the total variance
m = np.mean(mu, axis=0)
v = np.mean(mu**2 + var, axis=0) - m**2
# Clip negative variances and set them to the smallest
# positive float value
if v.shape[0] == 1:
v = np.clip(v, np.finfo(v.dtype).eps, np.inf)
else:
v = np.clip(v, np.finfo(v.dtype).eps, np.inf)
v[np.where((v < np.finfo(v.dtype).eps)
& (v > -np.finfo(v.dtype).eps))] = 0
if self.normalize_output:
m = zero_mean_unit_var_unnormalization(m, self.y_mean, self.y_std)
v *= self.y_std**2
return m, v
def test_zero_one_unit_var_unnormalization(self):
X_norm = np.random.randn(100, 3)
m = np.ones(X_norm.shape[1]) * 3
s = np.ones(X_norm.shape[1]) * 0.1
X = normalization.zero_mean_unit_var_unnormalization(X_norm, m, s)
np.testing.assert_almost_equal(np.mean(X, axis=0), m, decimal=1)
np.testing.assert_almost_equal(np.var(X, axis=0), s**2, decimal=1)
assert X_norm.shape == X.shape
def get_incumbent(self):
"""
Returns the best observed point and its function value
Returns
----------
incumbent: ndarray (D,)
current incumbent
incumbent_value: ndarray (N,)
the observed value of the incumbent
"""
inc, inc_value = super(DNGO, self).get_incumbent()
if self.normalize_input:
inc = zero_mean_unit_var_unnormalization(inc, self.X_mean, self.X_std)
if self.normalize_output:
inc_value = zero_mean_unit_var_unnormalization(inc_value, self.y_mean, self.y_std)
return inc, inc_value
def predict(self, X_test):
r"""
Returns the predictive mean and variance of the objective function at
the given test points.
Parameters
----------
X_test: np.ndarray (N, D)
N input test points
Returns
----------
np.array(N,)
predictive mean
np.array(N,)
predictive variance
"""
# Normalize inputs
if self.normalize_input:
X_, _, _ = zero_mean_unit_var_normalization(X_test, self.X_mean, self.X_std)
else:
X_ = X_test
# Get features from the net
layers = lasagne.layers.get_all_layers(self.network)
theta = lasagne.layers.get_output(layers[:-1], X_)[-1].eval()
# Marginalise predictions over hyperparameters of the BLR
mu = np.zeros([len(self.models), X_test.shape[0]])
var = np.zeros([len(self.models), X_test.shape[0]])
for i, m in enumerate(self.models):
mu[i], var[i] = m.predict(theta)
# See the algorithm runtime prediction paper by Hutter et al
# for the derivation of the total variance
m = np.mean(mu, axis=0)
v = np.mean(mu ** 2 + var, axis=0) - m ** 2
# Clip negative variances and set them to the smallest
# positive float value
if v.shape[0] == 1:
v = np.clip(v, np.finfo(v.dtype).eps, np.inf)
else:
v = np.clip(v, np.finfo(v.dtype).eps, np.inf)
v[np.where((v < np.finfo(v.dtype).eps) & (v > -np.finfo(v.dtype).eps))] = 0
if self.normalize_output:
m = zero_mean_unit_var_unnormalization(m, self.y_mean, self.y_std)
v *= self.y_std ** 2
return m, v
def get_incumbent(self):
"""
Returns the best observed point and its function value
Returns
----------
incumbent: ndarray (D,)
current incumbent
incumbent_value: ndarray (N,)
the observed value of the incumbent
"""
inc, inc_value = super(DNGO, self).get_incumbent()
if self.normalize_input:
inc = zero_mean_unit_var_unnormalization(inc, self.X_mean,
self.X_std)
if self.normalize_output:
inc_value = zero_mean_unit_var_unnormalization(
inc_value, self.y_mean, self.y_std)
return inc, inc_value
def predict(self, X_test, full_cov=False, **kwargs):
r"""
Returns the predictive mean and variance of the objective function at
the given test points.
Parameters
----------
X_test: np.ndarray (N, D)
Input test points
full_cov: bool
If set to true than the whole covariance matrix between the test points is returned
Returns
----------
np.array(N,)
predictive mean
np.array(N,) or np.array(N, N) if full_cov == True
predictive variance
"""
if not self.is_trained:
raise Exception('Model has to be trained first!')
if self.normalize_input:
X_test_norm, _, _ = normalization.zero_one_normalization(
X_test, self.lower, self.upper)
else:
X_test_norm = X_test
mu, var = self.gp.predict(self.y, X_test_norm)
if self.normalize_output:
mu = normalization.zero_mean_unit_var_unnormalization(
mu, self.y_mean, self.y_std)
var *= self.y_std**2
if not full_cov:
var = np.diag(var)
# Clip negative variances and set them to the smallest
# positive float value
if var.shape[0] == 1:
var = np.clip(var, np.finfo(var.dtype).eps, np.inf)
else:
var = np.clip(var, np.finfo(var.dtype).eps, np.inf)
var[np.where((var < np.finfo(var.dtype).eps)
& (var > -np.finfo(var.dtype).eps))] = 0
return mu, var
def predict(self, X_test, full_cov=False, **kwargs):
r"""
Returns the predictive mean and variance of the objective function at
the given test points.
Parameters
----------
X_test: np.ndarray (N, D)
Input test points
full_cov: bool
If set to true than the whole covariance matrix between the test points is returned
Returns
----------
np.array(N,)
predictive mean
np.array(N,) or np.array(N, N) if full_cov == True
predictive variance
"""
if not self.is_trained:
raise Exception('Model has to be trained first!')
if self.normalize_input:
X_test_norm, _, _ = normalization.zero_one_normalization(X_test, self.lower, self.upper)
else:
X_test_norm = X_test
mu, var = self.gp.predict(self.y, X_test_norm)
if self.normalize_output:
mu = normalization.zero_mean_unit_var_unnormalization(mu, self.y_mean, self.y_std)
var *= self.y_std ** 2
if not full_cov:
var = np.diag(var)
# Clip negative variances and set them to the smallest
# positive float value
if var.shape[0] == 1:
var = np.clip(var, np.finfo(var.dtype).eps, np.inf)
else:
var = np.clip(var, np.finfo(var.dtype).eps, np.inf)
var[np.where((var < np.finfo(var.dtype).eps) & (var > -np.finfo(var.dtype).eps))] = 0
return mu, var
def get_incumbent(self):
"""
Returns the best observed point and its function value
Returns
----------
incumbent: ndarray (D,)
current incumbent
incumbent_value: ndarray (N,)
the observed value of the incumbent
"""
inc, inc_value = super(GaussianProcessMCMC, self).get_incumbent()
if self.normalize_input:
inc = normalization.zero_one_unnormalization(inc, self.lower, self.upper)
if self.normalize_output:
inc_value = normalization.zero_mean_unit_var_unnormalization(inc_value, self.y_mean, self.y_std)
return inc, inc_value
def get_incumbent(self):
"""
Returns the best observed point and its function value
Returns
----------
incumbent: ndarray (D,)
current incumbent
incumbent_value: ndarray (N,)
the observed value of the incumbent
"""
inc, inc_value = super(GaussianProcess, self).get_incumbent()
if self.normalize_input:
inc = normalization.zero_one_unnormalization(inc, self.lower, self.upper)
if self.normalize_output:
inc_value = normalization.zero_mean_unit_var_unnormalization(inc_value, self.y_mean, self.y_std)
return inc, inc_value
rng = np.random.RandomState(42)
X = init_random_uniform(np.zeros(1), np.ones(1), 20, rng)
y = f(X)[:, 0]
model = DNGO()
model.train(X, y)
predictions = lasagne.layers.get_output(model.network,
zero_mean_unit_var_normalization(X, model.X_mean, model.X_std)[0],
deterministic=True).eval()
predictions = zero_mean_unit_var_unnormalization(predictions, model.y_mean, model.y_std)
X_test = np.linspace(0, 1, 100)[:, None]
X_test_norm = zero_mean_unit_var_normalization(X_test, model.X_mean, model.X_std)[0]
# Get features from the net
layers = lasagne.layers.get_all_layers(model.network)
basis_funcs = lasagne.layers.get_output(layers[:-1], X_test_norm)[-1].eval()
fvals = f(X_test)[:, 0]
m, v = model.predict(X_test)
colormap = plt.cm.gist_ncar
plt.gca().set_color_cycle([colormap(i) for i in np.linspace(0, 0.9, min(50, model.n_units_3))])
