def get_incumbent(self):
"""
Returns the best observed point and its function value
Returns
----------
incumbent: ndarray (D,)
current incumbent
incumbent_value: ndarray (N,)
the observed value of the incumbent
"""
if self.normalize_input:
X = zero_mean_unit_var_unnormalization(self.X, self.x_mean, self.x_std)
m = self.predict(X)[0]
else:
m = self.predict(self.X)[0]
best_idx = np.argmin(self.y)
inc = self.X[best_idx]
inc_value = m[best_idx]
if self.normalize_input:
inc = zero_mean_unit_var_unnormalization(inc, self.x_mean, self.x_std)
if self.normalize_output:
inc_value = zero_mean_unit_var_unnormalization(inc_value, self.y_mean, self.y_std)
return inc, inc_value
def predict(self, X_test, return_individual_predictions=False, *args, **kwargs):
"""
Returns the predictive mean and variance of the objective function at
the given test points.
Parameters
----------
X_test: np.ndarray (N, D)
Input test points
return_individual_predictions: bool
If set to true than the individual predictions of all samples are returned.
Returns
----------
np.array(N,)
predictive mean
np.array(N,)
predictive variance
"""
if not self.is_trained:
logging.error("Model is not trained!")
return
# Normalize input
if self.normalize_input:
X_, _, _ = zero_mean_unit_var_normalization(X_test, self.x_mean, self.x_std)
else:
X_ = X_test
f_out = []
theta_noise = []
for sample in self.samples:
lasagne.layers.set_all_param_values(self.net, sample)
out = self.single_predict(X_)
f_out.append(out[:, 0])
theta_noise.append(np.exp(out[:, 1]))
f_out = np.asarray(f_out)
theta_noise = np.asarray(theta_noise)
if return_individual_predictions:
if self.normalize_output:
f_out = zero_mean_unit_var_unnormalization(f_out, self.y_mean, self.y_std)
theta_noise *= self.y_std**2
return f_out, theta_noise
m = np.mean(f_out, axis=0)
# Total variance
# v = np.mean(f_out ** 2 + theta_noise, axis=0) - m ** 2
v = np.mean((f_out - m) ** 2, axis=0)
if self.normalize_output:
m = zero_mean_unit_var_unnormalization(m, self.y_mean, self.y_std)
v *= self.y_std ** 2
return m, v
def sample_functions(self, X_test, n_funcs=1):
"""
Samples F function values from the current posterior at the N
specified test point.
Parameters
----------
X_test: np.ndarray (N, D)
Input test points
n_funcs: int
Number of function values that are drawn at each test point.
Returns
----------
np.array(F, N)
The F function values drawn at the N test points.
"""
if self.normalize_input:
X_test_norm, _, _ = zero_mean_unit_var_normalization(X_test, self.x_mean, self.x_std)
else:
X_test_norm = X_test
f = np.zeros([n_funcs, X_test_norm.shape[0]])
for i in range(n_funcs):
lasagne.layers.set_all_param_values(self.net, self.samples[i])
out = self.single_predict(X_test_norm)[:, 0]
if self.normalize_output:
f[i, :] = zero_mean_unit_var_unnormalization(out, self.y_mean, self.y_std)
else:
f[i, :] = out
return f
def predict(self, X_test):
r"""
Returns the predictive mean and variance of the objective function at
the given test points.
Parameters
----------
X_test: np.ndarray (N, D)
N input test points
Returns
----------
np.array(N,)
predictive mean
np.array(N,)
predictive variance
"""
# Normalize inputs
if self.normalize_input:
X_, _, _ = zero_mean_unit_var_normalization(
X_test, self.X_mean, self.X_std)
else:
X_ = X_test
# Get features from the net
layers = lasagne.layers.get_all_layers(self.network)
theta = lasagne.layers.get_output(layers[:-1], X_)[-1].eval()
# Marginalise predictions over hyperparameters of the BLR
mu = np.zeros([len(self.models), X_test.shape[0]])
var = np.zeros([len(self.models), X_test.shape[0]])
for i, m in enumerate(self.models):
mu[i], var[i] = m.predict(theta)
# See the algorithm runtime prediction paper by Hutter et al
# for the derivation of the total variance
m = np.mean(mu, axis=0)
v = np.mean(mu**2 + var, axis=0) - m**2
# Clip negative variances and set them to the smallest
# positive float value
if v.shape[0] == 1:
v = np.clip(v, np.finfo(v.dtype).eps, np.inf)
else:
v = np.clip(v, np.finfo(v.dtype).eps, np.inf)
v[np.where((v < np.finfo(v.dtype).eps)
& (v > -np.finfo(v.dtype).eps))] = 0
if self.normalize_output:
m = zero_mean_unit_var_unnormalization(m, self.y_mean, self.y_std)
v *= self.y_std**2
return m, v
def get_incumbent(self):
"""
Returns the best observed point and its function value
Returns
----------
incumbent: ndarray (D,)
current incumbent
incumbent_value: ndarray (N,)
the observed value of the incumbent
"""
inc, inc_value = super(DNGO, self).get_incumbent()
if self.normalize_input:
inc = zero_mean_unit_var_unnormalization(inc, self.X_mean,
self.X_std)
if self.normalize_output:
inc_value = zero_mean_unit_var_unnormalization(
inc_value, self.y_mean, self.y_std)
return inc, inc_value
def predict(self, X_test, full_cov=False, **kwargs):
r"""
Returns the predictive mean and variance of the objective function at
the given test points.
Parameters
----------
X_test: np.ndarray (N, D)
Input test points
full_cov: bool
If set to true than the whole covariance matrix between the test points is returned
Returns
----------
np.array(N,)
predictive mean
np.array(N,) or np.array(N, N) if full_cov == True
predictive variance
"""
if not self.is_trained:
raise Exception('Model has to be trained first!')
if self.normalize_input:
X_test_norm, _, _ = normalization.zero_one_normalization(
X_test, self.lower, self.upper)
else:
X_test_norm = X_test
mu, var = self.gp.predict(self.y, X_test_norm)
if self.normalize_output:
mu = normalization.zero_mean_unit_var_unnormalization(
mu, self.y_mean, self.y_std)
var *= self.y_std**2
if not full_cov:
var = np.diag(var)
# Clip negative variances and set them to the smallest
# positive float value
if var.shape[0] == 1:
var = np.clip(var, np.finfo(var.dtype).eps, np.inf)
else:
var = np.clip(var, np.finfo(var.dtype).eps, np.inf)
var[np.where((var < np.finfo(var.dtype).eps)
& (var > -np.finfo(var.dtype).eps))] = 0
return mu, var
def get_incumbent(self):
"""
Returns the best observed point and its function value
Returns
----------
incumbent: ndarray (D,)
current incumbent
incumbent_value: ndarray (N,)
the observed value of the incumbent
"""
inc, inc_value = super(GaussianProcessMCMC, self).get_incumbent()
if self.normalize_input:
inc = normalization.zero_one_unnormalization(inc, self.lower, self.upper)
if self.normalize_output:
inc_value = normalization.zero_mean_unit_var_unnormalization(inc_value, self.y_mean, self.y_std)
return inc, inc_value
def sample_functions(self, X_test, n_funcs=1):
"""
Samples F function values from the current posterior at the N
specified test points.
Parameters
----------
X_test: np.ndarray (N, D)
Input test points
n_funcs: int
Number of function values that are drawn at each test point.
Returns
----------
function_samples: np.array(F, N)
The F function values drawn at the N test points.
"""
if self.normalize_input:
X_test_norm, _, _ = normalization.zero_one_normalization(
X_test, self.lower, self.upper)
else:
X_test_norm = X_test
if not self.is_trained:
raise Exception('Model has to be trained first!')
funcs = self.gp.sample_conditional(self.y, X_test_norm, n_funcs)
if self.normalize_output:
funcs = normalization.zero_mean_unit_var_unnormalization(
funcs, self.y_mean, self.y_std)
if len(funcs.shape) == 1:
return funcs[None, :]
else:
return funcs