def GP_stacking(y_train, X_train, X_test):
m = SparseGPRegression(X_train,
y_train,
num_inducing=20,
kernel=RBF(X_train.shape[1]))
m.optimize("bfgs", messages=0)
y_pred = m.predict(X_test)[0]
return y_pred.flatten()
def _update(self, new_X, new_y, eps=EPSILON):
X, y = self.data
X = deepcopy(X)
y = deepcopy(y)
if len(X) >= self._r_min:
X_vec = np.array([self.params2vec(x) for x in X])
y = np.array(y)
if self._is_normalize:
sig = np.sqrt(np.var(y))
sig = max(sig, eps)
mu = np.mean(y)
y = (y - mu) / sig
if self._backend == "sklearn":
if self._kernel is None:
self._kernel = Matern(nu=2.5)
self.model = GaussianProcessRegressor(
kernel=self._kernel,
n_restarts_optimizer=self._n_restarts_optimizer,
random_state=self._random_state,
normalize_y=False
)
self.model.fit(X_vec, y)
elif self._backend.lower() == "gpy":
y = np.array(y)[:, None]
if self.model is None:
self._create_model(X_vec, y)
else:
self.model.set_XY(X_vec, y)
self.model.optimize_restarts(self._n_restarts_optimizer,
optimizer=self._optimizer,
max_iters=self._max_iters,
messages=False,
verbose=False,
ipython_notebook=False)
def model_builder(X, Y, kernel):
if X.shape[0] < 1000 and not sparse:
tmp = GPRegression(X, Y, kernel=kernel)
else:
tmp = SparseGPRegression(X, Y, num_inducing=num_inducing, kernel=kernel)
if name is not None:
tmp.name = name
return tmp
def given_resample(self, data=[], dt=1):
"""
Resample the dynamics function given a list of inferred voltage and state trajectories
"""
# import pdb; pdb.set_trace()
uu = self.Z[:, 0]
V = self.Z[:, 1]
# Evaluate dz/dt using true Kdr dynamics
g = lambda x: x**4
ginv = lambda u: u**(1. / 4)
dg_dx = lambda x: 4 * x**3
logistic = sigma
logit = sigma_inv
dlogit = lambda x: 1. / (x * (1.0 - x))
u_to_x = lambda u: ginv(logistic(u))
x_to_u = lambda x: logit(g(x))
# uu = x_to_u(xx)
# Compute dynamics du/dt
alpha = lambda V: 0.01 * (V + 55.) / (1 - np.exp(-(V + 55.) / 10.))
beta = lambda V: 0.125 * np.exp(-(V + 65.) / 80.)
dx_dt = lambda x, V: alpha(V) * (1 - x) - beta(V) * x
du_dt = lambda u, V: dlogit(g(u_to_x(u))) * dg_dx(u_to_x(u)) * dx_dt(
u_to_x(u), V)
X = self.Z
Y = du_dt(uu, V)[:, None]
# Set up the sparse GP regression model with the sampled inputs and outputs
# gpr = SparseGPRegression(X, Y, self.kernel, Z=self.Z)
# gpr.likelihood.variance = 0.01
gpr = GPRegression(X, Y, self.kernel)
# HACK: Optimize the hyperparameters
# contrain all parameters to be positive
# gpr.constrain_positive('')
# optimize and plot
# gpr.ensure_default_constraints()
# gpr.optimize_restarts(num_restarts=10)
# gpr.plot()
# import pdb; pdb.set_trace()
# HACK: Rather than using a truly nonparametric approach, just sample
# the GP at the grid of inducing points and interpolate at the GP mean
self.h = gpr.posterior_samples_f(self.Z, size=1)
# HACK: Recreate the GP with the sampled function h
self.gp = SparseGPRegression(self.Z,
self.h,
self.kernel,
num_inducing=25)
def _create_model(self, X, y):
"""
Creates the GPy model given some input data X and Y.
"""
# Define kernel
input_dim = X.shape[1]
if self._kernel is None:
kern = GPy.kern.Matern52(input_dim, variance=1., ARD=self._ARD)
else:
kern = self._kernel
self._kernel = None
# Define model
noise_var = y.var() * 0.01
if not self._sparse:
self.model = GPRegression(X, y, kernel=kern, noise_var=noise_var)
else:
self.model = SparseGPRegression(X, y, kernel=kern,
num_inducing=self._num_inducing)
self.model.Gaussian_noise.constrain_bounded(1e-9, 1e6, warning=False)
def compute_runtimes(dimensions, n, num_inducing, kernel_class,
simulator_class, tag: str):
"""Compute runtimes at different dimensionalities for sample_size iterations at every dimension."""
sample_size = 10
kernel_creation = []
gp_creation = []
gp_optimization = []
for i in trange(len(dimensions)):
dim = dimensions[i]
kernel_creation_array = np.empty(sample_size)
gp_creation_array = np.empty(sample_size)
gp_optimization_array = np.empty(sample_size)
for j in range(sample_size):
simulator = simulator_class(n, random_state=j)
data = simulator.simulate(dim, n_informative=n)
a = time.time()
kernel = kernel_class(dim)
b = time.time()
m = SparseGPRegression(data.X_train,
data.y_train,
num_inducing=num_inducing,
kernel=kernel)
c = time.time()
m.optimize()
d = time.time()
kernel_creation_array[j] = b - a
gp_creation_array[j] = c - b
gp_optimization_array[j] = d - c
kernel_creation.append(kernel_creation_array.mean())
gp_creation.append(gp_creation_array.mean())
gp_optimization.append(gp_optimization_array.mean())
return RuntimeResult(kernel_creation, gp_creation, gp_optimization,
dimensions, tag)
def gp_surrogate(self,
Z=None,
Y=None,
kern_x=None,
kern_p=None,
num_inducing=None):
self.set_training_data(Z, Y)
assert self.Z is not None and self.Y is not None
self.set_kernels(kern_x, kern_p)
assert self.kern_x is not None and self.kern_p is not None
R, J = binary_dimensions(self.Z, self.binary_variables)
gps = []
for e in range(self.num_outputs):
gps.append([])
for r in R:
Jr = (J == r)
if not np.any(Jr):
gps[e].append(None)
continue
dim_xb = self.dim_x - self.dim_b
dim = self.dim_x + self.dim_p
kernx = self.kern_x(dim_xb, self.non_binary_variables, 'kernx')
kernp = self.kern_p(self.dim_p, range(self.dim_x, dim),
'kernp')
#Zr = self.Z[ np.ix_(Jr, I ) ]
Zr = self.Z[Jr]
Yr = self.Y[np.ix_(Jr, [e])]
numi = self.max_num_inducing if num_inducing is None \
else num_inducing
gp = SparseGPRegression(Zr,
Yr,
kernx * kernp,
num_inducing=numi)
gps[e].append(gp)
self.gps = gps
def _gp_regression(self, X, Y, kern, **kwargs):
num_inducing = kwargs.get('num_inducing', 100)
return SparseGPRegression(X, Y, kern, num_inducing=num_inducing)
# plt.scatter(X, y)
# plt.show()
gp_regression = GPRegression(X, y)
gp_regression.optimize(messages=True)
log_likelihood1 = gp_regression.log_likelihood()
model_output(gp_regression,
title="GP Regression with loglikelihood: " + str(log_likelihood1))
#################################
# inducing variables, u. Each inducing variable has its own associated input index, Z, which lives in the same space as X.
Z = np.hstack((np.linspace(2.5, 4., 3), np.linspace(7, 8.5, 3)))[:, None]
sparse_regression = SparseGPRegression(X, y, kernel=rbf, Z=Z)
sparse_regression.noise_var = noise_var
sparse_regression.inducing_inputs.constrain_fixed()
sparse_regression.optimize(messages=True)
log_likelihood2 = sparse_regression.log_likelihood()
#inducing variables fixed
model_output(
sparse_regression,
title="Inducing variables fixed parameters optimized with loglikelihood: "
+ str(log_likelihood2[0][0]))
#inducing variables optimized
sparse_regression.inducing_inputs.unconstrain()
class BayesSampler(BaseSampler):
"""Bayesian optimization sampler
Sample next location based on gaussian process
Parameters
----------
space: list(dict)
Define search space. Each element has to the following key
values: 'name', 'type', and 'domain' (,'num_grid' is optional).
init_X: array-like(float), shape=(n_samples, n_dim)
The list of parameters to initizlie sampler
init_y: array-like(float), shape(n_samples,)
The list of score of init_X
r_min: int
The number of random samples before starting using gaussian process
method: str
The name of acquisition functions
kernel: kernel object, optional
is_normalize: bool
If ture, normalized score values are used for optimization
n_restarts_optimizer: int
The number of trial to opimize GP hyperparameters
backend: str (default 'gpy')
Determine which GP package is used. That has to be
either of 'gpy' or 'sklearn'.
optimizer: str
The name of optimizers of hyperparameters of GP, which is valid when
backend='gpy'.
max_iters: int
The maximum number of iteration to optimize hyperparamters of GP,
which is valid when backend='gpy'.
ARD: bool
Wheather to use ARD for kernel, which is valid when backend='gpy'.
sparse: bool
If true, use sparse GP, which is valid when backend='gpy'.
num_inducing: int
The number of inducing inputs for sparse GP, which is valid
when backend='gpy' and sparse is True.
random_state: int
"""
sampler_name = "bayes"
def __init__(self, space, init_X=None, init_y=None, r_min=3, method="EI",
kernel=None, is_normalize=True, n_restarts_optimizer=10,
backend="gpy", optimizer="bfgs", max_iters=1000,
ARD=False, sparse=False,
num_inducing=10, random_state=RANDOM_STATE):
super(BayesSampler, self).__init__(space, init_X, init_y)
self._r_min = r_min
self.model = None
self.acquisition_func = self._get_acquisition_func(method)
self._is_normalize = is_normalize
self._ARD = ARD
self._kernel = kernel
self._sparse = sparse
self._num_inducing = num_inducing
self._optimizer = optimizer
self._max_iters = max_iters
self._backend = backend.lower()
self._n_restarts_optimizer = n_restarts_optimizer
self._random_state = random_state
def _update(self, new_X, new_y, eps=EPSILON):
X, y = self.data
X = deepcopy(X)
y = deepcopy(y)
if len(X) >= self._r_min:
X_vec = np.array([self.params2vec(x) for x in X])
y = np.array(y)
if self._is_normalize:
sig = np.sqrt(np.var(y))
sig = max(sig, eps)
mu = np.mean(y)
y = (y - mu) / sig
if self._backend == "sklearn":
if self._kernel is None:
self._kernel = Matern(nu=2.5)
self.model = GaussianProcessRegressor(
kernel=self._kernel,
n_restarts_optimizer=self._n_restarts_optimizer,
random_state=self._random_state,
normalize_y=False
)
self.model.fit(X_vec, y)
elif self._backend.lower() == "gpy":
y = np.array(y)[:, None]
if self.model is None:
self._create_model(X_vec, y)
else:
self.model.set_XY(X_vec, y)
self.model.optimize_restarts(self._n_restarts_optimizer,
optimizer=self._optimizer,
max_iters=self._max_iters,
messages=False,
verbose=False,
ipython_notebook=False)
def _create_model(self, X, y):
"""
Creates the GPy model given some input data X and Y.
"""
# Define kernel
input_dim = X.shape[1]
if self._kernel is None:
kern = GPy.kern.Matern52(input_dim, variance=1., ARD=self._ARD)
else:
kern = self._kernel
self._kernel = None
# Define model
noise_var = y.var() * 0.01
if not self._sparse:
self.model = GPRegression(X, y, kernel=kern, noise_var=noise_var)
else:
self.model = SparseGPRegression(X, y, kernel=kern,
num_inducing=self._num_inducing)
self.model.Gaussian_noise.constrain_bounded(1e-9, 1e6, warning=False)
def sample(self, num_samples=1, *args, **kwargs):
"""Sample next location to evaluate based on GP
Parameters
---------
num_samples: int
The number of samples
Returns
-------
Xs: list(dict), length is num_samples
"""
_num_data = self.num_data
if _num_data < self._r_min:
Xs = self._random_sample(num_samples)
else:
Xs = self._bayes_sample(num_samples)
return Xs
def _bayes_sample(self, num_samples, num_restarts=25):
num_restarts = max(num_samples, num_restarts)
init_params = self._random_sample(num_restarts)
init_xs = [self.params2vec(param) for param in init_params]
bounds = self.design_space.get_bounds()
if self._backend == "sklearn":
evaluated_loss = np.array(self.model.y_train_)
else:
evaluated_loss = np.array(self.model.Y)[:, 0]
ys = []
xs = []
def minus_ac(x):
return -self.acquisition_func(x, self.model,
evaluated_loss,
mode=self._backend)
for x0 in init_xs:
res = minimize(fun=minus_ac,
x0=x0,
bounds=bounds,
method='L-BFGS-B')
ys.append(-res.fun)
xs.append(res.x)
idx = np.argsort(ys)[::-1][:num_samples]
best_x = np.array(xs)[idx]
best_params = [self.vec2params(x) for x in best_x]
return best_params
def _random_sample(self, num_samples):
Xs = []
for i in range(num_samples):
x = random_sample(self.params_conf)
Xs.append(x)
return list(Xs)
def _get_acquisition_func(self, method):
if method == "EI":
return expected_improvement
else:
raise NotImplementedError(method)
def params2vec(self, params):
# Not include fixed params
vec = []
for conf in self.params_conf:
val = params[conf['name']]
vec.append(val)
vec = self.design_space.objective_to_model([vec])
return np.array(vec)
def vec2params(self, vec):
# Not include fixed params
params = {}
vec = self.design_space.model_to_objective([vec])
for i, val in enumerate(vec):
conf = self.params_conf[i]
params[conf["name"]] = val
return params