def optimize_s_sn_l(self, sn, s, l):
assert (isinstance(sn, float))
assert (l.shape == (self.fix_W.shape[1],))
# Create a GP
self.kernel.update_params(W=self.fix_W, s=s, l=l)
gp_reg = GPRegression(self.X, self.Y.reshape(-1, 1), self.kernel, noise_var=sn)
try:
gp_reg.optimize(optimizer="lbfgs", max_iters=config['max_iter_parameter_optimization'])
except Exception as e:
print(e)
print(gp_reg.kern.K(gp_reg.X))
print("Error above!")
# TODO: does this optimization work in the correct direction?
new_variance = gp_reg.kern.inner_kernel.variance
new_lengthscale = gp_reg.kern.inner_kernel.lengthscale
new_sn = gp_reg['Gaussian_noise.variance']
assert gp_reg.kern.inner_kernel.lengthscale is not None
assert gp_reg.kern.inner_kernel.variance is not None
# assert not np.isclose(np.asarray(new_lengthscale), np.zeros_like(new_lengthscale) ).all(), new_lengthscale
return float(new_variance), new_lengthscale.copy(), float(new_sn)
def permute(self, varb=None):
# get model input
x = self.x.copy()
y = self.y.copy()
# shuffle targeet variable
col = np.where(varb == self.x_keys)[0] # which one?
shuffled = np.random.choice(x[:, col].ravel(),
size=x.shape[0],
replace=False)
x[:, col] = shuffled[:, np.newaxis] # replace
# same steps as fit, below can be done more concisely, but I worry about minor differenes
x_dim = x.shape[1]
y_dim = y.shape[1]
kern = buildKernel(x_dim, ARD=self.ARD)
mcopy = GPRegression(x, y, kern)
if self.heteroscedastic:
kern = addFixedKernel(kern, y_dim, self.error)
mcopy = GPRegression(x, y, kern)
mcopy.optimize()
return mcopy.log_likelihood()
def maximize(self, iterations):
self.start_time2 = time()
for n in np.arange(iterations):
X = np.array(self.x).reshape(-1, 1) if self.dim == 1 else np.array(
self.x)
Y = np.array(self.y).reshape(-1, 1)
gpreg = GPRegression(X,
Y,
kernel=self.kernel,
noise_var=self.noise_var)
gpreg.optimize() # MAP for kernel hyper-parameters
vals, par = self.minimize_negative_acquisition(gpreg)
if self.dim == 1:
self.give_new_point(vals, par)
else:
self.x.append(par[vals.argmin()])
self.y.append(self.objective(self.x[-1]))
if self.log_info:
print(
"%s step in BO; objective value: %.4f at %.4f; time: %.2f s."
% (len(self.x), self.y[-1], self.x[-1], time() -
self.start_time2 + self.end_time1 - self.start_time1))
if n % 10 == 0:
self.a += 1
self.aquis_par.append(self.sample_aquis_param(self.a))
self.end_time2 = time()
def fit(self):
if self.model:
self.noise = self.model.Gaussian_noise.variance[0]
return None
x_dim = self.x.shape[1] # number of input dimensions, 1 if only time
y_dim = self.y.shape[
1] # number of ouptut dimensions, typically only 1 for log OD
kern = buildKernel(x_dim, ARD=self.ARD)
m = GPRegression(self.x, self.y, kern)
if self.heteroscedastic:
kern = addFixedKernel(kern, y_dim, self.error)
m = GPRegression(self.x, self.y, kern)
m.optimize()
self.noise = m.Gaussian_noise.variance[
0] # should be negligible (<1e-10) for full model
if self.heteroscedastic:
m.kern = m.kern.parts[
0] # cannot predict with fixed kernel, so remove it
self.model = m
def _model_chooser(self):
""" Initialize the model used for the optimization """
kernel = Matern52(len(self.variables_list), variance=1., ARD=False)
gpmodel = GPRegression(self.X, self.Y, kernel)
gpmodel.optimize()
self.model = GPyModelWrapper(gpmodel)
if self.noiseless:
gpmodel.Gaussian_noise.constrain_fixed(0.001)
self.model = GPyModelWrapper(gpmodel)
def _model_chooser(self):
""" Initialize the model used for the optimization """
kernel = Matern52(len(self.variables_list), variance=1., ARD=False)
gpmodel = GPRegression(self.X, self.Y, kernel)
gpmodel.optimize()
self.model = GPyModelWrapper(gpmodel)
if self.noiseless:
gpmodel.Gaussian_noise.constrain_fixed(0.001)
self.model = GPyModelWrapper(gpmodel)
def predict_y2(self):
x = self.x_new
y = self.y1
m = GPRegression(x, y)
m.optimize()
mu, cov = (m.predictive_gradients(x)[0], None)
self.y2 = mu[:, 0]
self.cov2 = cov
def learn_flow(X, y, lengthscales, variance=1.0):
dimensions = X.shape[1]
# lengthscales = [ l_scale for d in range(dimensions)]
kernel = GPy.kern.rbf(dimensions, ARD=True,
lengthscale=lengthscales, variance=variance)
m = GPRegression(X,y,kernel)
m.optimize('bfgs', max_iters=1000)
return m
def run_optimisation(current_range, freq_range, power_range):
parameter_space = ParameterSpace([\
ContinuousParameter('current', current_range[0], current_range[1]), \
ContinuousParameter('freq', freq_range[0], freq_range[1]), \
ContinuousParameter('power', power_range[0], power_range[1])
])
def function(X):
current = X[:, 0]
freq = X[:, 1]
power = X[:, 2]
out = np.zeros((len(current), 1))
for g in range(len(current)):
'''
Set JPA Current, Frequency & Power
'''
out[g, 0] = -get_SNR(
plot=False)[-1] #Negative as want to maximise SNR
return out
num_data_points = 10
design = RandomDesign(parameter_space)
X = design.get_samples(num_data_points)
Y = function(X)
model_gpy = GPRegression(X, Y)
model_gpy.optimize()
model_emukit = GPyModelWrapper(model_gpy)
exp_imprv = ExpectedImprovement(model=model_emukit)
optimizer = GradientAcquisitionOptimizer(space=parameter_space)
point_calc = SequentialPointCalculator(exp_imprv, optimizer)
coords = []
min = []
bayesopt_loop = BayesianOptimizationLoop(model=model_emukit,
space=parameter_space,
acquisition=exp_imprv,
batch_size=1)
stopping_condition = FixedIterationsStoppingCondition(i_max=100)
bayesopt_loop.run_loop(q, stopping_condition)
coord_results = bayesopt_loop.get_results().minimum_location
min_value = bayesopt_loop.get_results().minimum_value
step_results = bayesopt_loop.get_results().best_found_value_per_iteration
print(coord_results)
print(min_value)
return coord_results, abs(min_value)
def fit(self, X_train, Y_train):
num_samples = len(X_train)
if num_samples > self.max_sample:
print('num_samples > {}, splitting training; {} steps'.format(
self.max_sample, max(10, num_samples // self.max_sample)))
i = 0
while i < max(10, num_samples // self.max_sample):
#if self.verbose:
print('step {}/{}'.format(
i + 1, max(10, num_samples // self.max_sample)))
train_idx = np.random.choice(num_samples,
self.max_sample,
replace=False)
X = X_train[train_idx]
Y = Y_train[train_idx]
kernels = []
for dim in range(X_train.shape[1]):
kernels.append(self.kernel(1, active_dims=[dim]))
kern_multi = ft.reduce(lambda a, b: a + b, kernels)
try:
gpr = GPRegression(X,
Y,
kernel=kern_multi,
normalizer=self.normalizer)
gpr.optimize(messages=self.verbose, max_iters=200)
self.models.append(gpr)
i += 1
except (np.linalg.LinAlgError):
continue
else:
Repeat = True
print('num_samples <= 300, single training')
while Repeat:
kernels = []
for dim in range(X_train.shape[1]):
kernels.append(self.kernel(1, active_dims=[dim]))
kern_multi = ft.reduce(lambda a, b: a + b, kernels)
try:
gpr = GPRegression(X_train,
Y_train,
kernel=kern_multi,
normalizer=self.normalizer)
gpr.optimize(messages=self.verbose, max_iters=200)
self.models.append(gpr)
Repeat = False
except (np.linalg.LinAlgError):
continue
return self
def rmse_rbf(x_train: np.ndarray, y_train: np.ndarray, x_test: np.ndarray,
y_test: np.ndarray) -> float:
"""RMSE of a GPy RBF kernel.
:param x_train:
:param y_train:
:param x_test:
:param y_test:
:return:
"""
model = GPRegression(x_train,
y_train,
kernel=RBF(input_dim=x_train.shape[1]))
model.optimize()
return compute_gpy_model_rmse(model, x_test, y_test)
def maximize(self, iterations):
self.start_time2 = time()
for n in np.arange(iterations):
X = np.array(self.x).reshape(-1, 1) if self.dim == 1 else np.array(self.x)
Y = np.array(self.y).reshape(-1, 1)
gpreg = GPRegression(X, Y, kernel=self.kernel, noise_var=self.noise_var)
gpreg.optimize() # MAP for kernel hyper-parameters
vals, par = self.minimize_negative_acquisition(gpreg)
if self.dim == 1:
self.give_new_point(vals, par)
else:
self.x.append(par[vals.argmin()])
self.y.append(self.objective(self.x[-1]))
if self.log_info:
print("%s step in BO; objective value: %.4f at %.4f; time: %.2f s." %
(len(self.x), self.y[-1], self.x[-1],
time() - self.start_time2 + self.end_time1 - self.start_time1))
if n % 10 == 0:
self.a += 1
self.aquis_par.append(self.sample_aquis_param(self.a))
self.end_time2 = time()
class Kernel(object):
def __init__(self,
x0,
y0,
cons=None,
alpha=opt.ke_alpha,
beta=opt.ke_beta,
input_size=opt.ke_input_size,
hidden_size=opt.ke_hidden_size,
num_layers=opt.ke_num_layers,
bidirectional=opt.ke_bidirectional,
lr=opt.ke_lr,
weight_decay=opt.ke_weight_decay):
super(Kernel, self).__init__()
self.alpha = alpha
self.beta = beta
self.lstm = nn.LSTM(input_size=input_size,
hidden_size=hidden_size,
num_layers=num_layers,
bidirectional=bidirectional)
self.lstm = self.lstm.to(opt.device)
self.input_size = input_size
self.hidden_size = hidden_size
self.num_layers = num_layers
self.bidirectional = bidirectional
self.bi = 2 if bidirectional else 1
self.x = [x0]
self.y = torch.tensor([y0],
dtype=torch.float,
device=opt.device,
requires_grad=False)
self.cons = [cons]
inp, out = clean_x(self.x, self.cons)
self.model = GPRegression(inp, out)
self.model.Gaussian_noise.constrain_fixed(1e-6, warning=False)
self.model.optimize()
self.x_best = x0
self.y_best = y0
self.i_best = 0
self.n = 1
self.E = self.embedding(x0).view(1, -1)
self.K = self.kernel(self.E[0], self.E[0]).view(1, 1)
self.K_inv = torch.inverse(self.K + self.beta *
torch.eye(self.n, device=opt.device))
self.optimizer = optim.Adam(self.lstm.parameters(),
lr=lr,
weight_decay=weight_decay)
def embedding(self, xi):
inputs = xi.view(-1, 1, self.input_size)
outputs, (hn, cn) = self.lstm(inputs)
outputs = torch.mean(outputs.squeeze(1), dim=0)
outputs = outputs / torch.norm(outputs)
return outputs
def kernel(self, ei, ej):
d = ei - ej
d = torch.sum(d * d)
k = torch.exp(-d / (2 * self.alpha))
return k
def kernel_batch(self, en):
n = self.n
k = torch.zeros(n, device=opt.device)
for i in range(n):
k[i] = self.kernel(self.E[i], en)
return k
def predict(self, xn):
n = self.n
en = self.embedding(xn)
k = self.kernel_batch(en)
kn = self.kernel(en, en)
t = torch.mm(k.view(1, n), self.K_inv)
mu = torch.mm(t, self.y.view(n, 1))
sigma = kn - torch.mm(t, k.view(n, 1))
sigma = torch.sqrt(sigma + self.beta)
return mu, sigma
def acquisition_cons(self, xn):
with torch.no_grad():
xn_ = np.array([xn.cpu().numpy().flatten()])
mu_cons, sigma_cons = self.model.predict(xn_)
sigma_cons = sqrt(sigma_cons)
PoF = norm.cdf(0, mu_cons, sigma_cons)
mu, sigma = self.predict(xn)
mu = mu.item()
sigma = sigma.item()
y_best = self.y_best
z = (mu - y_best) / sigma
ei = (mu - y_best) * norm.cdf(z) + sigma * norm.pdf(z)
return ei * PoF
def acquisition(self, xn):
with torch.no_grad():
mu, sigma = self.predict(xn)
mu = mu.item()
sigma = sigma.item()
y_best = self.y_best
z = (mu - y_best) / sigma
ei = (mu - y_best) * norm.cdf(z) + sigma * norm.pdf(z)
return ei
def kernel_batch_ex(self, t):
n = self.n
k = torch.zeros(n - 1, device=opt.device)
for i in range(t):
k[i] = self.kernel(self.E[i], self.E[t])
for i in range(t + 1, n):
k[i - 1] = self.kernel(self.E[t], self.E[i])
return k
def predict_ex(self, t):
n = self.n
k = self.kernel_batch_ex(t)
kt = self.kernel(self.E[t], self.E[t])
indices = list(range(t)) + list(range(t + 1, n))
indices = torch.tensor(indices, dtype=torch.long, device=opt.device)
K = self.K
K = torch.index_select(K, 0, indices)
K = torch.index_select(K, 1, indices)
K_inv = torch.inverse(K +
self.beta * torch.eye(n - 1, device=opt.device))
y = torch.index_select(self.y, 0, indices)
t = torch.mm(k.view(1, n - 1), K_inv)
mu = torch.mm(t, y.view(n - 1, 1))
sigma = kt - torch.mm(t, k.view(n - 1, 1))
sigma = torch.sqrt(sigma + self.beta)
return mu, sigma
def add_sample(self, xn, yn, consn):
self.x.append(xn)
self.y = torch.cat((self.y,
torch.tensor([yn],
dtype=torch.float,
device=opt.device,
requires_grad=False)))
self.cons.append(consn)
inp, out = clean_x(self.x, self.cons)
self.model.set_XY(inp, out)
self.model.optimize()
n = self.n
if consn > 0:
if yn > self.y_best:
self.x_best = xn
self.y_best = yn
self.i_best = n
en = self.embedding(xn)
k = self.kernel_batch(en)
kn = self.kernel(en, en)
self.E = torch.cat((self.E, en.view(1, -1)), 0)
self.K = torch.cat(
(torch.cat((self.K, k.view(n, 1)),
1), torch.cat((k.view(1, n), kn.view(1, 1)), 1)), 0)
self.n += 1
self.K_inv = torch.inverse(self.K + self.beta *
torch.eye(self.n, device=opt.device))
def add_batch(self, x, y, cons):
self.x.extend(x)
self.y = torch.cat((self.y, y))
self.cons.extend(cons)
inp, out = clean_x(self.x, self.cons)
self.model.set_XY(inp, out)
self.model.optimize()
m = len(x)
for i in range(m):
n = self.n
if self.cons[i] > 0:
if y[i].item() > self.y_best:
self.x_best = x[i]
self.y_best = y[i].item()
self.i_best = n
en = self.embedding(x[i])
k = self.kernel_batch(en)
kn = self.kernel(en, en)
self.E = torch.cat((self.E, en.view(1, -1)), 0)
self.K = torch.cat(
(torch.cat((self.K, k.view(n, 1)),
1), torch.cat((k.view(1, n), kn.view(1, 1)), 1)), 0)
self.n += 1
self.K_inv = torch.inverse(self.K + self.beta *
torch.eye(self.n, device=opt.device))
def update_EK(self):
n = self.n
E_ = torch.zeros((n, self.E.size(1)), device=opt.device)
for i in range(n):
E_[i] = self.embedding(self.x[i])
self.E = E_
K_ = torch.zeros((n, n), device=opt.device)
for i in range(n):
for j in range(i, n):
k = self.kernel(self.E[i], self.E[j])
K_[i, j] = k
K_[j, i] = k
self.K = K_
self.K_inv = torch.inverse(self.K + self.beta *
torch.eye(self.n, device=opt.device))
def loss(self):
n = self.n
l = torch.zeros(n, device=opt.device)
for i in range(n):
mu, sigma = self.predict_ex(i)
d = self.y[i] - mu
l[i] = -(0.918939 + torch.log(sigma) + d * d / (2 * sigma * sigma))
l = -torch.mean(l)
return l
def opt_step(self):
if self.n < 2:
return 0.0
self.optimizer.zero_grad()
l = self.loss()
ll = -l.item()
l.backward()
self.optimizer.step()
self.update_EK()
return ll
def save(self, save_path):
path = os.path.dirname(save_path)
if not os.path.exists(path):
os.makedirs(path)
torch.save(self, save_path)
def _fit_model(self, X, Y):
model = GPRegression(X, Y, self.kernel)
model.optimize(messages=False, max_f_eval=self.max_feval)
self.model = model
def _fit_model(self, X, Y):
model = GPRegression(X, Y, self.kernel)
model.optimize(messages=False, max_f_eval=self.max_feval)
self.model = model
sum1 = 0
for i in range(0, len(x) - 1):
sum1 += (100 * (x[i + 1] - x[i]**2)**2 + (x[i] - 1)**2)
return sum1
if __name__ == "__main__":
np.random.seed(1)
dim = 50
f = sphere
X = np.random.uniform(-5, 5, (10, dim))
y = np.array([f(xi) for xi in X]).reshape(-1, 1)
gpy_kern = RBFg(input_dim=dim, ARD=False)
gpy_model = GPRegression(X, y, kernel=gpy_kern)
gpy_model.optimize()
mobo_kern = RBF(ARD=False)
mobo_model = moboGP(mobo_kern, X, y)
mobo_likelihood = Likelihood(mobo_model)
mobo_likelihood.evaluate()
optimizer = lbfgsb(mobo_model)
optimizer.opt()
print("gpy:", gpy_model.log_likelihood())
print("mobo:", mobo_model.log_likelihood)
Xtest = np.random.uniform(-5, 5, (1000, dim))
ytest = np.array([f(xi) for xi in Xtest]).reshape(-1, 1)
mu, _ = model.predict(x_new)
return mu[0, 0]
list_mse_gpr = []
list_mse_lgb = []
list_mse_svr = []
for i in range(1):
print(i)
x_train_m, y_train_m = a_gpr.sample_point(x_train, y_train, iter=80)
k_rbf = RBF(input_dim=n, variance=0.5, lengthscale=1)
gp_model = GPRegression(x_train_m,
np.reshape(y_train_m, (-1, 1)),
kernel=k_rbf)
gp_model.optimize(messages=False)
model_lgb = lgb.LGBMRegressor()
model_lgb.fit(x_train_m, y_train_m)
model_svr = SVR()
model_svr.fit(x_train_m, y_train_m)
y_pre_con = [
pre_gp_mu_var(np.reshape(x_test[i], (1, -1)), gp_model)
for i in range(np.shape(x_test)[0])
]
y_pre_lgb = [
model_lgb.predict(np.reshape(x_test[i], (1, -1)))
for i in range(np.shape(x_test)[0])
]
init_design = RandomDesign(space)
X_init = init_design.get_samples(2)
Y_init = np.array([b.objective_function(xi)["function_value"] for xi in X_init])[:, None]
if args.model_type == "bnn":
model = Bohamiann(X_init=X_init, Y_init=Y_init, verbose=True)
elif args.model_type == "rf":
model = RandomForest(X_init=X_init, Y_init=Y_init)
with_gradients = False
elif args.model_type == "gp":
kernel = Matern52(len(list_params), variance=1., ARD=True)
gpmodel = GPRegression(X_init, Y_init, kernel)
gpmodel.optimize()
model = GPyModelWrapper(gpmodel)
acquisition = ExpectedImprovement(model)
acquisition_optimizer = DirectOptimizer(space)
candidate_point_calculator = Sequential(acquisition, acquisition_optimizer)
bo = BayesianOptimizationLoop(model=model, space=space, X_init=X_init, Y_init=Y_init, acquisition=acquisition,
candidate_point_calculator=candidate_point_calculator)
overhead = []
st = time.time()
for i in range(args.num_iterations):
t = time.time()
bo.run_loop(user_function=obj, stopping_condition=FixedIterationsStoppingCondition(i + X_init.shape[0]))
X[:x_half, :] = np.linspace(0, 2,
x_half)[:,
None] # First cluster of inputs/covariates
X[x_half:, :] = np.linspace(
8, 10, x_half)[:, None] # Second cluster of inputs/covariates
rbf = RBF(input_dim=1)
mu = np.zeros(N)
cov = rbf.K(X) + np.eye(N) * np.sqrt(noise_var)
y = np.random.multivariate_normal(mu, cov).reshape(-1, 1)
# plt.scatter(X, y)
# plt.show()
gp_regression = GPRegression(X, y)
gp_regression.optimize(messages=True)
log_likelihood1 = gp_regression.log_likelihood()
model_output(gp_regression,
title="GP Regression with loglikelihood: " + str(log_likelihood1))
#################################
# inducing variables, u. Each inducing variable has its own associated input index, Z, which lives in the same space as X.
Z = np.hstack((np.linspace(2.5, 4., 3), np.linspace(7, 8.5, 3)))[:, None]
sparse_regression = SparseGPRegression(X, y, kernel=rbf, Z=Z)
sparse_regression.noise_var = noise_var
sparse_regression.inducing_inputs.constrain_fixed()
sparse_regression.optimize(messages=True)
def main():
print("######################")
global target, X0, Y0, values, frac_M, frac_X, bo_flag
#target_params = np.array([[0.14,0.4],[1.4,0.03]])
#target = LiX_wrapper(True,'LiF','Rocksalt','JC',
# target_params,False,False,eng)
target = np.array([[-764.5, 6.012 * 0.99, 6.012 * 0.99, 6.012 * 0.99]])
if focus == 'energy':
target_comp = target[0, 0].reshape(1, -1)
if focus == 'constant':
target_comp = target[0, 1].reshape(1, -1)
else:
target_comp = target[0, :4].reshape(1, -1)
print('Target initialized!')
latin_design = LatinDesign(parameter_space=parameter_space)
X0 = latin_design.get_samples(INIT_POINTS)
Y0 = np.array([])
for x in X0:
x = np.array([x])
Y0 = np.append(Y0, f.evaluate(x))
values = []
for y in Y0:
values.append(y.Y)
values = np.asarray(values, dtype=float)
### Redundancy check
if (values[:, 7:-1] == values[0, 7]).all():
values = values[:, :7]
frac_X = False
if (values[:, 4:7] == values[0, 4]).all():
values = values[:, :4]
frac_M = False
values = values.reshape(-1, np.max(np.shape(target)))
bo_flag = True
if focus == 'energy':
values = values[:, 0].reshape(-1, 1)
if focus == 'constant':
values = values[:, 1:4].reshape(-1, 3)
### BO Loop
kern = Matern52(X0.shape[1], variance=1)
model = GPRegression(X0,
values,
kernel=kern,
normalizer=True,
noise_var=NOISE) # Kernel = None: RBF default
model.optimize(optimizer='lbfgsb')
model.optimize_restarts(num_restarts=50, verbose=False)
model_wrapped = GPyModelWrapper(model)
acq = L2_LCB(model=model_wrapped, target=target_comp, beta=np.float64(1.))
# beta is the exploration constant
bayesopt_loop = BayesianOptimizationLoop(model=model_wrapped,
space=parameter_space,
acquisition=acq)
bayesopt_loop.run_loop(f, BO_ITER)
return save(bayesopt_loop)
class DCKE(RegressorMixin):
""" Dynamically Controlled Kernel Estimation
Computes the conditional expectation $E[Y \mid X=x]$ from
a training set $X_i$, $y_i$, $i=1, \ldots, N$ of joint
realizations of $X$ and $Y$ for an arbitrary prediction
set of $x$'s. The DCKE regressor first uses local regression
on a mesh grid to solve the problem on the mesh grid and then
uses GPR to evaluate in between the points on the mesh grid.
Optionally, a control variate $Z$ can be supplied together
with $\mu_Z = E[Z \mid X=x_k]$ for the points $x_k$ on the
mesh grid. In that case, the expectation
$E[Y +\beta (Z-\mu_Z) \mid X=x_k]$ is computed on the
mesh grid with variance reduced by the correlation between
$Y$ and $Z$.
"""
def __init__(self, locreg, gpr_kernel):
"""
Initializes the DCKE object.
:param locreg: an instance of LocalRegression
:param gpr_kernel: an instance of GPy.kern
"""
self.locreg = locreg
self.gpr_kernel = gpr_kernel
self.gpr_ = None
self.X_train_ = None
self.y_train_ = None
self.x_mesh_ = None
self.y_mesh_ = None
self.Z_ = None
self.mz_ = None
self.cov_ = None
self.var_ = None
self.beta_ = None
def fit(self, X, y, x_mesh, Z=None, mz=None, bandwidth=None):
"""
Fits the DCKE to training data.
:param X: a numpy array of shape (num_samples, num_dimensions)
:param y: a numpy array of shape (num_samples,)
:param Z: a numpy array of shape (num_samples,)
:param x_mesh: a numpy array of shape (num_meshes, num_dimensions)
:param mz: a numpy array of shape (num_meshes,) any any mz[k]
satisties mz[k] = E[Z \mid X=x_k]$ where x_k are the
points in x_mesh
:param bandwidth: bandwidth parameter for the local regression
:return:
"""
self.X_train_ = X
self.y_train_ = y
self.x_mesh_ = x_mesh
if Z is None and mz is None:
self.Z_ = np.zeros_like(self.y_train_)
self.mz_ = np.zeros(self.x_mesh_.shape[0])
elif (Z is None and mz is not None) or (Z is not None and mz is None):
raise ValueError(
'Parameter Z and mz have to be either both None or both not None.'
)
else:
self.Z_ = Z
self.mz_ = mz
self.locreg.warm_start = True
self.locreg.fit(X, y, bandwidth)
def _calculate_locregs(self):
"""
Uses the approximate conditional expectation operator
$\tilde E[_ \mid X=x]$ defined by the local regression in self.locreg
to compute the approximate optimal beta for the control variate $Z$
defined by $\beta_x = - \tfrac{\Cov[Y, Z \mid X=x]}{\Var[Z \mid X=x]}$
for all $x$ in self.x_mesh.
:return: beta, a numpy array of shape (num_mesh_points, )
"""
h = self.locreg.bandwidth
n = self.x_mesh_.shape[0]
self.cov_ = np.zeros(n)
self.var_ = np.zeros(n)
self.y_mesh_ = np.zeros(n)
self.beta_ = np.zeros(n)
m_y = np.zeros(n)
m_z = np.zeros(n)
for i in range(n):
m_y[i] = self.locreg.predict(np.atleast_2d(
self.x_mesh_[i]).T).squeeze()
self.locreg.fit_partial(np.atleast_2d(self.Z_).T, h)
m_z[i] = self.locreg.predict_partial().squeeze()
self.locreg.fit_partial(
(self.y_train_ - m_y[i]) * (self.Z_ - m_z[i]), h)
self.cov_[i] = self.locreg.predict_partial().squeeze()
self.locreg.fit_partial((self.Z_ - m_z[i])**2, h)
self.var_[i] = self.locreg.predict_partial().squeeze()
self.beta_[i] = -self.cov_[i] / self.var_[i]
self.locreg.fit_partial(
self.y_train_ + self.beta_[i] * (self.Z_ - self.mz_[i]), h)
self.y_mesh_[i] = self.locreg.predict_partial()
def predict(self, X):
"""
Predicts the conditional expectation $E[Y \mid X=x]$ for all x in $X$.
:param X: a numpy array of shape (num_predictions, num_dimensions)
:return: a numpy array of shape (num_predictions,)
"""
self._calculate_locregs()
self.gpr_ = GPRegression(self.x_mesh_,
np.atleast_2d(self.y_mesh_).T,
self.gpr_kernel)
self.gpr_.optimize(messages=False)
#self.gpr_.optimize_restarts(num_restarts = 10)
y_pred, self.gp_var_ = self.gpr_.predict(X)
self.gp_var_ = self.gp_var_.squeeze()
return y_pred.squeeze()
class EmpiricalStickBreakingGPModel(Model):
"""
Compute the empirical probability given the counts,
convert the empirical probability into a real valued
vector that can be modeled with a GP.
"""
def __init__(self, K, kernel, D=1, alpha=1):
self.alpha = alpha
self.K = K
self.D = D
self.kernel = kernel
def add_data(self, Z, X, optimize_hypers=True):
assert Z.ndim == 2 and Z.shape[1] == self.D
M = Z.shape[0]
assert X.shape == (M, self.K)
# Get the empirical probabilities (offset by 1 to ensure nonzero)
pi_emp_train = (self.alpha+X).astype(np.float) / \
(self.alpha + X).sum(axis=1)[:,None]
# Convert these to psi's
self.Z = Z
self.psi = np.array([pi_to_psi(pi) for pi in pi_emp_train])
# Compute the mean value of psi
self.mu = self.psi.mean(axis=0)
self.psi -= self.mu
# Create the GP Regression model
from GPy.models import GPRegression
self.model = GPRegression(Z, self.psi, self.kernel)
# Optimize the kernel parameters
if optimize_hypers:
self.model.optimize(messages=True)
def initialize_from_data(self, initialize_to_mle=False):
"For consistency"
pass
def generate(self, keep=True, **kwargs):
raise NotImplementedError
def collapsed_predict(self, Z_test):
psi_pred, psi_pred_var = self.model.predict(Z_test, full_cov=False)
psi_pred += self.mu
pi_pred = np.array([psi_to_pi(psi) for psi in psi_pred])
return pi_pred, psi_pred, psi_pred_var
def predict(self, Z_test):
return self.collapsed_predict(Z_test)
def predictive_log_likelihood(self, Z_test, X_test):
pi_pred, _, _ = self.predict(Z_test)
pll = 0
pll += gammaln(X_test.sum(axis=1)+1).sum() - gammaln(X_test+1).sum()
pll += np.nansum(X_test * np.log(pi_pred))
return pll, pi_pred
class EmpiricalStickBreakingGPModel(Model):
"""
Compute the empirical probability given the counts,
convert the empirical probability into a real valued
vector that can be modeled with a GP.
"""
def __init__(self, K, kernel, D=1, alpha=1):
self.alpha = alpha
self.K = K
self.D = D
self.kernel = kernel
def add_data(self, Z, X, optimize_hypers=True):
assert Z.ndim == 2 and Z.shape[1] == self.D
M = Z.shape[0]
assert X.shape == (M, self.K)
# Get the empirical probabilities (offset by 1 to ensure nonzero)
pi_emp_train = (self.alpha+X).astype(np.float) / \
(self.alpha + X).sum(axis=1)[:,None]
# Convert these to psi's
self.Z = Z
self.psi = np.array([pi_to_psi(pi) for pi in pi_emp_train])
# Compute the mean value of psi
self.mu = self.psi.mean(axis=0)
self.psi -= self.mu
# Create the GP Regression model
from GPy.models import GPRegression
self.model = GPRegression(Z, self.psi, self.kernel)
# Optimize the kernel parameters
if optimize_hypers:
self.model.optimize(messages=True)
def initialize_from_data(self, initialize_to_mle=False):
"For consistency"
pass
def generate(self, keep=True, **kwargs):
raise NotImplementedError
def collapsed_predict(self, Z_test):
psi_pred, psi_pred_var = self.model.predict(Z_test, full_cov=False)
psi_pred += self.mu
pi_pred = np.array([psi_to_pi(psi) for psi in psi_pred])
return pi_pred, psi_pred, psi_pred_var
def predict(self, Z_test):
return self.collapsed_predict(Z_test)
def predictive_log_likelihood(self, Z_test, X_test):
pi_pred, _, _ = self.predict(Z_test)
pll = 0
pll += gammaln(X_test.sum(axis=1)+1).sum() - gammaln(X_test+1).sum()
pll += np.nansum(X_test * np.log(pi_pred))
return pll, pi_pred
X_train = np.array([-4, -3, -2, -1, 3]).reshape(-1, 1) Y_train = np.sin(X_train) rbf = RBF(input_dim=1, variance=1.0, lengthscale=1.0) brownian = Brownian(input_dim=1, variance=1.0) periodic = PeriodicExponential(input_dim=1, variance=2.0, n_freq=100) cosine = Cosine(input_dim=1, variance=2) exponential = Exponential(input_dim=1, variance=2.0) integral = Integral(input_dim=1, variances=2.0) matern = Matern32(input_dim=1, variance=2.0) gpr = GPRegression(X_train, Y_train, matern) # Fix the noise variance to known value gpr.Gaussian_noise.variance = noise**2 gpr.Gaussian_noise.variance.fix() # Run optimization ret = gpr.optimize() print(ret) # Display optimized parameter values print(gpr) # Obtain optimized kernel parameters #l = gpr.rbf.lengthscale.values[0] #sigma_f = np.sqrt(gpr.rbf.variance.values[0]) # Plot the results with the built-in plot function gpr.plot() plt.show()
from deepgp import DeepGP
from visualization import plot_gp, model_output, pred_range, visualize_pinball
data = pods.datasets.olympic_marathon_men()
x = data['X']
y = data['Y']
offset = np.mean(y)
scale = np.sqrt(np.var(y))
xlim = (1875, 2030)
ylim = (2.5, 6.5)
yhat = (y - offset) / scale
gp_regression = GPRegression(x, yhat)
gp_regression.optimize()
model_output(gp_regression)
###################deep gp
hidden = 1
dgp = DeepGP(
[y.shape[1], hidden, x.shape[1]],
Y=yhat,
X=x,
inits=['PCA', 'PCA'],
kernels=[RBF(hidden, ARD=True),
RBF(x.shape[1], ARD=True)], # the kernels for each layer
num_inducing=50,
back_constraint=False)
data = pods.datasets.della_gatta_TRP63_gene_expression(data_set='della_gatta',
gene_number=937)
x = data['X']
y = data['Y']
offset = np.mean(y)
scale = np.sqrt(np.var(y))
yhat = (y - offset) / scale
#kernel = RBF(input_dim=1, variance=100)
#kernel = Matern32(input_dim=1, variance=2.0, lengthscale=200)
model = GPRegression(x, yhat)
model.kern.lengthscale = 20 #this will widen with 100, 200
#gp_regression.likelihood.variance = 0.001
print(model.log_likelihood())
model.optimize()
print(model.log_likelihood())
xt = np.linspace(-20, 260, 100)[:, np.newaxis]
yt_mean, yt_var = model.predict(xt)
plot_gp(yt_mean,
yt_var,
xt,
X_train=model.X.flatten(),
Y_train=model.Y.flatten())
# Check that resonator geometry is sensible:
if l_ind[g] > l_cap[g]/2:
out[g,0] = 10e20 #Large cost to bad geometry
else:
out[g,0] = -simulation_wrapper(host, COMSOL_model, paramfile, w[g], t, l_ind[g], pen, omega, gap_cap[g], w_cap[g], l_cap[g], w_mesa, h_mesa, gap_ind[g])[0]
return out
# Set up random seeding of parameter space
num_data_points = no_random_seeds
design = RandomDesign(parameter_space)
X = design.get_samples(num_data_points)
Y = q(X)
# Set up emukit model
model_gpy = GPRegression(X,Y)
model_gpy.optimize()
model_emukit = GPyModelWrapper(model_gpy)
# Set up Bayesian optimisation routine
exp_imprv = ExpectedImprovement(model = model_emukit)
optimizer = GradientAcquisitionOptimizer(space = parameter_space)
point_calc = SequentialPointCalculator(exp_imprv,optimizer)
# Bayesian optimisation routine
bayesopt_loop = BayesianOptimizationLoop(model = model_emukit,
space = parameter_space,
acquisition=exp_imprv,
batch_size=1)
stopping_condition = FixedIterationsStoppingCondition(i_max = no_BO_sims)
bayesopt_loop.run_loop(q, stopping_condition)
