def model(self, x_init, y_init, functions):
n_fidelities = len(functions)
base_kernels = [GPy.kern.RBF(1) for _ in range(len(functions))]
k = emukit.multi_fidelity.kernels.LinearMultiFidelityKernel(base_kernels)
# Train model
np.random.seed(123)
gpy_model = GPyLinearMultiFidelityModel(x_init, y_init, k, n_fidelities)
model = GPyMultiOutputWrapper(gpy_model, n_fidelities, n_optimization_restarts=5)
model.optimize()
return model
def log_linear_mfdgp(X_train, Y_train):
kernels = [GPy.kern.RBF(5), GPy.kern.RBF(5)]
lin_mf_kernel = emukit.multi_fidelity.kernels.LinearMultiFidelityKernel(
kernels)
gpy_lin_mf_model = GPyLinearMultiFidelityModel(
X_train, Y_train, lin_mf_kernel, n_fidelities=2)
gpy_lin_mf_model.mixed_noise.Gaussian_noise.fix(0)
gpy_lin_mf_model.mixed_noise.Gaussian_noise.fix(0)
lin_mf_model = GPyMultiOutputWrapper(
gpy_lin_mf_model, 2, n_optimization_restarts=5)
lin_mf_model.optimize()
return lin_mf_model
def train(self, x_l, y_l, x_h, y_h):
# Construct a linear multi-fidelity model
X_train, Y_train = convert_xy_lists_to_arrays([x_l, x_h], [y_l, y_h])
kernels = [GPy.kern.RBF(x_l.shape[1]), GPy.kern.RBF(x_h.shape[1])]
kernel = emukit.multi_fidelity.kernels.LinearMultiFidelityKernel(
kernels)
gpy_model = GPyLinearMultiFidelityModel(X_train,
Y_train,
kernel,
n_fidelities=2)
if self.noise is not None:
gpy_model.mixed_noise.Gaussian_noise.fix(self.noise)
gpy_model.mixed_noise.Gaussian_noise_1.fix(self.noise)
# Wrap the model using the given 'GPyMultiOutputWrapper'
self.model = GPyMultiOutputWrapper(
gpy_model, 2, n_optimization_restarts=self.n_optimization_restarts)
# Fit the model
self.model.optimize()
def linear_model(X_train, Y_train, save=False):
""" Returns GPy model of linear multi fidelity GP."""
kernels = [GPy.kern.RBF(1), GPy.kern.RBF(1)]
lin_mf_kernel = emukit.multi_fidelity.kernels.LinearMultiFidelityKernel(
kernels)
gpy_lin_mf_model = GPyLinearMultiFidelityModel(X_train,
Y_train,
lin_mf_kernel,
n_fidelities=2)
# Fix noise to kernel
gpy_lin_mf_model.mixed_noise.Gaussian_noise.fix(0)
gpy_lin_mf_model.mixed_noise.Gaussian_noise_1.fix(0)
# Optimise
lin_mf_model = GPyMultiOutputWrapper(gpy_lin_mf_model,
2,
n_optimization_restarts=5)
lin_mf_model.optimize()
if save is not False:
lin_mf_model.save_model(save)
return lin_mf_model
class LinearMFGP(object):
def __init__(self, noise=None, n_optimization_restarts=10):
self.noise = noise
self.n_optimization_restarts = n_optimization_restarts
self.model = None
def train(self, x_l, y_l, x_h, y_h):
# Construct a linear multi-fidelity model
X_train, Y_train = convert_xy_lists_to_arrays([x_l, x_h], [y_l, y_h])
kernels = [GPy.kern.RBF(x_l.shape[1]), GPy.kern.RBF(x_h.shape[1])]
kernel = emukit.multi_fidelity.kernels.LinearMultiFidelityKernel(
kernels)
gpy_model = GPyLinearMultiFidelityModel(X_train,
Y_train,
kernel,
n_fidelities=2)
if self.noise is not None:
gpy_model.mixed_noise.Gaussian_noise.fix(self.noise)
gpy_model.mixed_noise.Gaussian_noise_1.fix(self.noise)
# Wrap the model using the given 'GPyMultiOutputWrapper'
self.model = GPyMultiOutputWrapper(
gpy_model, 2, n_optimization_restarts=self.n_optimization_restarts)
# Fit the model
self.model.optimize()
def predict(self, x):
# Convert x_plot to its ndarray representation
X = convert_x_list_to_array([x, x])
X_l = X[:len(x)]
X_h = X[len(x):]
# Compute mean predictions and associated variance
lf_mean, lf_var = self.model.predict(X_l)
lf_std = np.sqrt(lf_var)
hf_mean, hf_var = self.model.predict(X_h)
hf_std = np.sqrt(hf_var)
return lf_mean, lf_std, hf_mean, hf_std
## Construct a linear multi-fidelity model
kernels = [GPy.kern.RBF(1), GPy.kern.RBF(1)]
lin_mf_kernel = emukit.multi_fidelity.kernels.LinearMultiFidelityKernel(
kernels)
gpy_lin_mf_model = GPyLinearMultiFidelityModel(X_train,
Y_train,
lin_mf_kernel,
n_fidelities=2)
# print(gpy_lin_mf_model)
gpy_lin_mf_model.mixed_noise.Gaussian_noise.fix(0)
gpy_lin_mf_model.mixed_noise.Gaussian_noise_1.fix(0)
lin_mf_model = model = GPyMultiOutputWrapper(gpy_lin_mf_model,
2,
n_optimization_restarts=5,
verbose_optimization=False)
## Fit the model
lin_mf_model.optimize()
####
## Convert test points to appropriate representation
# print(x_plot)
X_plot = convert_x_list_to_array([x_plot, x_plot])
# print(X_plot)
X_plot_low = X_plot[:200]
X_plot_high = X_plot[200:]
## Construct a linear multi-fidelity model
kernels = [GPy.kern.RBF(1), GPy.kern.RBF(1)]
lin_mf_kernel = emukit.multi_fidelity.kernels.LinearMultiFidelityKernel(
kernels)
gpy_lin_mf_model = GPyLinearMultiFidelityModel(X_train,
Y_train,
lin_mf_kernel,
n_fidelities=2)
gpy_lin_mf_model.mixed_noise.Gaussian_noise.fix(0)
gpy_lin_mf_model.mixed_noise.Gaussian_noise_1.fix(0)
## Wrap the model using the given 'GPyMultiOutputWrapper'
lin_mf_model = model = GPyMultiOutputWrapper(gpy_lin_mf_model,
2,
n_optimization_restarts=5)
## Fit the model
lin_mf_model.optimize()
## Convert x_plot to its ndarray representation
X_plot = convert_x_list_to_array([x_plot, x_plot])
X_plot_l = X_plot[:len(x_plot)]
X_plot_h = X_plot[len(x_plot):]
## Compute mean predictions and associated variance
lf_mean_lin_mf_model, lf_var_lin_mf_model = lin_mf_model.predict(X_plot_l)
# Train model
lin_mf_model = linear_mfdgp(X_train, Y_train)
# Training log likelihood
train_log_lik = lin_mf_model.log_likelihood()
hf_train_ll.append(train_log_lik)
# Heldout log likelihood
x_met = convert_x_list_to_array([x_val, x_val])
n = x_val.shape[0]
held_log_lik = test_log_likelihood(lin_mf_model, x_met[:n], y_val)
hf_heldout_ll.append(held_log_lik)
# R2
wrp_model = GPyMultiOutputWrapper(lin_mf_model,
2,
n_optimization_restarts=1)
lin_mf_l_y_pred, mf_l_y_std_pred = wrp_model.predict(x_met[:n])
lin_mf_h_y_pred, mf_h_y_std_pred = wrp_model.predict(x_met[n:])
R2_hf = r2_score(y_val, lin_mf_h_y_pred)
R2_lf = r2_score(y_val, lin_mf_l_y_pred)
R2_hf_list.append(R2_hf)
R2_lf_list.append(R2_lf)
plt.figure(figsize=(7, 7))
plt.scatter(R2_lf_list, R2_hf_list, s=50)
plt.scatter(0, 0, marker='+', color='r', s=100)
x = np.linspace(-2, 1, 2)
plt.plot(x, x, linestyle='--', color='grey')
plt.xlabel('Low-fidelity R2')
plt.ylabel('High-fidelity R2')
def denvsrecmain(n_fid, x_dim, n):
#################
### Functions ###
#################
f_exact = test_funs.ackley
def f_name():
return f_exact(0)[1]
def f_m(x):
return f_exact(x)[0]
a = 4 * (np.random.rand(n_fid - 1) - .5)
b = 4 * (np.random.rand(n_fid - 1) - .5)
c = 4 * (np.random.rand(n_fid - 1) - .5)
d = np.random.randint(-4, 5, size=(n_fid - 1, x_dim + 1))
def f(x, fid):
if fid == n_fid - 1:
return f_m(x)
else:
x1_ptb = np.array([x_i[0] - d[fid][0] for x_i in x])[:, None]
return a[fid] * f(x, fid + 1) + b[fid] * x1_ptb + c[fid]
###############
### x_dim-D ###
###############
### Plotting data ###
x_min = [-5] * x_dim
x_max = [5] * x_dim
x_plot_grid = [
np.linspace(x_min[j], x_max[j], 50)[:, None] for j in range(x_dim)
]
x_plot_mesh = np.meshgrid(*x_plot_grid)
x_plot_list = np.hstack([layer.reshape(-1, 1) for layer in x_plot_mesh])
X_plot_mf = convert_x_list_to_array([x_plot_list] * n_fid)
X_plot_mf_list = X_plot_mf.reshape((n_fid, len(x_plot_list), x_dim + 1))
### Training data ###
x_train = [x_plot_list[::len(x_plot_list) // n_i] for n_i in n
] # Include possibility to insert training data of choice.
y_train = [f(x_train[j], j) for j in range(n_fid)]
X_train_mf, Y_train_mf = convert_xy_lists_to_arrays(x_train, y_train)
############################
### DENSE GP CALCULATION ###
############################
n_opt_restarts = 3
kernels_mf = []
for k in range(n_fid):
kernels_mf.append(GPy.kern.RBF(input_dim=x_dim))
lin_mf_kernel = emukit.multi_fidelity.kernels.LinearMultiFidelityKernel(
kernels_mf)
# print(lin_mf_kernel)
# print(lin_mf_kernel.kernels[0])
start_den = time.time()
# print(X_train_mf)
gpy_m_den_mf = GPyLinearMultiFidelityModel(X_train_mf,
Y_train_mf,
lin_mf_kernel,
n_fidelities=n_fid)
# print(gpy_m_den_mf)
### Fixing kernel parameters ###
for k in range(n_fid):
gpy_m_den_mf.mixed_noise.likelihoods_list[k].fix(0)
# print(gpy_m_den_mf)
m_den_mf = GPyMultiOutputWrapper(gpy_m_den_mf,
n_fid,
n_optimization_restarts=n_opt_restarts,
verbose_optimization=False)
end_den_1 = time.time()
# print('Dense MFGPR construction', end_den_1 - start_den)
### Dense HPO ###
m_den_mf_pre_HPO = m_den_mf
m_den_mf.optimize()
print(gpy_m_den_mf)
# print(gpy_m_den_mf.kern)
# print(lin_mf_kernel)
# print(lin_mf_kernel.kernels[0])
# print(X_train_mf)
# test = lin_mf_kernel.K(X=X_train_mf)
# print(test)
# print(lin_mf_kernel)
end_den_2 = time.time()
# print('Dense MFGPR construction + HPO', end_den_2 - start_den)
# print(gpy_m_den_mf)
### Prediction ###
# for j in range(n_fid):
# a = time.time()
# test = m_den_mf.predict(X_plot_mf_list[j])
# b = time.time()
# print(b - a)
mu_den_mf = [m_den_mf.predict(X_plot_mf_list[j])[0] for j in range(n_fid)]
# print(X_plot_mf_list)
# print(mu_den_mf)
# sigma_den_mf = [m_den_mf.predict(X_plot_mf_list[j])[1] for j in range(n_fid)]
#
# end_den_3 = time.time()
# print('Dense MFGPR construction + HPO + prediction', end_den_3 - start_den)
################################
### RECURSIVE GP CALCULATION ###
################################
start_rec = time.time()
# m_rec_mf = GPy.models.multiGPRegression(x_train, y_train, kernel=[GPy.kern.RBF(x_dim) for i in
# range(n_fid)]) # Improve kernel selection...?
end_rec_1 = time.time()
# print('Recursive MFGPR construction', end_rec_1 - start_rec)
# for k in range(n_fid): m_rec_mf.models[k]['Gaussian_noise.variance'].fix(0)
### Recursive HPO ###
# m_rec_mf_pre_HPO = m_rec_mf
# m_rec_mf.optimize_restarts(restarts=n_opt_restarts, verbose=False)
# for j in range(n_fid): print(m_rec_mf.models[j])
end_rec_2 = time.time()
# print('Recursive MFGPR construction + HPO', end_rec_2 - start_rec)
# for k in range(m): print(m_rec_mf.models[k])
### Prediction ###
# mu_rec_mf, sigma_rec_mf = m_rec_mf.predict(x_plot_list)
# print(mu_rec_mf)
#
# end_rec_3 = time.time()
# print('Recursive MFGPR construction + HPO + prediction', end_rec_3 - start_rec)
# times = [np.array([end_den_1, end_den_2, end_den_3]) - start_den, np.array([end_rec_1, end_rec_2, end_rec_3]) - start_rec]
times = [end_den_2 - start_den, end_rec_2 - start_rec]
return times, n_fid, x_dim
gpy_m_den_mf = GPyLinearMultiFidelityModel(X_train_mf, Y_train_mf, lin_mf_kernel, n_fidelities=m)
# gpy_m_den = GPyLinearMultiFidelityModel(X_train, Y_train, lin_mf_kernel, n_fidelities=3)
end1 = time.time()
print('dense', end1 - start)
for k in range(m): gpy_m_den_mf.mixed_noise.likelihoods_list[k].fix(0)
# print(gpy_m_den_mf)
# for a in gpy_m_den_mf.mixed_noise: print(a)
# gpy_m_den.mixed_noise.Gaussian_noise.fix(0)
# gpy_m_den.mixed_noise.Gaussian_noise_1.fix(0)
# gpy_m_den.mixed_noise.Gaussian_noise_2.fix(0)
m_den_mf = GPyMultiOutputWrapper(gpy_m_den_mf, m, n_optimization_restarts=4, verbose_optimization=True)
# m_den = GPyMultiOutputWrapper(gpy_m_den, 3, n_optimization_restarts=4, verbose_optimization=False)
m_den_mf_pre_HPO = m_den_mf
m_den_mf.optimize()
print(gpy_m_den_mf)
# m_den.optimize()
end2 = time.time()
print('dense + HPO', end2 - start)
# print(gpy_m_den)
# Prediction
X_plot_mf = convert_x_list_to_array([x_plot] * m)
# X_plot = convert_x_list_to_array([x_plot, x_plot, x_plot])
# X_plot_0 = X_plot[:len(x_plot)]