def sklearn_fit_optimize(index, matrix):
raise NotImplementedError
from sklearn.gaussian_process import GaussianProcessRegressor
from sklearn.gaussian_process.kernels import RBF, ConstantKernel, WhiteKernel
y = matrix.loc[index, :].values
x = np.log2(1 +
matrix.columns.get_level_values('timepoint').str.replace(
"d", "").astype(int).values.reshape((y.shape[0], 1)))
kernel = 1 * (RBF(length_scale=1.0,
length_scale_bounds=(1e-05, 100000.0))) # +
# WhiteKernel(noise_level=1.0, noise_level_bounds=(1e-05, 100000.0)))# +
# ConstantKernel(constant_value=1.0, constant_value_bounds=(1e-05, 100000.0)))
white_kernel = 1 * ( # WhiteKernel(noise_level=1.0, noise_level_bounds=(1e-05, 100000.0)) )# +
ConstantKernel(constant_value=0.,
constant_value_bounds=(1e-05, 100000.0)))
# white_kernel = ConstantKernel(constant_value=1.0, constant_value_bounds=(1e-05, 100000.0))
m = GaussianProcessRegressor(kernel, normalize_y=False)
try:
m.fit(x, y)
except:
return np.nan, np.nan
w_m = GaussianProcessRegressor(white_kernel, normalize_y=False)
try:
w_m.fit(x, y)
except:
return np.nan, np.nan
return m.log_marginal_likelihood(), w_m.log_marginal_likelihood()
def test_gpr_lml_error():
"""Check that we raise the proper error in the LML method."""
gpr = GaussianProcessRegressor(kernel=RBF()).fit(X, y)
err_msg = "Gradient can only be evaluated for theta!=None"
with pytest.raises(ValueError, match=err_msg):
gpr.log_marginal_likelihood(eval_gradient=True)
def test_lml_without_cloning_kernel(kernel):
# Test that lml of optimized kernel is stored correctly.
gpr = GaussianProcessRegressor(kernel=kernel).fit(X, y)
input_theta = np.ones(gpr.kernel_.theta.shape, dtype=np.float64)
gpr.log_marginal_likelihood(input_theta, clone_kernel=False)
assert_almost_equal(gpr.kernel_.theta, input_theta, 7)
def test_lml_improving():
""" Test that hyperparameter-tuning improves log-marginal likelihood. """
for kernel in kernels:
if kernel == fixed_kernel: continue
gpr = GaussianProcessRegressor(kernel=kernel).fit(X, y)
assert_greater(gpr.log_marginal_likelihood(gpr.kernel_.theta),
gpr.log_marginal_likelihood(kernel.theta))
def test_lml_improving(kernel):
if sys.maxsize <= 2**32:
pytest.xfail("This test may fail on 32 bit Python")
# Test that hyperparameter-tuning improves log-marginal likelihood.
gpr = GaussianProcessRegressor(kernel=kernel).fit(X, y)
assert gpr.log_marginal_likelihood(
gpr.kernel_.theta) > gpr.log_marginal_likelihood(kernel.theta)
def test_lml_improving():
""" Test that hyperparameter-tuning improves log-marginal likelihood. """
for kernel in kernels:
if kernel == fixed_kernel:
continue
gpr = GaussianProcessRegressor(kernel=kernel).fit(X, y)
assert_greater(gpr.log_marginal_likelihood(gpr.kernel_.theta),
gpr.log_marginal_likelihood(kernel.theta))
def test_lml_gradient(kernel):
# Compare analytic and numeric gradient of log marginal likelihood.
gpr = GaussianProcessRegressor(kernel=kernel).fit(X, y)
lml, lml_gradient = gpr.log_marginal_likelihood(kernel.theta, True)
lml_gradient_approx = approx_fprime(
kernel.theta, lambda theta: gpr.log_marginal_likelihood(theta, False),
1e-10)
assert_almost_equal(lml_gradient, lml_gradient_approx, 3)
def test_lml_gradient():
""" Compare analytic and numeric gradient of log marginal likelihood. """
for kernel in kernels:
gpr = GaussianProcessRegressor(kernel=kernel).fit(X, y)
lml, lml_gradient = gpr.log_marginal_likelihood(kernel.theta, True)
lml_gradient_approx = approx_fprime(
kernel.theta, lambda theta: gpr.log_marginal_likelihood(theta, False), 1e-10
)
assert_almost_equal(lml_gradient, lml_gradient_approx, 3)
def sk_train(self,
X_train,
y_train,
kernels=['rbf', 'matern32', 'matern52'],
offset=False,
verboseQ=False,
saveQ=True):
for kern in (kernels):
if verboseQ: print('********* \n', kern)
sk_kernel = self.SK_kernel(kern, X_train.shape[1])
sk_kernel += WhiteKernel(noise_level=0.1**2,
noise_level_bounds=(1e-8, 10)) # noise
if offset: #add bias kernel
sk_kernel += Ck(1)
t0 = time.time()
gpr = GaussianProcessRegressor(kernel=sk_kernel,
n_restarts_optimizer=5)
gpr.fit(X_train, y_train)
self.sk_t = time.time() - t0
if verboseQ:
print('took ', self.sk_t, ' seconds')
print("Inital kernel: %s" % gpr.kernel)
print("Learned kernel: %s" % gpr.kernel_)
print("Log-marginal-likelihood: %.3f" %
gpr.log_marginal_likelihood(gpr.kernel_.theta))
if saveQ:
with open('SK_gpr_' + str(kern) + '.pickle', 'wb') as handle:
pickle.dump(gpr, handle, protocol=pickle.HIGHEST_PROTOCOL)
self.results['ll'] = gpr.log_marginal_likelihood(gpr.kernel_.theta)
try:
self.results['amp_param'] = gpr.kernel_.get_params(
)['k1__k1__k1__constant_value']
self.results['noise_param_variance'] = gpr.kernel_.get_params(
)['k1__k2__noise_level']
self.results['length_scale_param'] = gpr.kernel_.get_params(
)['k1__k1__k2__length_scale'].tolist()
self.results['offset_param'] = gpr.kernel_.get_params(
)['k2__constant_value']
except:
self.results['length_scale_param'] = gpr.kernel_.get_params(
)['k1__k2__length_scale'].tolist()
self.results['noise_param_variance'] = gpr.kernel_.get_params(
)['k2__noise_level']
self.results['amp_param'] = gpr.kernel_.get_params(
)['k1__k1__constant_value']
self.results['offset_param'] = None
def gene_expression_classifier(expr_data, profile_cols=None):
"""
The logic below is based on Kalaitzis & Lawrence 2011.
The implementation seems to work and the results that I'm
getting are close to what their R script is reporting, though
there are small discrepancies in the numbers.
:param expr_data: The expression data as a pandas dataframe
:param profile_cols: The profile column names (see example below).
:return: Return updated dataframe with likelihoods computed and compared.
"""
if profile_cols is None:
profile_cols = np.array(
[0, 20, 40, 60, 80, 100, 120, 140, 160, 180, 200, 220, 240],
dtype=np.float64)
profile_cols.shape = (len(profile_cols), 1)
for index, row in expr_data.iterrows():
profile = np.array(row, dtype=np.float64)
profile.shape = (len(profile), 1)
# Calculate the variance in the profile
profile_var = np.var(profile)
# Create noisy GP
noisy_kern = .001 * profile_var * RBF(
length_scale=1. / 1000) + WhiteKernel(.999 * profile_var)
noisy_m = GaussianProcessRegressor(kernel=noisy_kern,
n_restarts_optimizer=10)
# Fit GP to data
noisy_m.fit(profile_cols, profile)
# Compute log marginal likelihood of fitted GP
expr_data.loc[index,
'opt_likelihood'] = noisy_m.log_marginal_likelihood()
# Compute log marginal likelihood of hyperparameters that correspond to noise.
expr_data.loc[index,
'noisy_likelihood'] = noisy_m.log_marginal_likelihood(
theta=[
np.log(.999 * profile_var),
np.log(.05),
np.log(.001 * profile_var)
])
expr_data.loc[index, 'likelihood_ratio'] = expr_data.loc[index, 'opt_likelihood'] - \
expr_data.loc[index, 'noisy_likelihood']
return expr_data
def sk_kernel(self, hypers_dict):
amp = hypers_dict['amplitude_covar']
lengthscales = np.diag(hypers_dict['precisionMatrix'])**-0.5
noise_var = hypers_dict['noise_variance']
se_ard = Ck(amp) * RBF(length_scale=lengthscales,
length_scale_bounds=(1e-6, 10))
noise = WhiteKernel(noise_level=noise_var,
noise_level_bounds=(1e-9, 1)) # noise terms
sk_kernel = se_ard
if self.noiseQ:
sk_kernel += noise
t0 = time.time()
gpr = GaussianProcessRegressor(kernel=sk_kernel,
n_restarts_optimizer=5)
print("Initial kernel: %s" % gpr.kernel)
# self.ytrain = [y[0][0] for y in self.Y_obs]
gpr.fit(self.X_obs, np.array(self.Y_obs).flatten())
print('SK fit time is ', time.time() - t0)
print("Learned kernel: %s" % gpr.kernel_)
print("Log-marginal-likelihood: %.3f" %
gpr.log_marginal_likelihood(gpr.kernel_.theta))
#print(gpr.kernel_.get_params())
if self.noiseQ:
# RBF w/ noise
sk_ls = gpr.kernel_.get_params()['k1__k2__length_scale']
sk_amp = gpr.kernel_.get_params()['k1__k1__constant_value']
sk_loklik = gpr.log_marginal_likelihood(gpr.kernel_.theta)
sk_noise = gpr.kernel_.get_params()['k2__noise_level']
else:
#RBF w/o noise
sk_ls = gpr.kernel_.get_params()['k2__length_scale']
sk_amp = gpr.kernel_.get_params()['k1__constant_value']
sk_loklik = gpr.log_marginal_likelihood(gpr.kernel_.theta)
sk_noise = 0
# make dict
sk_hypers = {}
sk_hypers['precisionMatrix'] = np.diag(1. / (sk_ls**2))
sk_hypers['noise_variance'] = sk_noise
sk_hypers['amplitude_covar'] = sk_amp
return sk_loklik, sk_hypers
def test_gp_regression():
while True:
alpha = np.random.rand()
N = np.random.randint(2, 100)
M = np.random.randint(2, 100)
K = np.random.randint(1, N)
J = np.random.randint(1, 3)
X = np.random.rand(N, M)
y = np.random.rand(N, J)
X_test = np.random.rand(K, M)
gp = GPRegression(kernel="RBFKernel(sigma=1)", alpha=alpha)
gold = GaussianProcessRegressor(kernel=None,
alpha=alpha,
optimizer=None,
normalize_y=False)
gp.fit(X, y)
gold.fit(X, y)
preds, _ = gp.predict(X_test)
gold_preds = gold.predict(X_test)
np.testing.assert_almost_equal(preds, gold_preds)
mll = gp.marginal_log_likelihood()
gold_mll = gold.log_marginal_likelihood()
np.testing.assert_almost_equal(mll, gold_mll)
print("PASSED")
def fit_and_test_n_dim(samples, rewards, n, alpha=0.1, length_scale=5, use_nn = False):
if use_nn:
nn = Sequential()
nn.add(Dense(16, input_shape=(8,), activation="relu"))
nn.add(Dense(8, activation="relu"))
nn.add(Dense(1, activation="linear"))
opt = Adam(lr=0.01)
nn.compile(loss="mse", optimizer=opt)
else:
gp_kernel = C(12.0, (1e-3, 2e3)) * RBF(length_scale)
gp = GaussianProcessRegressor(kernel=gp_kernel, n_restarts_optimizer=8, alpha=alpha)
num_training = 300
relevant_samples_training = samples[:num_training,:n ]
relevant_rewards_training = rewards[:num_training]
relevant_samples_test = samples[num_training:,:n ]
relevant_rewards_test = rewards[num_training:]
if use_nn:
nn.fit(relevant_samples_training, relevant_rewards_training, epochs=10000, batch_size=800)
ll = 0
predictions = nn.predict_on_batch(relevant_samples_test)
else:
print("before fitting")
gp.fit(relevant_samples_training, relevant_rewards_training)
print("after fitting")
ll = gp.log_marginal_likelihood()
predictions = gp.predict(relevant_samples_test)
se = (predictions-relevant_rewards_test)**2
mse = np.mean(se)
plt.scatter(relevant_rewards_test, se)
plt.show()
return mse, ll
def gp(self,true_graph, training_idx):
training_X = []
training_Y = []
for idx in training_idx:
training_Y.append(true_graph.vertex_[idx].ig_)
nbs_p = self.adjacent_p(idx,true_graph)
print("The vert {}".format(idx))
print("The training X is {}".format(nbs_p))
print("The training Y is {}".format(training_Y[-1]))
print("===============================================")
training_X.append(nbs_p)
rbf = 1.0 * RBF(length_scale=1.0)
matern = 1.0 * Matern(length_scale=1.0, length_scale_bounds=(1e-1, 10.0),
nu=1.5)
gp_opt = GaussianProcessRegressor(kernel=rbf)
gp_opt.fit(training_X,training_Y)
print("The trained hyperparameter are {}".format((gp_opt.kernel_.theta)))
print("Log Marginal Likelihood (optimized): %.3f"
% gp_opt.log_marginal_likelihood(gp_opt.kernel_.theta))
# Contour 3d
x1_ = [i for i in range(self.rows_+1)]
x2_ = [j for j in range(self.cols_+1)]
X1_, X2_ = np.meshgrid(x1_,x2_)
y_mean = np.empty([self.rows_+1,self.cols_+1])
y_true = np.empty([self.rows_+1,self.cols_+1])
y_std = np.empty([self.rows_+1,self.cols_+1])
y_mean_u = np.empty([self.rows_+1,self.cols_+1])
y_mean_d = np.empty([self.rows_+1,self.cols_+1])
for i in range(self.rows_+1):
for j in range(self.cols_+1):
cur_idx = (self.rows_-X1_[i][j]-1)*self.cols_+X2_[i][j]
print("X: {}, Y: {}, idx: {}".format(X2_[i][j], X1_[i][j],cur_idx))
if X1_[i][j]<self.rows_ and X1_[i][j]>=0 and X2_[i][j]<self.cols_ and X2_[i][j]>=0 and cur_idx in nz_ig:
neigh_p = self.adjacent_p(cur_idx,true_graph)
y_mean[X2_[i][j],X1_[i][j]], y_std[X2_[i][j],X1_[i][j]] = gp_opt.predict([neigh_p], return_std=True)
print("Prediction ========================")
print("Vertex {}, X:{}, Y:{}".format(cur_idx,X2_[i][j],X1_[i][j]))
print("Testing data is {}".format(neigh_p))
print("Predicted IG {}".format(y_mean[X2_[i][j],X1_[i][j]]))
else:
y_mean[X2_[i][j] ,X1_[i][j]] = 0
y_std[X2_[i][j],X1_[i][j]] = 0.0
y_mean_u[X2_[i][j],X1_[i][j]] = y_mean[X2_[i][j],X1_[i][j]] + y_std[X2_[i][j],X1_[i][j]]
y_mean_d[X2_[i][j],X1_[i][j]] = y_mean[X2_[i][j],X1_[i][j]] - y_std[X2_[i][j],X1_[i][j]]
if X2_[i][j]<self.cols_ and X1_[i][j]<self.rows_ and X1_[i][j]>=0 and X2_[i][j]>=0:
idx_ = (self.rows_-X1_[i][j]-1)*self.cols_+X2_[i][j]
y_true[X2_[i][j],X1_[i][j]] = true_graph.vertex_[idx_].ig_
self.vertex_[idx_].ig_ = y_mean[X2_[i][j],X1_[i][j]]
true_graph.vertex_[idx_].L2_error_ = (true_graph.vertex_[idx_].ig_ - self.vertex_[idx_].ig_)**2
self.vertex_[idx_].ig_ub_ = y_mean[X2_[i][j],X1_[i][j]] + y_std[X2_[i][j],X1_[i][j]]
self.vertex_[idx_].ig_lb_ = y_mean[X2_[i][j],X1_[i][j]] - y_std[X2_[i][j],X1_[i][j]]
if round(true_graph.vertex_[idx_].ig_,3) <= round(self.vertex_[idx_].ig_ub_,3) and round(true_graph.vertex_[idx_].ig_,3) >= round(self.vertex_[idx_].ig_lb_,3):
self.vertex_[idx_].ig_pred_ = True
def test_random_starts():
"""
Test that an increasing number of random-starts of GP fitting only
increases the log marginal likelihood of the chosen theta.
"""
n_samples, n_features = 25, 2
np.random.seed(0)
rng = np.random.RandomState(0)
X = rng.randn(n_samples, n_features) * 2 - 1
y = np.sin(X).sum(axis=1) + np.sin(3 * X).sum(axis=1) \
+ rng.normal(scale=0.1, size=n_samples)
kernel = C(1.0, (1e-2, 1e2)) \
* RBF(length_scale=[1.0] * n_features,
length_scale_bounds=[(1e-4, 1e+2)] * n_features) \
+ WhiteKernel(noise_level=1e-5, noise_level_bounds=(1e-5, 1e1))
last_lml = -np.inf
for n_restarts_optimizer in range(5):
gp = GaussianProcessRegressor(
kernel=kernel,
n_restarts_optimizer=n_restarts_optimizer,
random_state=0,
).fit(X, y)
lml = gp.log_marginal_likelihood(gp.kernel_.theta)
assert_greater(lml, last_lml - np.finfo(np.float32).eps)
last_lml = lml
def fit_model(self, train_x, train_y):
"""
TODO: enter your code here
"""
""" Task 1 : Model selection"""
length_scales = [1, 2, 3, 4]
kernels = [Matern(), RationalQuadratic()]
# kernels = [ExpSineSquared()]
# Grid search:
# for kernel in kernels:
# gp = GaussianProcessRegressor(kernel=kernel, n_restarts_optimizer=9)
# gp.X_train_ = train_x
# gp.y_train_ = train_y
# gp.kernel_ = kernel
# for length_scale in length_scales:
# log_like_GS = gp.log_marginal_likelihood([length_scale])
# print(log_like_GS)
# Optimization method:
P_likelihood = []
Hyper_params = [None] * len(kernels)
for i, kernel in enumerate(kernels):
gp = GaussianProcessRegressor(kernel=kernel, alpha=1e-30)
gp.fit(train_x, train_y)
# print(gp._get_param_names())
# print(gp.get_params())
print("Kernel type:", kernel)
Hyper_params[i] = gp.kernel_.theta
print(f"Hyperparameters {i}:", Hyper_params[i])
P_likelihood.append(gp.log_marginal_likelihood())
print(f"kernel {i}:", P_likelihood[i])
pass
def main(n_samples):
np.random.seed(0)
print('player 0')
data0 = pd.read_csv("./saves/GP0.csv", index_col=0)
print('data\n%s' % (data0.tail(2)))
#pre = {'B':[0,0,0,0],'C':[1,0,1,0],'D':[2,0,1,0],'F':[1,2,1,1]}
#post = {'0': [1,0,0],'1': [0,1,0], '2':[0,0,1]}
raw = data0[-n_samples:]['name']
dic = {
'B': [0],
'C': [1],
'D': [2],
'F': [3],
'0': [0],
'1': [1],
'2': [2]
}
cols = ['pre0', 'post0']
translation = translate(0, raw, dic, cols)
print('translation\n%s' % (translation.tail(2)))
# kernel = W() + C(10, (1e-3, 1e1)) + C(1, (1e-3, 1e1)) * RBF(10, (1e-3, 1e1)) + C(1, (1e-3, 1e1)) * Matern(length_scale=1.0, length_scale_bounds=(1e-3, 1e1), nu=1.5) + C(1, (1e-3, 1e1)) * RationalQuadratic(length_scale=1.0, alpha=0.1)
# kernel = W() + C(1, (1e-3, 1e1)) + C(1, (1e-3, 1e1)) * RBF(1, (1e-3, 1e1)) + C(1, (1e-3, 1e1)) * ESS(1.0, 5.0, periodicity_bounds=(1e-2, 1e1)) + C(1, (1e-3, 1e1)) * DP() + C(1, (1e-3, 1e1)) * ESS(1.0, 5.0, periodicity_bounds=(1e-2, 1e1)) * RBF(1, (1e-3, 1e1)) + C(1, (1e-3, 1e1)) * ESS(1.0, 5.0, periodicity_bounds=(1e-2, 1e1)) * DP() + C(1, (1e-3, 1e1)) * RBF(1, (1e-3, 1e1)) * DP() + C(1, (1e-3, 1e1)) * ESS(1.0, 5.0, periodicity_bounds=(1e-2, 1e1)) * RBF(1, (1e-3, 1e1)) * DP()
# kernel = W() + C(1, (1e-3, 1e1)) + C(1, (1e-3, 1e1)) * RationalQuadratic(length_scale=1.0, alpha=0.1) + ESS(1.0, 2.0, periodicity_bounds=(1e-2, 1e2)) * DP() + C(1, (1e-3, 1e1)) * ESS(1.0, 2.0, periodicity_bounds=(1e-2, 1e2))
kernel = W() + C(1, (1e-3, 1e1)) + C(1, (1e-3, 1e1)) * RBF(
1, (1e-3, 1e1)) + ESS(1.0, 2.0, periodicity_bounds=(1e-2, 1e1)) * DP()
X0_train, X0_test, y0_train, y0_test = train_test_split(
translation, data0[-n_samples:][['va0', 'va1', 'va2']])
gp = GaussianProcessRegressor(kernel=kernel, n_restarts_optimizer=9)
stime = time.time()
gp.fit(X0_train, y0_train)
print("\nTime for GPR fitting: %.3f" % (time.time() - stime))
print("Posterior (kernel: %s)\n Log-Likelihood: %.3f" %
(gp.kernel_, gp.log_marginal_likelihood(gp.kernel_.theta)))
print('training result: %f' % r2_score(y0_train, gp.predict(X0_train)))
print('test result: %f' % r2_score(y0_test, gp.predict(X0_test)))
test = raw.loc[y0_test.index]
dic = {
'B': [[0, 0], [0, 0]],
'C': [[1, 0], [1, 0]],
'D': [[2, 0], [1, 0]],
'F': [[1, 2], [1, 1]],
'0': [[0]],
'1': [[1]],
'2': [[2]]
}
cols = ['action0', 'action1', 'hand']
original = translate(0, test, dic, cols)
M = model('p0')
M.restore()
stime = time.time()
prediction = []
for i in range(test.size):
I = original[['hand', 'action0', 'action1']].iloc[i]
prediction.append(M.predict(I)[0, :])
print("\nNeural Net: %.3f" % (time.time() - stime))
print('test result: %f' % r2_score(y0_test, prediction))
def getgpr(X, y):
kernel = ConstantKernel() + 1.0**2 * Matern(
length_scale=2.0, nu=1.5) + WhiteKernel(noise_level=1)
gp = GaussianProcessRegressor(kernel=kernel, n_restarts_optimizer=10)
gp.fit(X, y)
print(gp.kernel_)
print(gp.log_marginal_likelihood(gp.kernel_.theta))
return gp
def test_converged_to_local_maximum(kernel):
# Test that we are in local maximum after hyperparameter-optimization.
gpr = GaussianProcessRegressor(kernel=kernel).fit(X, y)
lml, lml_gradient = gpr.log_marginal_likelihood(gpr.kernel_.theta, True)
assert np.all((np.abs(lml_gradient) < 1e-4)
| (gpr.kernel_.theta == gpr.kernel_.bounds[:, 0])
| (gpr.kernel_.theta == gpr.kernel_.bounds[:, 1]))
def test_converged_to_local_maximum(kernel):
# Test that we are in local maximum after hyperparameter-optimization.
gpr = GaussianProcessRegressor(kernel=kernel).fit(X, y)
lml, lml_gradient = \
gpr.log_marginal_likelihood(gpr.kernel_.theta, True)
assert_true(np.all((np.abs(lml_gradient) < 1e-4) |
(gpr.kernel_.theta == gpr.kernel_.bounds[:, 0]) |
(gpr.kernel_.theta == gpr.kernel_.bounds[:, 1])))
def test_custom_optimizer():
""" Test that GPR can use externally defined optimizers. """
# Define a dummy optimizer that simply tests 50 random hyperparameters
def optimizer(obj_func, initial_theta, bounds):
rng = np.random.RandomState(0)
theta_opt, func_min = initial_theta, obj_func(initial_theta, eval_gradient=False)
for _ in range(50):
theta = np.atleast_1d(rng.uniform(np.maximum(-2, bounds[:, 0]), np.minimum(1, bounds[:, 1])))
f = obj_func(theta, eval_gradient=False)
if f < func_min:
theta_opt, func_min = theta, f
return theta_opt, func_min
for kernel in kernels:
if kernel == fixed_kernel:
continue
gpr = GaussianProcessRegressor(kernel=kernel, optimizer=optimizer)
gpr.fit(X, y)
# Checks that optimizer improved marginal likelihood
assert_greater(gpr.log_marginal_likelihood(gpr.kernel_.theta), gpr.log_marginal_likelihood(gpr.kernel.theta))
def fitness(self, candidate):
data = self.gym.state['data']
kernel = candidate.epigenesis()
gp = GaussianProcessRegressor(kernel=kernel,
alpha=0.001,
normalize_y=False)
try:
gp = gp.fit(*data)
except np.linalg.linalg.LinAlgError:
return -1e10
return gp.log_marginal_likelihood()
def test_custom_optimizer(kernel):
# Test that GPR can use externally defined optimizers.
# Define a dummy optimizer that simply tests 50 random hyperparameters
def optimizer(obj_func, initial_theta, bounds):
rng = np.random.RandomState(0)
theta_opt, func_min = \
initial_theta, obj_func(initial_theta, eval_gradient=False)
for _ in range(50):
theta = np.atleast_1d(rng.uniform(np.maximum(-2, bounds[:, 0]),
np.minimum(1, bounds[:, 1])))
f = obj_func(theta, eval_gradient=False)
if f < func_min:
theta_opt, func_min = theta, f
return theta_opt, func_min
gpr = GaussianProcessRegressor(kernel=kernel, optimizer=optimizer)
gpr.fit(X, y)
# Checks that optimizer improved marginal likelihood
assert (gpr.log_marginal_likelihood(gpr.kernel_.theta) >
gpr.log_marginal_likelihood(gpr.kernel.theta))
def gpr(self):
rbf = 1.0 * RBF(length_scale=1.0)
matern = 1.0 * Matern(
length_scale=1.0, length_scale_bounds=(1e-8, 5.0), nu=1.5)
gp_opt = GaussianProcessRegressor(kernel=matern)
gp_opt.fit(self.training_x_, self.training_y_)
self.gpr_ = gp_opt
print("The trained hyperparameter are {}".format(
(gp_opt.kernel_.theta)))
print("Log Marginal Likelihood (optimized): %.3f" %
gp_opt.log_marginal_likelihood(gp_opt.kernel_.theta))
def test_converged_to_local_maximum():
""" Test that we are in local maximum after hyperparameter-optimization."""
for kernel in kernels:
if kernel == fixed_kernel: continue
gpr = GaussianProcessRegressor(kernel=kernel).fit(X, y)
lml, lml_gradient = \
gpr.log_marginal_likelihood(gpr.kernel_.theta, True)
assert_true(np.all((np.abs(lml_gradient) < 1e-4)
| (gpr.kernel_.theta == gpr.kernel_.bounds[:, 0])
| (gpr.kernel_.theta == gpr.kernel_.bounds[:, 1])))
def sk_kernel(self, amp, noise_var, lengthscales):
se_ard = Ck(amp) * RBF(length_scale=lengthscales,
length_scale_bounds=(0.000001, 20))
noise = WhiteKernel(noise_level=noise_var,
noise_level_bounds=(1e-5, 100)) # noise terms
# sk_kernel = se_ard + noise + Ck(0.4) #with bias
sk_kernel = se_ard + noise
gpr = GaussianProcessRegressor(kernel=sk_kernel,
n_restarts_optimizer=5)
gpr.fit(self.X_obs, self.Y_obs)
print("Learned kernel: %s" % gpr.kernel_)
print("Log-marginal-likelihood: %.3f" %
gpr.log_marginal_likelihood(gpr.kernel_.theta))
sk_ls = gpr.kernel_.get_params()['k1__k1__k2__length_scale']
sk_noise = gpr.kernel_.get_params()['k1__k2__noise_level']
sk_amp = gpr.kernel_.get_params()['k1__k1__k1']
sk_loklik = gpr.log_marginal_likelihood(gpr.kernel_.theta)
return sk_loklik, sk_amp, sk_noise, sk_ls
class Gaussian_Process():
def __init__(self, kernel, mode):
self.kernel = kernel
if mode == 'OPT':
self.gp = GaussianProcessRegressor(kernel=kernel,
n_restarts_optimizer=10,
normalize_y=True)
elif mode == 'MAP':
self.gp = GaussianProcessRegressor(kernel=kernel,
optimizer = None,
n_restarts_optimizer=10,
normalize_y=True)
else:
raise Exception('Wrong GP initial mode!!!')
'''
def optimise__with_splice_sampling(self, initial_theta, num_iters, sigma, burn_in, X, y):
self.fit(X, y)
slice_sampler = Slice_sampler(num_iters = num_iters,
sigma = sigma,
burn_in = burn_in,
gp = self)
samples = slice_sampler.sample(init = initial_theta)
theta_opt = [np.mean(samples[0]),np.mean(samples[1])]
self.gp.set_params(**{"kernel__k1__length_scale": np.mean(samples[0]),
"kernel__k2__noise_level": np.power(np.mean(samples[0]),2)})
'''
def set_params(self, theta_set):
self.gp.set_params(**{"kernel__k1__noise_level": np.abs(theta_set[0]),
"kernel__k2__k1__constant_value": np.abs(theta_set[1]),
"kernel__k2__k2__length_scale": theta_set[2:]})
def log_marginal_likelihood(self,theta):
return self.gp.log_marginal_likelihood(theta)
def log_prior_parameters(self, theta):
return np.sum(np.log([ss.norm(0, 1).pdf(theta_k) for theta_k in theta]))
def log_joint_unnorm(self, theta):
return self.log_marginal_likelihood(theta) + self.log_prior_parameters(theta)
def fit(self, X, y):
self.gp.fit(X, y)
def predict(self, X):
mu, sigma = self.gp.predict(X, return_std =True)
return mu, sigma
def train_standard_gp(pch, snr, pch_pred):
pch = pch.reshape(-1, 1) # needed for SK learn input
snr = snr.reshape(-1, 1)
pch_pred = pch_pred.reshape(-1, 1)
kernel_sk = C(1, (1e-5, 1e5)) * RBF(1, (1e-5, 1e5)) + W(1, (1e-8, 1e5))
gpr = GaussianProcessRegressor(kernel=kernel_sk,
n_restarts_optimizer=20,
normalize_y=True)
gpr.fit(pch, snr)
mu_sk, std_sk = gpr.predict(pch_pred, return_std=True)
std_sk = np.reshape(std_sk, (np.size(std_sk), 1))
theta = gpr.kernel_.theta
lml = gpr.log_marginal_likelihood()
return mu_sk, std_sk, theta, lml
def regressor(X_train, Y_train):
kernel = 1.0 * RBF(length_scale=0.01, length_scale_bounds=(1e-1, 1e2)) + (
DotProduct()**3) * WhiteKernel(noise_level=2.e-8,
noise_level_bounds=(1e-10, 1e-1))
gp = GaussianProcessRegressor(kernel=kernel,
alpha=0.,
n_restarts_optimizer=15).fit(
X_train, Y_train)
print "kernel init: ", kernel
print "kernel init params: ", kernel.theta
print "kenel optimum: ", gp.kernel_
print "opt kernel params: ", gp.kernel_.theta
print "LML (opt): ", gp.log_marginal_likelihood()
return gp
def bic(self, parents, target, tau_max):
train_x, train_y = self._get_train_data(parents, target, tau_max)
kernel = RBF(length_scale=1, length_scale_bounds=(1e-2, 1e3)) \
+ WhiteKernel(noise_level=1e0, noise_level_bounds=(1e-10, 1e+1))
gp = GaussianProcessRegressor(kernel=kernel,
alpha=0.0,
normalize_y=True,
n_restarts_optimizer=2)
gp.fit(train_x[idx], train_y[idx])
eps = 1e-8
theta_opt = gp.kernel_.theta
n_params = theta_opt.shape[0]
H = np.zeros((n_params, n_params))
log_mll, mll_grad = gp.log_marginal_likelihood(theta_opt,
eval_gradient=True)
for i in range(n_params):
e_ii = np.zeros(n_params)
e_ii[i] = 1
log_mll_2, mll_grad_2 = gp.log_marginal_likelihood(
np.log(np.exp(theta_opt) + eps * e_ii), eval_gradient=True)
H[i, i] = (mll_grad_2[i] - mll_grad[i]) / eps
for i in range(n_params):
for j in range(i, n_params):
e_ij = np.zeros(n_params)
e_ij[i], e_ij[j] = 1, 1
log_mll_2, mll_grad_2 = gp.log_marginal_likelihood(
np.log(np.exp(theta_opt) + eps * e_ij), eval_gradient=True)
H[i, j] = 0.5 * ((np.dot(e_ij, mll_grad_2 - mll_grad)) / eps -
H[i, i] - H[j, j])
H[j, i] = H[i, j].copy()
detH = np.abs(np.linalg.det(H)) + 1e-2
print(H, detH, mll_grad, mll_grad_2, theta_opt)
return log_mll, log_mll - 0.5 * np.log(detH)
def gpregression(Xtrain, Ytrain, Nfeature):
cmean = [1.0] * Nfeature
cbound = [[1e-3, 10000]] * Nfeature
# kernel = C(1.0, [1e-3, 1e3]) * RBF(cmean, cbound)
kernel = C(1.0,
(1e-3, 1e3)) * matk(cmean, cbound, 2.5) + Wht(1.0, (1e-3, 1e3))
gp = GaussianProcessRegressor(kernel=kernel,
n_restarts_optimizer=50,
normalize_y=False)
gp.fit(Xtrain, Ytrain)
print("initial parameters:", kernel)
print("optimal parameters A:", gp.kernel_, "likelihood:",
gp.log_marginal_likelihood(gp.kernel_.theta))
# print("likelihood:", gp.log_marginal_likelihood(gp.kernel_.theta))
return gp
def train_phys_gp(pch, pch_phys, snr, snr_phys, pch_pred):
pch = pch.reshape(-1, 1) # needed for SK learn input
pch_phys = pch_phys.reshape(-1, 1)
pch_pred = pch_pred.reshape(-1, 1)
snr = snr.reshape(-1, 1)
snr_phys = snr_phys.reshape(-1, 1)
kernel_sk = C(1, (1e-5, 1e5)) * RBF(1, (1e-8, 1e5)) + W(1, (1e-5, 1e5))
gpr_phys = GaussianProcessRegressor(kernel=kernel_sk,
n_restarts_optimizer=20,
normalize_y=True)
gpr_phys.fit_phys(pch, pch_phys, snr, snr_phys)
mu_sk_phys, std_sk_phys = gpr_phys.predict(pch_pred, return_std=True)
std_sk_phys = np.reshape(std_sk_phys, (np.size(std_sk_phys), 1))
theta_phys = gpr_phys.kernel_.theta
lml_phys = gpr_phys.log_marginal_likelihood()
return mu_sk_phys, std_sk_phys, theta_phys, lml_phys
def makeGaussianProcess():
global y_t_pred, result
prefix = "%s_GP_FULL" % (name)
#kernel = RBF(1e1,(1e-5,1e7))
kernel = RationalQuadratic() #(1e1,(1e-5,1e7))
#kernel = ExpSineSquared()#(1e1,(1e-5,1e7))
model = GaussianProcessRegressor(kernel=kernel, n_restarts_optimizer=9)
x1 = x[:, 3:6:2]
x_t1 = x_t[:, 3:6:2]
y_t_pred = model.fit(x1, y).predict(x_t1)
r = model.score(x1, y)
print("score r = %s" % r)
print "Coefficients: %s" % model.get_params()
#print "Highest Coefficients: %s" % str(sorted(model.get_params(),key=lambda x:-x))
print str(
(model.kernel_, model.log_marginal_likelihood(model.kernel_.theta)))
return prefix, model
def fit_gp(x,
y,
sqrt=True,
in_lengthscale=1000,
in_noise=2.0,
std_band=1.96,
lgth_s=[500, 2000],
err_s=[2, 4]):
"""Fit Gaussian Process.
If sqrt: Take the square root for fit.
Return: x, y, y_max, y_min"""
if sqrt:
y = np.sqrt(y)
### Sort Values
idx = np.argsort(x)
x = x[idx]
y = y[idx]
X = x[:, np.newaxis]
kernel = 1.0 * RBF(length_scale=in_lengthscale, length_scale_bounds=lgth_s) \
+ WhiteKernel(noise_level=in_noise, noise_level_bounds=err_s)
gp = GaussianProcessRegressor(kernel=kernel, alpha=0.0).fit(X, y)
X_ = np.linspace(np.min(x), np.max(x), 1000)
y_mean, y_cov = gp.predict(X_[:, np.newaxis], return_cov=True)
title = "Initial: %s\nOptimum: %s\nLog-Marginal-Likelihood: %s" % (
kernel, gp.kernel_, gp.log_marginal_likelihood(gp.kernel_.theta))
print(title)
y_up = y_mean + std_band * np.sqrt(np.diag(y_cov))
y_down = y_mean - std_band * np.sqrt(np.diag(y_cov))
error_var = gp.kernel_.get_params()["k2__noise_level"]
stds = np.sqrt(np.diag(y_cov) - error_var)
assert (np.min(stds) >= 0) # Sanity Check that Error in Mean is >0:
y_up_mean = y_mean + std_band * stds
y_down_mean = y_mean - std_band * stds
if sqrt:
y_mean, y_up, y_down = np.maximum(y_mean, 0)**2, y_up**2, np.maximum(
y_down, 0)**2
y_up_mean, y_down_mean = y_up_mean**2, np.maximum(y_down_mean, 0)**2
return X_, y_mean, y_up, y_down, y_up_mean, y_down_mean
def test_random_starts():
# Test that an increasing number of random-starts of GP fitting only
# increases the log marginal likelihood of the chosen theta.
n_samples, n_features = 25, 2
rng = np.random.RandomState(0)
X = rng.randn(n_samples, n_features) * 2 - 1
y = np.sin(X).sum(axis=1) + np.sin(3 * X).sum(axis=1) \
+ rng.normal(scale=0.1, size=n_samples)
kernel = C(1.0, (1e-2, 1e2)) \
* RBF(length_scale=[1.0] * n_features,
length_scale_bounds=[(1e-4, 1e+2)] * n_features) \
+ WhiteKernel(noise_level=1e-5, noise_level_bounds=(1e-5, 1e1))
last_lml = -np.inf
for n_restarts_optimizer in range(5):
gp = GaussianProcessRegressor(
kernel=kernel, n_restarts_optimizer=n_restarts_optimizer,
random_state=0,).fit(X, y)
lml = gp.log_marginal_likelihood(gp.kernel_.theta)
assert_greater(lml, last_lml - np.finfo(np.float32).eps)
last_lml = lml
def plot(df, options):
UNIQ_GROUPS = df.group.unique()
UNIQ_GROUPS.sort()
sns.set_style("white")
grppal = sns.color_palette("Set2", len(UNIQ_GROUPS))
print '# UNIQ GROUPS', UNIQ_GROUPS
cent_stats = df.groupby(['position', 'group']).apply(stats_per_group)
cent_stats.reset_index(inplace=True)
print cent_stats
import time
from sklearn import preprocessing
from sklearn.gaussian_process import GaussianProcessRegressor
from sklearn.gaussian_process.kernels import Matern, WhiteKernel, ExpSineSquared, ConstantKernel, RBF
mean = cent_stats['values'].mean()
# kernel = ConstantKernel() + Matern(length_scale=mean, nu=3 / 2) + \
# WhiteKernel(noise_level=1e-10)
kernel = 1**2 * Matern(length_scale=1, nu=1.5) + \
WhiteKernel(noise_level=0.1)
figure = plt.figure(figsize=(10, 6))
palette = sns.color_palette('muted')
for i,GRP in enumerate(UNIQ_GROUPS):
groupDf = cent_stats[cent_stats['group'] == GRP]
X = groupDf['position'].values.reshape((-1, 1))
y = groupDf['values'].values.reshape((-1, 1))
y = preprocessing.scale(y)
N = groupDf['subj_count'].values.max()
# sns.lmplot(x="position", y="values", row="group",
# fit_reg=False, data=groupDf)
stime = time.time()
gp = GaussianProcessRegressor(kernel=kernel, normalize_y=True)
gp.fit(X, y)
print gp.kernel_
print gp.log_marginal_likelihood()
print("Time for GPR fitting: %.3f" % (time.time() - stime))
stime = time.time()
pred_x = np.linspace(0, 30, 100)
y_mean, y_std = gp.predict(pred_x.reshape((-1, 1)), return_std=True)
y_mean = y_mean[:, 0]
print("Time for GPR prediction: %.3f" % (time.time() - stime))
group_color = palette[i]
ci = y_std / math.sqrt(N) * 1.96
plt.scatter(X, y, color=group_color, alpha=0.1)
plt.plot(pred_x, y_mean, color=group_color)
plt.fill_between(pred_x, y_mean - ci, y_mean +
ci, color=group_color, alpha=0.3)
if options.title:
plt.suptitle(options.title)
if options.output:
plt.savefig(options.output, dpi=150)
if options.is_show:
plt.show()
def test_lml_precomputed():
""" Test that lml of optimized kernel is stored correctly. """
for kernel in kernels:
gpr = GaussianProcessRegressor(kernel=kernel).fit(X, y)
assert_equal(gpr.log_marginal_likelihood(gpr.kernel_.theta),
gpr.log_marginal_likelihood())
print "y: ", y
# initialize the kernel
kernel_test = RBF(length_scale=[2])
print "after kernel init"
# train the model
gp_test = GaussianProcessRegressor(kernel=kernel_test, alpha=0, normalize_y=True)
print "after GP regressor"
gp_test.fit(XT, y)
print ("GPML kernel: %s" % gp_test.kernel_)
print ("Log-marginal-likelihood: %.3f" % gp_test.log_marginal_likelihood(gp_test.kernel_.theta))
X_ = []
for i in range(15):
X_.append([i + 0.5])
XT_ = scaler.transform(X_)
print "X_ ", X_
y_pred, y_std = gp_test.predict(XT_, return_std=True)
# Plot the predict result
X = np.array(X)
y = np.array(y)
X_ = np.array(X_)
plt.scatter(X[:, 0], y, c="k")
plt.plot(X_[:, 0], y_pred)
plt.figure(0)
kernel = 1.0 * RBF(length_scale=100.0, length_scale_bounds=(1e-2, 1e3)) \
+ WhiteKernel(noise_level=1, noise_level_bounds=(1e-10, 1e+1))
gp = GaussianProcessRegressor(kernel=kernel,
alpha=0.0).fit(X, y)
X_ = np.linspace(0, 5, 100)
y_mean, y_cov = gp.predict(X_[:, np.newaxis], return_cov=True)
plt.plot(X_, y_mean, 'k', lw=3, zorder=9)
plt.fill_between(X_, y_mean - np.sqrt(np.diag(y_cov)),
y_mean + np.sqrt(np.diag(y_cov)),
alpha=0.5, color='k')
plt.plot(X_, 0.5*np.sin(3*X_), 'r', lw=3, zorder=9)
plt.scatter(X[:, 0], y, c='r', s=50, zorder=10)
plt.title("Initial: %s\nOptimum: %s\nLog-Marginal-Likelihood: %s"
% (kernel, gp.kernel_,
gp.log_marginal_likelihood(gp.kernel_.theta)))
plt.tight_layout()
# Second run
plt.figure(1)
kernel = 1.0 * RBF(length_scale=1.0, length_scale_bounds=(1e-2, 1e3)) \
+ WhiteKernel(noise_level=1e-5, noise_level_bounds=(1e-10, 1e+1))
gp = GaussianProcessRegressor(kernel=kernel,
alpha=0.0).fit(X, y)
X_ = np.linspace(0, 5, 100)
y_mean, y_cov = gp.predict(X_[:, np.newaxis], return_cov=True)
plt.plot(X_, y_mean, 'k', lw=3, zorder=9)
plt.fill_between(X_, y_mean - np.sqrt(np.diag(y_cov)),
y_mean + np.sqrt(np.diag(y_cov)),
alpha=0.5, color='k')
plt.plot(X_, 0.5*np.sin(3*X_), 'r', lw=3, zorder=9)
# Specify stationary and non-stationary kernel
kernel_matern = C(1.0, (1e-10, 1000)) \
* Matern(length_scale_bounds=(1e-1, 1e3), nu=1.5)
gp_matern = GaussianProcessRegressor(kernel=kernel_matern)
kernel_lls = C(1.0, (1e-10, 1000)) \
* LocalLengthScalesKernel.construct(X, l_L=0.1, l_U=2.0, l_samples=5)
gp_lls = GaussianProcessRegressor(kernel=kernel_lls, optimizer=de_optimizer)
# Fit GPs
gp_matern.fit(X, y)
gp_lls.fit(X, y)
print "Learned kernel Matern: %s" % gp_matern.kernel_
print "Log-marginal-likelihood Matern: %s" \
% gp_matern.log_marginal_likelihood(gp_matern.kernel_.theta)
print "Learned kernel LLS: %s" % gp_lls.kernel_
print "Log-marginal-likelihood LLS: %s" \
% gp_lls.log_marginal_likelihood(gp_lls.kernel_.theta)
# Compute GP mean and standard deviation on test data
X_ = np.linspace(-1, 1, 500)
y_mean_lls, y_std_lls = gp_lls.predict(X_[:, np.newaxis], return_std=True)
y_mean_matern, y_std_matern = \
gp_matern.predict(X_[:, np.newaxis], return_std=True)
plt.figure(figsize=(7, 7))
plt.subplot(2, 1, 1)
print 'after kernel init'
# train the model
gp_test = GaussianProcessRegressor(kernel=kernel_test, alpha=8, normalize_y=True)
print 'after GP regressor'
# print("GPML kernel: %s" % gp_test.kernel_)
# print("Log-marginal-likelihood: %.3f"
# % gp_test.log_marginal_likelihood(gp_test.kernel_.theta))
gp_test.fit(XT, y)
print("GPML kernel: %s" % gp_test.kernel_)
print("Log-marginal-likelihood: %.3f"
% gp_test.log_marginal_likelihood(gp_test.kernel_.theta))
X_ = []
for i in range(15):
X_.append([i+0.5, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1])
XT_ = scaler.transform(X_)
print 'X_ ', XT_
y_pred, y_std = gp_test.predict(XT_, return_std=True)
# Plot the predict result
X = np.array(X)
y = np.array(y)
X_ = np.array(X_)
plt.scatter(X[:, 0], y, c='k')
plt.plot(X_[:, 0], y_pred)
plt.plot(X_, y_samples, lw=1)
plt.xlim(0, 5)
plt.ylim(-3, 3)
plt.title("Prior (kernel: %s)" % kernel, fontsize=12)
# Generate data and fit GP
rng = np.random.RandomState(4)
X = rng.uniform(0, 5, 10)[:, np.newaxis]
y = np.sin((X[:, 0] - 2.5) ** 2)
gp.fit(X, y)
# Plot posterior
plt.subplot(2, 1, 2)
X_ = np.linspace(0, 5, 100)
y_mean, y_std = gp.predict(X_[:, np.newaxis], return_std=True)
plt.plot(X_, y_mean, "k", lw=3, zorder=9)
plt.fill_between(X_, y_mean - y_std, y_mean + y_std, alpha=0.5, color="k")
y_samples = gp.sample_y(X_[:, np.newaxis], 10)
plt.plot(X_, y_samples, lw=1)
plt.scatter(X[:, 0], y, c="r", s=50, zorder=10)
plt.xlim(0, 5)
plt.ylim(-3, 3)
plt.title(
"Posterior (kernel: %s)\n Log-Likelihood: %.3f" % (gp.kernel_, gp.log_marginal_likelihood(gp.kernel_.theta)),
fontsize=12,
)
plt.tight_layout()
plt.show()
plt.figure()
plt.scatter(inputs_x_array[:,0][index_y1::len_x2],y_pred[index_y1::len_x2]) #from x1_min to x1_max
plt.xlabel('X Label (radius)')
plt.ylabel('Z Label (density)')
plt.show()
index_y2 = 7 #only valid until len_x1
print("Radius is "+str(inputs_x_array[:,0][index_y2*len_x2:index_y2*len_x2 + len_x2][0])+"m")
plt.figure()
plt.scatter(inputs_x_array[:,1][index_y2*len_x2:index_y2*len_x2 + len_x2],y_pred[index_y2*len_x2:index_y2*len_x2 + len_x2]) #from x2_min to x2_max
plt.xlabel('Y Label (time)')
plt.ylabel('Z Label (density)')
plt.show()
print(gp.kernel_) #gives optimized hyperparameters
print(gp.log_marginal_likelihood(gp.kernel_.theta)) #log-likelihood
alpha=1e-10
input_prediction = gp.predict(X,return_std=True)
K, K_gradient = gp.kernel_(X, eval_gradient=True)
K[np.diag_indices_from(K)] += alpha
L = cholesky(K, lower=True) # Line 2
# Support multi-dimensional output of self.y_train_
if y.ndim == 1:
y = y[:, np.newaxis]
alpha = cho_solve((L, True), y)
log_likelihood_dims = -0.5 * np.einsum("ik,ik->k", y, alpha)
log_likelihood_dims -= np.log(np.diag(L)).sum()
def f(X):
# target function is just a linear relationship + heteroscadastic noise
return X + 0.5*np.random.multivariate_normal(np.zeros(X.shape[0]),
np.diag(X**2), 1)[0]
X = np.random.uniform(-7.5, 7.5, n_samples) # input data
y = f(X) # Generate target values by applying function to manifold
# Gaussian Process with RBF kernel and homoscedastic noise level
kernel_homo = C(1.0, (1e-10, 1000)) * RBF(1, (0.01, 100.0)) \
+ WhiteKernel(1e-3, (1e-10, 50.0))
gp_homoscedastic = GaussianProcessRegressor(kernel=kernel_homo, alpha=0)
gp_homoscedastic.fit(X[:, np.newaxis], y)
print "Homoscedastic kernel: %s" % gp_homoscedastic.kernel_
print "Homoscedastic LML: %.3f" \
% gp_homoscedastic.log_marginal_likelihood(gp_homoscedastic.kernel_.theta)
print
# Gaussian Process with RBF kernel and heteroscedastic noise level
prototypes = KMeans(n_clusters=10).fit(X[:, np.newaxis]).cluster_centers_
kernel_hetero = C(1.0, (1e-10, 1000)) * RBF(1, (0.01, 100.0)) \
+ HeteroscedasticKernel.construct(prototypes, 1e-3, (1e-10, 50.0),
gamma=5.0, gamma_bounds="fixed")
gp_heteroscedastic = GaussianProcessRegressor(kernel=kernel_hetero, alpha=0)
gp_heteroscedastic.fit(X[:, np.newaxis], y)
print "Heteroscedastic kernel: %s" % gp_heteroscedastic.kernel_
print "Heteroscedastic LML: %.3f" \
% gp_heteroscedastic.log_marginal_likelihood(gp_heteroscedastic.kernel_.theta)
# Plot result
def test_lml_improving(kernel):
# Test that hyperparameter-tuning improves log-marginal likelihood.
gpr = GaussianProcessRegressor(kernel=kernel).fit(X, y)
assert_greater(gpr.log_marginal_likelihood(gpr.kernel_.theta),
gpr.log_marginal_likelihood(kernel.theta))
architecture=((n_features, n_hidden, n_dim_manifold),)
kernel_nn = C(1.0, (1e-10, 100)) \
* ManifoldKernel.construct(base_kernel=RBF(0.1, (1.0, 100.0)),
architecture=architecture,
transfer_fct="tanh", max_nn_weight=1.0) \
+ WhiteKernel(1e-3, (1e-10, 1e-1))
gp_nn = GaussianProcessRegressor(kernel=kernel_nn, alpha=0,
n_restarts_optimizer=3)
# Fit GPs and create scatter plot on test data
gp.fit(X, y)
gp_nn.fit(X, y)
print "Initial kernel: %s" % gp_nn.kernel
print "Log-marginal-likelihood: %s" \
% gp_nn.log_marginal_likelihood(gp_nn.kernel.theta)
print "Learned kernel: %s" % gp_nn.kernel_
print "Log-marginal-likelihood: %s" \
% gp_nn.log_marginal_likelihood(gp_nn.kernel_.theta)
X_test_ = np.random.uniform(-5, 5, (1000, n_dim_manifold))
X_nn_test = X_test_.dot(A)
y_test = f(X_test_)
plt.figure(figsize=(8, 6))
plt.subplot(1, 2, 1)
plt.scatter(y_test, gp.predict(X_nn_test), c='b', label="GP RBF")
plt.scatter(y_test, gp_nn.predict(X_nn_test), c='r', label="GP NN")
plt.xlabel("True")
plt.ylabel("Predicted")
plt.legend(loc=0)
k2 = 2.4**2 * RBF(length_scale=90.0) \
* ExpSineSquared(length_scale=1.3, periodicity=1.0) # seasonal component
# medium term irregularity
k3 = 0.66**2 \
* RationalQuadratic(length_scale=1.2, alpha=0.78)
k4 = 0.18**2 * RBF(length_scale=0.134) \
+ WhiteKernel(noise_level=0.19**2) # noise terms
kernel_gpml = k1 + k2 + k3 + k4
gp = GaussianProcessRegressor(kernel=kernel_gpml, alpha=0,
optimizer=None, normalize_y=True)
gp.fit(X, y)
print("GPML kernel: %s" % gp.kernel_)
print("Log-marginal-likelihood: %.3f"
% gp.log_marginal_likelihood(gp.kernel_.theta))
# Kernel with optimized parameters
k1 = 50.0**2 * RBF(length_scale=50.0) # long term smooth rising trend
k2 = 2.0**2 * RBF(length_scale=100.0) \
* ExpSineSquared(length_scale=1.0, periodicity=1.0,
periodicity_bounds="fixed") # seasonal component
# medium term irregularities
k3 = 0.5**2 * RationalQuadratic(length_scale=1.0, alpha=1.0)
k4 = 0.1**2 * RBF(length_scale=0.1) \
+ WhiteKernel(noise_level=0.1**2,
noise_level_bounds=(1e-3, np.inf)) # noise terms
kernel = k1 + k2 + k3 + k4
gp = GaussianProcessRegressor(kernel=kernel, alpha=0,
normalize_y=True)
