def compare_against_mmd_test():
data = loadmat("../data/02-solar.mat")
X = data["X"]
y = data["y"]
X_train, y_train, X_test, y_test, N, N_test = prepare_dataset(X, y)
kernel = RBF(input_dim=1, variance=0.608, lengthscale=0.207)
m = GPRegression(X_train, y_train, kernel, noise_var=0.283)
m.optimize()
pred_mean, pred_std = m.predict(X_test)
s = GaussianQuadraticTest(None)
gradients = compute_gp_regression_gradients(y_test, pred_mean, pred_std)
U_matrix, stat = s.get_statistic_multiple_custom_gradient(y_test[:, 0], gradients[:, 0])
num_test_samples = 10000
null_samples = bootstrap_null(U_matrix, num_bootstrap=num_test_samples)
# null_samples = sample_null_simulated_gp(s, pred_mean, pred_std, num_test_samples)
p_value_ours = 1.0 - np.mean(null_samples <= stat)
y_rep = np.random.randn(len(X_test)) * pred_std.flatten() + pred_mean.flatten()
y_rep = np.atleast_2d(y_rep).T
A = np.hstack((X_test, y_test))
B = np.hstack((X_test, y_rep))
feats_p = RealFeatures(A.T)
feats_q = RealFeatures(B.T)
width = 1
kernel = GaussianKernel(10, width)
mmd = QuadraticTimeMMD()
mmd.set_kernel(kernel)
mmd.set_p(feats_p)
mmd.set_q(feats_q)
mmd_stat = mmd.compute_statistic()
# sample from null
num_null_samples = 10000
mmd_null_samples = np.zeros(num_null_samples)
for i in range(num_null_samples):
# fix y_rep from above, and change the other one (that would replace y_test)
y_rep2 = np.random.randn(len(X_test)) * pred_std.flatten() + pred_mean.flatten()
y_rep2 = np.atleast_2d(y_rep2).T
A = np.hstack((X_test, y_rep2))
feats_p = RealFeatures(A.T)
width = 1
kernel = GaussianKernel(10, width)
mmd = QuadraticTimeMMD()
mmd.set_kernel(kernel)
mmd.set_p(feats_p)
mmd.set_q(feats_q)
mmd_null_samples[i] = mmd.compute_statistic()
p_value_mmd = 1.0 - np.mean(mmd_null_samples <= mmd_stat)
return p_value_ours, p_value_mmd
y_rep = np.random.randn(len(X_test)) * pred_std.flatten() + pred_mean.flatten()
y_rep = np.atleast_2d(y_rep).T
# stack together (X_test,y_test) and (X_test, y_pred)
A = np.hstack((X_test, y_test))
B = np.hstack((X_test, y_rep))
# compute MMD between (X_test,y_test) and (X_test, y_pred)
feats_p = RealFeatures(A.T)
feats_q = RealFeatures(B.T)
width = 1
kernel = GaussianKernel(10, width)
mmd = QuadraticTimeMMD()
mmd.set_kernel(kernel)
mmd.set_p(feats_p)
mmd.set_q(feats_q)
mmd_stat = mmd.compute_statistic()
# sample from null
num_null_samples = 10000
mmd_null_samples = np.zeros(num_null_samples)
for i in range(num_null_samples):
# fix y_rep from above, and change the other one (that would replace y_test)
y_rep2 = np.random.randn(len(X_test)) * pred_std.flatten() + pred_mean.flatten()
y_rep2 = np.atleast_2d(y_rep2).T
A = np.hstack((X_test, y_rep2))
feats_p = RealFeatures(A.T)
width = 1
kernel = GaussianKernel(10, width)
