def test_gpr_regularization():
'''Training in the precense of duplicate input values.'''
class Kernel:
def __init__(self, v, L):
self.v = v
self.L = L
def __call__(self, X, Y=None, eval_gradient=False):
v = self.v
L = self.L
d = np.subtract.outer(X, Y if Y is not None else X)
f = v**2 * np.exp(-0.5 * d**2 / L**2)
if eval_gradient is False:
return f
else:
j1 = v**2 * np.exp(-0.5 * d**2 / L**2) * d**2 * L**-3
j2 = 2 * v * f
return f, np.stack((j1, j2), axis=2)
def diag(self, X):
return np.ones_like(X)
@property
def theta(self):
return np.log([self.v, self.L])
@theta.setter
def theta(self, t):
self.v, self.L = np.exp(t[:2])
@property
def bounds(self):
return np.log([[0.001, 10.0], [0.001, 10.0]])
def clone_with_theta(self, theta):
k = Kernel(1.0, 1.0)
k.theta = theta
return k
X = np.array([0, 1, 1, 2])
y = np.array([1, 0, 1, 0])
gpr1 = GaussianProcessRegressor(
kernel=Kernel(100.0, 1.0),
alpha=1e-6,
optimizer=False,
regularization='*'
)
gpr2 = GaussianProcessRegressor(
kernel=Kernel(100.0, 1.0),
alpha=1e-4,
optimizer=False,
regularization='+'
)
gpr1.fit(X, y, tol=1e-5)
gpr2.fit(X, y, tol=1e-5)
grid = np.linspace(0, 2, 9)
assert np.allclose(
gpr1.predict(grid), gpr2.predict(grid), rtol=1e-5, atol=1e-5
)
def test_gpr_squared_loocv_error(f):
class Kernel:
def __init__(self, L):
self.L = L
def __call__(self, X, Y=None, eval_gradient=False):
L = self.L
d = np.subtract.outer(X, Y if Y is not None else X)
f = np.exp(-0.5 * d**2 / L**2)
if eval_gradient is False:
return f
else:
j = np.exp(-0.5 * d**2 / L**2) * d**2 * L**-3
return f, np.stack((j, ), axis=2)
def diag(self, X):
return np.ones_like(X)
@property
def theta(self):
return np.log([self.L])
@theta.setter
def theta(self, t):
self.L = np.exp(t[0])
def clone_with_theta(self, theta):
k = Kernel(1.0)
k.theta = theta
return k
X = np.linspace(-1, 1, 6, endpoint=False)
y = f(X)
eps = 1e-4
for L in np.logspace(-2, 1, 20):
kernel = Kernel(L)
gpr = GaussianProcessRegressor(kernel=kernel, alpha=1e-10)
_, de = gpr.squared_loocv_error(X=X, y=y, eval_gradient=True)
theta0 = np.copy(kernel.theta)
e_pos = gpr.squared_loocv_error(
theta=theta0 + eps, X=X, y=y, eval_gradient=False
).item()
e_neg = gpr.squared_loocv_error(
theta=theta0 - eps, X=X, y=y, eval_gradient=False
).item()
de_diff = (e_pos - e_neg) / (2 * eps)
assert(de.item() == pytest.approx(de_diff, 1e-2, 1e-2))
def test_gpr_fit_mle(repeat, verbose):
'''test with a function with exactly two periods, and see if the GPR
can identify the frequency via hyperparameter optimization.'''
class Kernel:
def __init__(self, p, L):
self.p = p
self.L = L
def __call__(self, X, Y=None, eval_gradient=False):
d = np.subtract.outer(X, Y if Y is not None else X)
f = np.exp(-2 * np.sin(np.pi / self.p * d)**2 / self.L**2)
if eval_gradient is False:
return f
else:
s = np.sin(d * np.pi / self.p)
c = np.cos(d * np.pi / self.p)
j1 = 2.0 * np.pi * d * 2 * s * c * f / self.p**2 / self.L**2
j2 = 4.0 * s**2 * f / self.L**3
return f, np.stack((j1, j2), axis=2)
def diag(self, X):
return np.ones_like(X)
@property
def theta(self):
return np.log([self.p, self.L])
@theta.setter
def theta(self, t):
self.p, self.L = np.exp(t)
@property
def bounds(self):
return np.log([[1e-2, 10], [1e-2, 10]])
X = np.linspace(0, 1, 16, endpoint=False)
y = np.sin(X * 4 * np.pi)
kernel = Kernel(0.49, 0.1)
gpr = GaussianProcessRegressor(kernel=kernel, alpha=1e-10, optimizer=True)
gpr.fit(X, y, tol=1e-5, repeat=repeat, verbose=verbose)
assert(kernel.p == pytest.approx(0.5, 1e-2))
def test(chosen, label, color):
gpr = GaussianProcessRegressor(kernel, normalize_y=True)
gpr.fit(X[chosen], y[chosen])
plt.scatter(X[chosen], y[chosen], label=label, color=color)
plt.plot(grid, gpr.predict(grid), label=label, color=color)
print(f"RMSE of '{label}' model:", np.std(gpr.predict(X) - y))
print(f"{label} det: {np.prod(np.linalg.slogdet(kernel(X[chosen])))}")
def test_gpr_predict_periodic():
'''test with a function with exactly two periods, and see if the GPR
can use information across the periods to fill in the missing points.'''
class Kernel:
def __call__(self, X, Y=None):
d = np.subtract.outer(X, Y if Y is not None else X)
return np.exp(-2 * np.sin(np.pi / 0.5 * d)**2)
def diag(self, X):
return np.ones_like(X)
kernel = Kernel()
X = np.linspace(0, 1, 16, endpoint=False)
y = np.sin(X * 4 * np.pi)
mask = np.array([1, 0, 1, 0, 1, 0, 1, 0,
0, 1, 0, 1, 0, 1, 0, 1], dtype=np.bool_)
gpr = GaussianProcessRegressor(kernel=kernel, alpha=1e-10)
gpr.fit(X[mask], y[mask])
z = gpr.predict(X[~mask])
assert(z == pytest.approx(y[~mask], 1e-6))
def test_gpr_singular_kernel_matrix():
class Kernel:
def __init__(self, L):
self.L = L
def __call__(self, X, Y=None, eval_gradient=False):
L = self.L
d = np.subtract.outer(X, Y if Y is not None else X)
f = np.exp(-0.5 * d**2 / L**2)
if eval_gradient is False:
return f
else:
j = np.exp(-0.5 * d**2 / L**2) * d**2 * L**-3
return f, np.stack((j, ), axis=2)
def diag(self, X):
return np.ones_like(X)
@property
def theta(self):
return np.log([self.L])
@theta.setter
def theta(self, t):
self.L = np.exp(t[0])
def clone_with_theta(self, theta):
k = Kernel(1.0)
k.theta = theta
return k
X = np.ones(3)
y = np.random.rand(3)
gpr = GaussianProcessRegressor(kernel=Kernel(1.0), alpha=0)
gpr.fit(X, y)
z = gpr.predict(X)
assert(z == pytest.approx(np.mean(y)))
def test_gpr_fit_loocv_no_opt(normalize_y):
class Kernel:
def __call__(self, X, Y=None):
d = np.subtract.outer(X, Y if Y is not None else X)
return np.exp(-2 * np.sin(np.pi / 0.5 * d)**2)
def diag(self, X):
return np.ones_like(X)
X = np.linspace(0, 1, 16, endpoint=False)
y = np.sin(X * 4 * np.pi)
mask = np.array([1, 0, 1, 0, 1, 0, 1, 0,
0, 1, 0, 1, 0, 1, 0, 1], dtype=np.bool_)
kernel = Kernel()
gpr = GaussianProcessRegressor(
kernel=kernel,
alpha=1e-10,
normalize_y=normalize_y,
optimizer=False,
)
gpr.fit_loocv(X[mask], y[mask])
z1 = gpr.predict(X[~mask])
assert(z1 == pytest.approx(y[~mask], 1e-6))
def test_gpr_fit_self_consistency(X, y):
class Kernel:
def __call__(self, X, Y=None):
return np.exp(-np.subtract.outer(X, Y if Y is not None else X)**2)
def diag(self, X):
return np.ones_like(X)
kernel = Kernel()
gpr = GaussianProcessRegressor(kernel=kernel, alpha=1e-12)
with pytest.raises(RuntimeError):
gpr.predict(X)
gpr.fit(X, y)
z = gpr.predict(X)
assert(z == pytest.approx(y, 1e-3, 1e-3))
z, std = gpr.predict(X, return_std=True)
assert(z == pytest.approx(y, 1e-3, 1e-3))
assert(std == pytest.approx(np.zeros_like(y), 1e-3, 1e-3))
z, cov = gpr.predict(X, return_cov=True)
assert(z == pytest.approx(y, 1e-3, 1e-3))
assert(cov == pytest.approx(np.zeros((len(X), len(X))), 1e-3, 1e-3))
def test_gpr_predict_loocv(f):
class Kernel:
def __call__(self, X, Y=None):
return np.exp(-np.subtract.outer(X, Y if Y is not None else X)**2)
def diag(self, X):
return np.ones_like(X)
kernel = Kernel()
gpr = GaussianProcessRegressor(kernel=kernel, alpha=1e-12)
X = np.linspace(-1, 1, 6)
y = f(X)
y_loocv, std_loocv = gpr.predict_loocv(X, y, return_std=True)
assert(y_loocv == pytest.approx(gpr.predict_loocv(X, y, return_std=False)))
for i, _ in enumerate(X):
Xi = np.delete(X, i)
yi = np.delete(y, i)
gpr_loocv = GaussianProcessRegressor(kernel=kernel, alpha=1e-12)
gpr_loocv.fit(Xi, yi)
y_loocv_i, std_loocv_i = gpr_loocv.predict(X[[i]], return_std=True)
assert(y_loocv_i.item() == pytest.approx(y_loocv[i], 1e-7))
assert(std_loocv_i.item() == pytest.approx(std_loocv[i], 1e-7))
def test_gpr_fit_loocv_opt(normalize_y, repeat):
class Kernel:
def __init__(self, L):
self.L = L
def __call__(self, X, Y=None, eval_gradient=False):
L = self.L
d = np.subtract.outer(X, Y if Y is not None else X)
f = np.exp(-0.5 * d**2 / L**2)
if eval_gradient is False:
return f
else:
j = np.exp(-0.5 * d**2 / L**2) * d**2 * L**-3
return f, np.stack((j, ), axis=2)
def diag(self, X):
return np.ones_like(X)
@property
def theta(self):
return np.log([self.L])
@theta.setter
def theta(self, t):
self.L = np.exp(t[0])
@property
def bounds(self):
return np.log([[1e-2, 10]])
def clone_with_theta(self, theta):
k = Kernel(1.0)
k.theta = theta
return k
X = np.linspace(-1, 1, 6, endpoint=False)
y = np.sin(X * np.pi)
kernel = Kernel(0.1)
gpr = GaussianProcessRegressor(
kernel=kernel,
alpha=1e-10,
normalize_y=normalize_y,
optimizer=True
)
e1 = gpr.squared_loocv_error(X=X, y=y)
gpr.fit_loocv(X, y, repeat=repeat, verbose=True)
e2 = gpr.squared_loocv_error(X=X, y=y)
assert(e1 > e2)
assert(kernel.L == pytest.approx(0.86, 0.01))
def test_gpr_fit_duplicate_x(loss):
'''Training in the precense of duplicate input values.'''
class Kernel:
def __init__(self, L):
self.L = L
def __call__(self, X, Y=None, eval_gradient=False):
L = self.L
d = np.subtract.outer(X, Y if Y is not None else X)
f = np.exp(-0.5 * d**2 / L**2)
if eval_gradient is False:
return f
else:
j = np.exp(-0.5 * d**2 / L**2) * d**2 * L**-3
return f, np.stack((j, ), axis=2)
def diag(self, X):
return np.ones_like(X)
@property
def theta(self):
return np.log([self.L])
@theta.setter
def theta(self, t):
self.L = np.exp(t[0])
@property
def bounds(self):
return np.log([[0.001, 10.0]])
def clone_with_theta(self, theta):
k = Kernel(1.0)
k.theta = theta
return k
X = np.array([0, 1, 1, 2, 3, 3.995, 4, 6, 7, 8, 8.0001, 9])
y = np.array([1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 0, 1])
kernel = Kernel(1.0)
gpr = GaussianProcessRegressor(kernel=kernel, alpha=0, optimizer=True)
gpr.fit(X, y, tol=1e-5, loss=loss)
assert gpr.predict([1]) == pytest.approx(0.5)
assert gpr.predict([4]) == pytest.approx(1.0)
assert gpr.predict([8]) == pytest.approx(0.0)
# Dummy rewriter of a real numbers
class RandomJitter(AbstractRewriter):
def __init__(self, s, n):
self.s = s
self.n = n
def __call__(self, x, rng):
return np.minimum(3, np.maximum(0, rng.normal(x.g, self.s, self.n)))
# Function to be learned
def f(x):
return np.sin(x) + 2e-4 * x**3 - 2.0 * np.exp(-x**2)
# return 2 * x
# GPR surrogate model
x = np.linspace(0, 3, 13)
y = f(x)
gpr = GaussianProcessRegressor(kernel=Kernel(0.5))
gpr.fit(x, y)
# Run MCTS
mcts = MCTSGraphTransformer(
rewriter=RandomJitter(0.333, 9),
surrogate=gpr,
)
# print(mcts.seek(g0=0.5, target=2.0))
print(mcts.seek(g0=0.5, target=0.0, maxiter=20))
def test_gpr_fit_masked_target():
class Kernel:
def __call__(self, X, Y=None):
return np.exp(-np.subtract.outer(X, Y if Y is not None else X)**2)
def diag(self, X):
return np.ones_like(X)
X = np.array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])
y = np.random.randn(10)
bad = [1, 4, 7]
y[bad] = None
kernel = Kernel()
gpr = GaussianProcessRegressor(kernel=kernel, alpha=1e-12)
gpr.fit(X, y)
assert(np.all(np.isfinite(gpr.predict(X))))
baseline = GaussianProcessRegressor(kernel=kernel, alpha=1e-12)
baseline.fit(X[~np.isnan(y)], y[~np.isnan(y)])
grid = np.linspace(-1, 10, 100)
assert(np.allclose(gpr.predict(grid), baseline.predict(grid)))