def test_embedding_2():
np.random.seed(1)
# Create a categorical version of branin
def branin(x,y,z):
x = (x*15)-5
y = y*15
if z == 1 or z == 2:
return np.square(y - (5.1/(4*np.square(np.pi)))*np.square(x) + (5/np.pi)*x - 6) + 10*(1-(1./(8*np.pi)))*np.cos(x) + 10*z;
elif z == 3:
return np.square((y-1) - (5.1/(4*np.square(np.pi)))*np.square(x-1) + (5/np.pi)*(x-1) - 6) + 10*(1-(1./(8*np.pi)))*np.cos(x-1) + 10*z;
else:
if np.array(x).ndim:
return 10*npr.randn(x.shape[0])
else:
return 10*npr.randn()
N = 100
D = 5
groups = [(0,1,2,3)]
c = len(groups[0])
# Spray random points around a 5-D grid (3 categorical and 2 continuous)
X = np.random.rand(N,D)
a = X[:,:c].argmax(1)
X[:,:c] = 0*X[:,:c]
X[:,:c][range(N),a] = 1
# Get the values from Branin and make the categorical ones 1-of-k
x = X[:,-2]
y = X[:,-1]
z = X[:,:c].argmax(1) + 1
# Compute and standardize the function values
vals = np.zeros(N)
for i in xrange(N):
vals[i] = branin(x[i],y[i],z[i])
vals = (vals - vals.mean()) / vals.std()
# Set up the model
mgp = MixedGP(D,groups)
mgp.fit(X, vals)
w = np.zeros((c,2))
for i in xrange(mgp.num_states):
mgp.set_state(i)
w += mgp.params['norm_lin_0_weights'].value.reshape((c,2))
w /= mgp.num_states
# Compute the pairwise distances implies by the embedding
dists = cdist(w,w)
# Make sure they're embedded with the right relative ordering.
assert dists[0,1] < dists[0,2] and dists[0,1] < dists[0,3] and dists[1,2] < dists[1,3]
def test_fit():
npr.seed(1)
N = 12
D = 10
groups = [[1,3],[5,6,9]]
mgp = MixedGP(D, groups, burnin=5)
inputs = npr.rand(N,D)
pending = npr.rand(3,D)
W = npr.randn(D,1)
vals = inputs.dot(W).flatten() + np.sqrt(1e-3)*npr.randn(N)
mgp.fit(inputs, vals, pending)
assert mgp.chain_length == 15
assert all([np.all(p.value != p.initial_value) for p in mgp.params.values()])
assert len(mgp._cache_list) == 10
assert len(mgp._hypers_list) == 10
assert len(mgp._fantasy_values_list) == 10
def test_fit():
npr.seed(1)
N = 12
D = 10
groups = [[1, 3], [5, 6, 9]]
mgp = MixedGP(D, groups, burnin=5)
inputs = npr.rand(N, D)
pending = npr.rand(3, D)
W = npr.randn(D, 1)
vals = inputs.dot(W).flatten() + np.sqrt(1e-3) * npr.randn(N)
mgp.fit(inputs, vals, pending)
assert mgp.chain_length == 15
assert all(
[np.all(p.value != p.initial_value) for p in mgp.params.values()])
assert len(mgp._cache_list) == 10
assert len(mgp._hypers_list) == 10
assert len(mgp._fantasy_values_list) == 10
def test_embedding():
np.random.seed(1)
# Create a categorical version of branin (actually an integer, but we'll treat it as categorical)
def branin(x, y, z):
x = (x * 15) - 5
y = y * 15
return np.square(y - (5.1 / (4 * np.square(np.pi))) * np.square(x) +
(5 / np.pi) * x -
6) + 10 * (1 - (1. /
(8 * np.pi))) * np.cos(x) + 10 * z
N = 100
D = 5
groups = [(0, 1, 2, 3)]
c = len(groups[0])
# Spray random points around a 5-D grid (3 categorical and 2 continuous)
X = np.random.rand(N, D)
a = X[:, :c].argmax(1)
X[:, :c] = 0 * X[:, :c]
X[:, :c][range(N), a] = 1
# Get the values from Branin and make the categorical ones 1-of-k
x = X[:, -2]
y = X[:, -1]
z = X[:, :c].argmax(1) + 1
# Compute and standardize the function values
vals = branin(x, y, z)
vals = (vals - vals.mean()) / vals.std()
# Set up the model
mgp = MixedGP(D, groups)
mgp.fit(X, vals)
w = np.zeros((c, 2))
for i in xrange(mgp.num_states):
mgp.set_state(i)
w += mgp.params['norm_lin_0_weights'].value.reshape((c, 2))
w /= mgp.num_states
# Compute the pairwise distances implies by the embedding
dists = cdist(w, w)
# Make sure they're embedded with the right relative ordering.
assert dists[0, 1] < dists[0, 2] and dists[0, 1] < dists[0, 3] and dists[
1, 2] < dists[1, 3]
def test_predict():
npr.seed(1)
N = 12
Npend = 3
Ntest = 2
D = 10
groups = [[1, 3], [5, 6, 9]]
mgp = MixedGP(D, groups, burnin=5, num_fantasies=7)
pred = npr.rand(Ntest, D)
# Test with 0 points
mu, v = mgp.predict(pred)
np.testing.assert_allclose(mu,
0,
rtol=1e-7,
atol=0,
err_msg='',
verbose=True)
np.testing.assert_allclose(v,
1 + 1e-6,
rtol=1e-7,
atol=0,
err_msg='',
verbose=True)
#Test with 1 point
X = np.zeros((1, D))
W = npr.randn(D, 1)
val = X.dot(W).flatten() + np.sqrt(1e-3) * npr.randn()
mgp.fit(X, val, fit_hypers=False)
mu, v = mgp.predict(pred)
# Points closer to the origin will have less variance
if np.linalg.norm(pred[0] - X) < np.linalg.norm(pred[1] - X):
assert v[0] < v[1]
else:
assert v[0] > v[1]
# Predict at the point itself
mu, v = mgp.predict(X)
np.testing.assert_allclose(mu,
val,
rtol=1e-5,
atol=0,
err_msg='',
verbose=True)
# Now let's make sure it doesn't break with more data and pending
inputs = npr.rand(N, D)
vals = inputs.dot(W).flatten() + np.sqrt(1e-3) * npr.randn(N)
pending = npr.rand(Npend, D)
mgp.fit(inputs, vals, pending)
mu, v = mgp.predict(pred)
# Now let's check the gradients
eps = 1e-5
mu, v, dmu, dv = mgp.predict(pred, compute_grad=True)
# The implied loss is np.sum(mu**2) + np.sum(v**2)
dloss = 2 * (dmu * mu[:, np.newaxis, :]).sum(2) + 2 * (
v[:, np.newaxis, np.newaxis] * dv).sum(2)
dloss_est = np.zeros(dloss.shape)
for i in xrange(Ntest):
for j in xrange(D):
pred[i, j] += eps
mu, v = mgp.predict(pred)
loss_1 = np.sum(mu**2) + np.sum(v**2)
pred[i, j] -= 2 * eps
mu, v = mgp.predict(pred)
loss_2 = np.sum(mu**2) + np.sum(v**2)
pred[i, j] += eps
dloss_est[i, j] = ((loss_1 - loss_2) / (2 * eps))
assert np.linalg.norm(dloss - dloss_est) < 1e-6
def test_init():
groups = [[1, 3], [5, 6, 9]]
mgp = MixedGP(10, groups)
def test_predict():
npr.seed(1)
N = 12
Npend = 3
Ntest = 2
D = 10
groups = [[1,3],[5,6,9]]
mgp = MixedGP(D, groups, burnin=5, num_fantasies=7)
pred = npr.rand(Ntest,D)
# Test with 0 points
mu, v = mgp.predict(pred)
np.testing.assert_allclose(mu, 0, rtol=1e-7, atol=0, err_msg='', verbose=True)
np.testing.assert_allclose(v, 1+1e-6, rtol=1e-7, atol=0, err_msg='', verbose=True)
#Test with 1 point
X = np.zeros((1,D))
W = npr.randn(D,1)
val = X.dot(W).flatten() + np.sqrt(1e-3)*npr.randn()
mgp.fit(X, val, fit_hypers=False)
mu, v = mgp.predict(pred)
# Points closer to the origin will have less variance
if np.linalg.norm(pred[0] - X) < np.linalg.norm(pred[1] - X):
assert v[0] < v[1]
else:
assert v[0] > v[1]
# Predict at the point itself
mu, v = mgp.predict(X)
np.testing.assert_allclose(mu, val, rtol=1e-5, atol=0, err_msg='', verbose=True)
# Now let's make sure it doesn't break with more data and pending
inputs = npr.rand(N,D)
vals = inputs.dot(W).flatten() + np.sqrt(1e-3)*npr.randn(N)
pending = npr.rand(Npend,D)
mgp.fit(inputs, vals, pending)
mu, v = mgp.predict(pred)
# Now let's check the gradients
eps = 1e-5
mu, v, dmu, dv = mgp.predict(pred, compute_grad=True)
# The implied loss is np.sum(mu**2) + np.sum(v**2)
dloss = 2*(dmu*mu[:,np.newaxis,:]).sum(2) + 2*(v[:,np.newaxis,np.newaxis]*dv).sum(2)
dloss_est = np.zeros(dloss.shape)
for i in xrange(Ntest):
for j in xrange(D):
pred[i,j] += eps
mu, v = mgp.predict(pred)
loss_1 = np.sum(mu**2) + np.sum(v**2)
pred[i,j] -= 2*eps
mu, v = mgp.predict(pred)
loss_2 = np.sum(mu**2) + np.sum(v**2)
pred[i,j] += eps
dloss_est[i,j] = ((loss_1 - loss_2) / (2*eps))
assert np.linalg.norm(dloss - dloss_est) < 1e-6
