def test_embedding_2(): np.random.seed(1) # Create a categorical version of branin def branin(x,y,z): x = (x*15)-5 y = y*15 if z == 1 or z == 2: return np.square(y - (5.1/(4*np.square(np.pi)))*np.square(x) + (5/np.pi)*x - 6) + 10*(1-(1./(8*np.pi)))*np.cos(x) + 10*z; elif z == 3: return np.square((y-1) - (5.1/(4*np.square(np.pi)))*np.square(x-1) + (5/np.pi)*(x-1) - 6) + 10*(1-(1./(8*np.pi)))*np.cos(x-1) + 10*z; else: if np.array(x).ndim: return 10*npr.randn(x.shape[0]) else: return 10*npr.randn() N = 100 D = 5 groups = [(0,1,2,3)] c = len(groups[0]) # Spray random points around a 5-D grid (3 categorical and 2 continuous) X = np.random.rand(N,D) a = X[:,:c].argmax(1) X[:,:c] = 0*X[:,:c] X[:,:c][range(N),a] = 1 # Get the values from Branin and make the categorical ones 1-of-k x = X[:,-2] y = X[:,-1] z = X[:,:c].argmax(1) + 1 # Compute and standardize the function values vals = np.zeros(N) for i in xrange(N): vals[i] = branin(x[i],y[i],z[i]) vals = (vals - vals.mean()) / vals.std() # Set up the model mgp = MixedGP(D,groups) mgp.fit(X, vals) w = np.zeros((c,2)) for i in xrange(mgp.num_states): mgp.set_state(i) w += mgp.params['norm_lin_0_weights'].value.reshape((c,2)) w /= mgp.num_states # Compute the pairwise distances implies by the embedding dists = cdist(w,w) # Make sure they're embedded with the right relative ordering. assert dists[0,1] < dists[0,2] and dists[0,1] < dists[0,3] and dists[1,2] < dists[1,3]
def test_fit(): npr.seed(1) N = 12 D = 10 groups = [[1,3],[5,6,9]] mgp = MixedGP(D, groups, burnin=5) inputs = npr.rand(N,D) pending = npr.rand(3,D) W = npr.randn(D,1) vals = inputs.dot(W).flatten() + np.sqrt(1e-3)*npr.randn(N) mgp.fit(inputs, vals, pending) assert mgp.chain_length == 15 assert all([np.all(p.value != p.initial_value) for p in mgp.params.values()]) assert len(mgp._cache_list) == 10 assert len(mgp._hypers_list) == 10 assert len(mgp._fantasy_values_list) == 10
def test_fit(): npr.seed(1) N = 12 D = 10 groups = [[1, 3], [5, 6, 9]] mgp = MixedGP(D, groups, burnin=5) inputs = npr.rand(N, D) pending = npr.rand(3, D) W = npr.randn(D, 1) vals = inputs.dot(W).flatten() + np.sqrt(1e-3) * npr.randn(N) mgp.fit(inputs, vals, pending) assert mgp.chain_length == 15 assert all( [np.all(p.value != p.initial_value) for p in mgp.params.values()]) assert len(mgp._cache_list) == 10 assert len(mgp._hypers_list) == 10 assert len(mgp._fantasy_values_list) == 10
def test_embedding(): np.random.seed(1) # Create a categorical version of branin (actually an integer, but we'll treat it as categorical) def branin(x, y, z): x = (x * 15) - 5 y = y * 15 return np.square(y - (5.1 / (4 * np.square(np.pi))) * np.square(x) + (5 / np.pi) * x - 6) + 10 * (1 - (1. / (8 * np.pi))) * np.cos(x) + 10 * z N = 100 D = 5 groups = [(0, 1, 2, 3)] c = len(groups[0]) # Spray random points around a 5-D grid (3 categorical and 2 continuous) X = np.random.rand(N, D) a = X[:, :c].argmax(1) X[:, :c] = 0 * X[:, :c] X[:, :c][range(N), a] = 1 # Get the values from Branin and make the categorical ones 1-of-k x = X[:, -2] y = X[:, -1] z = X[:, :c].argmax(1) + 1 # Compute and standardize the function values vals = branin(x, y, z) vals = (vals - vals.mean()) / vals.std() # Set up the model mgp = MixedGP(D, groups) mgp.fit(X, vals) w = np.zeros((c, 2)) for i in xrange(mgp.num_states): mgp.set_state(i) w += mgp.params['norm_lin_0_weights'].value.reshape((c, 2)) w /= mgp.num_states # Compute the pairwise distances implies by the embedding dists = cdist(w, w) # Make sure they're embedded with the right relative ordering. assert dists[0, 1] < dists[0, 2] and dists[0, 1] < dists[0, 3] and dists[ 1, 2] < dists[1, 3]
def test_predict(): npr.seed(1) N = 12 Npend = 3 Ntest = 2 D = 10 groups = [[1, 3], [5, 6, 9]] mgp = MixedGP(D, groups, burnin=5, num_fantasies=7) pred = npr.rand(Ntest, D) # Test with 0 points mu, v = mgp.predict(pred) np.testing.assert_allclose(mu, 0, rtol=1e-7, atol=0, err_msg='', verbose=True) np.testing.assert_allclose(v, 1 + 1e-6, rtol=1e-7, atol=0, err_msg='', verbose=True) #Test with 1 point X = np.zeros((1, D)) W = npr.randn(D, 1) val = X.dot(W).flatten() + np.sqrt(1e-3) * npr.randn() mgp.fit(X, val, fit_hypers=False) mu, v = mgp.predict(pred) # Points closer to the origin will have less variance if np.linalg.norm(pred[0] - X) < np.linalg.norm(pred[1] - X): assert v[0] < v[1] else: assert v[0] > v[1] # Predict at the point itself mu, v = mgp.predict(X) np.testing.assert_allclose(mu, val, rtol=1e-5, atol=0, err_msg='', verbose=True) # Now let's make sure it doesn't break with more data and pending inputs = npr.rand(N, D) vals = inputs.dot(W).flatten() + np.sqrt(1e-3) * npr.randn(N) pending = npr.rand(Npend, D) mgp.fit(inputs, vals, pending) mu, v = mgp.predict(pred) # Now let's check the gradients eps = 1e-5 mu, v, dmu, dv = mgp.predict(pred, compute_grad=True) # The implied loss is np.sum(mu**2) + np.sum(v**2) dloss = 2 * (dmu * mu[:, np.newaxis, :]).sum(2) + 2 * ( v[:, np.newaxis, np.newaxis] * dv).sum(2) dloss_est = np.zeros(dloss.shape) for i in xrange(Ntest): for j in xrange(D): pred[i, j] += eps mu, v = mgp.predict(pred) loss_1 = np.sum(mu**2) + np.sum(v**2) pred[i, j] -= 2 * eps mu, v = mgp.predict(pred) loss_2 = np.sum(mu**2) + np.sum(v**2) pred[i, j] += eps dloss_est[i, j] = ((loss_1 - loss_2) / (2 * eps)) assert np.linalg.norm(dloss - dloss_est) < 1e-6
def test_init(): groups = [[1, 3], [5, 6, 9]] mgp = MixedGP(10, groups)
def test_predict(): npr.seed(1) N = 12 Npend = 3 Ntest = 2 D = 10 groups = [[1,3],[5,6,9]] mgp = MixedGP(D, groups, burnin=5, num_fantasies=7) pred = npr.rand(Ntest,D) # Test with 0 points mu, v = mgp.predict(pred) np.testing.assert_allclose(mu, 0, rtol=1e-7, atol=0, err_msg='', verbose=True) np.testing.assert_allclose(v, 1+1e-6, rtol=1e-7, atol=0, err_msg='', verbose=True) #Test with 1 point X = np.zeros((1,D)) W = npr.randn(D,1) val = X.dot(W).flatten() + np.sqrt(1e-3)*npr.randn() mgp.fit(X, val, fit_hypers=False) mu, v = mgp.predict(pred) # Points closer to the origin will have less variance if np.linalg.norm(pred[0] - X) < np.linalg.norm(pred[1] - X): assert v[0] < v[1] else: assert v[0] > v[1] # Predict at the point itself mu, v = mgp.predict(X) np.testing.assert_allclose(mu, val, rtol=1e-5, atol=0, err_msg='', verbose=True) # Now let's make sure it doesn't break with more data and pending inputs = npr.rand(N,D) vals = inputs.dot(W).flatten() + np.sqrt(1e-3)*npr.randn(N) pending = npr.rand(Npend,D) mgp.fit(inputs, vals, pending) mu, v = mgp.predict(pred) # Now let's check the gradients eps = 1e-5 mu, v, dmu, dv = mgp.predict(pred, compute_grad=True) # The implied loss is np.sum(mu**2) + np.sum(v**2) dloss = 2*(dmu*mu[:,np.newaxis,:]).sum(2) + 2*(v[:,np.newaxis,np.newaxis]*dv).sum(2) dloss_est = np.zeros(dloss.shape) for i in xrange(Ntest): for j in xrange(D): pred[i,j] += eps mu, v = mgp.predict(pred) loss_1 = np.sum(mu**2) + np.sum(v**2) pred[i,j] -= 2*eps mu, v = mgp.predict(pred) loss_2 = np.sum(mu**2) + np.sum(v**2) pred[i,j] += eps dloss_est[i,j] = ((loss_1 - loss_2) / (2*eps)) assert np.linalg.norm(dloss - dloss_est) < 1e-6