def check_lower_bound(shape_mu, shape_alpha, plates_mu=(), **kwargs): M = GaussianARD(np.ones(plates_mu + shape_mu), np.ones(plates_mu + shape_mu), shape=shape_mu, plates=plates_mu) if not ('ndim' in kwargs or 'shape' in kwargs): kwargs['ndim'] = len(shape_mu) X = GaussianARD(M, 2*np.ones(shape_alpha), **kwargs) Y = GaussianARD(X, 3*np.ones(X.get_shape(0)), **kwargs) Y.observe(4*np.ones(Y.get_shape(0))) X.update() Cov = 1/(2+3) mu = Cov * (2*1 + 3*4) x2 = mu**2 + Cov logH_X = (+ 0.5*(1+np.log(2*np.pi)) + 0.5*np.log(Cov)) logp_X = (- 0.5*np.log(2*np.pi) + 0.5*np.log(2) - 0.5*2*(x2 - 2*mu*1 + 1**2+1)) r = np.prod(X.get_shape(0)) self.assertAllClose(r * (logp_X + logH_X), X.lower_bound_contribution())
def test_rotate_plates(self): # Basic test for Gaussian vectors X = GaussianARD(np.random.randn(3,2), np.random.rand(3,2), shape=(2,), plates=(3,)) (u0, u1) = X.get_moments() Cov = u1 - linalg.outer(u0, u0, ndim=1) Q = np.random.randn(3,3) Qu0 = np.einsum('ik,kj->ij', Q, u0) QCov = np.einsum('k,kij->kij', np.sum(Q, axis=0)**2, Cov) Qu1 = QCov + linalg.outer(Qu0, Qu0, ndim=1) X.rotate_plates(Q, plate_axis=-1) (u0, u1) = X.get_moments() self.assertAllClose(u0, Qu0) self.assertAllClose(u1, Qu1) # Test full covariance, that is, with observations X = GaussianARD(np.random.randn(3,2), np.random.rand(3,2), shape=(2,), plates=(3,)) Y = Gaussian(X, [[2.0, 1.5], [1.5, 3.0]], plates=(3,)) Y.observe(np.random.randn(3,2)) X.update() (u0, u1) = X.get_moments() Cov = u1 - linalg.outer(u0, u0, ndim=1) Q = np.random.randn(3,3) Qu0 = np.einsum('ik,kj->ij', Q, u0) QCov = np.einsum('k,kij->kij', np.sum(Q, axis=0)**2, Cov) Qu1 = QCov + linalg.outer(Qu0, Qu0, ndim=1) X.rotate_plates(Q, plate_axis=-1) (u0, u1) = X.get_moments() self.assertAllClose(u0, Qu0) self.assertAllClose(u1, Qu1) pass
def check(indices, plates, shape, axis=-1, use_mask=False): mu = np.random.rand(*(plates+shape)) alpha = np.random.rand(*(plates+shape)) X = GaussianARD(mu, alpha, shape=shape, plates=plates) Y = Take(X, indices, plate_axis=axis) Z = GaussianARD(Y, 1, shape=shape) z = np.random.randn(*(Z.get_shape(0))) if use_mask: mask = np.mod(np.reshape(np.arange(np.prod(Z.plates)), Z.plates), 2) != 0 else: mask = True Z.observe(z, mask=mask) X.update() (x0, x1) = X.get_moments() # For comparison, build the same model brute force X = GaussianARD(mu, alpha, shape=shape, plates=plates) # Number of trailing plate axes before the take axis N = len(X.plates) + axis # Reshape the take axes into a single axis z_shape = X.plates[:axis] + (-1,) if axis < -1: z_shape = z_shape + X.plates[(axis+1):] z_shape = z_shape + shape z = np.reshape(z, z_shape) # Reshape the take axes into a single axis if use_mask: mask_shape = X.plates[:axis] + (-1,) if axis < -1: mask_shape = mask_shape + X.plates[(axis+1):] mask = np.reshape(mask, mask_shape) for (j, i) in enumerate(range(np.size(indices))): ind = np.array(indices).flatten()[i] index_x = N*(slice(None),) + (ind,) index_z = N*(slice(None),) + (j,) # print(index) Xi = X[index_x] zi = z[index_z] Zi = GaussianARD(Xi, 1, ndim=len(shape)) if use_mask: maski = mask[index_z] else: maski = True Zi.observe(zi, mask=maski) X.update() self.assertAllClose( x0, X.get_moments()[0], ) self.assertAllClose( x1, X.get_moments()[1], ) return
def test_lowerbound(self): """ Test the variational Bayesian lower bound term for GaussianARD. """ # Test vector formula with full noise covariance m = np.random.randn(2) alpha = np.random.rand(2) y = np.random.randn(2) X = GaussianARD(m, alpha, ndim=1) V = np.array([[3, 1], [1, 3]]) Y = Gaussian(X, V) Y.observe(y) X.update() Cov = np.linalg.inv(np.diag(alpha) + V) mu = np.dot(Cov, np.dot(V, y) + alpha * m) x2 = np.outer(mu, mu) + Cov logH_X = (+2 * 0.5 * (1 + np.log(2 * np.pi)) + 0.5 * np.log(np.linalg.det(Cov))) logp_X = ( -2 * 0.5 * np.log(2 * np.pi) + 0.5 * np.log(np.linalg.det(np.diag(alpha))) - 0.5 * np.sum( np.diag(alpha) * (x2 - np.outer(mu, m) - np.outer(m, mu) + np.outer(m, m)))) self.assertAllClose(logp_X + logH_X, X.lower_bound_contribution()) def check_lower_bound(shape_mu, shape_alpha, plates_mu=(), **kwargs): M = GaussianARD(np.ones(plates_mu + shape_mu), np.ones(plates_mu + shape_mu), shape=shape_mu, plates=plates_mu) X = GaussianARD(M, 2 * np.ones(shape_alpha), **kwargs) Y = GaussianARD(X, 3 * np.ones(X.get_shape(0)), **kwargs) Y.observe(4 * np.ones(Y.get_shape(0))) X.update() Cov = 1 / (2 + 3) mu = Cov * (2 * 1 + 3 * 4) x2 = mu**2 + Cov logH_X = (+0.5 * (1 + np.log(2 * np.pi)) + 0.5 * np.log(Cov)) logp_X = (-0.5 * np.log(2 * np.pi) + 0.5 * np.log(2) - 0.5 * 2 * (x2 - 2 * mu * 1 + 1**2 + 1)) r = np.prod(X.get_shape(0)) self.assertAllClose(r * (logp_X + logH_X), X.lower_bound_contribution()) # Test scalar formula check_lower_bound((), ()) # Test array formula check_lower_bound((2, 3), (2, 3)) # Test dim-broadcasting of mu check_lower_bound((3, 1), (2, 3, 4)) # Test dim-broadcasting of alpha check_lower_bound((2, 3, 4), (3, 1)) # Test dim-broadcasting of mu and alpha check_lower_bound((3, 1), (3, 1), shape=(2, 3, 4)) # Test dim-broadcasting of mu with plates check_lower_bound((), (), plates_mu=(), shape=(), plates=(5, )) # BUG: Scalar parents for array variable caused einsum error check_lower_bound((), (), shape=(3, )) # BUG: Log-det was summed over plates check_lower_bound((), (), shape=(3, ), plates=(4, )) pass
def test_message_to_parent_alpha(self): """ Test the message from GaussianARD the 2nd parent (alpha). """ # Check formula with uncertain parent mu mu = GaussianARD(1, 1) tau = Gamma(0.5 * 1e10, 1e10) X = GaussianARD(mu, tau) X.observe(3) (m0, m1) = tau._message_from_children() self.assertAllClose(m0, -0.5 * (3**2 - 2 * 3 * 1 + 1**2 + 1)) self.assertAllClose(m1, 0.5) # Check formula with uncertain node tau = Gamma(1e10, 1e10) X = GaussianARD(2, tau) Y = GaussianARD(X, 1) Y.observe(5) X.update() (m0, m1) = tau._message_from_children() self.assertAllClose(m0, -0.5 * (1 / (1 + 1) + 3.5**2 - 2 * 3.5 * 2 + 2**2)) self.assertAllClose(m1, 0.5) # Check alpha larger than mu alpha = Gamma(np.ones((3, 2, 3)) * 1e10, 1e10) X = GaussianARD(np.ones((2, 3)), alpha, ndim=3) X.observe(2 * np.ones((3, 2, 3))) (m0, m1) = alpha._message_from_children() self.assertAllClose( m0 * np.ones((3, 2, 3)), -0.5 * (2**2 - 2 * 2 * 1 + 1**2) * np.ones((3, 2, 3))) self.assertAllClose(m1 * np.ones((3, 2, 3)), 0.5 * np.ones((3, 2, 3))) # Check mu larger than alpha tau = Gamma(np.ones((2, 3)) * 1e10, 1e10) X = GaussianARD(np.ones((3, 2, 3)), tau, ndim=3) X.observe(2 * np.ones((3, 2, 3))) (m0, m1) = tau._message_from_children() self.assertAllClose( m0, -0.5 * (2**2 - 2 * 2 * 1 + 1**2) * 3 * np.ones((2, 3))) self.assertAllClose(m1 * np.ones((2, 3)), 0.5 * 3 * np.ones((2, 3))) # Check node larger than mu and alpha tau = Gamma(np.ones((3, )) * 1e10, 1e10) X = GaussianARD(np.ones((2, 3)), tau, shape=(3, 2, 3)) X.observe(2 * np.ones((3, 2, 3))) (m0, m1) = tau._message_from_children() self.assertAllClose( m0 * np.ones(3), -0.5 * (2**2 - 2 * 2 * 1 + 1**2) * 6 * np.ones( (3, ))) self.assertAllClose(m1 * np.ones(3), 0.5 * 6 * np.ones(3)) # Check plates for smaller mu than node tau = Gamma(np.ones((4, 1, 2, 3)) * 1e10, 1e10) X = GaussianARD(GaussianARD(1, 1, shape=(3, ), plates=(4, 1, 1)), tau, shape=(2, 3), plates=(4, 5)) X.observe(2 * np.ones((4, 5, 2, 3))) (m0, m1) = tau._message_from_children() self.assertAllClose(m0 * np.ones( (4, 1, 2, 3)), (-0.5 * (2**2 - 2 * 2 * 1 + 1**2 + 1) * 5 * np.ones( (4, 1, 2, 3)))) self.assertAllClose(m1 * np.ones((4, 1, 2, 3)), 5 * 0.5 * np.ones( (4, 1, 2, 3))) # Check mask tau = Gamma(np.ones((4, 3)) * 1e10, 1e10) X = GaussianARD(np.ones(3), tau, shape=(3, ), plates=( 2, 4, )) X.observe(2 * np.ones((2, 4, 3)), mask=[[True, False, True, False], [False, True, True, False]]) (m0, m1) = tau._message_from_children() self.assertAllClose(m0 * np.ones((4, 3)), (-0.5 * (2**2 - 2 * 2 * 1 + 1**2) * np.ones( (4, 3)) * np.array([[1], [1], [2], [0]]))) self.assertAllClose( m1 * np.ones((4, 3)), 0.5 * np.array([[1], [1], [2], [0]]) * np.ones((4, 3))) # Check non-ARD Gaussian child mu = np.array([1, 2]) alpha = np.array([3, 4]) Alpha = Gamma(alpha * 1e10, 1e10) Lambda = np.array([[1, 0.5], [0.5, 1]]) X = GaussianARD(mu, Alpha, ndim=1) Y = Gaussian(X, Lambda) y = np.array([5, 6]) Y.observe(y) X.update() (m0, m1) = Alpha._message_from_children() Cov = np.linalg.inv(np.diag(alpha) + Lambda) mean = np.dot(Cov, np.dot(np.diag(alpha), mu) + np.dot(Lambda, y)) self.assertAllClose( m0 * np.ones(2), -0.5 * np.diag( np.outer(mean, mean) + Cov - np.outer(mean, mu) - np.outer(mu, mean) + np.outer(mu, mu))) self.assertAllClose(m1 * np.ones(2), 0.5 * np.ones(2)) pass
def test_message_to_parent_mu(self): """ Test that GaussianARD computes the message to the 1st parent correctly. """ # Check formula with uncertain parent alpha mu = GaussianARD(0, 1) alpha = Gamma(2, 1) X = GaussianARD(mu, alpha) X.observe(3) (m0, m1) = mu._message_from_children() #(m0, m1) = X._message_to_parent(0) self.assertAllClose(m0, 2 * 3) self.assertAllClose(m1, -0.5 * 2) # Check formula with uncertain node mu = GaussianARD(1, 1e10) X = GaussianARD(mu, 2) Y = GaussianARD(X, 1) Y.observe(5) X.update() (m0, m1) = mu._message_from_children() self.assertAllClose(m0, 2 * 1 / (2 + 1) * (2 * 1 + 1 * 5)) self.assertAllClose(m1, -0.5 * 2) # Check alpha larger than mu mu = GaussianARD(np.zeros((2, 3)), 1e10, shape=(2, 3)) X = GaussianARD(mu, 2 * np.ones((3, 2, 3))) X.observe(3 * np.ones((3, 2, 3))) (m0, m1) = mu._message_from_children() self.assertAllClose(m0, 2 * 3 * 3 * np.ones((2, 3))) self.assertAllClose(m1, -0.5 * 3 * 2 * misc.identity(2, 3)) # Check mu larger than alpha mu = GaussianARD(np.zeros((3, 2, 3)), 1e10, shape=(3, 2, 3)) X = GaussianARD(mu, 2 * np.ones((2, 3))) X.observe(3 * np.ones((3, 2, 3))) (m0, m1) = mu._message_from_children() self.assertAllClose(m0, 2 * 3 * np.ones((3, 2, 3))) self.assertAllClose(m1, -0.5 * 2 * misc.identity(3, 2, 3)) # Check node larger than mu and alpha mu = GaussianARD(np.zeros((2, 3)), 1e10, shape=(2, 3)) X = GaussianARD(mu, 2 * np.ones((3, )), shape=(3, 2, 3)) X.observe(3 * np.ones((3, 2, 3))) (m0, m1) = mu._message_from_children() self.assertAllClose(m0, 2 * 3 * 3 * np.ones((2, 3))) self.assertAllClose(m1, -0.5 * 2 * 3 * misc.identity(2, 3)) # Check broadcasting of dimensions mu = GaussianARD(np.zeros((2, 1)), 1e10, shape=(2, 1)) X = GaussianARD(mu, 2 * np.ones((2, 3)), shape=(2, 3)) X.observe(3 * np.ones((2, 3))) (m0, m1) = mu._message_from_children() self.assertAllClose(m0, 2 * 3 * 3 * np.ones((2, 1))) self.assertAllClose(m1, -0.5 * 2 * 3 * misc.identity(2, 1)) # Check plates for smaller mu than node mu = GaussianARD(0, 1, shape=(3, ), plates=(4, 1, 1)) X = GaussianARD(mu, 2 * np.ones((3, )), shape=(2, 3), plates=(4, 5)) X.observe(3 * np.ones((4, 5, 2, 3))) (m0, m1) = mu._message_from_children() self.assertAllClose(m0 * np.ones((4, 1, 1, 3)), 2 * 3 * 5 * 2 * np.ones((4, 1, 1, 3))) self.assertAllClose( m1 * np.ones((4, 1, 1, 3, 3)), -0.5 * 2 * 5 * 2 * misc.identity(3) * np.ones((4, 1, 1, 3, 3))) # Check mask mu = GaussianARD(np.zeros((2, 1, 3)), 1e10, shape=(3, )) X = GaussianARD(mu, 2 * np.ones((2, 4, 3)), shape=(3, ), plates=( 2, 4, )) X.observe(3 * np.ones((2, 4, 3)), mask=[[True, True, True, False], [False, True, False, True]]) (m0, m1) = mu._message_from_children() self.assertAllClose(m0, (2 * 3 * np.ones( (2, 1, 3)) * np.array([[[3]], [[2]]]))) self.assertAllClose(m1, (-0.5 * 2 * misc.identity(3) * np.ones( (2, 1, 1, 1)) * np.array([[[[3]]], [[[2]]]]))) # Check mask with different shapes mu = GaussianARD(np.zeros((2, 1, 3)), 1e10, shape=()) X = GaussianARD(mu, 2 * np.ones((2, 4, 3)), shape=(3, ), plates=( 2, 4, )) mask = np.array([[True, True, True, False], [False, True, False, True]]) X.observe(3 * np.ones((2, 4, 3)), mask=mask) (m0, m1) = mu._message_from_children() self.assertAllClose( m0, 2 * 3 * np.sum( np.ones((2, 4, 3)) * mask[..., None], axis=-2, keepdims=True)) self.assertAllClose(m1, (-0.5 * 2 * np.sum( np.ones((2, 4, 3)) * mask[..., None], axis=-2, keepdims=True))) # Check non-ARD Gaussian child mu = np.array([1, 2]) Mu = GaussianARD(mu, 1e10, shape=(2, )) alpha = np.array([3, 4]) Lambda = np.array([[1, 0.5], [0.5, 1]]) X = GaussianARD(Mu, alpha) Y = Gaussian(X, Lambda) y = np.array([5, 6]) Y.observe(y) X.update() (m0, m1) = Mu._message_from_children() mean = np.dot(np.linalg.inv(np.diag(alpha) + Lambda), np.dot(np.diag(alpha), mu) + np.dot(Lambda, y)) self.assertAllClose(m0, np.dot(np.diag(alpha), mean)) self.assertAllClose(m1, -0.5 * np.diag(alpha)) # Check broadcasted variable axes mu = GaussianARD(np.zeros(1), 1e10, shape=(1, )) X = GaussianARD(mu, 2, shape=(3, )) X.observe(3 * np.ones(3)) (m0, m1) = mu._message_from_children() self.assertAllClose(m0, 2 * 3 * np.sum(np.ones(3), axis=-1, keepdims=True)) self.assertAllClose( m1, -0.5 * 2 * np.sum(np.identity(3), axis=(-1, -2), keepdims=True)) pass
def test_message_to_child(self): """ Test moments of GaussianARD. """ # Check that moments have full shape when broadcasting X = GaussianARD(np.zeros((2, )), np.ones((3, 2)), shape=(4, 3, 2)) (u0, u1) = X._message_to_child() self.assertEqual(np.shape(u0), (4, 3, 2)) self.assertEqual(np.shape(u1), (4, 3, 2, 4, 3, 2)) # Check the formula X = GaussianARD(2, 3) (u0, u1) = X._message_to_child() self.assertAllClose(u0, 2) self.assertAllClose(u1, 2**2 + 1 / 3) # Check the formula for multidimensional arrays X = GaussianARD(2 * np.ones((2, 1, 4)), 3 * np.ones((2, 3, 1)), ndim=3) (u0, u1) = X._message_to_child() self.assertAllClose(u0, 2 * np.ones((2, 3, 4))) self.assertAllClose( u1, 2**2 * np.ones( (2, 3, 4, 2, 3, 4)) + 1 / 3 * misc.identity(2, 3, 4)) # Check the formula for dim-broadcasted mu X = GaussianARD(2 * np.ones((3, 1)), 3 * np.ones((2, 3, 4)), ndim=3) (u0, u1) = X._message_to_child() self.assertAllClose(u0, 2 * np.ones((2, 3, 4))) self.assertAllClose( u1, 2**2 * np.ones( (2, 3, 4, 2, 3, 4)) + 1 / 3 * misc.identity(2, 3, 4)) # Check the formula for dim-broadcasted alpha X = GaussianARD(2 * np.ones((2, 3, 4)), 3 * np.ones((3, 1)), ndim=3) (u0, u1) = X._message_to_child() self.assertAllClose(u0, 2 * np.ones((2, 3, 4))) self.assertAllClose( u1, 2**2 * np.ones( (2, 3, 4, 2, 3, 4)) + 1 / 3 * misc.identity(2, 3, 4)) # Check the formula for dim-broadcasted mu and alpha X = GaussianARD(2 * np.ones((3, 1)), 3 * np.ones((3, 1)), shape=(2, 3, 4)) (u0, u1) = X._message_to_child() self.assertAllClose(u0, 2 * np.ones((2, 3, 4))) self.assertAllClose( u1, 2**2 * np.ones( (2, 3, 4, 2, 3, 4)) + 1 / 3 * misc.identity(2, 3, 4)) # Check the formula for dim-broadcasted mu with plates mu = GaussianARD(2 * np.ones((5, 1, 3, 4)), np.ones((5, 1, 3, 4)), shape=(3, 4), plates=(5, 1)) X = GaussianARD(mu, 3 * np.ones((5, 2, 3, 4)), shape=(2, 3, 4), plates=(5, )) (u0, u1) = X._message_to_child() self.assertAllClose(u0, 2 * np.ones((5, 2, 3, 4))) self.assertAllClose( u1, 2**2 * np.ones( (5, 2, 3, 4, 2, 3, 4)) + 1 / 3 * misc.identity(2, 3, 4)) # Check posterior X = GaussianARD(2, 3) Y = GaussianARD(X, 1) Y.observe(10) X.update() (u0, u1) = X._message_to_child() self.assertAllClose(u0, 1 / (3 + 1) * (3 * 2 + 1 * 10)) self.assertAllClose(u1, (1 / (3 + 1) * (3 * 2 + 1 * 10))**2 + 1 / (3 + 1)) pass
def test_lowerbound(self): """ Test the variational Bayesian lower bound term for GaussianARD. """ # Test vector formula with full noise covariance m = np.random.randn(2) alpha = np.random.rand(2) y = np.random.randn(2) X = GaussianARD(m, alpha, ndim=1) V = np.array([[3,1],[1,3]]) Y = Gaussian(X, V) Y.observe(y) X.update() Cov = np.linalg.inv(np.diag(alpha) + V) mu = np.dot(Cov, np.dot(V, y) + alpha*m) x2 = np.outer(mu, mu) + Cov logH_X = (+ 2*0.5*(1+np.log(2*np.pi)) + 0.5*np.log(np.linalg.det(Cov))) logp_X = (- 2*0.5*np.log(2*np.pi) + 0.5*np.log(np.linalg.det(np.diag(alpha))) - 0.5*np.sum(np.diag(alpha) * (x2 - np.outer(mu,m) - np.outer(m,mu) + np.outer(m,m)))) self.assertAllClose(logp_X + logH_X, X.lower_bound_contribution()) def check_lower_bound(shape_mu, shape_alpha, plates_mu=(), **kwargs): M = GaussianARD(np.ones(plates_mu + shape_mu), np.ones(plates_mu + shape_mu), shape=shape_mu, plates=plates_mu) if not ('ndim' in kwargs or 'shape' in kwargs): kwargs['ndim'] = len(shape_mu) X = GaussianARD(M, 2*np.ones(shape_alpha), **kwargs) Y = GaussianARD(X, 3*np.ones(X.get_shape(0)), **kwargs) Y.observe(4*np.ones(Y.get_shape(0))) X.update() Cov = 1/(2+3) mu = Cov * (2*1 + 3*4) x2 = mu**2 + Cov logH_X = (+ 0.5*(1+np.log(2*np.pi)) + 0.5*np.log(Cov)) logp_X = (- 0.5*np.log(2*np.pi) + 0.5*np.log(2) - 0.5*2*(x2 - 2*mu*1 + 1**2+1)) r = np.prod(X.get_shape(0)) self.assertAllClose(r * (logp_X + logH_X), X.lower_bound_contribution()) # Test scalar formula check_lower_bound((), ()) # Test array formula check_lower_bound((2,3), (2,3)) # Test dim-broadcasting of mu check_lower_bound((3,1), (2,3,4)) # Test dim-broadcasting of alpha check_lower_bound((2,3,4), (3,1)) # Test dim-broadcasting of mu and alpha check_lower_bound((3,1), (3,1), shape=(2,3,4)) # Test dim-broadcasting of mu with plates check_lower_bound((), (), plates_mu=(), shape=(), plates=(5,)) # BUG: Scalar parents for array variable caused einsum error check_lower_bound((), (), shape=(3,)) # BUG: Log-det was summed over plates check_lower_bound((), (), shape=(3,), plates=(4,)) pass
def test_message_to_parent_alpha(self): """ Test the message from GaussianARD the 2nd parent (alpha). """ # Check formula with uncertain parent mu mu = GaussianARD(1,1) tau = Gamma(0.5*1e10, 1e10) X = GaussianARD(mu, tau) X.observe(3) (m0, m1) = tau._message_from_children() self.assertAllClose(m0, -0.5*(3**2 - 2*3*1 + 1**2+1)) self.assertAllClose(m1, 0.5) # Check formula with uncertain node tau = Gamma(1e10, 1e10) X = GaussianARD(2, tau) Y = GaussianARD(X, 1) Y.observe(5) X.update() (m0, m1) = tau._message_from_children() self.assertAllClose(m0, -0.5*(1/(1+1)+3.5**2 - 2*3.5*2 + 2**2)) self.assertAllClose(m1, 0.5) # Check alpha larger than mu alpha = Gamma(np.ones((3,2,3))*1e10, 1e10) X = GaussianARD(np.ones((2,3)), alpha, ndim=3) X.observe(2*np.ones((3,2,3))) (m0, m1) = alpha._message_from_children() self.assertAllClose(m0 * np.ones((3,2,3)), -0.5*(2**2 - 2*2*1 + 1**2) * np.ones((3,2,3))) self.assertAllClose(m1*np.ones((3,2,3)), 0.5*np.ones((3,2,3))) # Check mu larger than alpha tau = Gamma(np.ones((2,3))*1e10, 1e10) X = GaussianARD(np.ones((3,2,3)), tau, ndim=3) X.observe(2*np.ones((3,2,3))) (m0, m1) = tau._message_from_children() self.assertAllClose(m0, -0.5*(2**2 - 2*2*1 + 1**2) * 3 * np.ones((2,3))) self.assertAllClose(m1 * np.ones((2,3)), 0.5 * 3 * np.ones((2,3))) # Check node larger than mu and alpha tau = Gamma(np.ones((3,))*1e10, 1e10) X = GaussianARD(np.ones((2,3)), tau, shape=(3,2,3)) X.observe(2*np.ones((3,2,3))) (m0, m1) = tau._message_from_children() self.assertAllClose(m0 * np.ones(3), -0.5*(2**2 - 2*2*1 + 1**2) * 6 * np.ones((3,))) self.assertAllClose(m1 * np.ones(3), 0.5 * 6 * np.ones(3)) # Check plates for smaller mu than node tau = Gamma(np.ones((4,1,2,3))*1e10, 1e10) X = GaussianARD(GaussianARD(1, 1, shape=(3,), plates=(4,1,1)), tau, shape=(2,3), plates=(4,5)) X.observe(2*np.ones((4,5,2,3))) (m0, m1) = tau._message_from_children() self.assertAllClose(m0 * np.ones((4,1,2,3)), (-0.5 * (2**2 - 2*2*1 + 1**2+1) * 5*np.ones((4,1,2,3)))) self.assertAllClose(m1 * np.ones((4,1,2,3)), 5*0.5 * np.ones((4,1,2,3))) # Check mask tau = Gamma(np.ones((4,3))*1e10, 1e10) X = GaussianARD(np.ones(3), tau, shape=(3,), plates=(2,4,)) X.observe(2*np.ones((2,4,3)), mask=[[True, False, True, False], [False, True, True, False]]) (m0, m1) = tau._message_from_children() self.assertAllClose(m0 * np.ones((4,3)), (-0.5 * (2**2 - 2*2*1 + 1**2) * np.ones((4,3)) * np.array([[1], [1], [2], [0]]))) self.assertAllClose(m1 * np.ones((4,3)), 0.5 * np.array([[1], [1], [2], [0]]) * np.ones((4,3))) # Check non-ARD Gaussian child mu = np.array([1,2]) alpha = np.array([3,4]) Alpha = Gamma(alpha*1e10, 1e10) Lambda = np.array([[1, 0.5], [0.5, 1]]) X = GaussianARD(mu, Alpha, ndim=1) Y = Gaussian(X, Lambda) y = np.array([5,6]) Y.observe(y) X.update() (m0, m1) = Alpha._message_from_children() Cov = np.linalg.inv(np.diag(alpha)+Lambda) mean = np.dot(Cov, np.dot(np.diag(alpha), mu) + np.dot(Lambda, y)) self.assertAllClose(m0 * np.ones(2), -0.5 * np.diag( np.outer(mean, mean) + Cov - np.outer(mean, mu) - np.outer(mu, mean) + np.outer(mu, mu))) self.assertAllClose(m1 * np.ones(2), 0.5 * np.ones(2)) pass
def test_message_to_parent_mu(self): """ Test that GaussianARD computes the message to the 1st parent correctly. """ # Check formula with uncertain parent alpha mu = GaussianARD(0, 1) alpha = Gamma(2,1) X = GaussianARD(mu, alpha) X.observe(3) (m0, m1) = mu._message_from_children() #(m0, m1) = X._message_to_parent(0) self.assertAllClose(m0, 2*3) self.assertAllClose(m1, -0.5*2) # Check formula with uncertain node mu = GaussianARD(1, 1e10) X = GaussianARD(mu, 2) Y = GaussianARD(X, 1) Y.observe(5) X.update() (m0, m1) = mu._message_from_children() self.assertAllClose(m0, 2 * 1/(2+1)*(2*1+1*5)) self.assertAllClose(m1, -0.5*2) # Check alpha larger than mu mu = GaussianARD(np.zeros((2,3)), 1e10, shape=(2,3)) X = GaussianARD(mu, 2*np.ones((3,2,3))) X.observe(3*np.ones((3,2,3))) (m0, m1) = mu._message_from_children() self.assertAllClose(m0, 2*3 * 3 * np.ones((2,3))) self.assertAllClose(m1, -0.5 * 3 * 2*misc.identity(2,3)) # Check mu larger than alpha mu = GaussianARD(np.zeros((3,2,3)), 1e10, shape=(3,2,3)) X = GaussianARD(mu, 2*np.ones((2,3))) X.observe(3*np.ones((3,2,3))) (m0, m1) = mu._message_from_children() self.assertAllClose(m0, 2 * 3 * np.ones((3,2,3))) self.assertAllClose(m1, -0.5 * 2*misc.identity(3,2,3)) # Check node larger than mu and alpha mu = GaussianARD(np.zeros((2,3)), 1e10, shape=(2,3)) X = GaussianARD(mu, 2*np.ones((3,)), shape=(3,2,3)) X.observe(3*np.ones((3,2,3))) (m0, m1) = mu._message_from_children() self.assertAllClose(m0, 2*3 * 3*np.ones((2,3))) self.assertAllClose(m1, -0.5 * 2 * 3*misc.identity(2,3)) # Check broadcasting of dimensions mu = GaussianARD(np.zeros((2,1)), 1e10, shape=(2,1)) X = GaussianARD(mu, 2*np.ones((2,3)), shape=(2,3)) X.observe(3*np.ones((2,3))) (m0, m1) = mu._message_from_children() self.assertAllClose(m0, 2*3 * 3*np.ones((2,1))) self.assertAllClose(m1, -0.5 * 2 * 3*misc.identity(2,1)) # Check plates for smaller mu than node mu = GaussianARD(0,1, shape=(3,), plates=(4,1,1)) X = GaussianARD(mu, 2*np.ones((3,)), shape=(2,3), plates=(4,5)) X.observe(3*np.ones((4,5,2,3))) (m0, m1) = mu._message_from_children() self.assertAllClose(m0 * np.ones((4,1,1,3)), 2*3 * 5*2*np.ones((4,1,1,3))) self.assertAllClose(m1 * np.ones((4,1,1,3,3)), -0.5*2 * 5*2*misc.identity(3) * np.ones((4,1,1,3,3))) # Check mask mu = GaussianARD(np.zeros((2,1,3)), 1e10, shape=(3,)) X = GaussianARD(mu, 2*np.ones((2,4,3)), shape=(3,), plates=(2,4,)) X.observe(3*np.ones((2,4,3)), mask=[[True, True, True, False], [False, True, False, True]]) (m0, m1) = mu._message_from_children() self.assertAllClose(m0, (2*3 * np.ones((2,1,3)) * np.array([[[3]], [[2]]]))) self.assertAllClose(m1, (-0.5*2 * misc.identity(3) * np.ones((2,1,1,1)) * np.array([[[[3]]], [[[2]]]]))) # Check mask with different shapes mu = GaussianARD(np.zeros((2,1,3)), 1e10, shape=()) X = GaussianARD(mu, 2*np.ones((2,4,3)), shape=(3,), plates=(2,4,)) mask = np.array([[True, True, True, False], [False, True, False, True]]) X.observe(3*np.ones((2,4,3)), mask=mask) (m0, m1) = mu._message_from_children() self.assertAllClose(m0, 2*3 * np.sum(np.ones((2,4,3))*mask[...,None], axis=-2, keepdims=True)) self.assertAllClose(m1, (-0.5*2 * np.sum(np.ones((2,4,3))*mask[...,None], axis=-2, keepdims=True))) # Check non-ARD Gaussian child mu = np.array([1,2]) Mu = GaussianARD(mu, 1e10, shape=(2,)) alpha = np.array([3,4]) Lambda = np.array([[1, 0.5], [0.5, 1]]) X = GaussianARD(Mu, alpha, ndim=1) Y = Gaussian(X, Lambda) y = np.array([5,6]) Y.observe(y) X.update() (m0, m1) = Mu._message_from_children() mean = np.dot(np.linalg.inv(np.diag(alpha)+Lambda), np.dot(np.diag(alpha), mu) + np.dot(Lambda, y)) self.assertAllClose(m0, np.dot(np.diag(alpha), mean)) self.assertAllClose(m1, -0.5*np.diag(alpha)) # Check broadcasted variable axes mu = GaussianARD(np.zeros(1), 1e10, shape=(1,)) X = GaussianARD(mu, 2, shape=(3,)) X.observe(3*np.ones(3)) (m0, m1) = mu._message_from_children() self.assertAllClose(m0, 2*3 * np.sum(np.ones(3), axis=-1, keepdims=True)) self.assertAllClose(m1, -0.5*2 * np.sum(np.identity(3), axis=(-1,-2), keepdims=True)) pass
def test_message_to_child(self): """ Test moments of GaussianARD. """ # Check that moments have full shape when broadcasting X = GaussianARD(np.zeros((2,)), np.ones((3,2)), shape=(4,3,2)) (u0, u1) = X._message_to_child() self.assertEqual(np.shape(u0), (4,3,2)) self.assertEqual(np.shape(u1), (4,3,2,4,3,2)) # Check the formula X = GaussianARD(2, 3) (u0, u1) = X._message_to_child() self.assertAllClose(u0, 2) self.assertAllClose(u1, 2**2 + 1/3) # Check the formula for multidimensional arrays X = GaussianARD(2*np.ones((2,1,4)), 3*np.ones((2,3,1)), ndim=3) (u0, u1) = X._message_to_child() self.assertAllClose(u0, 2*np.ones((2,3,4))) self.assertAllClose(u1, 2**2 * np.ones((2,3,4,2,3,4)) + 1/3 * misc.identity(2,3,4)) # Check the formula for dim-broadcasted mu X = GaussianARD(2*np.ones((3,1)), 3*np.ones((2,3,4)), ndim=3) (u0, u1) = X._message_to_child() self.assertAllClose(u0, 2*np.ones((2,3,4))) self.assertAllClose(u1, 2**2 * np.ones((2,3,4,2,3,4)) + 1/3 * misc.identity(2,3,4)) # Check the formula for dim-broadcasted alpha X = GaussianARD(2*np.ones((2,3,4)), 3*np.ones((3,1)), ndim=3) (u0, u1) = X._message_to_child() self.assertAllClose(u0, 2*np.ones((2,3,4))) self.assertAllClose(u1, 2**2 * np.ones((2,3,4,2,3,4)) + 1/3 * misc.identity(2,3,4)) # Check the formula for dim-broadcasted mu and alpha X = GaussianARD(2*np.ones((3,1)), 3*np.ones((3,1)), shape=(2,3,4)) (u0, u1) = X._message_to_child() self.assertAllClose(u0, 2*np.ones((2,3,4))) self.assertAllClose(u1, 2**2 * np.ones((2,3,4,2,3,4)) + 1/3 * misc.identity(2,3,4)) # Check the formula for dim-broadcasted mu with plates mu = GaussianARD(2*np.ones((5,1,3,4)), np.ones((5,1,3,4)), shape=(3,4), plates=(5,1)) X = GaussianARD(mu, 3*np.ones((5,2,3,4)), shape=(2,3,4), plates=(5,)) (u0, u1) = X._message_to_child() self.assertAllClose(u0, 2*np.ones((5,2,3,4))) self.assertAllClose(u1, 2**2 * np.ones((5,2,3,4,2,3,4)) + 1/3 * misc.identity(2,3,4)) # Check posterior X = GaussianARD(2, 3) Y = GaussianARD(X, 1) Y.observe(10) X.update() (u0, u1) = X._message_to_child() self.assertAllClose(u0, 1/(3+1) * (3*2 + 1*10)) self.assertAllClose(u1, (1/(3+1) * (3*2 + 1*10))**2 + 1/(3+1)) pass
def test_annealing(self): X = GaussianARD(3, 4) X.initialize_from_parameters(-1, 6) Q = VB(X) Q.set_annealing(0.1) # # Check that the gradient is correct # # Initial parameters phi0 = X.phi # Gradient rg = X.get_riemannian_gradient() g = X.get_gradient(rg) # Numerical gradient of the first parameter eps = 1e-6 p0 = X.get_parameters() l0 = Q.compute_lowerbound(ignore_masked=False) g_num = [(), ()] e = eps p1 = p0[0] + e X.set_parameters([p1, p0[1]]) l1 = Q.compute_lowerbound(ignore_masked=False) g_num[0] = (l1 - l0) / eps # Numerical gradient of the second parameter p1 = p0[1] + e X.set_parameters([p0[0], p1]) l1 = Q.compute_lowerbound(ignore_masked=False) g_num[1] = (l1 - l0) / (eps) # Check self.assertAllClose(g[0], g_num[0]) self.assertAllClose(g[1], g_num[1]) # # Gradient should be zero after updating # X.update() # Initial parameters phi0 = X.phi # Numerical gradient of the first parameter eps = 1e-8 p0 = X.get_parameters() l0 = Q.compute_lowerbound(ignore_masked=False) g_num = [(), ()] e = eps p1 = p0[0] + e X.set_parameters([p1, p0[1]]) l1 = Q.compute_lowerbound(ignore_masked=False) g_num[0] = (l1 - l0) / eps # Numerical gradient of the second parameter p1 = p0[1] + e X.set_parameters([p0[0], p1]) l1 = Q.compute_lowerbound(ignore_masked=False) g_num[1] = (l1 - l0) / (eps) # Check self.assertAllClose(0, g_num[0], atol=1e-5) self.assertAllClose(0, g_num[1], atol=1e-5) # Not at the optimum X.initialize_from_parameters(-1, 6) # Initial parameters phi0 = X.phi # Gradient g = X.get_riemannian_gradient() # Parameters after VB-EM update X.update() phi1 = X.phi # Check self.assertAllClose(g[0], phi1[0] - phi0[0]) self.assertAllClose(g[1], phi1[1] - phi0[1]) pass