def test_mask_to_parent(self):
"""
Test the mask handling in Mixture node
"""
K = 3
Z = Categorical(np.ones(K)/K,
plates=(4,5,1))
Mu = GaussianARD(0, 1,
shape=(2,),
plates=(4,K,5))
Alpha = Gamma(1, 1,
plates=(4,K,5,2))
X = Mixture(Z, GaussianARD, Mu, Alpha, cluster_plate=-3)
Y = GaussianARD(X, 1, ndim=1)
mask = np.reshape((np.mod(np.arange(4*5), 2) == 0),
(4,5))
Y.observe(np.ones((4,5,2)),
mask=mask)
self.assertArrayEqual(Z.mask,
mask[:,:,None])
self.assertArrayEqual(Mu.mask,
mask[:,None,:])
self.assertArrayEqual(Alpha.mask,
mask[:,None,:,None])
pass
def check_lower_bound(shape_mu, shape_alpha, plates_mu=(), **kwargs):
M = GaussianARD(np.ones(plates_mu + shape_mu),
np.ones(plates_mu + shape_mu),
shape=shape_mu,
plates=plates_mu)
if not ('ndim' in kwargs or 'shape' in kwargs):
kwargs['ndim'] = len(shape_mu)
X = GaussianARD(M,
2*np.ones(shape_alpha),
**kwargs)
Y = GaussianARD(X,
3*np.ones(X.get_shape(0)),
**kwargs)
Y.observe(4*np.ones(Y.get_shape(0)))
X.update()
Cov = 1/(2+3)
mu = Cov * (2*1 + 3*4)
x2 = mu**2 + Cov
logH_X = (+ 0.5*(1+np.log(2*np.pi))
+ 0.5*np.log(Cov))
logp_X = (- 0.5*np.log(2*np.pi)
+ 0.5*np.log(2)
- 0.5*2*(x2 - 2*mu*1 + 1**2+1))
r = np.prod(X.get_shape(0))
self.assertAllClose(r * (logp_X + logH_X),
X.lower_bound_contribution())
def _setup_linear_regression():
"""
Setup code for the pdf and contour tests.
This code is from http://www.bayespy.org/examples/regression.html
"""
np.random.seed(1)
k = 2 # slope
c = 5 # bias
s = 2 # noise standard deviation
x = np.arange(10)
y = k * x + c + s * np.random.randn(10)
X = np.vstack([x, np.ones(len(x))]).T
B = GaussianARD(0, 1e-6, shape=(2, ))
F = SumMultiply('i,i', B, X)
tau = Gamma(1e-3, 1e-3)
Y = GaussianARD(F, tau)
Y.observe(y)
Q = VB(Y, B, tau)
Q.update(repeat=1000)
xh = np.linspace(-5, 15, 100)
Xh = np.vstack([xh, np.ones(len(xh))]).T
Fh = SumMultiply('i,i', B, Xh)
return locals()
def model(M=10, N=100, D=3):
"""
Construct linear state-space model.
See, for instance, the following publication:
"Fast variational Bayesian linear state-space model"
Luttinen (ECML 2013)
"""
# Dynamics matrix with ARD
alpha = Gamma(1e-5, 1e-5, plates=(D, ), name='alpha')
A = GaussianARD(0,
alpha,
shape=(D, ),
plates=(D, ),
plotter=bpplt.GaussianHintonPlotter(rows=0,
cols=1,
scale=0),
name='A')
A.initialize_from_value(np.identity(D))
# Latent states with dynamics
X = GaussianMarkovChain(
np.zeros(D), # mean of x0
1e-3 * np.identity(D), # prec of x0
A, # dynamics
np.ones(D), # innovation
n=N, # time instances
plotter=bpplt.GaussianMarkovChainPlotter(scale=2),
name='X')
X.initialize_from_value(np.random.randn(N, D))
# Mixing matrix from latent space to observation space using ARD
gamma = Gamma(1e-5, 1e-5, plates=(D, ), name='gamma')
gamma.initialize_from_value(1e-2 * np.ones(D))
C = GaussianARD(0,
gamma,
shape=(D, ),
plates=(M, 1),
plotter=bpplt.GaussianHintonPlotter(rows=0,
cols=2,
scale=0),
name='C')
C.initialize_from_value(np.random.randn(M, 1, D))
# Observation noise
tau = Gamma(1e-5, 1e-5, name='tau')
tau.initialize_from_value(1e2)
# Underlying noiseless function
F = SumMultiply('i,i', C, X, name='F')
# Noisy observations
Y = GaussianARD(F, tau, name='Y')
Q = VB(Y, F, C, gamma, X, A, alpha, tau, C)
return Q
def fit(self, X, y):
self.weights = GaussianARD(0, 1e-6, shape=(X.shape[-1], ))
y_mean = SumMultiply('i,i', self.weights, X)
precision = Gamma(1, .1)
y_obs = GaussianARD(y_mean, precision)
y_obs.observe(y)
Q = VB(y_obs, self.weights, precision)
Q.update(repeat=self.n_iter, tol=self.tolerance, verbose=False)
def test_message_to_child(self):
"""
Test the message to child of Mixture node.
"""
K = 3
#
# Estimate moments from parents only
#
# Simple case
mu = GaussianARD([0,2,4], 1,
ndim=0,
plates=(K,))
alpha = Gamma(1, 1,
plates=(K,))
z = Categorical(np.ones(K)/K)
X = Mixture(z, GaussianARD, mu, alpha)
self.assertEqual(X.plates, ())
self.assertEqual(X.dims, ( (), () ))
u = X._message_to_child()
self.assertAllClose(u[0],
2)
self.assertAllClose(u[1],
2**2+1)
# Broadcasting the moments on the cluster axis
mu = GaussianARD(2, 1,
ndim=0,
plates=(K,))
alpha = Gamma(1, 1,
plates=(K,))
z = Categorical(np.ones(K)/K)
X = Mixture(z, GaussianARD, mu, alpha)
self.assertEqual(X.plates, ())
self.assertEqual(X.dims, ( (), () ))
u = X._message_to_child()
self.assertAllClose(u[0],
2)
self.assertAllClose(u[1],
2**2+1)
#
# Estimate moments with observed children
#
pass
def check_init(true_plates, true_shape, mu, alpha, **kwargs):
X = GaussianARD(mu, alpha, **kwargs)
self.assertEqual(X.dims, (true_shape, true_shape + true_shape),
msg="Constructed incorrect dimensionality")
self.assertEqual(X.plates,
true_plates,
msg="Constructed incorrect plates")
def __init__(self, prior, precision=1e-6, n_iter=1000, tolerance=1e-8):
super().__init__()
self.n_iter = n_iter
self.prior = prior
self.precision = precision
self.tolerance = tolerance
self.weights = GaussianARD(prior, precision, shape=prior.shape)
def test_plates_multiplier_from_parent(self):
X = GaussianARD(np.random.randn(3, 2), 1, ndim=1)
Y = Take(X, [0, 1, 2, 1, 1])
self.assertEqual(Y._plates_multiplier_from_parent(0), ())
pass
def test_riemannian_gradient(self):
"""Test Riemannian gradient of a Gamma node."""
#
# Without observations
#
# Construct model
a = np.random.rand()
b = np.random.rand()
tau = Gamma(a, b)
# Random initialization
tau.initialize_from_parameters(np.random.rand(),
np.random.rand())
# Initial parameters
phi0 = tau.phi
# Gradient
g = tau.get_riemannian_gradient()
# Parameters after VB-EM update
tau.update()
phi1 = tau.phi
# Check
self.assertAllClose(g[0],
phi1[0] - phi0[0])
self.assertAllClose(g[1],
phi1[1] - phi0[1])
#
# With observations
#
# Construct model
a = np.random.rand()
b = np.random.rand()
tau = Gamma(a, b)
mu = np.random.randn()
Y = GaussianARD(mu, tau)
Y.observe(np.random.randn())
# Random initialization
tau.initialize_from_parameters(np.random.rand(),
np.random.rand())
# Initial parameters
phi0 = tau.phi
# Gradient
g = tau.get_riemannian_gradient()
# Parameters after VB-EM update
tau.update()
phi1 = tau.phi
# Check
self.assertAllClose(g[0],
phi1[0] - phi0[0])
self.assertAllClose(g[1],
phi1[1] - phi0[1])
pass
def pca():
np.random.seed(41)
M = 10
N = 3000
D = 5
# Construct the PCA model
alpha = Gamma(1e-3, 1e-3, plates=(D, ), name='alpha')
W = GaussianARD(0, alpha, plates=(M, 1), shape=(D, ), name='W')
X = GaussianARD(0, 1, plates=(1, N), shape=(D, ), name='X')
tau = Gamma(1e-3, 1e-3, name='tau')
W.initialize_from_random()
F = SumMultiply('d,d->', W, X)
Y = GaussianARD(F, tau, name='Y')
# Observe data
data = np.sum(np.random.randn(M, 1, D - 1) * np.random.randn(1, N, D - 1),
axis=-1) + 1e-1 * np.random.randn(M, N)
Y.observe(data)
# Initialize VB engine
Q = VB(Y, X, W, alpha, tau)
# Take one update step (so phi is ok)
Q.update(repeat=1)
Q.save()
# Run VB-EM
Q.update(repeat=200)
bpplt.pyplot.plot(np.cumsum(Q.cputime), Q.L, 'k-')
# Restore the state
Q.load()
# Run Riemannian conjugate gradient
#Q.optimize(X, alpha, maxiter=100, collapsed=[W, tau])
Q.optimize(W, tau, maxiter=100, collapsed=[X, alpha])
bpplt.pyplot.plot(np.cumsum(Q.cputime), Q.L, 'r:')
bpplt.pyplot.show()
def test_init(self):
"""
Test the creation of Mixture node
"""
# Do not accept non-negative cluster plates
z = Categorical(np.random.dirichlet([1, 1]))
self.assertRaises(ValueError,
Mixture,
z,
GaussianARD,
GaussianARD(0, 1, plates=(2, )),
Gamma(1, 1, plates=(2, )),
cluster_plate=0)
# Try constructing a mixture without any of the parents having the
# cluster plate axis
z = Categorical(np.random.dirichlet([1, 1]))
self.assertRaises(ValueError, Mixture, z, GaussianARD,
GaussianARD(0, 1, plates=()), Gamma(1, 1, plates=()))
def test_lower_bound_contribution(self):
a = 15
b = 21
y = 4
x = Gamma(a, b)
x.observe(y)
testing.assert_allclose(
x.lower_bound_contribution(),
(
a * np.log(b) +
(a - 1) * np.log(y) -
b * y -
special.gammaln(a)
)
)
# Just one latent node so we'll get exact marginal likelihood
#
# p(Y) = p(Y,X)/p(X|Y) = p(Y|X) * p(X) / p(X|Y)
a = 2.3
b = 4.1
x = 1.9
y = 4.8
tau = Gamma(a, b)
Y = GaussianARD(x, tau)
Y.observe(y)
mu = x
nu = 2 * a
s2 = b / a
a_post = a + 0.5
b_post = b + 0.5*(y - x)**2
tau.update()
testing.assert_allclose(
[-b_post, a_post],
tau.phi
)
testing.assert_allclose(
Y.lower_bound_contribution() + tau.lower_bound_contribution(), # + tau.g,
(
special.gammaln((nu+1)/2)
- special.gammaln(nu/2)
- 0.5 * np.log(nu)
- 0.5 * np.log(np.pi)
- 0.5 * np.log(s2)
- 0.5 * (nu + 1) * np.log(
1 + (y - mu)**2 / (nu * s2)
)
)
)
return
def test_rotate_plates(self):
# Basic test for Gaussian vectors
X = GaussianARD(np.random.randn(3,2),
np.random.rand(3,2),
shape=(2,),
plates=(3,))
(u0, u1) = X.get_moments()
Cov = u1 - linalg.outer(u0, u0, ndim=1)
Q = np.random.randn(3,3)
Qu0 = np.einsum('ik,kj->ij', Q, u0)
QCov = np.einsum('k,kij->kij', np.sum(Q, axis=0)**2, Cov)
Qu1 = QCov + linalg.outer(Qu0, Qu0, ndim=1)
X.rotate_plates(Q, plate_axis=-1)
(u0, u1) = X.get_moments()
self.assertAllClose(u0, Qu0)
self.assertAllClose(u1, Qu1)
# Test full covariance, that is, with observations
X = GaussianARD(np.random.randn(3,2),
np.random.rand(3,2),
shape=(2,),
plates=(3,))
Y = Gaussian(X, [[2.0, 1.5], [1.5, 3.0]],
plates=(3,))
Y.observe(np.random.randn(3,2))
X.update()
(u0, u1) = X.get_moments()
Cov = u1 - linalg.outer(u0, u0, ndim=1)
Q = np.random.randn(3,3)
Qu0 = np.einsum('ik,kj->ij', Q, u0)
QCov = np.einsum('k,kij->kij', np.sum(Q, axis=0)**2, Cov)
Qu1 = QCov + linalg.outer(Qu0, Qu0, ndim=1)
X.rotate_plates(Q, plate_axis=-1)
(u0, u1) = X.get_moments()
self.assertAllClose(u0, Qu0)
self.assertAllClose(u1, Qu1)
pass
def test_message_to_parents(self):
""" Check gradient passed to inputs parent node """
D = 3
X = Gaussian(np.random.randn(D), random.covariance(D))
a = Gamma(np.random.rand(D), np.random.rand(D))
Y = GaussianARD(X, a)
Y.observe(np.random.randn(D))
self.assert_message_to_parent(Y, X)
self.assert_message_to_parent(Y, a)
pass
def fit(self, X, y):
self._init_weights()
# self.cost,
# self.myopic_voc(action, state),
# self.vpi_action(action, state),
# self.vpi(state),
# self.expected_term_reward(state)
self.tau = Gamma(self.prior_a, self.prior_b)
F = SumMultiply('i,i', self.weights, X)
y_obs = GaussianARD(F, self.tau)
y_obs.observe(y)
Q = VB(y_obs, self.weights)
Q.update(repeat=10, tol=1e-4, verbose=False)
def check(shape, plates, einsum_x, einsum_xx, axis=-1):
# TODO/FIXME: Improve by having non-diagonal precision/covariance
# parameter for the Gaussian X
D = shape[axis]
X = GaussianARD(np.random.randn(*(plates + shape)),
np.random.rand(*(plates + shape)),
shape=shape,
plates=plates)
(x, xx) = X.get_moments()
R = np.random.randn(D, D)
X.rotate(R, axis=axis)
(rx, rxxr) = X.get_moments()
self.assertAllClose(rx, np.einsum(einsum_x, R, x))
self.assertAllClose(rxxr, np.einsum(einsum_xx, R, xx, R))
pass
def test_mask_to_parent(self):
"""
Test the mask handling in Gate node
"""
X = GaussianARD(2, 1, shape=(4, 5), plates=(3, 2))
F = Gate([0, 0, 1], X)
self.assertAllClose(
F._compute_weights_to_parent(0, [True, False, False]),
[True, False, False])
self.assertAllClose(
F._compute_weights_to_parent(1, [True, False, False]),
[[True], [False], [False]])
pass
def test_initialization(self):
"""
Test initialization methods of GaussianARD
"""
X = GaussianARD(1, 2, shape=(2, ), plates=(3, ))
# Prior initialization
mu = 1 * np.ones((3, 2))
alpha = 2 * np.ones((3, 2))
X.initialize_from_prior()
u = X._message_to_child()
self.assertAllClose(u[0] * np.ones((3, 2)), mu)
self.assertAllClose(
u[1] * np.ones((3, 2, 2)),
linalg.outer(mu, mu, ndim=1) + misc.diag(1 / alpha, ndim=1))
# Parameter initialization
mu = np.random.randn(3, 2)
alpha = np.random.rand(3, 2)
X.initialize_from_parameters(mu, alpha)
u = X._message_to_child()
self.assertAllClose(u[0], mu)
self.assertAllClose(
u[1],
linalg.outer(mu, mu, ndim=1) + misc.diag(1 / alpha, ndim=1))
# Value initialization
x = np.random.randn(3, 2)
X.initialize_from_value(x)
u = X._message_to_child()
self.assertAllClose(u[0], x)
self.assertAllClose(u[1], linalg.outer(x, x, ndim=1))
# Random initialization
X.initialize_from_random()
pass
def test_message_to_child(self):
"""
Test the message to child of Concatenate node.
"""
# Two parents without shapes
X1 = GaussianARD(0, 1, plates=(2, ), shape=())
X2 = GaussianARD(0, 1, plates=(3, ), shape=())
Y = Concatenate(X1, X2)
u1 = X1.get_moments()
u2 = X2.get_moments()
u = Y.get_moments()
self.assertAllClose((u[0] * np.ones((5, )))[:2], u1[0] * np.ones(
(2, )))
self.assertAllClose((u[1] * np.ones((5, )))[:2], u1[1] * np.ones(
(2, )))
self.assertAllClose((u[0] * np.ones((5, )))[2:], u2[0] * np.ones(
(3, )))
self.assertAllClose((u[1] * np.ones((5, )))[2:], u2[1] * np.ones(
(3, )))
# Two parents with shapes
X1 = GaussianARD(0, 1, plates=(2, ), shape=(4, ))
X2 = GaussianARD(0, 1, plates=(3, ), shape=(4, ))
Y = Concatenate(X1, X2)
u1 = X1.get_moments()
u2 = X2.get_moments()
u = Y.get_moments()
self.assertAllClose((u[0] * np.ones((5, 4)))[:2], u1[0] * np.ones(
(2, 4)))
self.assertAllClose((u[1] * np.ones((5, 4, 4)))[:2], u1[1] * np.ones(
(2, 4, 4)))
self.assertAllClose((u[0] * np.ones((5, 4)))[2:], u2[0] * np.ones(
(3, 4)))
self.assertAllClose((u[1] * np.ones((5, 4, 4)))[2:], u2[1] * np.ones(
(3, 4, 4)))
# Test with non-constant axis
X1 = GaussianARD(0, 1, plates=(2, 4), shape=())
X2 = GaussianARD(0, 1, plates=(3, 4), shape=())
Y = Concatenate(X1, X2, axis=-2)
u1 = X1.get_moments()
u2 = X2.get_moments()
u = Y.get_moments()
self.assertAllClose((u[0] * np.ones((5, 4)))[:2], u1[0] * np.ones(
(2, 4)))
self.assertAllClose((u[1] * np.ones((5, 4)))[:2], u1[1] * np.ones(
(2, 4)))
self.assertAllClose((u[0] * np.ones((5, 4)))[2:], u2[0] * np.ones(
(3, 4)))
self.assertAllClose((u[1] * np.ones((5, 4)))[2:], u2[1] * np.ones(
(3, 4)))
# Test with constant parent
X1 = np.random.randn(2, 4)
X2 = GaussianARD(0, 1, plates=(3, ), shape=(4, ))
Y = Concatenate(X1, X2)
u1 = Y.parents[0].get_moments()
u2 = X2.get_moments()
u = Y.get_moments()
self.assertAllClose((u[0] * np.ones((5, 4)))[:2], u1[0] * np.ones(
(2, 4)))
self.assertAllClose((u[1] * np.ones((5, 4, 4)))[:2], u1[1] * np.ones(
(2, 4, 4)))
self.assertAllClose((u[0] * np.ones((5, 4)))[2:], u2[0] * np.ones(
(3, 4)))
self.assertAllClose((u[1] * np.ones((5, 4, 4)))[2:], u2[1] * np.ones(
(3, 4, 4)))
pass
def test_init(self):
"""
Test the creation of Concatenate node
"""
# One parent only
X = GaussianARD(0, 1, plates=(3, ), shape=())
Y = Concatenate(X)
self.assertEqual(Y.plates, (3, ))
self.assertEqual(Y.dims, ((), ()))
X = GaussianARD(0, 1, plates=(3, ), shape=(2, 4))
Y = Concatenate(X)
self.assertEqual(Y.plates, (3, ))
self.assertEqual(Y.dims, ((2, 4), (2, 4, 2, 4)))
# Two parents
X1 = GaussianARD(0, 1, plates=(2, ), shape=())
X2 = GaussianARD(0, 1, plates=(3, ), shape=())
Y = Concatenate(X1, X2)
self.assertEqual(Y.plates, (5, ))
self.assertEqual(Y.dims, ((), ()))
# Two parents with shapes
X1 = GaussianARD(0, 1, plates=(2, ), shape=(4, 6))
X2 = GaussianARD(0, 1, plates=(3, ), shape=(4, 6))
Y = Concatenate(X1, X2)
self.assertEqual(Y.plates, (5, ))
self.assertEqual(Y.dims, ((4, 6), (4, 6, 4, 6)))
# Two parents with non-default axis
X1 = GaussianARD(0, 1, plates=(2, 4), shape=())
X2 = GaussianARD(0, 1, plates=(3, 4), shape=())
Y = Concatenate(X1, X2, axis=-2)
self.assertEqual(Y.plates, (5, 4))
self.assertEqual(Y.dims, ((), ()))
# Three parents
X1 = GaussianARD(0, 1, plates=(2, ), shape=())
X2 = GaussianARD(0, 1, plates=(3, ), shape=())
X3 = GaussianARD(0, 1, plates=(4, ), shape=())
Y = Concatenate(X1, X2, X3)
self.assertEqual(Y.plates, (9, ))
self.assertEqual(Y.dims, ((), ()))
# Constant parent
X1 = [7.2, 3.5]
X2 = GaussianARD(0, 1, plates=(3, ), shape=())
Y = Concatenate(X1, X2)
self.assertEqual(Y.plates, (5, ))
self.assertEqual(Y.dims, ((), ()))
# Different moments
X1 = GaussianARD(0, 1, plates=(3, ))
X2 = Gamma(1, 1, plates=(4, ))
self.assertRaises(ValueError, Concatenate, X1, X2)
# Incompatible shapes
X1 = GaussianARD(0, 1, plates=(3, ), shape=(2, ))
X2 = GaussianARD(0, 1, plates=(2, ), shape=())
self.assertRaises(ValueError, Concatenate, X1, X2)
# Incompatible plates
X1 = GaussianARD(0, 1, plates=(4, 3), shape=())
X2 = GaussianARD(0, 1, plates=(
5,
2,
), shape=())
self.assertRaises(ValueError, Concatenate, X1, X2)
pass
def test_message_to_parent(self):
"""
Test the message to parents of Concatenate node.
"""
# Two parents without shapes
X1 = GaussianARD(0, 1, plates=(2, ), shape=())
X2 = GaussianARD(0, 1, plates=(3, ), shape=())
Z = Concatenate(X1, X2)
Y = GaussianARD(Z, 1)
Y.observe(np.random.randn(*Y.get_shape(0)))
m1 = X1._message_from_children()
m2 = X2._message_from_children()
m = Z._message_from_children()
self.assertAllClose((m[0] * np.ones((5, )))[:2], m1[0] * np.ones(
(2, )))
self.assertAllClose((m[1] * np.ones((5, )))[:2], m1[1] * np.ones(
(2, )))
self.assertAllClose((m[0] * np.ones((5, )))[2:], m2[0] * np.ones(
(3, )))
self.assertAllClose((m[1] * np.ones((5, )))[2:], m2[1] * np.ones(
(3, )))
# Two parents with shapes
with warnings.catch_warnings():
warnings.simplefilter("ignore", FutureWarning)
X1 = GaussianARD(0, 1, plates=(2, ), shape=(4, 6))
X2 = GaussianARD(0, 1, plates=(3, ), shape=(4, 6))
Z = Concatenate(X1, X2)
Y = GaussianARD(Z, 1)
Y.observe(np.random.randn(*Y.get_shape(0)))
m1 = X1._message_from_children()
m2 = X2._message_from_children()
m = Z._message_from_children()
self.assertAllClose((m[0] * np.ones((5, 4, 6)))[:2],
m1[0] * np.ones((2, 4, 6)))
self.assertAllClose((m[1] * np.ones((5, 4, 6, 4, 6)))[:2],
m1[1] * np.ones((2, 4, 6, 4, 6)))
self.assertAllClose((m[0] * np.ones((5, 4, 6)))[2:],
m2[0] * np.ones((3, 4, 6)))
self.assertAllClose((m[1] * np.ones((5, 4, 6, 4, 6)))[2:],
m2[1] * np.ones((3, 4, 6, 4, 6)))
# Two parents with non-default concatenation axis
X1 = GaussianARD(0, 1, plates=(2, 4), shape=())
X2 = GaussianARD(0, 1, plates=(3, 4), shape=())
Z = Concatenate(X1, X2, axis=-2)
Y = GaussianARD(Z, 1)
Y.observe(np.random.randn(*Y.get_shape(0)))
m1 = X1._message_from_children()
m2 = X2._message_from_children()
m = Z._message_from_children()
self.assertAllClose((m[0] * np.ones((5, 4)))[:2], m1[0] * np.ones(
(2, 4)))
self.assertAllClose((m[1] * np.ones((5, 4)))[:2], m1[1] * np.ones(
(2, 4)))
self.assertAllClose((m[0] * np.ones((5, 4)))[2:], m2[0] * np.ones(
(3, 4)))
self.assertAllClose((m[1] * np.ones((5, 4)))[2:], m2[1] * np.ones(
(3, 4)))
# Constant parent
X1 = np.random.randn(2, 4, 6)
X2 = GaussianARD(0, 1, plates=(3, ), shape=(4, 6))
Z = Concatenate(X1, X2)
Y = GaussianARD(Z, 1)
Y.observe(np.random.randn(*Y.get_shape(0)))
m1 = Z._message_to_parent(0)
m2 = X2._message_from_children()
m = Z._message_from_children()
self.assertAllClose((m[0] * np.ones((5, 4, 6)))[:2],
m1[0] * np.ones((2, 4, 6)))
self.assertAllClose((m[1] * np.ones((5, 4, 6, 4, 6)))[:2],
m1[1] * np.ones((2, 4, 6, 4, 6)))
self.assertAllClose((m[0] * np.ones((5, 4, 6)))[2:],
m2[0] * np.ones((3, 4, 6)))
self.assertAllClose((m[1] * np.ones((5, 4, 6, 4, 6)))[2:],
m2[1] * np.ones((3, 4, 6, 4, 6)))
pass
import numpy
numpy.random.seed(1)
M = 20
N = 100
import numpy as np
x = np.random.randn(N, 2)
w = np.random.randn(M, 2)
f = np.einsum('ik,jk->ij', w, x)
y = f + 0.1 * np.random.randn(M, N)
D = 10
from bayespy.nodes import GaussianARD, Gamma, SumMultiply
X = GaussianARD(0, 1, plates=(1, N), shape=(D, ))
alpha = Gamma(1e-5, 1e-5, plates=(D, ))
C = GaussianARD(0, alpha, plates=(M, 1), shape=(D, ))
F = SumMultiply('d,d->', X, C)
tau = Gamma(1e-5, 1e-5)
Y = GaussianARD(F, tau)
Y.observe(y)
from bayespy.inference import VB
Q = VB(Y, X, C, alpha, tau)
C.initialize_from_random()
from bayespy.inference.vmp.transformations import RotateGaussianARD
rot_X = RotateGaussianARD(X)
rot_C = RotateGaussianARD(C, alpha)
from bayespy.inference.vmp.transformations import RotationOptimizer
R = RotationOptimizer(rot_X, rot_C, D)
Q.set_callback(R.rotate)
Q.update(repeat=1000)
import bayespy.plot as bpplt
bpplt.plot(F)
bpplt.plot(f, color='r', marker='x', linestyle='None')
def test_gradient(self):
"""Test standard gradient of a Gamma node."""
D = 3
np.random.seed(42)
#
# Without observations
#
# Construct model
a = np.random.rand(D)
b = np.random.rand(D)
tau = Gamma(a, b)
Q = VB(tau)
# Random initialization
tau.initialize_from_parameters(np.random.rand(D),
np.random.rand(D))
# Initial parameters
phi0 = tau.phi
# Gradient
rg = tau.get_riemannian_gradient()
g = tau.get_gradient(rg)
# Numerical gradient
eps = 1e-8
p0 = tau.get_parameters()
l0 = Q.compute_lowerbound(ignore_masked=False)
g_num = [np.zeros(D), np.zeros(D)]
for i in range(D):
e = np.zeros(D)
e[i] = eps
p1 = p0[0] + e
tau.set_parameters([p1, p0[1]])
l1 = Q.compute_lowerbound(ignore_masked=False)
g_num[0][i] = (l1 - l0) / eps
for i in range(D):
e = np.zeros(D)
e[i] = eps
p1 = p0[1] + e
tau.set_parameters([p0[0], p1])
l1 = Q.compute_lowerbound(ignore_masked=False)
g_num[1][i] = (l1 - l0) / eps
# Check
self.assertAllClose(g[0],
g_num[0])
self.assertAllClose(g[1],
g_num[1])
#
# With observations
#
# Construct model
a = np.random.rand(D)
b = np.random.rand(D)
tau = Gamma(a, b)
mu = np.random.randn(D)
Y = GaussianARD(mu, tau)
Y.observe(np.random.randn(D))
Q = VB(Y, tau)
# Random initialization
tau.initialize_from_parameters(np.random.rand(D),
np.random.rand(D))
# Initial parameters
phi0 = tau.phi
# Gradient
rg = tau.get_riemannian_gradient()
g = tau.get_gradient(rg)
# Numerical gradient
eps = 1e-8
p0 = tau.get_parameters()
l0 = Q.compute_lowerbound(ignore_masked=False)
g_num = [np.zeros(D), np.zeros(D)]
for i in range(D):
e = np.zeros(D)
e[i] = eps
p1 = p0[0] + e
tau.set_parameters([p1, p0[1]])
l1 = Q.compute_lowerbound(ignore_masked=False)
g_num[0][i] = (l1 - l0) / eps
for i in range(D):
e = np.zeros(D)
e[i] = eps
p1 = p0[1] + e
tau.set_parameters([p0[0], p1])
l1 = Q.compute_lowerbound(ignore_masked=False)
g_num[1][i] = (l1 - l0) / eps
# Check
self.assertAllClose(g[0],
g_num[0])
self.assertAllClose(g[1],
g_num[1])
pass
def test_message_to_parent(self):
"""
Test the message to parents of Mixture node.
"""
K = 3
# Broadcasting the moments on the cluster axis
Mu = GaussianARD(2, 1, ndim=0, plates=(K, ))
(mu, mumu) = Mu._message_to_child()
Alpha = Gamma(3, 1, plates=(K, ))
(alpha, logalpha) = Alpha._message_to_child()
z = Categorical(np.ones(K) / K)
X = Mixture(z, GaussianARD, Mu, Alpha)
tau = 4
Y = GaussianARD(X, tau)
y = 5
Y.observe(y)
(x, xx) = X._message_to_child()
m = z._message_from_children()
self.assertAllClose(
m[0] * np.ones(K),
random.gaussian_logpdf(xx * alpha, x * alpha * mu, mumu * alpha,
logalpha, 0) * np.ones(K))
m = Mu._message_from_children()
self.assertAllClose(m[0], 1 / K * (alpha * x) * np.ones(3))
self.assertAllClose(m[1], -0.5 * 1 / K * alpha * np.ones(3))
# Some parameters do not have cluster plate axis
Mu = GaussianARD(2, 1, ndim=0, plates=(K, ))
(mu, mumu) = Mu._message_to_child()
Alpha = Gamma(3, 1) # Note: no cluster plate axis!
(alpha, logalpha) = Alpha._message_to_child()
z = Categorical(np.ones(K) / K)
X = Mixture(z, GaussianARD, Mu, Alpha)
tau = 4
Y = GaussianARD(X, tau)
y = 5
Y.observe(y)
(x, xx) = X._message_to_child()
m = z._message_from_children()
self.assertAllClose(
m[0] * np.ones(K),
random.gaussian_logpdf(xx * alpha, x * alpha * mu, mumu * alpha,
logalpha, 0) * np.ones(K))
m = Mu._message_from_children()
self.assertAllClose(m[0], 1 / K * (alpha * x) * np.ones(3))
self.assertAllClose(m[1], -0.5 * 1 / K * alpha * np.ones(3))
# Cluster assignments do not have as many plate axes as parameters.
M = 2
Mu = GaussianARD(2, 1, ndim=0, plates=(K, M))
(mu, mumu) = Mu._message_to_child()
Alpha = Gamma(3, 1, plates=(K, M))
(alpha, logalpha) = Alpha._message_to_child()
z = Categorical(np.ones(K) / K)
X = Mixture(z, GaussianARD, Mu, Alpha, cluster_plate=-2)
tau = 4
Y = GaussianARD(X, tau)
y = 5 * np.ones(M)
Y.observe(y)
(x, xx) = X._message_to_child()
m = z._message_from_children()
self.assertAllClose(
m[0] * np.ones(K),
np.sum(random.gaussian_logpdf(xx * alpha, x * alpha * mu,
mumu * alpha, logalpha, 0) * np.ones(
(K, M)),
axis=-1))
m = Mu._message_from_children()
self.assertAllClose(m[0] * np.ones((K, M)),
1 / K * (alpha * x) * np.ones((K, M)))
self.assertAllClose(m[1] * np.ones((K, M)),
-0.5 * 1 / K * alpha * np.ones((K, M)))
# Mixed distribution broadcasts g
# This tests for a found bug. The bug caused an error.
Z = Categorical([0.3, 0.5, 0.2])
X = Mixture(Z, Categorical, [[0.2, 0.8], [0.1, 0.9], [0.3, 0.7]])
m = Z._message_from_children()
pass
def model(M=20, N=100, D=10, K=3):
"""
Construct the linear state-space model with switching dynamics.
"""
#
# Switching dynamics (HMM)
#
# Prior for initial state probabilities
rho = Dirichlet(1e-3 * np.ones(K), name='rho')
# Prior for state transition probabilities
V = Dirichlet(1e-3 * np.ones(K), plates=(K, ), name='V')
v = 10 * np.identity(K) + 1 * np.ones((K, K))
v /= np.sum(v, axis=-1, keepdims=True)
V.initialize_from_value(v)
# Hidden states (with unknown initial state probabilities and state
# transition probabilities)
Z = CategoricalMarkovChain(rho,
V,
states=N - 1,
name='Z',
plotter=bpplt.CategoricalMarkovChainPlotter(),
initialize=False)
Z.u[0] = np.random.dirichlet(np.ones(K))
Z.u[1] = np.reshape(
np.random.dirichlet(0.5 * np.ones(K * K), size=(N - 2)), (N - 2, K, K))
#
# Linear state-space models
#
# Dynamics matrix with ARD
# (K,D) x ()
alpha = Gamma(1e-5, 1e-5, plates=(K, 1, D), name='alpha')
# (K,1,1,D) x (D)
A = GaussianARD(0,
alpha,
shape=(D, ),
plates=(K, D),
name='A',
plotter=bpplt.GaussianHintonPlotter())
A.initialize_from_value(
np.identity(D) * np.ones((K, D, D)) + 0.1 * np.random.randn(K, D, D))
# Latent states with dynamics
# (K,1) x (N,D)
X = SwitchingGaussianMarkovChain(
np.zeros(D), # mean of x0
1e-3 * np.identity(D), # prec of x0
A, # dynamics
Z, # dynamics selection
np.ones(D), # innovation
n=N, # time instances
name='X',
plotter=bpplt.GaussianMarkovChainPlotter())
X.initialize_from_value(10 * np.random.randn(N, D))
# Mixing matrix from latent space to observation space using ARD
# (K,1,1,D) x ()
gamma = Gamma(1e-5, 1e-5, plates=(D, ), name='gamma')
# (K,M,1) x (D)
C = GaussianARD(0,
gamma,
shape=(D, ),
plates=(M, 1),
name='C',
plotter=bpplt.GaussianHintonPlotter(rows=-3, cols=-1))
C.initialize_from_value(np.random.randn(M, 1, D))
# Underlying noiseless function
# (K,M,N) x ()
F = SumMultiply('i,i', C, X, name='F')
#
# Mixing the models
#
# Observation noise
tau = Gamma(1e-5, 1e-5, name='tau')
tau.initialize_from_value(1e2)
# Emission/observation distribution
Y = GaussianARD(F, tau, name='Y')
Q = VB(Y, F, Z, rho, V, C, gamma, X, A, alpha, tau)
return Q
def check(indices, plates, shape, axis=-1, use_mask=False):
mu = np.random.rand(*(plates+shape))
alpha = np.random.rand(*(plates+shape))
X = GaussianARD(mu, alpha, shape=shape, plates=plates)
Y = Take(X, indices, plate_axis=axis)
Z = GaussianARD(Y, 1, shape=shape)
z = np.random.randn(*(Z.get_shape(0)))
if use_mask:
mask = np.mod(np.reshape(np.arange(np.prod(Z.plates)), Z.plates), 2) != 0
else:
mask = True
Z.observe(z, mask=mask)
X.update()
(x0, x1) = X.get_moments()
# For comparison, build the same model brute force
X = GaussianARD(mu, alpha, shape=shape, plates=plates)
# Number of trailing plate axes before the take axis
N = len(X.plates) + axis
# Reshape the take axes into a single axis
z_shape = X.plates[:axis] + (-1,)
if axis < -1:
z_shape = z_shape + X.plates[(axis+1):]
z_shape = z_shape + shape
z = np.reshape(z, z_shape)
# Reshape the take axes into a single axis
if use_mask:
mask_shape = X.plates[:axis] + (-1,)
if axis < -1:
mask_shape = mask_shape + X.plates[(axis+1):]
mask = np.reshape(mask, mask_shape)
for (j, i) in enumerate(range(np.size(indices))):
ind = np.array(indices).flatten()[i]
index_x = N*(slice(None),) + (ind,)
index_z = N*(slice(None),) + (j,)
# print(index)
Xi = X[index_x]
zi = z[index_z]
Zi = GaussianARD(Xi, 1, ndim=len(shape))
if use_mask:
maski = mask[index_z]
else:
maski = True
Zi.observe(zi, mask=maski)
X.update()
self.assertAllClose(
x0,
X.get_moments()[0],
)
self.assertAllClose(
x1,
X.get_moments()[1],
)
return
def test_parent_validity(self):
"""
Test that the parent nodes are validated properly
"""
# Test scalar index, no shape
X = GaussianARD(1, 1, plates=(2,), shape=())
Y = Take(X, 1)
self.assertEqual(
Y.plates,
(),
)
self.assertEqual(
Y.dims,
( (), () )
)
# Test vector indices, no shape
X = GaussianARD(1, 1, plates=(2,), shape=())
Y = Take(X, [1, 1, 0, 1])
self.assertEqual(
Y.plates,
(4,),
)
self.assertEqual(
Y.dims,
( (), () )
)
# Test matrix indices, no shape
X = GaussianARD(1, 1, plates=(2,), shape=())
Y = Take(X, [[1, 1, 0], [1, 0, 1]])
self.assertEqual(
Y.plates,
(2, 3),
)
self.assertEqual(
Y.dims,
( (), () )
)
# Test scalar index, with shape
X = GaussianARD(1, 1, plates=(3,), shape=(2,))
Y = Take(X, 2)
self.assertEqual(
Y.plates,
(),
)
self.assertEqual(
Y.dims,
( (2,), (2, 2) )
)
# Test vector indices, with shape
X = GaussianARD(1, 1, plates=(3,), shape=(2,))
Y = Take(X, [1, 1, 0, 2])
self.assertEqual(
Y.plates,
(4,),
)
self.assertEqual(
Y.dims,
( (2,), (2, 2) )
)
# Test matrix indices, no shape
X = GaussianARD(1, 1, plates=(3,), shape=(2,))
Y = Take(X, np.ones((4, 5), dtype=np.int))
self.assertEqual(
Y.plates,
(4, 5),
)
self.assertEqual(
Y.dims,
( (2,), (2, 2) )
)
# Test scalar indices with more plate axes
X = GaussianARD(1, 1, plates=(4, 2), shape=())
Y = Take(X, 1)
self.assertEqual(
Y.plates,
(4,),
)
self.assertEqual(
Y.dims,
( (), () )
)
# Test vector indices with more plate axes
X = GaussianARD(1, 1, plates=(4, 2), shape=())
Y = Take(X, np.ones(3, dtype=np.int))
self.assertEqual(
Y.plates,
(4, 3),
)
self.assertEqual(
Y.dims,
( (), () )
)
# Test take on other plate axis
X = GaussianARD(1, 1, plates=(4, 2), shape=())
Y = Take(X, np.ones(3, dtype=np.int), plate_axis=-2)
self.assertEqual(
Y.plates,
(3, 2),
)
self.assertEqual(
Y.dims,
( (), () )
)
# Test positive plate axis
X = GaussianARD(1, 1, plates=(4, 2), shape=())
self.assertRaises(
ValueError,
Take,
X,
np.ones(3, dtype=np.int),
plate_axis=0,
)
# Test indices out of bounds
X = GaussianARD(1, 1, plates=(2,), shape=())
self.assertRaises(
ValueError,
Take,
X,
[0, -3],
)
X = GaussianARD(1, 1, plates=(2,), shape=())
self.assertRaises(
ValueError,
Take,
X,
[0, 2],
)
# Test non-integer indices
X = GaussianARD(1, 1, plates=(2,), shape=())
self.assertRaises(
ValueError,
Take,
X,
[0, 1.5],
)
pass
def test_moments(self):
"""
Test moments computation in Take node
"""
# Test scalar index, no shape
X = GaussianARD([1, 2], [1, 0.5], shape=())
Y = Take(X, 1)
self.assertAllClose(
Y.get_moments()[0],
2,
)
self.assertAllClose(
Y.get_moments()[1],
6,
)
# Test vector indices, no shape
X = GaussianARD([1, 2], [1, 0.5], shape=())
Y = Take(X, [1, 1, 0, 1])
self.assertAllClose(
Y.get_moments()[0],
[2, 2, 1, 2],
)
self.assertAllClose(
Y.get_moments()[1],
[6, 6, 2, 6],
)
# Test matrix indices, no shape
X = GaussianARD([1, 2], [1, 0.5], shape=())
Y = Take(X, [[1, 1, 0], [1, 0, 1]])
self.assertAllClose(
Y.get_moments()[0],
[[2, 2, 1], [2, 1, 2]],
)
self.assertAllClose(
Y.get_moments()[1],
[[6, 6, 2], [6, 2, 6]],
)
# Test scalar index, with shape
X = GaussianARD([[1, 2], [3, 4], [5, 6]], [[1, 1/2], [1/3, 1/4], [1/5, 1/6]], shape=(2,))
Y = Take(X, 2)
self.assertAllClose(
Y.get_moments()[0],
[5, 6],
)
self.assertAllClose(
Y.get_moments()[1],
[[25+5, 30], [30, 36+6]],
)
# Test vector indices, with shape
X = GaussianARD([[1, 2], [3, 4], [5, 6]], [[1, 1/2], [1/3, 1/4], [1/5, 1/6]], shape=(2,))
Y = Take(X, [1, 1, 0, 2])
self.assertAllClose(
Y.get_moments()[0],
[[3, 4], [3, 4], [1, 2], [5, 6]],
)
self.assertAllClose(
Y.get_moments()[1],
[
[[9+3, 12], [12, 16+4]],
[[9+3, 12], [12, 16+4]],
[[1+1, 2], [2, 4+2]],
[[25+5, 30], [30, 36+6]]
],
)
# Test matrix indices, no shape
X = GaussianARD([[1, 2], [3, 4], [5, 6]], [[1, 1/2], [1/3, 1/4], [1/5, 1/6]], shape=(2,))
Y = Take(X, [[1, 1], [0, 2]])
self.assertAllClose(
Y.get_moments()[0],
[[[3, 4], [3, 4]], [[1, 2], [5, 6]]],
)
self.assertAllClose(
Y.get_moments()[1],
[
[[[9+3, 12], [12, 16+4]],
[[9+3, 12], [12, 16+4]]],
[[[1+1, 2], [2, 4+2]],
[[25+5, 30], [30, 36+6]]],
],
)
# Test with more plate axes
X = GaussianARD([[1, 2], [3, 4], [5, 6]], [[1, 1/2], [1/3, 1/4], [1/5, 1/6]], shape=())
Y = Take(X, [1, 0, 1])
self.assertAllClose(
Y.get_moments()[0],
[[2, 1, 2], [4, 3, 4], [6, 5, 6]],
)
self.assertAllClose(
Y.get_moments()[1],
[[4+2, 1+1, 4+2], [16+4, 9+3, 16+4], [36+6, 25+5, 36+6]],
)
# Test take on other plate axis
X = GaussianARD([[1, 2], [3, 4], [5, 6]], [[1, 1/2], [1/3, 1/4], [1/5, 1/6]], shape=())
Y = Take(X, [2, 0], plate_axis=-2)
self.assertAllClose(
Y.get_moments()[0],
[[5, 6], [1, 2]],
)
self.assertAllClose(
Y.get_moments()[1],
[[25+5, 36+6], [1+1, 4+2]],
)
# Test parent broadcasting
X = GaussianARD([1, 2], [1, 1/2], plates=(3,), shape=(2,))
Y = Take(X, [1, 1, 0, 1])
self.assertAllClose(
Y.get_moments()[0],
[[1, 2], [1, 2], [1, 2], [1, 2]],
)
self.assertAllClose(
Y.get_moments()[1],
[
[[1+1, 2], [2, 4+2]],
[[1+1, 2], [2, 4+2]],
[[1+1, 2], [2, 4+2]],
[[1+1, 2], [2, 4+2]],
]
)
pass
firstline = True
for xx in x2:
if firstline:
y = np.mean(xx)
firstline = False
continue
y = np.vstack([y, (k * np.mean(xx) + c + s * np.random.randn(1))])
y = y.reshape(y.shape[0], )
X = x2.reshape(x2.shape[0], 1)
from bayespy.nodes import GaussianARD
B = GaussianARD(0, 1e-6, shape=(X.shape[1], ))
from bayespy.nodes import SumMultiply
F = SumMultiply('i,i', B, X)
from bayespy.nodes import Gamma
tau = Gamma(1e-3, 1e-3)
Y = GaussianARD(F, tau)
Y.observe(y)
from bayespy.inference import VB
Q = VB(Y, B, tau)
#Q.update(repeat=100990)
distribution = []
result = []
distribution = F.get_moments()
for min_val, max_val in zip(distribution[0], distribution[1]):
#mean = []
