def test_gaussian_mixture_plot():
"""
Test the gaussian_mixture plotting function.
The code is from http://www.bayespy.org/examples/gmm.html
"""
np.random.seed(1)
y0 = np.random.multivariate_normal([0, 0], [[1, 0], [0, 0.02]], size=50)
y1 = np.random.multivariate_normal([0, 0], [[0.02, 0], [0, 1]], size=50)
y2 = np.random.multivariate_normal([2, 2], [[1, -0.9], [-0.9, 1]], size=50)
y3 = np.random.multivariate_normal([-2, -2], [[0.1, 0], [0, 0.1]], size=50)
y = np.vstack([y0, y1, y2, y3])
bpplt.pyplot.plot(y[:, 0], y[:, 1], 'rx')
N = 200
D = 2
K = 10
alpha = Dirichlet(1e-5 * np.ones(K), name='alpha')
Z = Categorical(alpha, plates=(N, ), name='z')
mu = Gaussian(np.zeros(D), 1e-5 * np.identity(D), plates=(K, ), name='mu')
Lambda = Wishart(D, 1e-5 * np.identity(D), plates=(K, ), name='Lambda')
Y = Mixture(Z, Gaussian, mu, Lambda, name='Y')
Z.initialize_from_random()
Q = VB(Y, mu, Lambda, Z, alpha)
Y.observe(y)
Q.update(repeat=1000)
bpplt.gaussian_mixture_2d(Y, scale=2)
def _setup_bernoulli_mixture():
"""
Setup code for the hinton tests.
This code is from http://www.bayespy.org/examples/bmm.html
"""
np.random.seed(1)
p0 = [0.1, 0.9, 0.1, 0.9, 0.1, 0.9, 0.1, 0.9, 0.1, 0.9]
p1 = [0.1, 0.1, 0.1, 0.1, 0.1, 0.9, 0.9, 0.9, 0.9, 0.9]
p2 = [0.9, 0.9, 0.9, 0.9, 0.9, 0.1, 0.1, 0.1, 0.1, 0.1]
p = np.array([p0, p1, p2])
z = random.categorical([1 / 3, 1 / 3, 1 / 3], size=100)
x = random.bernoulli(p[z])
N = 100
D = 10
K = 10
R = Dirichlet(K * [1e-5], name='R')
Z = Categorical(R, plates=(N, 1), name='Z')
P = Beta([0.5, 0.5], plates=(D, K), name='P')
X = Mixture(Z, Bernoulli, P)
Q = VB(Z, R, X, P)
P.initialize_from_random()
X.observe(x)
Q.update(repeat=1000)
return (R, P, Z)
def test_mask_to_parent(self):
"""
Test the mask handling in Mixture node
"""
K = 3
Z = Categorical(np.ones(K)/K,
plates=(4,5))
Mu = GaussianARD(0, 1,
shape=(2,),
plates=(4,K,5))
Alpha = Gamma(1, 1,
plates=(4,K,5,2))
X = Mixture(Z, GaussianARD, Mu, Alpha, cluster_plate=-2)
Y = GaussianARD(X, 1)
mask = np.reshape((np.mod(np.arange(4*5), 2) == 0),
(4,5))
Y.observe(np.ones((4,5,2)),
mask=mask)
self.assertArrayEqual(X._mask_to_parent(0),
mask)
self.assertArrayEqual(X._mask_to_parent(1),
mask[:,None,:])
self.assertArrayEqual(X._mask_to_parent(2),
mask[:,None,:,None])
pass
def test_mixture(self):
"""
Test mixture of Bernoulli
"""
P = Mixture([2,0,0], Bernoulli, [0.1, 0.2, 0.3])
u = P._message_to_child()
self.assertEqual(len(u), 1)
self.assertAllClose(u[0], [0.3, 0.1, 0.1])
pass
def test_mixture(self):
"""
Test mixture of Bernoulli
"""
P = Mixture([2, 0, 0], Bernoulli, [0.1, 0.2, 0.3])
u = P._message_to_child()
self.assertEqual(len(u), 1)
self.assertAllClose(u[0], [0.3, 0.1, 0.1])
pass
def test_mixture(self):
"""
Test binomial mixture
"""
X = Mixture(2, Binomial, 10, [0.1, 0.2, 0.3, 0.4])
u = X._message_to_child()
self.assertAllClose(u[0], 3.0)
pass
def test_mixture(self):
"""
Test binomial mixture
"""
X = Mixture(2, Binomial, 10, [0.1, 0.2, 0.3, 0.4])
u = X._message_to_child()
self.assertAllClose(u[0], 3.0)
pass
def test_constant(self):
"""
Test constant categorical nodes
"""
# Basic test
Y = Mixture(2, Gamma, [1, 2, 3], [1, 1, 1])
u = Y._message_to_child()
self.assertAllClose(u[0],
3/1)
# Test with one plate axis
alpha = [[1, 2, 3],
[4, 5, 6]]
Y = Mixture([2, 1], Gamma, alpha, 1)
u = Y._message_to_child()
self.assertAllClose(u[0],
[3, 5])
# Test with two plate axes
alpha = [ [[1, 2, 3],
[4, 5, 6]],
[[7, 8, 9],
[10, 11, 12]] ]
Y = Mixture([[2, 1], [0, 2]], Gamma, alpha, 1)
u = Y._message_to_child()
self.assertAllClose(u[0],
[[3, 5],
[7, 12]])
pass
def test_nans(self):
"""
Test multinomial mixture
"""
# The probabilities p1 cause problems
p0 = [0.1, 0.9]
p1 = [1.0 - 1e-50, 1e-50]
Z = Categorical([1 - 1e-10, 1e-10])
X = Mixture(Z, Multinomial, 10, [p0, p1])
u = X._message_to_child()
self.assertAllClose(u[0], [1, 9])
p0 = [0.1, 0.9]
p1 = [1.0 - 1e-10, 1e-10]
Z = Categorical([1 - 1e-50, 1e-50])
X = Mixture(Z, Multinomial, 10, [p0, p1])
u = X._message_to_child()
self.assertAllClose(u[0], [1, 9])
with warnings.catch_warnings():
warnings.simplefilter("ignore", RuntimeWarning)
warnings.simplefilter("ignore", UserWarning)
p0 = [0.1, 0.9]
p1 = [1.0, 0.0]
X = Mixture(0, Multinomial, 10, [p0, p1])
u = X._message_to_child()
self.assertAllClose(u[0], np.nan * np.ones(2))
pass
def test_lowerbound(self):
"""
Test log likelihood lower bound for Mixture node
"""
# Mixed distribution broadcasts g
# This tests for a found bug. The bug caused an error.
Z = Categorical([0.3, 0.5, 0.2])
X = Mixture(Z, Categorical, [[0.2,0.8], [0.1,0.9], [0.3,0.7]])
X.lower_bound_contribution()
pass
def test_lowerbound(self):
"""
Test log likelihood lower bound for Mixture node
"""
# Mixed distribution broadcasts g
# This tests for a found bug. The bug caused an error.
Z = Categorical([0.3, 0.5, 0.2])
X = Mixture(Z, Categorical, [[0.2, 0.8], [0.1, 0.9], [0.3, 0.7]])
X.lower_bound_contribution()
pass
def test_random(self):
"""
Test random sampling of mixture node
"""
o = 1e-20
X = Mixture([1, 0, 2], Categorical,
[[o, o, o, 1], [o, o, 1, o], [1, o, o, o]])
x = X.random()
self.assertAllClose(x, [2, 3, 0])
pass
def test_random(self):
"""
Test random sampling of mixture node
"""
o = 1e-20
X = Mixture([1, 0, 2], Categorical, [ [o, o, o, 1],
[o, o, 1, o],
[1, o, o, o] ])
x = X.random()
self.assertAllClose(x, [2, 3, 0])
pass
def test_mixture(self):
"""
Test multinomial mixture
"""
p0 = [0.1, 0.5, 0.2, 0.2]
p1 = [0.5, 0.1, 0.1, 0.3]
p2 = [0.3, 0.2, 0.1, 0.4]
X = Mixture(2, Multinomial, 10, [p0, p1, p2])
u = X._message_to_child()
self.assertAllClose(u[0], 10 * np.array(p2))
pass
def _run(self, x, K=25, beta=0.5, alpha=0.00001, hinton_plot=False, end=False):
'''Only to be used when doing parameter optimization.'''
self.participant_list = x[0]
N = len(x[0]) #number of data points (i.e. WCS participants)
D = np.shape(x[1])[1] #number of features
#K = 20 #number of initial clusters
R = Dirichlet(K*[alpha],
name='R')
Z = Categorical(R,
plates=(N,1),
name='Z')
P = Beta([beta, beta],
plates=(D,K),
name='P')
X = Mixture(Z, Bernoulli, P)
Q = VB(Z, R, X, P)
P.initialize_from_random()
X.observe(x[1])
Q.update(repeat=1000)
log_likelihood = Q.L[Q.iter-1]
if hinton_plot:
bpplt.hinton(Z)
bpplt.pyplot.show()
bpplt.hinton(R)
bpplt.pyplot.show()
#Get the weight matrix stored in Z (weights determine which cluster data point belongs to)
z = Z._message_to_child()[0]
z = z * np.ones(Z.plates+(1,))
z = np.squeeze(z)
self.z = z
#Get the weights stored in R (proportional to the size of the clusters)
r = np.exp(R._message_to_child()[0])
r = r * np.ones(R.plates+(1,))
r = np.squeeze(r)
self.r = r
#Get the cluster assignment of each data point
self.c_assign = np.argmax(self.z, axis=1)
return log_likelihood
def test_mixture(self):
"""
Test multinomial mixture
"""
p0 = [0.1, 0.5, 0.2, 0.2]
p1 = [0.5, 0.1, 0.1, 0.3]
p2 = [0.3, 0.2, 0.1, 0.4]
X = Mixture(2, Multinomial, 10, [p0, p1, p2])
u = X._message_to_child()
self.assertAllClose(u[0],
10*np.array(p2))
pass
def test_message_to_child(self):
"""
Test the message to child of Mixture node.
"""
K = 3
#
# Estimate moments from parents only
#
# Simple case
mu = GaussianARD([0,2,4], 1,
ndim=0,
plates=(K,))
alpha = Gamma(1, 1,
plates=(K,))
z = Categorical(np.ones(K)/K)
X = Mixture(z, GaussianARD, mu, alpha)
self.assertEqual(X.plates, ())
self.assertEqual(X.dims, ( (), () ))
u = X._message_to_child()
self.assertAllClose(u[0],
2)
self.assertAllClose(u[1],
2**2+1)
# Broadcasting the moments on the cluster axis
mu = GaussianARD(2, 1,
ndim=0,
plates=(K,))
alpha = Gamma(1, 1,
plates=(K,))
z = Categorical(np.ones(K)/K)
X = Mixture(z, GaussianARD, mu, alpha)
self.assertEqual(X.plates, ())
self.assertEqual(X.dims, ( (), () ))
u = X._message_to_child()
self.assertAllClose(u[0],
2)
self.assertAllClose(u[1],
2**2+1)
#
# Estimate moments with observed children
#
pass
def test_mask_to_parent(self):
"""
Test the mask handling in Mixture node
"""
K = 3
Z = Categorical(np.ones(K)/K,
plates=(4,5,1))
Mu = GaussianARD(0, 1,
shape=(2,),
plates=(4,K,5))
Alpha = Gamma(1, 1,
plates=(4,K,5,2))
X = Mixture(Z, GaussianARD, Mu, Alpha, cluster_plate=-3)
Y = GaussianARD(X, 1, ndim=1)
mask = np.reshape((np.mod(np.arange(4*5), 2) == 0),
(4,5))
Y.observe(np.ones((4,5,2)),
mask=mask)
self.assertArrayEqual(Z.mask,
mask[:,:,None])
self.assertArrayEqual(Mu.mask,
mask[:,None,:])
self.assertArrayEqual(Alpha.mask,
mask[:,None,:,None])
pass
def test_gaussian_mixture_plot():
"""
Test the gaussian_mixture plotting function.
The code is from http://www.bayespy.org/examples/gmm.html
"""
np.random.seed(1)
y0 = np.random.multivariate_normal([0, 0], [[1, 0], [0, 0.02]], size=50)
y1 = np.random.multivariate_normal([0, 0], [[0.02, 0], [0, 1]], size=50)
y2 = np.random.multivariate_normal([2, 2], [[1, -0.9], [-0.9, 1]], size=50)
y3 = np.random.multivariate_normal([-2, -2], [[0.1, 0], [0, 0.1]], size=50)
y = np.vstack([y0, y1, y2, y3])
bpplt.pyplot.plot(y[:,0], y[:,1], 'rx')
N = 200
D = 2
K = 10
alpha = Dirichlet(1e-5*np.ones(K),
name='alpha')
Z = Categorical(alpha,
plates=(N,),
name='z')
mu = Gaussian(np.zeros(D), 1e-5*np.identity(D),
plates=(K,),
name='mu')
Lambda = Wishart(D, 1e-5*np.identity(D),
plates=(K,),
name='Lambda')
Y = Mixture(Z, Gaussian, mu, Lambda,
name='Y')
Z.initialize_from_random()
Q = VB(Y, mu, Lambda, Z, alpha)
Y.observe(y)
Q.update(repeat=1000)
bpplt.gaussian_mixture_2d(Y, scale=2)
# Have to define these limits because on some particular environments these
# may otherwise differ and thus result in an image comparsion failure
bpplt.pyplot.xlim([-3, 6])
bpplt.pyplot.ylim([-3, 5])
def test_deterministic_mappings(self):
x = Categorical([0.8, 0.2])
y = Mixture(
x,
Categorical,
[
[0.10, 0.90],
[0.00, 1.00],
]
)
y.observe(0)
x.update()
self.assertAllClose(x.u[0], [1, 0])
y.observe(1)
x.update()
p = np.array([0.8*0.9, 0.2*1.0])
self.assertAllClose(x.u[0], p / np.sum(p))
pass
def test_constant(self):
"""
Test constant categorical nodes
"""
# Basic test
Y = Mixture(2, Gamma, [1, 2, 3], [1, 1, 1])
u = Y._message_to_child()
self.assertAllClose(u[0],
3/1)
# Test with one plate axis
alpha = [[1, 2, 3],
[4, 5, 6]]
Y = Mixture([2, 1], Gamma, alpha, 1)
u = Y._message_to_child()
self.assertAllClose(u[0],
[3, 5])
# Test with two plate axes
alpha = [ [[1, 2, 3],
[4, 5, 6]],
[[7, 8, 9],
[10, 11, 12]] ]
Y = Mixture([[2, 1], [0, 2]], Gamma, alpha, 1)
u = Y._message_to_child()
self.assertAllClose(u[0],
[[3, 5],
[7, 12]])
pass
def test_nans(self):
"""
Test multinomial mixture
"""
# The probabilities p1 cause problems
p0 = [0.1, 0.9]
p1 = [1.0-1e-50, 1e-50]
Z = Categorical([1-1e-10, 1e-10])
X = Mixture(Z, Multinomial, 10, [p0, p1])
u = X._message_to_child()
self.assertAllClose(u[0],
[1, 9])
p0 = [0.1, 0.9]
p1 = [1.0-1e-10, 1e-10]
Z = Categorical([1-1e-50, 1e-50])
X = Mixture(Z, Multinomial, 10, [p0, p1])
u = X._message_to_child()
self.assertAllClose(u[0],
[1, 9])
with warnings.catch_warnings():
warnings.simplefilter("ignore", RuntimeWarning)
warnings.simplefilter("ignore", UserWarning)
p0 = [0.1, 0.9]
p1 = [1.0, 0.0]
X = Mixture(0, Multinomial, 10, [p0, p1])
u = X._message_to_child()
self.assertAllClose(u[0],
np.nan*np.ones(2))
pass
def test_gradient(self):
"""
Check the Euclidean gradient of the categorical node
"""
Z = Categorical([[0.3, 0.5, 0.2], [0.1, 0.6, 0.3]])
Y = Mixture(Z, Gamma, [2, 3, 4], [5, 6, 7])
Y.observe([4.2, 0.2])
def f(x):
Z.set_parameters([np.reshape(x, Z.get_shape(0))])
return Z.lower_bound_contribution() + Y.lower_bound_contribution()
def df(x):
Z.set_parameters([np.reshape(x, Z.get_shape(0))])
g = Z.get_riemannian_gradient()
return Z.get_gradient(g)[0]
x0 = np.ravel(np.log([[2, 3, 7], [0.1, 3, 1]]))
self.assertAllClose(
misc.gradient(f, x0),
np.ravel(df(x0))
)
pass
def mixture_model(distribution, *args, K=3, N=100):
# Prior for state probabilities
alpha = Dirichlet(1e-3 * np.ones(K), name='alpha')
# Cluster assignments
Z = Categorical(alpha, plates=(N, ), name='Z')
# Observation distribution
Y = Mixture(Z, distribution, *args, name='Y')
Q = VB(Y, Z, alpha)
return Q
def test_mixture_with_count_array(self):
"""
Test binomial mixture with varying number of trials
"""
p0 = 0.6
p1 = 0.9
p2 = 0.3
counts = [[10], [5], [3]]
X = Mixture(2, Binomial, counts, [p0, p1, p2])
u = X._message_to_child()
self.assertAllClose(u[0], np.array(counts)[:, 0] * np.array(p2))
# Multi-mixture and count array
# Shape(p) = (2, 1, 3) + ()
p = [
[[0.6, 0.9, 0.8]],
[[0.1, 0.2, 0.3]],
]
# Shape(Z1) = (1, 3) + (2,) -> () + (2,)
Z1 = 1
# Shape(Z2) = (1,) + (3,) -> () + (3,)
Z2 = 2
# Shape(counts) = (5, 1)
counts = [[10], [5], [3], [2], [4]]
# Shape(X) = (5,) + ()
X = Mixture(
Z1,
Mixture,
Z2,
Binomial,
counts,
p,
# NOTE: We mix over axes -3 and -1. But as we first mix over the
# default (-1), then the next mixing happens over -2 (because one
# axis was already dropped).
cluster_plate=-2,
)
self.assertAllClose(X._message_to_child()[0],
np.array(counts)[:, 0] * np.array(p)[Z1, :, Z2])
# Can't have non-singleton axis in counts over the mixed axis
p0 = 0.6
p1 = 0.9
p2 = 0.3
counts = [10, 5, 3]
self.assertRaises(
ValueError,
Mixture,
2,
Binomial,
counts,
[p0, p1, p2],
)
return
def _setup_bernoulli_mixture():
"""
Setup code for the hinton tests.
This code is from http://www.bayespy.org/examples/bmm.html
"""
np.random.seed(1)
p0 = [0.1, 0.9, 0.1, 0.9, 0.1, 0.9, 0.1, 0.9, 0.1, 0.9]
p1 = [0.1, 0.1, 0.1, 0.1, 0.1, 0.9, 0.9, 0.9, 0.9, 0.9]
p2 = [0.9, 0.9, 0.9, 0.9, 0.9, 0.1, 0.1, 0.1, 0.1, 0.1]
p = np.array([p0, p1, p2])
z = random.categorical([1/3, 1/3, 1/3], size=100)
x = random.bernoulli(p[z])
N = 100
D = 10
K = 10
R = Dirichlet(K*[1e-5],
name='R')
Z = Categorical(R,
plates=(N,1),
name='Z')
P = Beta([0.5, 0.5],
plates=(D,K),
name='P')
X = Mixture(Z, Bernoulli, P)
Q = VB(Z, R, X, P)
P.initialize_from_random()
X.observe(x)
Q.update(repeat=1000)
return (R,P,Z)
def hidden_markov_model(distribution, *args, K=3, N=100):
# Prior for initial state probabilities
alpha = Dirichlet(1e-3 * np.ones(K), name='alpha')
# Prior for state transition probabilities
A = Dirichlet(1e-3 * np.ones(K), plates=(K, ), name='A')
# Hidden states (with unknown initial state probabilities and state
# transition probabilities)
Z = CategoricalMarkovChain(alpha, A, states=N, name='Z')
# Emission/observation distribution
Y = Mixture(Z, distribution, *args, name='Y')
Q = VB(Y, Z, alpha, A)
return Q
def test_message_to_child(self):
"""
Test the message to child of Mixture node.
"""
K = 3
#
# Estimate moments from parents only
#
# Simple case
mu = GaussianARD([0,2,4], 1,
ndim=0,
plates=(K,))
alpha = Gamma(1, 1,
plates=(K,))
z = Categorical(np.ones(K)/K)
X = Mixture(z, GaussianARD, mu, alpha)
self.assertEqual(X.plates, ())
self.assertEqual(X.dims, ( (), () ))
u = X._message_to_child()
self.assertAllClose(u[0],
2)
self.assertAllClose(u[1],
2**2+1)
# Broadcasting the moments on the cluster axis
mu = GaussianARD(2, 1,
ndim=0,
plates=(K,))
alpha = Gamma(1, 1,
plates=(K,))
z = Categorical(np.ones(K)/K)
X = Mixture(z, GaussianARD, mu, alpha)
self.assertEqual(X.plates, ())
self.assertEqual(X.dims, ( (), () ))
u = X._message_to_child()
self.assertAllClose(u[0],
2)
self.assertAllClose(u[1],
2**2+1)
#
# Estimate moments with observed children
#
pass
def test_deterministic_mappings(self):
x = Categorical([0.8, 0.2])
y = Mixture(x, Categorical, [
[0.10, 0.90],
[0.00, 1.00],
])
y.observe(0)
x.update()
self.assertAllClose(x.u[0], [1, 0])
y.observe(1)
x.update()
p = np.array([0.8 * 0.9, 0.2 * 1.0])
self.assertAllClose(x.u[0], p / np.sum(p))
pass
import numpy numpy.random.seed(1) p0 = [0.1, 0.9, 0.1, 0.9, 0.1, 0.9, 0.1, 0.9, 0.1, 0.9] p1 = [0.1, 0.1, 0.1, 0.1, 0.1, 0.9, 0.9, 0.9, 0.9, 0.9] p2 = [0.9, 0.9, 0.9, 0.9, 0.9, 0.1, 0.1, 0.1, 0.1, 0.1] import numpy as np p = np.array([p0, p1, p2]) from bayespy.utils import random z = random.categorical([1 / 3, 1 / 3, 1 / 3], size=100) x = random.bernoulli(p[z]) N = 100 D = 10 K = 10 from bayespy.nodes import Categorical, Dirichlet R = Dirichlet(K * [1e-5], name='R') Z = Categorical(R, plates=(N, 1), name='Z') from bayespy.nodes import Beta P = Beta([0.5, 0.5], plates=(D, K), name='P') from bayespy.nodes import Mixture, Bernoulli X = Mixture(Z, Bernoulli, P) from bayespy.inference import VB Q = VB(Z, R, X, P) P.initialize_from_random() X.observe(x) Q.update(repeat=1000) import bayespy.plot as bpplt bpplt.hinton(P) bpplt.pyplot.show()
def run(self, K=25, beta=0.5, alpha=0.00001, foci_thresh=0, num_neigh=4, hinton_plot=False, end=False):
'''Performs one run of the BBDP according to the specified parameters.'''
print("Transforming WCS participant data into binary vectors...")
x = u.transform_data_all(self.langs, norm=False, end=end, foci=True, foci_thresh=foci_thresh, num_neigh=num_neigh)
print("Finished transforming participant data")
self.participant_list = x[0]
N = len(x[0]) #number of data points (i.e. WCS participants)
D = np.shape(x[1])[1] #number of features
#K = 20 #number of initial clusters
R = Dirichlet(K*[alpha],
name='R')
Z = Categorical(R,
plates=(N,1),
name='Z')
P = Beta([beta, beta],
plates=(D,K),
name='P')
X = Mixture(Z, Bernoulli, P)
Q = VB(Z, R, X, P)
P.initialize_from_random()
X.observe(x[1])
Q.update(repeat=1000)
if hinton_plot:
bpplt.hinton(Z)
bpplt.pyplot.show()
bpplt.hinton(R)
bpplt.pyplot.show()
#Get the weight matrix stored in Z (weights determine which cluster data point belongs to)
z = Z._message_to_child()[0]
z = z * np.ones(Z.plates+(1,))
z = np.squeeze(z)
self.z = z
#Get the weights stored in R (proportional to the size of the clusters)
r = np.exp(R._message_to_child()[0])
r = r * np.ones(R.plates+(1,))
r = np.squeeze(r)
self.r = r
#Get the cluster assignment of each data point
self.c_assign = np.argmax(self.z, axis=1)
#Write cluster results to a file
if self.write_to_file:
if end:
save_path = "cluster_results_end_K={}_B={}_a={}_t={}_nn={}".format(K, beta, alpha, foci_thresh, num_neigh)
else:
save_path = "cluster_results_K={}_B={}_a={}_t={}_nn={}".format(K, beta, alpha, foci_thresh, num_neigh)
while path.exists(save_path+".txt"):
#save_path already exists
try:
old_file_num = int(save_path[save_path.find('(')+1:-1])
new_file_num = old_file_num + 1
save_path = save_path[0:save_path.find('(')] + '(' + str(new_file_num) + ')'
except ValueError:
save_path = save_path + " (1)"
self.save_path = save_path
file = open(path.abspath(self.save_path+".txt"), 'w')
#Write cluster assignment matrix Z (gives the probability that observation i belongs to cluster j)
if 'Z' not in self.in_file:
for i in range(len(self.z)):
line = "\t".join([str(x) for x in self.z[i]]) + "\n"
file.write(line)
file.write('---Z\n')
self.in_file.append('Z')
#Write cluster weights matrix R (proportional to the size of the resulting clusters)
if 'R' not in self.in_file:
line = "\t".join([str(x) for x in self.r]) + "\n"
file.write(line)
file.write('---R\n')
self.in_file.append('R')
#Write deterministic cluster assignments with the corresponding participant key
if 'C' not in self.in_file:
line1 = "\t".join([str(x) for x in self.participant_list]) + "\n"
line2 = "\t".join([str(x) for x in self.c_assign]) + "\n"
file.write(line1)
file.write(line2)
file.write('---C\n')
self.in_file.append('C')
file.close()
return self.c_assign
def generateCPD(self, term): #, X_train, y_train, X_test, y_test, X_validation, y_validation, g_train, g_test, g_validation):
clf = loadClf(self.ontology[term]['name'], self.fold, self.clfName)
posTrain = sum(clf.y_train == POSTIVE_LABEL)
negTrain = sum(clf.y_train == NEGATIVE_LABEL)
totalTrain = posTrain + negTrain
children = sorted(self.ontology[term]['children'])
parents = sorted(self.ontology[term]['parents'])
labels = {l : PRIOR for l in product(*((POSTIVE_LABEL,NEGATIVE_LABEL),)*(len(children)+1))}
if children:
childNodes = [self.ontology[child]['node'][self.fold][self.clfName] for child in children]
for gene,y in zip(clf.g_train, clf.y_train):
event = []
for child in children:
event.append(POSTIVE_LABEL if gene in self.ontology.associations[child] else NEGATIVE_LABEL)
event.append(POSTIVE_LABEL if gene in self.ontology.associations[term] else NEGATIVE_LABEL)
assert (gene in self.ontology.associations[term]) == (y == POSTIVE_LABEL)
event = tuple(event)
labels[event] += 1
def countBoth(event):
return labels[event[:-1]+(POSTIVE_LABEL,)] + labels[event[:-1]+(NEGATIVE_LABEL,)]
cprior = PRIOR * (2 ** len(children))
types = [Mixture]*(len(children)-1) + [Categorical]
mixparams = [i for s in zip(childNodes, types) for i in s]
cpd = numpy.empty((2,)*(len(children)+1))
for event, counted in labels.items():
v=cpd
for b in event[:-1]:
v = v[b]
hid = event[-1]
print("Event: ", event)
if POSTIVE_LABEL not in event[:-1]: # Všichni potomci označeni "ne"
v[hid] = counted/countBoth(event)
print("Stored %d / %d" % (counted,countBoth(event)))
else:
v[hid] = {POSTIVE_LABEL: 0.99, NEGATIVE_LABEL:0.01}[hid]
print("Stored %d : %d" % (hid, v[hid]))
#print(term,"<-",",".join(children))
print(cpd)
#print(labels)
hidden = Mixture(*mixparams, cpd)
hidden.params = cpd
else: #No children
#hidden = DiscreteDistribution({'0': posTrain / totalTrain, '1': negTrain / totalTrain})
params = (posTrain / totalTrain, negTrain / totalTrain)
hidden = Categorical(params)
hidden.params = params
#print("Hidden node %s:" % term)
#print(repr(hidden))
#print([p for p in hidden.parents if isinstance(p, Stochastic)])
#print(hidden.get_moments())
conf = clf.conf + PRIOR
#posTest, negTest = numpy.sum(conf, 1)
posTest, negTest = numpy.sum(conf, 0)
#print("Confusion matrix:")
#print(conf)
try:
assert term != self.ontology.root
pos_decisions = clf.decision_function(clf.X_test[clf.y_test==POSTIVE_LABEL])
neg_decisions = clf.decision_function(clf.X_test[clf.y_test==NEGATIVE_LABEL])
means = [numpy.mean(pos_decisions)], [numpy.mean(neg_decisions)]
maxprec = 100.0
precs = [[numpy.min((1/numpy.var(pos_decisions), maxprec))]], [[numpy.min((1/numpy.var(neg_decisions), maxprec))]]
#else:
except (ValueError, AssertionError):
means = [-1.], [1.]
precs = [[1.]], [[1.]]
print("Gaussian params:", term, self.ontology[term]['name'], means, precs)
observed = Mixture(hidden, Gaussian, means, precs)
#observed = ConditionalProbabilityTable([
# ['0', '0', conf[0][0] / posTest], # if term != root else 1.],
# ['0', '1', conf[0][1] / posTest], # if term != root else 0.],
# ['1', '0', conf[1][0] / negTest], # if term != root else 0.],
# ['1', '1', conf[1][1] / negTest]], #if term != root else 1.]],
# [hidden.distribution])
#print("Observed node %s - %s:" % (term, self.ontology[term]['name']))
#print(repr(observed))
#print([p for p in observed.parents if isinstance(p, Stochastic)])
self.ontology[term]['node'][self.fold][self.clfName] = hidden
#self.ontology[term]['clf'][self.fold][self.clfName] = clf, X_validation, y_validation, g_validation
assert self.lenValidation is None or self.lenValidation == len(clf.y_validation)
self.lenValidation = len(clf.y_validation)
self.allobserved[term] = observed
self.allhidden[term] = hidden
self.extranodes.update((p for p in hidden.parents if isinstance(p, Stochastic)))
D = 2 K = 5 from bayespy.nodes import Dirichlet, Categorical alpha = Dirichlet(1e-5 * np.ones(K), name='alpha') Zi = Categorical(alpha, plates=(N, ), name='zi') from bayespy.nodes import Gaussian, Wishart mui = Gaussian(np.zeros(D), 1e-5 * np.identity(D), plates=(K, ), name='mui') Lambdai = Wishart(D, 1e-5 * np.identity(D), plates=(K, ), name='Lambdai') from bayespy.nodes import Mixture Y = Mixture(Zi, Gaussian, mui, Lambdai, name='Y') Zi.initialize_from_random() from bayespy.inference import VB Q = VB(Y, mui, Lambdai, Zi, alpha) Y.observe(np.reshape(C_mat, (-1, 2))) Q.update(repeat=10) #%% K = 5 #hyperparameter neta = 1e-6 * np.ones(K) #hyperparameter print(neta.shape) print(neta)
def run(N=100000, N_batch=50, seed=42, maxiter=100, plot=True):
"""
Run deterministic annealing demo for 1-D Gaussian mixture.
"""
if seed is not None:
np.random.seed(seed)
# Number of clusters in the model
K = 20
# Dimensionality of the data
D = 5
# Generate data
K_true = 10
spread = 5
means = spread * np.random.randn(K_true, D)
z = random.categorical(np.ones(K_true), size=N)
data = np.empty((N, D))
for n in range(N):
data[n] = means[z[n]] + np.random.randn(D)
#
# Standard VB-EM algorithm
#
# Full model
mu = Gaussian(np.zeros(D), np.identity(D), plates=(K, ), name='means')
alpha = Dirichlet(np.ones(K), name='class probabilities')
Z = Categorical(alpha, plates=(N, ), name='classes')
Y = Mixture(Z, Gaussian, mu, np.identity(D), name='observations')
# Break symmetry with random initialization of the means
mu.initialize_from_random()
# Put the data in
Y.observe(data)
# Run inference
Q = VB(Y, Z, mu, alpha)
Q.save(mu)
Q.update(repeat=maxiter)
if plot:
bpplt.pyplot.plot(np.cumsum(Q.cputime), Q.L, 'k-')
max_cputime = np.sum(Q.cputime[~np.isnan(Q.cputime)])
#
# Stochastic variational inference
#
# Construct smaller model (size of the mini-batch)
mu = Gaussian(np.zeros(D), np.identity(D), plates=(K, ), name='means')
alpha = Dirichlet(np.ones(K), name='class probabilities')
Z = Categorical(alpha,
plates=(N_batch, ),
plates_multiplier=(N / N_batch, ),
name='classes')
Y = Mixture(Z, Gaussian, mu, np.identity(D), name='observations')
# Break symmetry with random initialization of the means
mu.initialize_from_random()
# Inference engine
Q = VB(Y, Z, mu, alpha, autosave_filename=Q.autosave_filename)
Q.load(mu)
# Because using mini-batches, messages need to be multiplied appropriately
print("Stochastic variational inference...")
Q.ignore_bound_checks = True
maxiter *= int(N / N_batch)
delay = 1
forgetting_rate = 0.7
for n in range(maxiter):
# Observe a mini-batch
subset = np.random.choice(N, N_batch)
Y.observe(data[subset, :])
# Learn intermediate variables
Q.update(Z)
# Set step length
step = (n + delay)**(-forgetting_rate)
# Stochastic gradient for the global variables
Q.gradient_step(mu, alpha, scale=step)
if np.sum(Q.cputime[:n]) > max_cputime:
break
if plot:
bpplt.pyplot.plot(np.cumsum(Q.cputime), Q.L, 'r:')
bpplt.pyplot.xlabel('CPU time (in seconds)')
bpplt.pyplot.ylabel('VB lower bound')
bpplt.pyplot.legend(['VB-EM', 'Stochastic inference'],
loc='lower right')
bpplt.pyplot.title('VB for Gaussian mixture model')
return
from bayespy.nodes import Dirichlet, Categorical from bayespy.nodes import Gaussian, Wishart from bayespy.nodes import Mixture from bayespy.inference import VB y0 = np.random.multivariate_normal([0, 0], [[2, 0], [0, 0.1]], size=50) y1 = np.random.multivariate_normal([0, 0], [[0.1, 0], [0, 2]], size=50) y2 = np.random.multivariate_normal([2, 2], [[2, -1.5], [-1.5, 2]], size=50) y3 = np.random.multivariate_normal([-2, -2], [[0.5, 0], [0, 0.5]], size=50) y = np.vstack([y0, y1, y2, y3]) N = 200 D = 2 K = 10 alpha = Dirichlet(1e-5*np.ones(K), name='alpha') Z = Categorical(alpha, plates=(N,),name='z') mu = Gaussian(np.zeros(D),1e-5*np.identity(D),plates=(K,),name='mu') Lambda = Wishart(D,1e-5*np.identity(D),plates=(K,),name='Lambda') Y = Mixture(Z, Gaussian, mu, Lambda, name='Y') Z.initialize_from_random() Q = VB(Y, mu, Lambda, Z, alpha) Y.observe(y) Q.update(repeat=1000) bpplt.gaussian_mixture_2d(Y, alpha=alpha, scale=2)
from bayespy.nodes import Categorical, Mixture, Gaussian
from bayespy.inference import VB
import copy
import numpy as np
import bayespy.plot as bpplt
hidden2 = Categorical((0.7, 0.3))
hidden1 = Mixture(hidden2, Categorical, ((0.6, 0.4), (0.1, 0.9)))
observed1 = Mixture(hidden1, Gaussian, ([-1.0], [0.9]), ([[1.3]], [[0.8]]))
observed2 = Mixture(hidden2, Gaussian, ([-0.9], [1.1]), ([[1.2]], [[0.7]]))
observed_1, observed_2, hidden_1, hidden_2 = copy.deepcopy(
(observed1, observed2, hidden1, hidden2))
observed_1.observe((-1.2, ))
observed_2.observe((1.2, ))
Q = VB(hidden_1, hidden_2, observed_1, observed_2, tol=1e-10)
Q.update(repeat=100)
print(hidden_1.get_moments())
print(hidden_2.get_moments())
observed_1, observed_2, hidden_1, hidden_2 = copy.deepcopy(
(observed1, observed2, hidden1, hidden2))
observed_1.observe((-0.2, ))
observed_2.observe((1.2, ))
Q = VB(hidden_1, hidden_2, observed_1, observed_2)
Q.update(repeat=100)
print(hidden_1.get_moments())
print(hidden_2.get_moments())
observed_1, observed_2, hidden_1, hidden_2 = copy.deepcopy(
def test_message_to_parent(self):
"""
Test the message to parents of Mixture node.
"""
K = 3
# Broadcasting the moments on the cluster axis
Mu = GaussianARD(2, 1,
ndim=0,
plates=(K,))
(mu, mumu) = Mu._message_to_child()
Alpha = Gamma(3, 1,
plates=(K,))
(alpha, logalpha) = Alpha._message_to_child()
z = Categorical(np.ones(K)/K)
X = Mixture(z, GaussianARD, Mu, Alpha)
tau = 4
Y = GaussianARD(X, tau)
y = 5
Y.observe(y)
(x, xx) = X._message_to_child()
m = z._message_from_children()
self.assertAllClose(m[0] * np.ones(K),
random.gaussian_logpdf(xx*alpha,
x*alpha*mu,
mumu*alpha,
logalpha,
0)
* np.ones(K))
m = Mu._message_from_children()
self.assertAllClose(m[0],
1/K * (alpha*x) * np.ones(3))
self.assertAllClose(m[1],
-0.5 * 1/K * alpha * np.ones(3))
# Some parameters do not have cluster plate axis
Mu = GaussianARD(2, 1,
ndim=0,
plates=(K,))
(mu, mumu) = Mu._message_to_child()
Alpha = Gamma(3, 1) # Note: no cluster plate axis!
(alpha, logalpha) = Alpha._message_to_child()
z = Categorical(np.ones(K)/K)
X = Mixture(z, GaussianARD, Mu, Alpha)
tau = 4
Y = GaussianARD(X, tau)
y = 5
Y.observe(y)
(x, xx) = X._message_to_child()
m = z._message_from_children()
self.assertAllClose(m[0] * np.ones(K),
random.gaussian_logpdf(xx*alpha,
x*alpha*mu,
mumu*alpha,
logalpha,
0)
* np.ones(K))
m = Mu._message_from_children()
self.assertAllClose(m[0],
1/K * (alpha*x) * np.ones(3))
self.assertAllClose(m[1],
-0.5 * 1/K * alpha * np.ones(3))
# Cluster assignments do not have as many plate axes as parameters.
M = 2
Mu = GaussianARD(2, 1,
ndim=0,
plates=(K,M))
(mu, mumu) = Mu._message_to_child()
Alpha = Gamma(3, 1,
plates=(K,M))
(alpha, logalpha) = Alpha._message_to_child()
z = Categorical(np.ones(K)/K)
X = Mixture(z, GaussianARD, Mu, Alpha, cluster_plate=-2)
tau = 4
Y = GaussianARD(X, tau)
y = 5 * np.ones(M)
Y.observe(y)
(x, xx) = X._message_to_child()
m = z._message_from_children()
self.assertAllClose(m[0]*np.ones(K),
np.sum(random.gaussian_logpdf(xx*alpha,
x*alpha*mu,
mumu*alpha,
logalpha,
0) *
np.ones((K,M)),
axis=-1))
m = Mu._message_from_children()
self.assertAllClose(m[0] * np.ones((K,M)),
1/K * (alpha*x) * np.ones((K,M)))
self.assertAllClose(m[1] * np.ones((K,M)),
-0.5 * 1/K * alpha * np.ones((K,M)))
# Mixed distribution broadcasts g
# This tests for a found bug. The bug caused an error.
Z = Categorical([0.3, 0.5, 0.2])
X = Mixture(Z, Categorical, [[0.2,0.8], [0.1,0.9], [0.3,0.7]])
m = Z._message_from_children()
pass
p0 = [0.1, 0.1, 0.1, 0.9, 0.9, 0.9, 0.1, 0.1, 0.1, 0.1]
p1 = [0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.9, 0.9, 0.9, 0.9]
p2 = [0.9, 0.9, 0.9, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1]
p = np.array([p0, p1, p2])
z = random.categorical([1/3, 1/3, 1/3], size=100)
x = random.bernoulli(p[z])
N = 100
D = 10
K = 3
R = Dirichlet(K*[1e-5],name='R')
Z = Categorical(R,plates=(N,1),name='Z')
P = Beta([0.5, 0.5],plates=(D,K),name='P')
X = Mixture(Z, Bernoulli, P)
Q = VB(Z, R, X, P)
P.initialize_from_random()
X.observe(x)
Q.update(repeat=1000)
#print(" P:")
#print( P.get_moments() )
#print(" R:")
#print( R.get_moments() )
print(" Z:")
print( Z.get_moments() )
import numpy
numpy.random.seed(1)
from bayespy.nodes import CategoricalMarkovChain
a0 = [0.6, 0.4] # p(rainy)=0.6, p(sunny)=0.4
A = [
[0.7, 0.3], # p(rainy->rainy)=0.7, p(rainy->sunny)=0.3
[0.4, 0.6]
] # p(sunny->rainy)=0.4, p(sunny->sunny)=0.6
N = 100
Z = CategoricalMarkovChain(a0, A, states=N)
from bayespy.nodes import Categorical, Mixture
P = [[0.1, 0.4, 0.5], [0.6, 0.3, 0.1]]
Y = Mixture(Z, Categorical, P)
weather = Z.random()
from bayespy.inference import VB
import bayespy.plot as bpplt
import numpy as np
mu = np.array([[0, 0], [3, 4], [6, 0]])
std = 2.0
K = 3
N = 200
p0 = np.ones(K) / K
q = 0.9
r = (1 - q) / (K - 1)
P = q * np.identity(K) + r * (np.ones((3, 3)) - np.identity(3))
y = np.zeros((N, 2))
z = np.zeros(N)
def test_message_to_parent(self):
"""
Test the message to parents of Mixture node.
"""
K = 3
# Broadcasting the moments on the cluster axis
Mu = GaussianARD(2, 1,
ndim=0,
plates=(K,))
(mu, mumu) = Mu._message_to_child()
Alpha = Gamma(3, 1,
plates=(K,))
(alpha, logalpha) = Alpha._message_to_child()
z = Categorical(np.ones(K)/K)
X = Mixture(z, GaussianARD, Mu, Alpha)
tau = 4
Y = GaussianARD(X, tau)
y = 5
Y.observe(y)
(x, xx) = X._message_to_child()
m = z._message_from_children()
self.assertAllClose(m[0] * np.ones(K),
random.gaussian_logpdf(xx*alpha,
x*alpha*mu,
mumu*alpha,
logalpha,
0)
* np.ones(K))
m = Mu._message_from_children()
self.assertAllClose(m[0],
1/K * (alpha*x) * np.ones(3))
self.assertAllClose(m[1],
-0.5 * 1/K * alpha * np.ones(3))
# Some parameters do not have cluster plate axis
Mu = GaussianARD(2, 1,
ndim=0,
plates=(K,))
(mu, mumu) = Mu._message_to_child()
Alpha = Gamma(3, 1) # Note: no cluster plate axis!
(alpha, logalpha) = Alpha._message_to_child()
z = Categorical(np.ones(K)/K)
X = Mixture(z, GaussianARD, Mu, Alpha)
tau = 4
Y = GaussianARD(X, tau)
y = 5
Y.observe(y)
(x, xx) = X._message_to_child()
m = z._message_from_children()
self.assertAllClose(m[0] * np.ones(K),
random.gaussian_logpdf(xx*alpha,
x*alpha*mu,
mumu*alpha,
logalpha,
0)
* np.ones(K))
m = Mu._message_from_children()
self.assertAllClose(m[0],
1/K * (alpha*x) * np.ones(3))
self.assertAllClose(m[1],
-0.5 * 1/K * alpha * np.ones(3))
# Cluster assignments do not have as many plate axes as parameters.
M = 2
Mu = GaussianARD(2, 1,
ndim=0,
plates=(K,M))
(mu, mumu) = Mu._message_to_child()
Alpha = Gamma(3, 1,
plates=(K,M))
(alpha, logalpha) = Alpha._message_to_child()
z = Categorical(np.ones(K)/K)
X = Mixture(z, GaussianARD, Mu, Alpha, cluster_plate=-2)
tau = 4
Y = GaussianARD(X, tau)
y = 5 * np.ones(M)
Y.observe(y)
(x, xx) = X._message_to_child()
m = z._message_from_children()
self.assertAllClose(m[0]*np.ones(K),
np.sum(random.gaussian_logpdf(xx*alpha,
x*alpha*mu,
mumu*alpha,
logalpha,
0) *
np.ones((K,M)),
axis=-1))
m = Mu._message_from_children()
self.assertAllClose(m[0] * np.ones((K,M)),
1/K * (alpha*x) * np.ones((K,M)))
self.assertAllClose(m[1] * np.ones((K,M)),
-0.5 * 1/K * alpha * np.ones((K,M)))
# Mixed distribution broadcasts g
# This tests for a found bug. The bug caused an error.
Z = Categorical([0.3, 0.5, 0.2])
X = Mixture(Z, Categorical, [[0.2,0.8], [0.1,0.9], [0.3,0.7]])
m = Z._message_from_children()
#
# Test nested mixtures
#
t1 = [1, 1, 0, 3, 3]
t2 = [2]
p = Dirichlet([1, 1], plates=(4, 3))
X = Mixture(t1, Mixture, t2, Categorical, p)
X.observe([1, 1, 0, 0, 0])
p.update()
self.assertAllClose(
p.phi[0],
[
[[1, 1], [1, 1], [2, 1]],
[[1, 1], [1, 1], [1, 3]],
[[1, 1], [1, 1], [1, 1]],
[[1, 1], [1, 1], [3, 1]],
]
)
# Test sample plates in nested mixtures
t1 = Categorical([0.3, 0.7], plates=(5,))
t2 = [[1], [1], [0], [3], [3]]
t3 = 2
p = Dirichlet([1, 1], plates=(2, 4, 3))
X = Mixture(t1, Mixture, t2, Mixture, t3, Categorical, p)
X.observe([1, 1, 0, 0, 0])
p.update()
self.assertAllClose(
p.phi[0],
[
[
[[1, 1], [1, 1], [1.3, 1]],
[[1, 1], [1, 1], [1, 1.6]],
[[1, 1], [1, 1], [1, 1]],
[[1, 1], [1, 1], [1.6, 1]],
],
[
[[1, 1], [1, 1], [1.7, 1]],
[[1, 1], [1, 1], [1, 2.4]],
[[1, 1], [1, 1], [1, 1]],
[[1, 1], [1, 1], [2.4, 1]],
]
]
)
# Check that Gate and nested Mixture are equal
t1 = Categorical([0.3, 0.7], plates=(5,))
t2 = Categorical([0.1, 0.3, 0.6], plates=(5, 1))
p = Dirichlet([1, 2, 3, 4], plates=(2, 3))
X = Mixture(t1, Mixture, t2, Categorical, p)
X.observe([3, 3, 1, 2, 2])
t1_msg = t1._message_from_children()
t2_msg = t2._message_from_children()
p_msg = p._message_from_children()
t1 = Categorical([0.3, 0.7], plates=(5,))
t2 = Categorical([0.1, 0.3, 0.6], plates=(5, 1))
p = Dirichlet([1, 2, 3, 4], plates=(2, 3))
X = Categorical(Gate(t1, Gate(t2, p)))
X.observe([3, 3, 1, 2, 2])
t1_msg2 = t1._message_from_children()
t2_msg2 = t2._message_from_children()
p_msg2 = p._message_from_children()
self.assertAllClose(t1_msg[0], t1_msg2[0])
self.assertAllClose(t2_msg[0], t2_msg2[0])
self.assertAllClose(p_msg[0], p_msg2[0])
pass
FALSE = 0
TRUE = 1
def _or(p_false, p_true):
"""
Build probability table for OR-operation of two parents
p_false: Probability table to use if both are FALSE
p_true: Probability table to use if one or both is TRUE
"""
return np.take([p_false, p_true], [[FALSE, TRUE], [TRUE, TRUE]], axis=0)
asia = Categorical([0.5, 0.5])
tuberculosis = Mixture(asia, Categorical, [[0.99, 0.01], [0.8, 0.2]])
smoking = Categorical([0.5, 0.5])
lung = Mixture(smoking, Categorical, [[0.98, 0.02], [0.25, 0.75]])
bronchitis = Mixture(smoking, Categorical, [[0.97, 0.03], [0.08, 0.92]])
xray = Mixture(tuberculosis, Mixture, lung, Categorical,
_or([0.96, 0.04], [0.115, 0.885]))
dyspnea = Mixture(bronchitis, Mixture, tuberculosis, Mixture, lung, Categorical,
[_or([0.6, 0.4], [0.18, 0.82]),
_or([0.11, 0.89], [0.04, 0.96])])
# Mark observations
def run(N=500, seed=42, maxiter=100, plot=True):
"""
Run deterministic annealing demo for 1-D Gaussian mixture.
"""
if seed is not None:
np.random.seed(seed)
mu = GaussianARD(0, 1,
plates=(2,),
name='means')
Z = Categorical([0.3, 0.7],
plates=(N,),
name='classes')
Y = Mixture(Z, GaussianARD, mu, 1,
name='observations')
# Generate data
z = Z.random()
data = np.empty(N)
for n in range(N):
data[n] = [4, -4][z[n]]
Y.observe(data)
# Initialize means closer to the inferior local optimum in which the
# cluster means are swapped
mu.initialize_from_value([0, 6])
Q = VB(Y, Z, mu)
Q.save()
#
# Standard VB-EM algorithm
#
Q.update(repeat=maxiter)
mu_vbem = mu.u[0].copy()
L_vbem = Q.compute_lowerbound()
#
# VB-EM with deterministic annealing
#
Q.load()
beta = 0.01
while beta < 1.0:
beta = min(beta*1.2, 1.0)
print("Set annealing to %.2f" % beta)
Q.set_annealing(beta)
Q.update(repeat=maxiter, tol=1e-4)
mu_anneal = mu.u[0].copy()
L_anneal = Q.compute_lowerbound()
print("==============================")
print("RESULTS FOR VB-EM vs ANNEALING")
print("Fixed component probabilities:", np.array([0.3, 0.7]))
print("True component means:", np.array([4, -4]))
print("VB-EM component means:", mu_vbem)
print("VB-EM lower bound:", L_vbem)
print("Annealed VB-EM component means:", mu_anneal)
print("Annealed VB-EM lower bound:", L_anneal)
return
def run(N=100000, N_batch=50, seed=42, maxiter=100, plot=True):
"""
Run deterministic annealing demo for 1-D Gaussian mixture.
"""
if seed is not None:
np.random.seed(seed)
# Number of clusters in the model
K = 20
# Dimensionality of the data
D = 5
# Generate data
K_true = 10
spread = 5
means = spread * np.random.randn(K_true, D)
z = random.categorical(np.ones(K_true), size=N)
data = np.empty((N,D))
for n in range(N):
data[n] = means[z[n]] + np.random.randn(D)
#
# Standard VB-EM algorithm
#
# Full model
mu = Gaussian(np.zeros(D), np.identity(D),
plates=(K,),
name='means')
alpha = Dirichlet(np.ones(K),
name='class probabilities')
Z = Categorical(alpha,
plates=(N,),
name='classes')
Y = Mixture(Z, Gaussian, mu, np.identity(D),
name='observations')
# Break symmetry with random initialization of the means
mu.initialize_from_random()
# Put the data in
Y.observe(data)
# Run inference
Q = VB(Y, Z, mu, alpha)
Q.save(mu)
Q.update(repeat=maxiter)
if plot:
bpplt.pyplot.plot(np.cumsum(Q.cputime), Q.L, 'k-')
max_cputime = np.sum(Q.cputime[~np.isnan(Q.cputime)])
#
# Stochastic variational inference
#
# Construct smaller model (size of the mini-batch)
mu = Gaussian(np.zeros(D), np.identity(D),
plates=(K,),
name='means')
alpha = Dirichlet(np.ones(K),
name='class probabilities')
Z = Categorical(alpha,
plates=(N_batch,),
plates_multiplier=(N/N_batch,),
name='classes')
Y = Mixture(Z, Gaussian, mu, np.identity(D),
name='observations')
# Break symmetry with random initialization of the means
mu.initialize_from_random()
# Inference engine
Q = VB(Y, Z, mu, alpha, autosave_filename=Q.autosave_filename)
Q.load(mu)
# Because using mini-batches, messages need to be multiplied appropriately
print("Stochastic variational inference...")
Q.ignore_bound_checks = True
maxiter *= int(N/N_batch)
delay = 1
forgetting_rate = 0.7
for n in range(maxiter):
# Observe a mini-batch
subset = np.random.choice(N, N_batch)
Y.observe(data[subset,:])
# Learn intermediate variables
Q.update(Z)
# Set step length
step = (n + delay) ** (-forgetting_rate)
# Stochastic gradient for the global variables
Q.gradient_step(mu, alpha, scale=step)
if np.sum(Q.cputime[:n]) > max_cputime:
break
if plot:
bpplt.pyplot.plot(np.cumsum(Q.cputime), Q.L, 'r:')
bpplt.pyplot.xlabel('CPU time (in seconds)')
bpplt.pyplot.ylabel('VB lower bound')
bpplt.pyplot.legend(['VB-EM', 'Stochastic inference'], loc='lower right')
bpplt.pyplot.title('VB for Gaussian mixture model')
return
from bayespy.nodes import Categorical, Mixture
from bayespy.inference import VB
import numpy as np
TRUE, FALSE = 1, 0
def _or(p_false, p_true):
return np.take([p_false, p_true], [[FALSE, TRUE], [TRUE, TRUE]], axis=0)
A = Categorical([0.5, 0.5])
T = Mixture(A, Categorical, [[0.99, 0.01], [0.8, 0.2]])
S = Categorical([0.5, 0.5])
L = Mixture(S, Categorical, [[0.98, 0.02], [0.75, 0.25]])
B = Mixture(S, Categorical, [[0.97, 0.03], [0.70, 0.30]])
X = Mixture(T, Mixture, L, Categorical, _or([0.96, 0.04], [0.115, 0.885]))
D = Mixture(B, Mixture, X, Categorical, _or([0.115, 0.885], [0.04, 0.96]))
T.observe(TRUE)
S.observe(FALSE)
B.observe(TRUE)
Q = VB(A, T, S, L, B, X, D)
def test_message_to_parent(self):
"""
Test the message to parents of Mixture node.
"""
K = 3
# Broadcasting the moments on the cluster axis
Mu = GaussianARD(2, 1,
ndim=0,
plates=(K,))
(mu, mumu) = Mu._message_to_child()
Alpha = Gamma(3, 1,
plates=(K,))
(alpha, logalpha) = Alpha._message_to_child()
z = Categorical(np.ones(K)/K)
X = Mixture(z, GaussianARD, Mu, Alpha)
tau = 4
Y = GaussianARD(X, tau)
y = 5
Y.observe(y)
(x, xx) = X._message_to_child()
m = X._message_to_parent(0)
self.assertAllClose(m[0],
random.gaussian_logpdf(xx*alpha,
x*alpha*mu,
mumu*alpha,
logalpha,
0))
m = X._message_to_parent(1)
self.assertAllClose(m[0],
1/K * (alpha*x) * np.ones(3))
self.assertAllClose(m[1],
-0.5 * 1/K * alpha * np.ones(3))
# Some parameters do not have cluster plate axis
Mu = GaussianARD(2, 1,
ndim=0,
plates=(K,))
(mu, mumu) = Mu._message_to_child()
Alpha = Gamma(3, 1) # Note: no cluster plate axis!
(alpha, logalpha) = Alpha._message_to_child()
z = Categorical(np.ones(K)/K)
X = Mixture(z, GaussianARD, Mu, Alpha)
tau = 4
Y = GaussianARD(X, tau)
y = 5
Y.observe(y)
(x, xx) = X._message_to_child()
m = X._message_to_parent(0)
self.assertAllClose(m[0],
random.gaussian_logpdf(xx*alpha,
x*alpha*mu,
mumu*alpha,
logalpha,
0))
m = X._message_to_parent(1)
self.assertAllClose(m[0],
1/K * (alpha*x) * np.ones(3))
self.assertAllClose(m[1],
-0.5 * 1/K * alpha * np.ones(3))
# Cluster assignments do not have as many plate axes as parameters.
M = 2
Mu = GaussianARD(2, 1,
ndim=0,
plates=(K,M))
(mu, mumu) = Mu._message_to_child()
Alpha = Gamma(3, 1,
plates=(K,M))
(alpha, logalpha) = Alpha._message_to_child()
z = Categorical(np.ones(K)/K)
X = Mixture(z, GaussianARD, Mu, Alpha, cluster_plate=-2)
tau = 4
Y = GaussianARD(X, tau)
y = 5 * np.ones(M)
Y.observe(y)
(x, xx) = X._message_to_child()
m = X._message_to_parent(0)
self.assertAllClose(m[0]*np.ones(K),
np.sum(random.gaussian_logpdf(xx*alpha,
x*alpha*mu,
mumu*alpha,
logalpha,
0) *
np.ones((K,M)),
axis=-1))
m = X._message_to_parent(1)
self.assertAllClose(m[0] * np.ones((K,M)),
1/K * (alpha*x) * np.ones((K,M)))
self.assertAllClose(m[1] * np.ones((K,M)),
-0.5 * 1/K * alpha * np.ones((K,M)))
pass
def test_message_to_parent(self):
"""
Test the message to parents of Mixture node.
"""
K = 3
# Broadcasting the moments on the cluster axis
Mu = GaussianARD(2, 1, ndim=0, plates=(K, ))
(mu, mumu) = Mu._message_to_child()
Alpha = Gamma(3, 1, plates=(K, ))
(alpha, logalpha) = Alpha._message_to_child()
z = Categorical(np.ones(K) / K)
X = Mixture(z, GaussianARD, Mu, Alpha)
tau = 4
Y = GaussianARD(X, tau)
y = 5
Y.observe(y)
(x, xx) = X._message_to_child()
m = z._message_from_children()
self.assertAllClose(
m[0] * np.ones(K),
random.gaussian_logpdf(xx * alpha, x * alpha * mu, mumu * alpha,
logalpha, 0) * np.ones(K))
m = Mu._message_from_children()
self.assertAllClose(m[0], 1 / K * (alpha * x) * np.ones(3))
self.assertAllClose(m[1], -0.5 * 1 / K * alpha * np.ones(3))
# Some parameters do not have cluster plate axis
Mu = GaussianARD(2, 1, ndim=0, plates=(K, ))
(mu, mumu) = Mu._message_to_child()
Alpha = Gamma(3, 1) # Note: no cluster plate axis!
(alpha, logalpha) = Alpha._message_to_child()
z = Categorical(np.ones(K) / K)
X = Mixture(z, GaussianARD, Mu, Alpha)
tau = 4
Y = GaussianARD(X, tau)
y = 5
Y.observe(y)
(x, xx) = X._message_to_child()
m = z._message_from_children()
self.assertAllClose(
m[0] * np.ones(K),
random.gaussian_logpdf(xx * alpha, x * alpha * mu, mumu * alpha,
logalpha, 0) * np.ones(K))
m = Mu._message_from_children()
self.assertAllClose(m[0], 1 / K * (alpha * x) * np.ones(3))
self.assertAllClose(m[1], -0.5 * 1 / K * alpha * np.ones(3))
# Cluster assignments do not have as many plate axes as parameters.
M = 2
Mu = GaussianARD(2, 1, ndim=0, plates=(K, M))
(mu, mumu) = Mu._message_to_child()
Alpha = Gamma(3, 1, plates=(K, M))
(alpha, logalpha) = Alpha._message_to_child()
z = Categorical(np.ones(K) / K)
X = Mixture(z, GaussianARD, Mu, Alpha, cluster_plate=-2)
tau = 4
Y = GaussianARD(X, tau)
y = 5 * np.ones(M)
Y.observe(y)
(x, xx) = X._message_to_child()
m = z._message_from_children()
self.assertAllClose(
m[0] * np.ones(K),
np.sum(random.gaussian_logpdf(xx * alpha, x * alpha * mu,
mumu * alpha, logalpha, 0) * np.ones(
(K, M)),
axis=-1))
m = Mu._message_from_children()
self.assertAllClose(m[0] * np.ones((K, M)),
1 / K * (alpha * x) * np.ones((K, M)))
self.assertAllClose(m[1] * np.ones((K, M)),
-0.5 * 1 / K * alpha * np.ones((K, M)))
# Mixed distribution broadcasts g
# This tests for a found bug. The bug caused an error.
Z = Categorical([0.3, 0.5, 0.2])
X = Mixture(Z, Categorical, [[0.2, 0.8], [0.1, 0.9], [0.3, 0.7]])
m = Z._message_from_children()
pass
def generateCPD(
self, term
): #, X_train, y_train, X_test, y_test, X_validation, y_validation, g_train, g_test, g_validation):
clf = loadClf(self.ontology[term]['name'], self.fold, self.clfName)
posTrain = sum(clf.y_train == POSTIVE_LABEL)
negTrain = sum(clf.y_train == NEGATIVE_LABEL)
totalTrain = posTrain + negTrain
children = sorted(self.ontology[term]['children'])
parents = sorted(self.ontology[term]['parents'])
labels = {
l: PRIOR
for l in product(*((POSTIVE_LABEL, NEGATIVE_LABEL), ) *
(len(children) + 1))
}
if children:
childNodes = [
self.ontology[child]['node'][self.fold][self.clfName]
for child in children
]
for gene, y in zip(clf.g_train, clf.y_train):
event = []
for child in children:
event.append(POSTIVE_LABEL if gene in self.ontology.
associations[child] else NEGATIVE_LABEL)
event.append(POSTIVE_LABEL if gene in self.ontology.
associations[term] else NEGATIVE_LABEL)
assert (gene in self.ontology.associations[term]) == (
y == POSTIVE_LABEL)
event = tuple(event)
labels[event] += 1
def countBoth(event):
return labels[event[:-1] +
(POSTIVE_LABEL, )] + labels[event[:-1] +
(NEGATIVE_LABEL, )]
cprior = PRIOR * (2**len(children))
types = [Mixture] * (len(children) - 1) + [Categorical]
mixparams = [i for s in zip(childNodes, types) for i in s]
cpd = numpy.empty((2, ) * (len(children) + 1))
for event, counted in labels.items():
v = cpd
for b in event[:-1]:
v = v[b]
hid = event[-1]
print("Event: ", event)
if POSTIVE_LABEL not in event[:
-1]: # Všichni potomci označeni "ne"
v[hid] = counted / countBoth(event)
print("Stored %d / %d" % (counted, countBoth(event)))
else:
v[hid] = {POSTIVE_LABEL: 0.99, NEGATIVE_LABEL: 0.01}[hid]
print("Stored %d : %d" % (hid, v[hid]))
#print(term,"<-",",".join(children))
print(cpd)
#print(labels)
hidden = Mixture(*mixparams, cpd)
hidden.params = cpd
else: #No children
#hidden = DiscreteDistribution({'0': posTrain / totalTrain, '1': negTrain / totalTrain})
params = (posTrain / totalTrain, negTrain / totalTrain)
hidden = Categorical(params)
hidden.params = params
#print("Hidden node %s:" % term)
#print(repr(hidden))
#print([p for p in hidden.parents if isinstance(p, Stochastic)])
#print(hidden.get_moments())
conf = clf.conf + PRIOR
#posTest, negTest = numpy.sum(conf, 1)
posTest, negTest = numpy.sum(conf, 0)
#print("Confusion matrix:")
#print(conf)
try:
assert term != self.ontology.root
pos_decisions = clf.decision_function(
clf.X_test[clf.y_test == POSTIVE_LABEL])
neg_decisions = clf.decision_function(
clf.X_test[clf.y_test == NEGATIVE_LABEL])
means = [numpy.mean(pos_decisions)], [numpy.mean(neg_decisions)]
maxprec = 100.0
precs = [[numpy.min((1 / numpy.var(pos_decisions), maxprec))]
], [[numpy.min((1 / numpy.var(neg_decisions), maxprec))]]
#else:
except (ValueError, AssertionError):
means = [-1.], [1.]
precs = [[1.]], [[1.]]
print("Gaussian params:", term, self.ontology[term]['name'], means,
precs)
observed = Mixture(hidden, Gaussian, means, precs)
#observed = ConditionalProbabilityTable([
# ['0', '0', conf[0][0] / posTest], # if term != root else 1.],
# ['0', '1', conf[0][1] / posTest], # if term != root else 0.],
# ['1', '0', conf[1][0] / negTest], # if term != root else 0.],
# ['1', '1', conf[1][1] / negTest]], #if term != root else 1.]],
# [hidden.distribution])
#print("Observed node %s - %s:" % (term, self.ontology[term]['name']))
#print(repr(observed))
#print([p for p in observed.parents if isinstance(p, Stochastic)])
self.ontology[term]['node'][self.fold][self.clfName] = hidden
#self.ontology[term]['clf'][self.fold][self.clfName] = clf, X_validation, y_validation, g_validation
assert self.lenValidation is None or self.lenValidation == len(
clf.y_validation)
self.lenValidation = len(clf.y_validation)
self.allobserved[term] = observed
self.allhidden[term] = hidden
self.extranodes.update(
(p for p in hidden.parents if isinstance(p, Stochastic)))
def _or(p_false, p_true):
"""
Build probability table for OR-operation of two parents
p_false: Probability table to use if both are FALSE
p_true: Probability table to use if one or both is TRUE
"""
return np.take([p_false, p_true], [[FALSE, TRUE], [TRUE, TRUE]], axis=0)
asia = Categorical([0.5, 0.5])
tuberculosis = Mixture(asia, Categorical, [[0.99, 0.01], [0.8, 0.2]])
smoking = Categorical([0.5, 0.5])
lung = Mixture(smoking, Categorical, [[0.98, 0.02], [0.25, 0.75]])
bronchitis = Mixture(smoking, Categorical, [[0.97, 0.03], [0.08, 0.92]])
xray = Mixture(tuberculosis, Mixture, lung, Categorical,
_or([0.96, 0.04], [0.115, 0.885]))
dyspnea = Mixture(
bronchitis, Mixture, tuberculosis, Mixture, lung, Categorical,
[_or([0.6, 0.4], [0.18, 0.82]),
_or([0.11, 0.89], [0.04, 0.96])])
import numpy
numpy.random.seed(1)
from bayespy.nodes import CategoricalMarkovChain
a0 = [0.6, 0.4] # p(rainy)=0.6, p(sunny)=0.4
A = [
[0.7, 0.3], # p(rainy->rainy)=0.7, p(rainy->sunny)=0.3
[0.4, 0.6]
] # p(sunny->rainy)=0.4, p(sunny->sunny)=0.6
N = 100
Z = CategoricalMarkovChain(a0, A, states=N)
from bayespy.nodes import Categorical, Mixture
P = [[0.1, 0.4, 0.5], [0.6, 0.3, 0.1]]
Y = Mixture(Z, Categorical, P)
weather = Z.random()
from bayespy.inference import VB
import bayespy.plot as bpplt
import numpy as np
mu = np.array([[0, 0], [3, 4], [6, 0]])
std = 2.0
K = 3
N = 200
p0 = np.ones(K) / K
q = 0.9
r = (1 - q) / (K - 1)
P = q * np.identity(K) + r * (np.ones((3, 3)) - np.identity(3))
y = np.zeros((N, 2))
z = np.zeros(N)
state = np.random.choice(K, p=p0)
for n in range(N):
z[n] = state
y[n, :] = std * np.random.randn(2) + mu[state]
