def _create_model(self, d, k, mode, nframes, emiter):
#+++++++++++++++++++++++++++++++++++++++++++++++++
# Generate a model with k components, d dimensions
#+++++++++++++++++++++++++++++++++++++++++++++++++
w, mu, va = GM.gen_param(d, k, mode, spread=1.5)
gm = GM.fromvalues(w, mu, va)
# Sample nframes frames from the model
data = gm.sample(nframes)
#++++++++++++++++++++++++++++++++++++++++++
# Approximate the models with classical EM
#++++++++++++++++++++++++++++++++++++++++++
# Init the model
lgm = GM(d, k, mode)
gmm = GMM(lgm, 'kmean')
gmm.init(data, niter=KM_ITER)
self.gm0 = copy.copy(gmm.gm)
# The actual EM, with likelihood computation
for i in range(emiter):
g, tgd = gmm.compute_responsabilities(data)
gmm.update_em(data, g)
self.data = data
self.gm = lgm
def generate_dataset(d, k, mode, nframes):
"""Generate a dataset useful for EM anf GMM testing.
returns:
data : ndarray
data from the true model.
tgm : GM
the true model (randomly generated)
gm0 : GM
the initial model
gm : GM
the trained model
"""
# Generate a model
w, mu, va = GM.gen_param(d, k, mode, spread=2.0)
tgm = GM.fromvalues(w, mu, va)
# Generate data from the model
data = tgm.sample(nframes)
# Run EM on the model, by running the initialization separetely.
gmm = GMM(GM(d, k, mode), 'test')
gmm.init_random(data)
gm0 = copy.copy(gmm.gm)
gmm = GMM(copy.copy(gmm.gm), 'test')
em = EM()
em.train(data, gmm)
return data, tgm, gm0, gmm.gm
def test_conf_ellip(self):
"""Only test whether the call succeed. To check wether the result is
OK, you have to plot the results."""
d = 3
k = 3
w, mu, va = GM.gen_param(d, k)
gm = GM.fromvalues(w, mu, va)
gm.conf_ellipses()
def test_1d_bogus(self):
"""Check that functions which do not make sense for 1d fail nicely."""
d = 1
k = 2
w, mu, va = GM.gen_param(d, k)
gm = GM.fromvalues(w, mu, va)
try:
gm.conf_ellipses()
raise AssertionError("This should not work !")
except ValueError, e:
print "Ok, conf_ellipses failed as expected (with msg: " + str(
e) + ")"
def cluster(data, k, mode='full'):
d = data.shape[1]
gm = GM(d, k, mode)
gmm = GMM(gm)
em = EM()
em.train(data, gmm, maxiter=20)
return gm
def cluster(data, k, mode='full'):
d = data.shape[1]
gm = GM(d, k, mode)
gmm = GMM(gm, 'random')
em = RegularizedEM(pcnt=pcnt, pval=pval)
em.train(data, gmm, maxiter=20)
return gm, gmm.bic(data)
def _run_pure_online(self, d, k, mode, nframes):
#++++++++++++++++++++++++++++++++++++++++
# Approximate the models with online EM
#++++++++++++++++++++++++++++++++++++++++
ogm = GM(d, k, mode)
ogmm = OnGMM(ogm, 'kmean')
init_data = self.data[0:nframes / 20, :]
ogmm.init(init_data)
# Forgetting param
ku = 0.005
t0 = 200
lamb = 1 - 1 / (N.arange(-1, nframes - 1) * ku + t0)
nu0 = 0.2
nu = N.zeros((len(lamb), 1))
nu[0] = nu0
for i in range(1, len(lamb)):
nu[i] = 1. / (1 + lamb[i] / nu[i - 1])
# object version of online EM
for t in range(nframes):
# the assert are here to check we do not create copies
# unvoluntary for parameters
assert ogmm.pw is ogmm.cw
assert ogmm.pmu is ogmm.cmu
assert ogmm.pva is ogmm.cva
ogmm.compute_sufficient_statistics_frame(self.data[t], nu[t])
ogmm.update_em_frame()
ogmm.gm.set_param(ogmm.cw, ogmm.cmu, ogmm.cva)
return ogmm.gm
def cluster(data, k):
d = data.shape[1]
gm = GM(d, k)
gmm = GMM(gm)
em = EM()
em.train(data, gmm, maxiter=20)
return gm, gmm.bic(data)
def _check(self, d, k, mode, nframes, emiter):
#++++++++++++++++++++++++++++++++++++++++
# Approximate the models with online EM
#++++++++++++++++++++++++++++++++++++++++
# Learn the model with Online EM
ogm = GM(d, k, mode)
ogmm = OnGMM(ogm, 'kmean')
init_data = self.data
ogmm.init(init_data, niter=KM_ITER)
# Check that online kmean init is the same than kmean offline init
ogm0 = copy.copy(ogm)
assert_array_equal(ogm0.w, self.gm0.w)
assert_array_equal(ogm0.mu, self.gm0.mu)
assert_array_equal(ogm0.va, self.gm0.va)
# Forgetting param
lamb = N.ones((nframes, 1))
lamb[0] = 0
nu0 = 1.0
nu = N.zeros((len(lamb), 1))
nu[0] = nu0
for i in range(1, len(lamb)):
nu[i] = 1. / (1 + lamb[i] / nu[i - 1])
# object version of online EM: the p* arguments are updated only at each
# epoch, which is equivalent to on full EM iteration on the
# classic EM algorithm
ogmm.pw = ogmm.cw.copy()
ogmm.pmu = ogmm.cmu.copy()
ogmm.pva = ogmm.cva.copy()
for e in range(emiter):
for t in range(nframes):
ogmm.compute_sufficient_statistics_frame(self.data[t], nu[t])
ogmm.update_em_frame()
# Change pw args only a each epoch
ogmm.pw = ogmm.cw.copy()
ogmm.pmu = ogmm.cmu.copy()
ogmm.pva = ogmm.cva.copy()
# For equivalence between off and on, we allow a margin of error,
# because of round-off errors.
print " Checking precision of equivalence with offline EM trainer "
maxtestprec = 18
try:
for i in range(maxtestprec):
assert_array_almost_equal(self.gm.w, ogmm.pw, decimal=i)
assert_array_almost_equal(self.gm.mu, ogmm.pmu, decimal=i)
assert_array_almost_equal(self.gm.va, ogmm.pva, decimal=i)
print "\t !! Precision up to %d decimals !! " % i
except AssertionError:
if i < AR_AS_PREC:
print """\t !!NOT OK: Precision up to %d decimals only,
outside the allowed range (%d) !! """ % (i, AR_AS_PREC)
raise AssertionError
else:
print "\t !!OK: Precision up to %d decimals !! " % i
def test_get_va(self):
"""Test _get_va for diag and full mode."""
d = 3
k = 2
ld = 2
dim = [0, 2]
w, mu, va = GM.gen_param(d, k, 'full')
va = N.arange(d * d * k).reshape(d * k, d)
gm = GM.fromvalues(w, mu, va)
tva = N.empty(ld * ld * k)
for i in range(k * ld * ld):
tva[i] = dim[i %
ld] + (i % 4) / ld * dim[1] * d + d * d * (i /
(ld * ld))
tva = tva.reshape(ld * k, ld)
sva = gm._get_va(dim)
assert N.all(sva == tva)
def _create_model_and_run_em(self, d, k, mode, nframes):
#+++++++++++++++++++++++++++++++++++++++++++++++++
# Generate a model with k components, d dimensions
#+++++++++++++++++++++++++++++++++++++++++++++++++
w, mu, va = GM.gen_param(d, k, mode, spread = 1.5)
gm = GM.fromvalues(w, mu, va)
# Sample nframes frames from the model
data = gm.sample(nframes)
#++++++++++++++++++++++++++++++++++++++++++
# Approximate the models with classical EM
#++++++++++++++++++++++++++++++++++++++++++
# Init the model
lgm = GM(d, k, mode)
gmm = GMM(lgm, 'kmean')
em = EM()
lk = em.train(data, gmm)
def _test(self, dataset, log):
dic = load_dataset(dataset)
gm = GM.fromvalues(dic['w0'], dic['mu0'], dic['va0'])
gmm = GMM(gm, 'test')
EM().train(dic['data'], gmm, log = log)
assert_array_almost_equal(gmm.gm.w, dic['w'], DEF_DEC)
assert_array_almost_equal(gmm.gm.mu, dic['mu'], DEF_DEC)
assert_array_almost_equal(gmm.gm.va, dic['va'], DEF_DEC)
def test_2d_diag_logpdf(self):
d = 2
w = N.array([0.4, 0.6])
mu = N.array([[0., 2], [-1, -2]])
va = N.array([[1, 0.5], [0.5, 1]])
x = N.random.randn(100, 2)
gm = GM.fromvalues(w, mu, va)
y1 = N.sum(multiple_gauss_den(x, mu, va) * w, 1)
y2 = gm.pdf(x, log=True)
assert_array_almost_equal(N.log(y1), y2)
def _test_common(self, d, k, mode):
dic = load_dataset('%s_%dd_%dk.mat' % (mode, d, k))
gm = GM.fromvalues(dic['w0'], dic['mu0'], dic['va0'])
gmm = GMM(gm, 'test')
a, na = gmm.compute_responsabilities(dic['data'])
la, nla = gmm.compute_log_responsabilities(dic['data'])
ta = N.log(a)
tna = N.log(na)
if not N.all(N.isfinite(ta)):
print "precision problem for %s, %dd, %dk, test need fixing" % (mode, d, k)
else:
assert_array_almost_equal(ta, la, DEF_DEC)
if not N.all(N.isfinite(tna)):
print "precision problem for %s, %dd, %dk, test need fixing" % (mode, d, k)
else:
assert_array_almost_equal(tna, nla, DEF_DEC)
#------------------------------------------------------- # Values for weights, mean and (diagonal) variances # - the weights are an array of rank 1 # - mean is expected to be rank 2 with one row for one component # - variances are also expteced to be rank 2. For diagonal, one row # is one diagonal, for full, the first d rows are the first variance, # etc... In this case, the variance matrix should be k*d rows and d # colums w = N.array([0.2, 0.45, 0.35]) mu = N.array([[4.1, 3], [1, 5], [-2, -3]]) va = N.array([[1, 1.5], [3, 4], [2, 3.5]]) #----------------------------------------- # First method: directly from parameters: # Both methods are equivalents. gm = GM.fromvalues(w, mu, va) #------------------------------------- # Second method to build a GM instance: gm = GM(d, k, mode = 'diag') # The set_params checks that w, mu, and va corresponds to k, d and m gm.set_param(w, mu, va) # Once set_params is called, both methods are equivalent. The 2d # method is useful when using a GM object for learning (where # the learner class will set the params), whereas the first one # is useful when there is a need to quickly sample a model # from existing values, without a need to give the hyper parameters # Create a Gaussian Mixture from the parameters, and sample # 1000 items from it (one row = one 2 dimension sample)
#+++++++++++++++++++++++++++++ # Meta parameters of the model # - k: Number of components # - d: dimension of each Gaussian # - mode: Mode of covariance matrix: full or diag (string) # - nframes: number of frames (frame = one data point = one # row of d elements) k = 2 d = 2 mode = 'diag' nframes = 1e3 #+++++++++++++++++++++++++++++++++++++++++++ # Create an artificial GM model, samples it #+++++++++++++++++++++++++++++++++++++++++++ w, mu, va = GM.gen_param(d, k, mode, spread = 1.5) gm = GM.fromvalues(w, mu, va) # Sample nframes frames from the model data = gm.sample(nframes) #++++++++++++++++++++++++ # Learn the model with EM #++++++++++++++++++++++++ # Create a Model from a Gaussian mixture with kmean initialization lgm = GM(d, k, mode) gmm = GMM(lgm, 'kmean') # The actual EM, with likelihood computation. The threshold # is compared to the (linearly appromixated) derivative of the likelihood
#------------------------------------------------------- # Values for weights, mean and (diagonal) variances # - the weights are an array of rank 1 # - mean is expected to be rank 2 with one row for one component # - variances are also expteced to be rank 2. For diagonal, one row # is one diagonal, for full, the first d rows are the first variance, # etc... In this case, the variance matrix should be k*d rows and d # colums w = N.array([0.2, 0.45, 0.35]) mu = N.array([[4.1, 3], [1, 5], [-2, -3]]) va = N.array([[1, 1.5], [3, 4], [2, 3.5]]) #----------------------------------------- # First method: directly from parameters: # Both methods are equivalents. gm = GM.fromvalues(w, mu, va) #------------------------------------- # Second method to build a GM instance: gm = GM(d, k, mode='diag') # The set_params checks that w, mu, and va corresponds to k, d and m gm.set_param(w, mu, va) # Once set_params is called, both methods are equivalent. The 2d # method is useful when using a GM object for learning (where # the learner class will set the params), whereas the first one # is useful when there is a need to quickly sample a model # from existing values, without a need to give the hyper parameters # Create a Gaussian Mixture from the parameters, and sample # 1000 items from it (one row = one 2 dimension sample)
# This is a simple test to check whether plotting ellipsoides of confidence and
# isodensity contours match
import numpy as N
from numpy.testing import set_package_path, restore_path
import pylab as P
from scikits.learn.machine.em import EM, GM, GMM
# Generate a simple mixture model, plot its confidence ellipses + isodensity
# curves for both diagonal and full covariance matrices
d = 3
k = 3
dim = [0, 2]
# diag model
w, mu, va = GM.gen_param(d, k)
dgm = GM.fromvalues(w, mu, va)
# full model
w, mu, va = GM.gen_param(d, k, 'full', spread = 1)
fgm = GM.fromvalues(w, mu, va)
def plot_model(gm, dim):
X, Y, Z, V = gm.density_on_grid(dim = dim)
h = gm.plot(dim = dim)
[i.set_linestyle('-.') for i in h]
P.contour(X, Y, Z, V)
data = gm.sample(200)
P.plot(data[:, dim[0]], data[:,dim[1]], '.')
# Plot the contours and the ellipsoids of confidence
P.subplot(2, 1, 1)
