def test_gaussian_mixture_verbose():
rng = np.random.RandomState(0)
rand_data = RandomData(rng)
n_components = rand_data.n_components
for covar_type in COVARIANCE_TYPE:
X = rand_data.X[covar_type]
g = GaussianMixture(n_components=n_components,
n_init=1,
reg_covar=0,
random_state=rng,
covariance_type=covar_type,
verbose=1)
h = GaussianMixture(n_components=n_components,
n_init=1,
reg_covar=0,
random_state=rng,
covariance_type=covar_type,
verbose=2)
old_stdout = sys.stdout
sys.stdout = StringIO()
try:
g.fit(X)
h.fit(X)
finally:
sys.stdout = old_stdout
def test_gaussian_mixture_estimate_log_prob_resp():
# test whether responsibilities are normalized
rng = np.random.RandomState(0)
rand_data = RandomData(rng, scale=5)
n_samples = rand_data.n_samples
n_features = rand_data.n_features
n_components = rand_data.n_components
X = rng.rand(n_samples, n_features)
for covar_type in COVARIANCE_TYPE:
weights = rand_data.weights
means = rand_data.means
precisions = rand_data.precisions[covar_type]
g = GaussianMixture(n_components=n_components,
random_state=rng,
weights_init=weights,
means_init=means,
precisions_init=precisions,
covariance_type=covar_type)
g.fit(X)
resp = g.predict_proba(X)
assert_array_almost_equal(resp.sum(axis=1), np.ones(n_samples))
assert_array_equal(g.weights_init, weights)
assert_array_equal(g.means_init, means)
assert_array_equal(g.precisions_init, precisions)
def test_score():
covar_type = 'full'
rng = np.random.RandomState(0)
rand_data = RandomData(rng, scale=7)
n_components = rand_data.n_components
X = rand_data.X[covar_type]
# Check the error message if we don't call fit
gmm1 = GaussianMixture(n_components=n_components, n_init=1,
max_iter=1, reg_covar=0, random_state=rng,
covariance_type=covar_type)
assert_raise_message(NotFittedError,
"This GaussianMixture instance is not fitted "
"yet. Call 'fit' with appropriate arguments "
"before using this method.", gmm1.score, X)
# Check score value
with warnings.catch_warnings():
warnings.simplefilter("ignore", ConvergenceWarning)
gmm1.fit(X)
gmm_score = gmm1.score(X)
gmm_score_proba = gmm1.score_samples(X).mean()
assert_almost_equal(gmm_score, gmm_score_proba)
# Check if the score increase
gmm2 = GaussianMixture(n_components=n_components, n_init=1, reg_covar=0,
random_state=rng,
covariance_type=covar_type).fit(X)
assert_greater(gmm2.score(X), gmm1.score(X))
def test_score():
covar_type = 'full'
rng = np.random.RandomState(0)
rand_data = RandomData(rng, scale=7)
n_components = rand_data.n_components
X = rand_data.X[covar_type]
# Check the error message if we don't call fit
gmm1 = GaussianMixture(n_components=n_components,
n_init=1,
max_iter=1,
reg_covar=0,
random_state=rng,
covariance_type=covar_type)
assert_raise_message(
NotFittedError, "This GaussianMixture instance is not fitted "
"yet. Call 'fit' with appropriate arguments "
"before using this method.", gmm1.score, X)
# Check score value
with warnings.catch_warnings():
warnings.simplefilter("ignore", ConvergenceWarning)
gmm1.fit(X)
gmm_score = gmm1.score(X)
gmm_score_proba = gmm1.score_samples(X).mean()
assert_almost_equal(gmm_score, gmm_score_proba)
# Check if the score increase
gmm2 = GaussianMixture(n_components=n_components,
n_init=1,
reg_covar=0,
random_state=rng,
covariance_type=covar_type).fit(X)
assert_greater(gmm2.score(X), gmm1.score(X))
def create_mask(img, background_probability=0.75, use_triangle=False, use_otsu=False):
test_mask = None
if use_triangle:
test_mask = sitk.GetArrayFromImage(sitk.TriangleThreshold(img, 0, 1))
elif use_otsu:
test_mask = sitk.GetArrayFromImage(sitk.OtsuThreshold(img, 0, 1))
else:
if type(img) is sitk.SimpleITK.Image:
img = sitk.GetArrayFromImage(img)
gmix = GaussianMixture(n_components=3, covariance_type='full', init_params='kmeans', verbose=0)
gmix.fit(img.ravel().reshape(-1, 1))
covariances = gmix.covariances_
mean_background = gmix.means_.min()
covariance_background = covariances[np.where( gmix.means_ == mean_background ) ][0][0]
z_score = st.norm.ppf(background_probability)
threshold = z_score * np.sqrt(covariance_background) + mean_background
test_mask = (img > threshold)
eroded_im = morphology.opening(test_mask, selem=morphology.ball(2))
connected_comp = skimage.measure.label(eroded_im)
out = skimage.measure.regionprops(connected_comp)
area_max = 0.0
idx_max = 0
for i in range(len(out)):
if out[i].area > area_max:
area_max = out[i].area
idx_max = i+1
connected_comp[ connected_comp != idx_max ] = 0
mask = connected_comp
mask_sitk = sitk.GetImageFromArray(mask)
mask_sitk.CopyInformation(img)
return mask_sitk
def macro_clusters(self, n_clusters, max_iters=1000, **kwargs):
data = np.concatenate([
m.sample(np.int32(np.ceil(m.n / 100))) for m in self.micro_clusters
])
gmm = GaussianMixture(n_components=n_clusters)
gmm.fit(data)
self.macro_centroids = gmm.means_
self.gmm_model = gmm
def test_check_precisions():
rng = np.random.RandomState(0)
rand_data = RandomData(rng)
n_components, n_features = rand_data.n_components, rand_data.n_features
# Define the bad precisions for each covariance_type
precisions_bad_shape = {
'full': np.ones((n_components + 1, n_features, n_features)),
'tied': np.ones((n_features + 1, n_features + 1)),
'diag': np.ones((n_components + 1, n_features)),
'spherical': np.ones((n_components + 1))
}
# Define not positive-definite precisions
precisions_not_pos = np.ones((n_components, n_features, n_features))
precisions_not_pos[0] = np.eye(n_features)
precisions_not_pos[0, 0, 0] = -1.
precisions_not_positive = {
'full': precisions_not_pos,
'tied': precisions_not_pos[0],
'diag': -1. * np.ones((n_components, n_features)),
'spherical': -1. * np.ones(n_components)
}
not_positive_errors = {
'full': 'symmetric, positive-definite',
'tied': 'symmetric, positive-definite',
'diag': 'positive',
'spherical': 'positive'
}
for covar_type in COVARIANCE_TYPE:
X = RandomData(rng).X[covar_type]
g = GaussianMixture(n_components=n_components,
covariance_type=covar_type,
random_state=rng)
# Check precisions with bad shapes
g.precisions_init = precisions_bad_shape[covar_type]
assert_raise_message(
ValueError, "The parameter '%s precision' should have "
"the shape of" % covar_type, g.fit, X)
# Check not positive precisions
g.precisions_init = precisions_not_positive[covar_type]
assert_raise_message(
ValueError, "'%s precision' should be %s" %
(covar_type, not_positive_errors[covar_type]), g.fit, X)
# Check the correct init of precisions_init
g.precisions_init = rand_data.precisions[covar_type]
g.fit(X)
assert_array_equal(rand_data.precisions[covar_type], g.precisions_init)
def test_check_precisions():
rng = np.random.RandomState(0)
rand_data = RandomData(rng)
n_components, n_features = rand_data.n_components, rand_data.n_features
# Define the bad precisions for each covariance_type
precisions_bad_shape = {
'full': np.ones((n_components + 1, n_features, n_features)),
'tied': np.ones((n_features + 1, n_features + 1)),
'diag': np.ones((n_components + 1, n_features)),
'spherical': np.ones((n_components + 1))}
# Define not positive-definite precisions
precisions_not_pos = np.ones((n_components, n_features, n_features))
precisions_not_pos[0] = np.eye(n_features)
precisions_not_pos[0, 0, 0] = -1.
precisions_not_positive = {
'full': precisions_not_pos,
'tied': precisions_not_pos[0],
'diag': np.full((n_components, n_features), -1.),
'spherical': np.full(n_components, -1.)}
not_positive_errors = {
'full': 'symmetric, positive-definite',
'tied': 'symmetric, positive-definite',
'diag': 'positive',
'spherical': 'positive'}
for covar_type in COVARIANCE_TYPE:
X = RandomData(rng).X[covar_type]
g = GaussianMixture(n_components=n_components,
covariance_type=covar_type,
random_state=rng)
# Check precisions with bad shapes
g.precisions_init = precisions_bad_shape[covar_type]
assert_raise_message(ValueError,
"The parameter '%s precision' should have "
"the shape of" % covar_type,
g.fit, X)
# Check not positive precisions
g.precisions_init = precisions_not_positive[covar_type]
assert_raise_message(ValueError,
"'%s precision' should be %s"
% (covar_type, not_positive_errors[covar_type]),
g.fit, X)
# Check the correct init of precisions_init
g.precisions_init = rand_data.precisions[covar_type]
g.fit(X)
assert_array_equal(rand_data.precisions[covar_type], g.precisions_init)
def main():
#Import data from file
trainData, trainLabel = importTrainData()
testData, testLabel = importTestData()
#PCA procedure
#For train data
rawlowDivTrainData = []
pca = PCA(n_components=537)
#pca = PCA(n_components=537, whiten="True")
rawlowDivTrainData = pca.fit_transform(trainData)
#For test data which sum up to 10000
lowDivTestData = pca.fit_transform(testData)
#Classify manually, divide the data into 10 parts by labels
numTrainData = [[], [], [], [], [], [], [], [], [], []]
for i in range(60000):
numTrainData[trainLabel[i]].append(rawlowDivTrainData[i])
#Train Gmm for every mode/Each mode represent a number.
myGmms = []
#GMM procedure
for num in range(10):
print("GMM", num)
gmmClassifier = GaussianMixture(n_components=25,
init_params='kmeans',
max_iter=1000)
gmmClassifier.fit(numTrainData[num])
myGmms.append(gmmClassifier)
#Validate
# We need to calculate all the total probability under each GMM, and choose the biggest one.
correctCount = 0
allProb = []
for num in range(10):
allProb.append(np.array(myGmms[num].score_samples(lowDivTestData)))
allProb = np.array(allProb)
#If the result seem strange, print the predict label
for i in range(10000):
predictLabel = 0
max = np.max(allProb[:, i])
for num in range(10):
if max == allProb[num, i]:
predictLabel = num
if predictLabel == testLabel[i]:
correctCount += 1
#Output
print("The correct rate is:", correctCount / 10000)
def test_gaussian_mixture_fit():
# recover the ground truth
rng = np.random.RandomState(0)
rand_data = RandomData(rng)
n_features = rand_data.n_features
n_components = rand_data.n_components
for covar_type in COVARIANCE_TYPE:
X = rand_data.X[covar_type]
g = GaussianMixture(n_components=n_components,
n_init=20,
reg_covar=0,
random_state=rng,
covariance_type=covar_type)
g.fit(X)
# needs more data to pass the test with rtol=1e-7
assert_allclose(np.sort(g.weights_),
np.sort(rand_data.weights),
rtol=0.1,
atol=1e-2)
arg_idx1 = g.means_[:, 0].argsort()
arg_idx2 = rand_data.means[:, 0].argsort()
assert_allclose(g.means_[arg_idx1],
rand_data.means[arg_idx2],
rtol=0.1,
atol=1e-2)
if covar_type == 'full':
prec_pred = g.precisions_
prec_test = rand_data.precisions['full']
elif covar_type == 'tied':
prec_pred = np.array([g.precisions_] * n_components)
prec_test = np.array([rand_data.precisions['tied']] * n_components)
elif covar_type == 'spherical':
prec_pred = np.array(
[np.eye(n_features) * c for c in g.precisions_])
prec_test = np.array([
np.eye(n_features) * c
for c in rand_data.precisions['spherical']
])
elif covar_type == 'diag':
prec_pred = np.array([np.diag(d) for d in g.precisions_])
prec_test = np.array(
[np.diag(d) for d in rand_data.precisions['diag']])
arg_idx1 = np.trace(prec_pred, axis1=1, axis2=2).argsort()
arg_idx2 = np.trace(prec_test, axis1=1, axis2=2).argsort()
for k, h in zip(arg_idx1, arg_idx2):
ecov = EmpiricalCovariance()
ecov.covariance_ = prec_test[h]
# the accuracy depends on the number of data and randomness, rng
assert_allclose(ecov.error_norm(prec_pred[k]), 0, atol=0.1)
def test_check_covariances():
rng = np.random.RandomState(0)
rand_data = RandomData(rng)
n_components, n_features = rand_data.n_components, rand_data.n_features
# Define the bad covariances for each covariance_type
covariances_bad_shape = {
'full': rng.rand(n_components + 1, n_features, n_features),
'tied': rng.rand(n_features + 1, n_features + 1),
'diag': rng.rand(n_components + 1, n_features),
'spherical': rng.rand(n_components + 1)}
# Define not positive-definite covariances
covariances_not_pos = rng.rand(n_components, n_features, n_features)
covariances_not_pos[0] = np.eye(n_features)
covariances_not_pos[0, 0, 0] = -1.
covariances_not_positive = {
'full': covariances_not_pos,
'tied': covariances_not_pos[0],
'diag': -1. * np.ones((n_components, n_features)),
'spherical': -1. * np.ones(n_components)}
not_positive_errors = {
'full': 'symmetric, positive-definite',
'tied': 'symmetric, positive-definite',
'diag': 'positive',
'spherical': 'positive'}
for cov_type in ['full', 'tied', 'diag', 'spherical']:
X = rand_data.X[cov_type]
g = GaussianMixture(n_components=n_components,
covariance_type=cov_type)
# Check covariance with bad shapes
g.covariances_init = covariances_bad_shape[cov_type]
assert_raise_message(ValueError,
"The parameter '%s covariance' should have "
"the shape of" % cov_type,
g.fit, X)
# Check not positive covariances
g.covariances_init = covariances_not_positive[cov_type]
assert_raise_message(ValueError,
"'%s covariance' should be %s"
% (cov_type, not_positive_errors[cov_type]),
g.fit, X)
# Check the correct init of covariances_init
g.covariances_init = rand_data.covariances[cov_type]
g.fit(X)
assert_array_equal(rand_data.covariances[cov_type], g.covariances_init)
def test_sample():
rng = np.random.RandomState(0)
rand_data = RandomData(rng, scale=7, n_components=3)
n_features, n_components = rand_data.n_features, rand_data.n_components
for covar_type in COVARIANCE_TYPE:
X = rand_data.X[covar_type]
gmm = GaussianMixture(n_components=n_components,
covariance_type=covar_type,
random_state=rng)
# To sample we need that GaussianMixture is fitted
assert_raise_message(NotFittedError, "This GaussianMixture instance "
"is not fitted", gmm.sample, 0)
gmm.fit(X)
assert_raise_message(ValueError, "Invalid value for 'n_samples",
gmm.sample, 0)
# Just to make sure the class samples correctly
n_samples = 20000
X_s, y_s = gmm.sample(n_samples)
for k in range(n_components):
if covar_type == 'full':
assert_array_almost_equal(gmm.covariances_[k],
np.cov(X_s[y_s == k].T),
decimal=1)
elif covar_type == 'tied':
assert_array_almost_equal(gmm.covariances_,
np.cov(X_s[y_s == k].T),
decimal=1)
elif covar_type == 'diag':
assert_array_almost_equal(gmm.covariances_[k],
np.diag(np.cov(X_s[y_s == k].T)),
decimal=1)
else:
assert_array_almost_equal(gmm.covariances_[k],
np.var(X_s[y_s == k] -
gmm.means_[k]),
decimal=1)
means_s = np.array(
[np.mean(X_s[y_s == k], 0) for k in range(n_components)])
assert_array_almost_equal(gmm.means_, means_s, decimal=1)
# Check shapes of sampled data, see
# https://github.com/scikit-learn/scikit-learn/issues/7701
assert_equal(X_s.shape, (n_samples, n_features))
for sample_size in range(1, 100):
X_s, _ = gmm.sample(sample_size)
assert_equal(X_s.shape, (sample_size, n_features))
def test_warm_start(seed):
random_state = seed
rng = np.random.RandomState(random_state)
n_samples, n_features, n_components = 500, 2, 2
X = rng.rand(n_samples, n_features)
# Assert the warm_start give the same result for the same number of iter
g = GaussianMixture(n_components=n_components,
n_init=1,
max_iter=2,
reg_covar=0,
random_state=random_state,
warm_start=False)
h = GaussianMixture(n_components=n_components,
n_init=1,
max_iter=1,
reg_covar=0,
random_state=random_state,
warm_start=True)
g.fit(X)
score1 = h.fit(X).score(X)
score2 = h.fit(X).score(X)
assert_almost_equal(g.weights_, h.weights_)
assert_almost_equal(g.means_, h.means_)
assert_almost_equal(g.precisions_, h.precisions_)
assert score2 > score1
# Assert that by using warm_start we can converge to a good solution
g = GaussianMixture(n_components=n_components,
n_init=1,
max_iter=5,
reg_covar=0,
random_state=random_state,
warm_start=False,
tol=1e-6)
h = GaussianMixture(n_components=n_components,
n_init=1,
max_iter=5,
reg_covar=0,
random_state=random_state,
warm_start=True,
tol=1e-6)
g.fit(X)
assert not g.converged_
h.fit(X)
# depending on the data there is large variability in the number of
# refit necessary to converge due to the complete randomness of the
# data
for _ in range(1000):
h.fit(X)
if h.converged_:
break
assert h.converged_
def test_sample():
rng = np.random.RandomState(0)
rand_data = RandomData(rng, scale=7, n_components=3)
n_features, n_components = rand_data.n_features, rand_data.n_components
for covar_type in COVARIANCE_TYPE:
X = rand_data.X[covar_type]
gmm = GaussianMixture(n_components=n_components,
covariance_type=covar_type, random_state=rng)
# To sample we need that GaussianMixture is fitted
assert_raise_message(NotFittedError, "This GaussianMixture instance "
"is not fitted", gmm.sample, 0)
gmm.fit(X)
assert_raise_message(ValueError, "Invalid value for 'n_samples",
gmm.sample, 0)
# Just to make sure the class samples correctly
n_samples = 20000
X_s, y_s = gmm.sample(n_samples)
for k in range(n_components):
if covar_type == 'full':
assert_array_almost_equal(gmm.covariances_[k],
np.cov(X_s[y_s == k].T), decimal=1)
elif covar_type == 'tied':
assert_array_almost_equal(gmm.covariances_,
np.cov(X_s[y_s == k].T), decimal=1)
elif covar_type == 'diag':
assert_array_almost_equal(gmm.covariances_[k],
np.diag(np.cov(X_s[y_s == k].T)),
decimal=1)
else:
assert_array_almost_equal(
gmm.covariances_[k], np.var(X_s[y_s == k] - gmm.means_[k]),
decimal=1)
means_s = np.array([np.mean(X_s[y_s == k], 0)
for k in range(n_components)])
assert_array_almost_equal(gmm.means_, means_s, decimal=1)
# Check shapes of sampled data, see
# https://github.com/scikit-learn/scikit-learn/issues/7701
assert_equal(X_s.shape, (n_samples, n_features))
for sample_size in range(1, 100):
X_s, _ = gmm.sample(sample_size)
assert_equal(X_s.shape, (sample_size, n_features))
def test_convergence_detected_with_warm_start():
# We check that convergence is detected when warm_start=True
rng = np.random.RandomState(0)
rand_data = RandomData(rng)
n_components = rand_data.n_components
X = rand_data.X['full']
for max_iter in (1, 2, 50):
gmm = GaussianMixture(n_components=n_components, warm_start=True,
max_iter=max_iter, random_state=rng)
for _ in range(100):
gmm.fit(X)
if gmm.converged_:
break
assert gmm.converged_
assert max_iter >= gmm.n_iter_
def test_convergence_detected_with_warm_start():
# We check that convergence is detected when warm_start=True
rng = np.random.RandomState(0)
rand_data = RandomData(rng)
n_components = rand_data.n_components
X = rand_data.X['full']
for max_iter in (1, 2, 50):
gmm = GaussianMixture(n_components=n_components, warm_start=True,
max_iter=max_iter, random_state=rng)
for _ in range(100):
gmm.fit(X)
if gmm.converged_:
break
assert gmm.converged_
assert max_iter >= gmm.n_iter_
def test_monotonic_likelihood():
# We check that each step of the EM without regularization improve
# monotonically the training set likelihood
rng = np.random.RandomState(0)
rand_data = RandomData(rng, scale=7)
n_components = rand_data.n_components
for covar_type in COVARIANCE_TYPE:
X = rand_data.X[covar_type]
gmm = GaussianMixture(n_components=n_components,
covariance_type=covar_type,
reg_covar=0,
warm_start=True,
max_iter=1,
random_state=rng,
tol=1e-7)
current_log_likelihood = -np.infty
with warnings.catch_warnings():
warnings.simplefilter("ignore", ConvergenceWarning)
# Do one training iteration at a time so we can make sure that the
# training log likelihood increases after each iteration.
for _ in range(600):
prev_log_likelihood = current_log_likelihood
try:
current_log_likelihood = gmm.fit(X).score(X)
except ConvergenceWarning:
pass
assert_greater_equal(current_log_likelihood,
prev_log_likelihood)
if gmm.converged_:
break
assert_true(gmm.converged_)
def test_monotonic_likelihood():
# We check that each step of the EM without regularization improve
# monotonically the training set likelihood
rng = np.random.RandomState(0)
rand_data = RandomData(rng, scale=7)
n_components = rand_data.n_components
for covar_type in COVARIANCE_TYPE:
X = rand_data.X[covar_type]
gmm = GaussianMixture(n_components=n_components,
covariance_type=covar_type, reg_covar=0,
warm_start=True, max_iter=1, random_state=rng,
tol=1e-7)
current_log_likelihood = -np.infty
with warnings.catch_warnings():
warnings.simplefilter("ignore", ConvergenceWarning)
# Do one training iteration at a time so we can make sure that the
# training log likelihood increases after each iteration.
for _ in range(600):
prev_log_likelihood = current_log_likelihood
try:
current_log_likelihood = gmm.fit(X).score(X)
except ConvergenceWarning:
pass
assert_greater_equal(current_log_likelihood,
prev_log_likelihood)
if gmm.converged_:
break
assert gmm.converged_
def test_gaussian_mixture_fit():
# recover the ground truth
rng = np.random.RandomState(0)
rand_data = RandomData(rng)
n_features = rand_data.n_features
n_components = rand_data.n_components
for covar_type in COVARIANCE_TYPE:
X = rand_data.X[covar_type]
g = GaussianMixture(n_components=n_components, n_init=20,
reg_covar=0, random_state=rng,
covariance_type=covar_type)
g.fit(X)
# needs more data to pass the test with rtol=1e-7
assert_allclose(np.sort(g.weights_), np.sort(rand_data.weights),
rtol=0.1, atol=1e-2)
arg_idx1 = g.means_[:, 0].argsort()
arg_idx2 = rand_data.means[:, 0].argsort()
assert_allclose(g.means_[arg_idx1], rand_data.means[arg_idx2],
rtol=0.1, atol=1e-2)
if covar_type == 'full':
prec_pred = g.precisions_
prec_test = rand_data.precisions['full']
elif covar_type == 'tied':
prec_pred = np.array([g.precisions_] * n_components)
prec_test = np.array([rand_data.precisions['tied']] * n_components)
elif covar_type == 'spherical':
prec_pred = np.array([np.eye(n_features) * c
for c in g.precisions_])
prec_test = np.array([np.eye(n_features) * c for c in
rand_data.precisions['spherical']])
elif covar_type == 'diag':
prec_pred = np.array([np.diag(d) for d in g.precisions_])
prec_test = np.array([np.diag(d) for d in
rand_data.precisions['diag']])
arg_idx1 = np.trace(prec_pred, axis1=1, axis2=2).argsort()
arg_idx2 = np.trace(prec_test, axis1=1, axis2=2).argsort()
for k, h in zip(arg_idx1, arg_idx2):
ecov = EmpiricalCovariance()
ecov.covariance_ = prec_test[h]
# the accuracy depends on the number of data and randomness, rng
assert_allclose(ecov.error_norm(prec_pred[k]), 0, atol=0.1)
def test_check_weights():
rng = np.random.RandomState(0)
rand_data = RandomData(rng)
n_components = rand_data.n_components
X = rand_data.X['full']
g = GaussianMixture(n_components=n_components)
# Check bad shape
weights_bad_shape = rng.rand(n_components, 1)
g.weights_init = weights_bad_shape
assert_raise_message(ValueError,
"The parameter 'weights' should have the shape of "
"(%d,), "
"but got %s" % (n_components,
str(weights_bad_shape.shape)),
g.fit, X)
# Check bad range
weights_bad_range = rng.rand(n_components) + 1
g.weights_init = weights_bad_range
assert_raise_message(ValueError,
"The parameter 'weights' should be in the range "
"[0, 1], but got max value %.5f, min value %.5f"
% (np.min(weights_bad_range),
np.max(weights_bad_range)),
g.fit, X)
# Check bad normalization
weights_bad_norm = rng.rand(n_components)
weights_bad_norm = weights_bad_norm / (weights_bad_norm.sum() + 1)
g.weights_init = weights_bad_norm
assert_raise_message(ValueError,
"The parameter 'weights' should be normalized, "
"but got sum(weights) = %.5f"
% np.sum(weights_bad_norm),
g.fit, X)
# Check good weights matrix
weights = rand_data.weights
g = GaussianMixture(weights_init=weights, n_components=n_components)
g.fit(X)
assert_array_equal(weights, g.weights_init)
def test_gaussian_mixture_aic_bic():
# Test the aic and bic criteria
rng = np.random.RandomState(0)
n_samples, n_features, n_components = 50, 3, 2
X = rng.randn(n_samples, n_features)
# standard gaussian entropy
sgh = 0.5 * (fast_logdet(np.cov(X.T, bias=1)) +
n_features * (1 + np.log(2 * np.pi)))
for cv_type in COVARIANCE_TYPE:
g = GaussianMixture(
n_components=n_components, covariance_type=cv_type,
random_state=rng, max_iter=200)
g.fit(X)
aic = 2 * n_samples * sgh + 2 * g._n_parameters()
bic = (2 * n_samples * sgh +
np.log(n_samples) * g._n_parameters())
bound = n_features / np.sqrt(n_samples)
assert (g.aic(X) - aic) / n_samples < bound
assert (g.bic(X) - bic) / n_samples < bound
def test_gaussian_mixture_verbose():
rng = np.random.RandomState(0)
rand_data = RandomData(rng)
n_components = rand_data.n_components
for covar_type in COVARIANCE_TYPE:
X = rand_data.X[covar_type]
g = GaussianMixture(n_components=n_components, n_init=1, reg_covar=0,
random_state=rng, covariance_type=covar_type,
verbose=1)
h = GaussianMixture(n_components=n_components, n_init=1, reg_covar=0,
random_state=rng, covariance_type=covar_type,
verbose=2)
old_stdout = sys.stdout
sys.stdout = StringIO()
try:
g.fit(X)
h.fit(X)
finally:
sys.stdout = old_stdout
def test_gaussian_mixture_aic_bic():
# Test the aic and bic criteria
rng = np.random.RandomState(0)
n_samples, n_features, n_components = 50, 3, 2
X = rng.randn(n_samples, n_features)
# standard gaussian entropy
sgh = 0.5 * (fast_logdet(np.cov(X.T, bias=1)) + n_features *
(1 + np.log(2 * np.pi)))
for cv_type in COVARIANCE_TYPE:
g = GaussianMixture(n_components=n_components,
covariance_type=cv_type,
random_state=rng,
max_iter=200)
g.fit(X)
aic = 2 * n_samples * sgh + 2 * g._n_parameters()
bic = (2 * n_samples * sgh + np.log(n_samples) * g._n_parameters())
bound = n_features / np.sqrt(n_samples)
assert_true((g.aic(X) - aic) / n_samples < bound)
assert_true((g.bic(X) - bic) / n_samples < bound)
def test_gaussian_mixture_fit_best_params():
rng = np.random.RandomState(0)
rand_data = RandomData(rng)
n_components = rand_data.n_components
n_init = 10
for covar_type in COVARIANCE_TYPE:
X = rand_data.X[covar_type]
g = GaussianMixture(n_components=n_components, n_init=1, reg_covar=0,
random_state=rng, covariance_type=covar_type)
ll = []
for _ in range(n_init):
g.fit(X)
ll.append(g.score(X))
ll = np.array(ll)
g_best = GaussianMixture(n_components=n_components,
n_init=n_init, reg_covar=0, random_state=rng,
covariance_type=covar_type)
g_best.fit(X)
assert_almost_equal(ll.min(), g_best.score(X))
def test_gaussian_mixture_fit_best_params():
rng = np.random.RandomState(0)
rand_data = RandomData(rng)
n_components = rand_data.n_components
n_init = 10
for covar_type in COVARIANCE_TYPE:
X = rand_data.X[covar_type]
g = GaussianMixture(n_components=n_components, n_init=1, reg_covar=0,
random_state=rng, covariance_type=covar_type)
ll = []
for _ in range(n_init):
g.fit(X)
ll.append(g.score(X))
ll = np.array(ll)
g_best = GaussianMixture(n_components=n_components,
n_init=n_init, reg_covar=0, random_state=rng,
covariance_type=covar_type)
g_best.fit(X)
assert_almost_equal(ll.min(), g_best.score(X))
def test_property():
rng = np.random.RandomState(0)
rand_data = RandomData(rng, scale=7)
n_components = rand_data.n_components
for covar_type in COVARIANCE_TYPE:
X = rand_data.X[covar_type]
gmm = GaussianMixture(n_components=n_components,
covariance_type=covar_type, random_state=rng,
n_init=5)
gmm.fit(X)
if covar_type == 'full':
for prec, covar in zip(gmm.precisions_, gmm.covariances_):
assert_array_almost_equal(linalg.inv(prec), covar)
elif covar_type == 'tied':
assert_array_almost_equal(linalg.inv(gmm.precisions_),
gmm.covariances_)
else:
assert_array_almost_equal(gmm.precisions_, 1. / gmm.covariances_)
def test_property():
rng = np.random.RandomState(0)
rand_data = RandomData(rng, scale=7)
n_components = rand_data.n_components
for covar_type in COVARIANCE_TYPE:
X = rand_data.X[covar_type]
gmm = GaussianMixture(n_components=n_components,
covariance_type=covar_type, random_state=rng,
n_init=5)
gmm.fit(X)
if covar_type == 'full':
for prec, covar in zip(gmm.precisions_, gmm.covariances_):
assert_array_almost_equal(linalg.inv(prec), covar)
elif covar_type == 'tied':
assert_array_almost_equal(linalg.inv(gmm.precisions_),
gmm.covariances_)
else:
assert_array_almost_equal(gmm.precisions_, 1. / gmm.covariances_)
def test_check_means():
rng = np.random.RandomState(0)
rand_data = RandomData(rng)
n_components, n_features = rand_data.n_components, rand_data.n_features
X = rand_data.X['full']
g = GaussianMixture(n_components=n_components)
# Check means bad shape
means_bad_shape = rng.rand(n_components + 1, n_features)
g.means_init = means_bad_shape
assert_raise_message(ValueError,
"The parameter 'means' should have the shape of ",
g.fit, X)
# Check good means matrix
means = rand_data.means
g.means_init = means
g.fit(X)
assert_array_equal(means, g.means_init)
def test_check_means():
rng = np.random.RandomState(0)
rand_data = RandomData(rng)
n_components, n_features = rand_data.n_components, rand_data.n_features
X = rand_data.X['full']
g = GaussianMixture(n_components=n_components)
# Check means bad shape
means_bad_shape = rng.rand(n_components + 1, n_features)
g.means_init = means_bad_shape
assert_raise_message(ValueError,
"The parameter 'means' should have the shape of ",
g.fit, X)
# Check good means matrix
means = rand_data.means
g.means_init = means
g.fit(X)
assert_array_equal(means, g.means_init)
class ScikitLL(LikelihoodEvaluator):
"""
Fastest Single Core Version so far!
"""
def __init__(self, Xpoints, numMixtures):
super().__init__(Xpoints, numMixtures)
self.evaluator = GaussianMixture(numMixtures, 'diag')
self.Xpoints = Xpoints
self.evaluator.fit(Xpoints)
def __str__(self):
return "SciKit's learn implementation Implementation"
def loglikelihood(self, means, diagCovs, weights):
self.evaluator.weights_ = weights
self.evaluator.covariances_ = diagCovs
self.evaluator.means_ = means
self.evaluator.precisions_cholesky_ = _compute_precision_cholesky(
diagCovs, "diag")
return self.numPoints * np.sum(self.evaluator.score(self.Xpoints))
def test_warm_start():
random_state = 0
rng = np.random.RandomState(random_state)
n_samples, n_features, n_components = 500, 2, 2
X = rng.rand(n_samples, n_features)
# Assert the warm_start give the same result for the same number of iter
g = GaussianMixture(n_components=n_components,
n_init=1,
max_iter=2,
reg_covar=0,
random_state=random_state,
warm_start=False)
h = GaussianMixture(n_components=n_components,
n_init=1,
max_iter=1,
reg_covar=0,
random_state=random_state,
warm_start=True)
with warnings.catch_warnings():
warnings.simplefilter("ignore", ConvergenceWarning)
g.fit(X)
score1 = h.fit(X).score(X)
score2 = h.fit(X).score(X)
assert_almost_equal(g.weights_, h.weights_)
assert_almost_equal(g.means_, h.means_)
assert_almost_equal(g.precisions_, h.precisions_)
assert_greater(score2, score1)
# Assert that by using warm_start we can converge to a good solution
g = GaussianMixture(n_components=n_components,
n_init=1,
max_iter=5,
reg_covar=0,
random_state=random_state,
warm_start=False,
tol=1e-6)
h = GaussianMixture(n_components=n_components,
n_init=1,
max_iter=5,
reg_covar=0,
random_state=random_state,
warm_start=True,
tol=1e-6)
with warnings.catch_warnings():
warnings.simplefilter("ignore", ConvergenceWarning)
g.fit(X)
h.fit(X).fit(X)
assert_true(not g.converged_)
assert_true(h.converged_)
def test_sample():
rng = np.random.RandomState(0)
rand_data = RandomData(rng, scale=7)
n_features, n_components = rand_data.n_features, rand_data.n_components
for covar_type in COVARIANCE_TYPE:
X = rand_data.X[covar_type]
gmm = GaussianMixture(n_components=n_components,
covariance_type=covar_type, random_state=rng)
# To sample we need that GaussianMixture is fitted
assert_raise_message(NotFittedError, "This GaussianMixture instance "
"is not fitted", gmm.sample, 0)
gmm.fit(X)
assert_raise_message(ValueError, "Invalid value for 'n_samples",
gmm.sample, 0)
# Just to make sure the class samples correctly
X_s, y_s = gmm.sample(20000)
for k in range(n_features):
if covar_type == 'full':
assert_array_almost_equal(gmm.covariances_[k],
np.cov(X_s[y_s == k].T), decimal=1)
elif covar_type == 'tied':
assert_array_almost_equal(gmm.covariances_,
np.cov(X_s[y_s == k].T), decimal=1)
elif covar_type == 'diag':
assert_array_almost_equal(gmm.covariances_[k],
np.diag(np.cov(X_s[y_s == k].T)),
decimal=1)
else:
assert_array_almost_equal(
gmm.covariances_[k], np.var(X_s[y_s == k] - gmm.means_[k]),
decimal=1)
means_s = np.array([np.mean(X_s[y_s == k], 0)
for k in range(n_features)])
assert_array_almost_equal(gmm.means_, means_s, decimal=1)
def test_warm_start(seed):
random_state = seed
rng = np.random.RandomState(random_state)
n_samples, n_features, n_components = 500, 2, 2
X = rng.rand(n_samples, n_features)
# Assert the warm_start give the same result for the same number of iter
g = GaussianMixture(n_components=n_components, n_init=1, max_iter=2,
reg_covar=0, random_state=random_state,
warm_start=False)
h = GaussianMixture(n_components=n_components, n_init=1, max_iter=1,
reg_covar=0, random_state=random_state,
warm_start=True)
g.fit(X)
score1 = h.fit(X).score(X)
score2 = h.fit(X).score(X)
assert_almost_equal(g.weights_, h.weights_)
assert_almost_equal(g.means_, h.means_)
assert_almost_equal(g.precisions_, h.precisions_)
assert score2 > score1
# Assert that by using warm_start we can converge to a good solution
g = GaussianMixture(n_components=n_components, n_init=1,
max_iter=5, reg_covar=0, random_state=random_state,
warm_start=False, tol=1e-6)
h = GaussianMixture(n_components=n_components, n_init=1,
max_iter=5, reg_covar=0, random_state=random_state,
warm_start=True, tol=1e-6)
g.fit(X)
assert not g.converged_
h.fit(X)
# depending on the data there is large variability in the number of
# refit necessary to converge due to the complete randomness of the
# data
for _ in range(1000):
h.fit(X)
if h.converged_:
break
assert h.converged_
def test_gaussian_mixture_predict_predict_proba():
rng = np.random.RandomState(0)
rand_data = RandomData(rng)
for covar_type in COVARIANCE_TYPE:
X = rand_data.X[covar_type]
Y = rand_data.Y
g = GaussianMixture(n_components=rand_data.n_components,
random_state=rng, weights_init=rand_data.weights,
means_init=rand_data.means,
precisions_init=rand_data.precisions[covar_type],
covariance_type=covar_type)
# Check a warning message arrive if we don't do fit
assert_raise_message(NotFittedError,
"This GaussianMixture instance is not fitted "
"yet. Call 'fit' with appropriate arguments "
"before using this method.", g.predict, X)
g.fit(X)
Y_pred = g.predict(X)
Y_pred_proba = g.predict_proba(X).argmax(axis=1)
assert_array_equal(Y_pred, Y_pred_proba)
assert_greater(adjusted_rand_score(Y, Y_pred), .95)
def test_gaussian_mixture_predict_predict_proba():
rng = np.random.RandomState(0)
rand_data = RandomData(rng)
for covar_type in COVARIANCE_TYPE:
X = rand_data.X[covar_type]
Y = rand_data.Y
g = GaussianMixture(n_components=rand_data.n_components,
random_state=rng, weights_init=rand_data.weights,
means_init=rand_data.means,
precisions_init=rand_data.precisions[covar_type],
covariance_type=covar_type)
# Check a warning message arrive if we don't do fit
assert_raise_message(NotFittedError,
"This GaussianMixture instance is not fitted "
"yet. Call 'fit' with appropriate arguments "
"before using this method.", g.predict, X)
g.fit(X)
Y_pred = g.predict(X)
Y_pred_proba = g.predict_proba(X).argmax(axis=1)
assert_array_equal(Y_pred, Y_pred_proba)
assert_greater(adjusted_rand_score(Y, Y_pred), .95)
def test_gaussian_mixture_estimate_log_prob_resp():
# test whether responsibilities are normalized
rng = np.random.RandomState(0)
rand_data = RandomData(rng, scale=5)
n_samples = rand_data.n_samples
n_features = rand_data.n_features
n_components = rand_data.n_components
X = rng.rand(n_samples, n_features)
for covar_type in COVARIANCE_TYPE:
weights = rand_data.weights
means = rand_data.means
precisions = rand_data.precisions[covar_type]
g = GaussianMixture(n_components=n_components, random_state=rng,
weights_init=weights, means_init=means,
precisions_init=precisions,
covariance_type=covar_type)
g.fit(X)
resp = g.predict_proba(X)
assert_array_almost_equal(resp.sum(axis=1), np.ones(n_samples))
assert_array_equal(g.weights_init, weights)
assert_array_equal(g.means_init, means)
assert_array_equal(g.precisions_init, precisions)
def test_score_samples():
covar_type = 'full'
rng = np.random.RandomState(0)
rand_data = RandomData(rng, scale=7)
n_components = rand_data.n_components
X = rand_data.X[covar_type]
# Check the error message if we don't call fit
gmm = GaussianMixture(n_components=n_components, n_init=1, reg_covar=0,
random_state=rng, covariance_type=covar_type)
assert_raise_message(NotFittedError,
"This GaussianMixture instance is not fitted "
"yet. Call 'fit' with appropriate arguments "
"before using this method.", gmm.score_samples, X)
gmm_score_samples = gmm.fit(X).score_samples(X)
assert_equal(gmm_score_samples.shape[0], rand_data.n_samples)
def test_score_samples():
covar_type = 'full'
rng = np.random.RandomState(0)
rand_data = RandomData(rng, scale=7)
n_components = rand_data.n_components
X = rand_data.X[covar_type]
# Check the error message if we don't call fit
gmm = GaussianMixture(n_components=n_components, n_init=1, reg_covar=0,
random_state=rng, covariance_type=covar_type)
assert_raise_message(NotFittedError,
"This GaussianMixture instance is not fitted "
"yet. Call 'fit' with appropriate arguments "
"before using this method.", gmm.score_samples, X)
gmm_score_samples = gmm.fit(X).score_samples(X)
assert_equal(gmm_score_samples.shape[0], rand_data.n_samples)
def test_warm_start():
random_state = 0
rng = np.random.RandomState(random_state)
n_samples, n_features, n_components = 500, 2, 2
X = rng.rand(n_samples, n_features)
# Assert the warm_start give the same result for the same number of iter
g = GaussianMixture(n_components=n_components, n_init=1, max_iter=2,
reg_covar=0, random_state=random_state,
warm_start=False)
h = GaussianMixture(n_components=n_components, n_init=1, max_iter=1,
reg_covar=0, random_state=random_state,
warm_start=True)
with warnings.catch_warnings():
warnings.simplefilter("ignore", ConvergenceWarning)
g.fit(X)
score1 = h.fit(X).score(X)
score2 = h.fit(X).score(X)
assert_almost_equal(g.weights_, h.weights_)
assert_almost_equal(g.means_, h.means_)
assert_almost_equal(g.precisions_, h.precisions_)
assert_greater(score2, score1)
# Assert that by using warm_start we can converge to a good solution
g = GaussianMixture(n_components=n_components, n_init=1,
max_iter=5, reg_covar=0, random_state=random_state,
warm_start=False, tol=1e-6)
h = GaussianMixture(n_components=n_components, n_init=1,
max_iter=5, reg_covar=0, random_state=random_state,
warm_start=True, tol=1e-6)
with warnings.catch_warnings():
warnings.simplefilter("ignore", ConvergenceWarning)
g.fit(X)
h.fit(X).fit(X)
assert_true(not g.converged_)
assert_true(h.converged_)
def trainGmmEM(x,
t,
K,
cov_type='full',
n_max_iter=500,
n_restarts_random=20,
n_restarts_kmeans=20,
regularize=1e-2,
uniform_class_prior=True):
'''
Trains a GMM for each class of the given data. If EM is run for several
times, the models with the largest log-likelihood are kept.
x: The input features
t: The target values
K: List containing the number of components per class
cov_type: Which covariance type should be used ('full' or 'diag')
n_max_iter: Maximum number of iterations used for EM training
n_restarts_random: Number of random restarts with random initial values
n_restarts_kmeans: Number of random restarts where initial values are
computed with the k-means algorithm.
regularize: Regularizer for the diagonal of the covariance matrices
uniform_class_prior: If True, the class prior is set to 1/C for each class,
otherwise it is computed as the fraction of samples per class.
Returns a dictionary containing all parameters.
'''
assert n_restarts_random + n_restarts_kmeans > 0
C = int(np.max(t) + 1)
params = {}
for c in range(C):
print 'EM training for class %d/%d with %d components (N=%d, D=%d)' % (
c, C, K[c], np.sum(t == c), x.shape[1])
t_start = time()
if n_restarts_random > 0:
gmm1 = GaussianMixture(n_components=K[c],
covariance_type=cov_type,
reg_covar=regularize,
max_iter=n_max_iter,
n_init=n_restarts_random,
init_params='random',
verbose=2)
gmm1.fit(x[t == c, :])
if n_restarts_kmeans > 0:
gmm2 = GaussianMixture(n_components=K[c],
covariance_type=cov_type,
reg_covar=regularize,
max_iter=n_max_iter,
n_init=n_restarts_kmeans,
init_params='kmeans',
verbose=2)
gmm2.fit(x[t == c, :])
t_elapsed = time() - t_start
print 'EM training for class %d/%d finished in %f seconds' % (
c, C, t_elapsed)
# Select the better model of gmm1 and gmm2
# Don't use gmm.lower_bound_, it returns the last logl and not the best
score1 = gmm1.score(x[t == c, :]) if n_restarts_random > 0 else -np.Inf
score2 = gmm2.score(x[t == c, :]) if n_restarts_kmeans > 0 else -np.Inf
gmm = gmm1 if score1 > score2 else gmm2
params['alpha_%d' % (c)] = gmm.weights_
params['mu_%d' % (c)] = gmm.means_
params['Sigma_%d' % (c)] = gmm.covariances_
params['Lambda_%d' % (c)] = gmm.precisions_
if uniform_class_prior == True:
params['prior'] = np.full((C, ), 1. / C, 'float32')
else:
_, counts = np.unique(t, return_counts=True)
counts = np.asarray(counts, 'float32')
params['prior'] = counts / np.sum(counts)
return params
estimator = GaussianMixture(n_components=2, max_iter=1000, random_state=0, init_params='random')
plt.figure()
plt.plot(X_train[:, 0], X_train[:, 1], 'k.', markersize=25)
plt.savefig('Figure_6-datapoints.png')
plt.close()
colors = ['r', 'b']
# initial means
estimator.means_init = np.array([[0, 0.5], [0.5, 0]])
# Train the other parameters using the EM algorithm.
estimator.fit(X_train)
classes = estimator.predict(X_train)
plt.figure()
for i in range(2):
plt.plot(X_train[classes == i, 0], X_train[classes == i, 1], colors[i]+'.', markersize=25, label='class'+str(i+1))
plt.plot(estimator.means_init[i, 0], estimator.means_init[i, 1], colors[i]+'*', markersize=20, label='mean_init')
plt.plot(np.mean(X_train[classes == i, 0]), np.mean(X_train[classes == i, 1]), colors[i] + 'P', markersize=15, label='mean_init')
plt.title('mean_init='+str(estimator.means_init))
plt.legend()
plt.savefig('Figure_6-EM1.png')
plt.close()
# initial means
estimator.means_init = np.array([[0, 1], [0, -1]])
# Train the other parameters using the EM algorithm.
def wemd_from_pred_samples(y_pred, ):
gmm = GMM(covariance_type="diag")
gmm = gmm.fit(y_pred)
y_s, _ = gmm.sample(len(y_pred))
return wemd_from_samples(y_s, y_pred)
if args.plot:
plot_sample(x, y, pdfpath="temp/gmm_%ssamples.pdf" % sample_type)
# to plot probability contours
xx, yy = np.meshgrid(np.linspace(np.min(x[:, 0]), np.max(x[:, 0]), 50),
np.linspace(np.min(x[:, 1]), np.max(x[:, 1]), 50))
x_grid = np.c_[xx.ravel(), yy.ravel()]
cov_type = 'full' # 'spherical', 'diag', 'tied', 'full'
n_classes = 3
max_iters = 20 if sample_type == "" else 100
gmm = GMM(n_components=n_classes,
covariance_type=cov_type,
max_iter=max_iters,
random_state=0)
gmm.fit(x)
logger.debug("Means:\n%s" % str(gmm.means_))
logger.debug("Covariances:\n%s" % str(gmm.covariances_))
scores = gmm.score_samples(x)
top_anoms = np.argsort(scores)[np.arange(10)]
if args.plot:
# colors = ["red", "blue"]
colors = ['navy', 'turquoise', 'darkorange']
pdfpath = "temp/gmm_%scontours.pdf" % sample_type
dp = DataPlotter(pdfpath=pdfpath, rows=1, cols=2)
pl = dp.get_next_plot()
hemispheres_index = np.load(HEMI_INDEXES)
side_index = np.mod(hemispheres_index, 2)
labels = np.load(DBSCAN_LABELS)
g = GaussianMixture(n_components=N, n_init=N_INIT, init_params=INIT_METHOD)
for j, side in enumerate(SIDES.keys()):
data_ = data[side_index == j]
labels_ = labels[side_index == j]
final_labels = labels_.copy()
# reencode the labels to take into account the added clusters
final_labels[labels_ == 2] = 4
# select central cluster
data_ = data_[labels_ == 1]
g.fit(data_)
w = g.weights_
m = g.means_
c = g.covariances_
pred_labels = g.predict(data_)
# directly sort the labels according to the precentral coordinates of the cluster
t = np.argsort(m, axis=0)[:, 0]
sorted_means = m[t]
sorted_cov = c[t]
sorted_weigth = w[t]
sorted_pred_labels = pred_labels.copy()
for z, s in enumerate(t):
# s+1 to account for the existence of ventral cluster
sorted_pred_labels[pred_labels == s] = z + 1