def test_suffstat_sk_tied():
# use equation Nk * Sk / N = S_tied
rng = np.random.RandomState(0)
n_samples, n_features, n_components = 500, 2, 2
resp = rng.rand(n_samples, n_components)
resp = resp / resp.sum(axis=1)[:, np.newaxis]
X = rng.rand(n_samples, n_features)
nk = resp.sum(axis=0)
xk = np.dot(resp.T, X) / nk[:, np.newaxis]
covars_pred_full = _estimate_gaussian_covariances_full(resp, X, nk, xk, 0)
covars_pred_full = np.sum(nk[:, np.newaxis, np.newaxis] * covars_pred_full,
0) / n_samples
covars_pred_tied = _estimate_gaussian_covariances_tied(resp, X, nk, xk, 0)
ecov = EmpiricalCovariance()
ecov.covariance_ = covars_pred_full
assert_almost_equal(ecov.error_norm(covars_pred_tied, norm='frobenius'), 0)
assert_almost_equal(ecov.error_norm(covars_pred_tied, norm='spectral'), 0)
# check the precision computation
precs_chol_pred = _compute_precision_cholesky(covars_pred_tied, 'tied')
precs_pred = np.dot(precs_chol_pred, precs_chol_pred.T)
precs_est = linalg.inv(covars_pred_tied)
assert_array_almost_equal(precs_est, precs_pred)
def _initialize1(self):
# n_samples, _ = X.shape
# weights, means, covariances = _estimate_gaussian_parameters(
# X, resp, self.reg_covar, self.covariance_type)
# weights /= n_samples
# self.weights_ = (weights if self.weights_init is None
# else self.weights_init)
# self.means_ = means if self.means_init is None else self.means_init
if self.precisions_init is None:
# self.covariances_ = covariances
self.precisions_cholesky_ = _compute_precision_cholesky(
self.covariances_, self.covariance_type)
elif self.covariance_type == 'full':
self.precisions_cholesky_ = np.array([
linalg.cholesky(prec_init, lower=True)
for prec_init in self.precisions_init
])
elif self.covariance_type == 'tied':
self.precisions_cholesky_ = linalg.cholesky(self.precisions_init,
lower=True)
else:
self.precisions_cholesky_ = self.precisions_init
def _onehot_to_initial_params(
X: np.ndarray, onehot: np.ndarray,
cov_type: CovarianceType) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:
"""
Computes cluster weights, cluster means and cluster precisions from
a given clustering.
Parameters
----------
X : array-like, shape (n_samples, n_features)
List of n_features-dimensional data points. Each row
corresponds to a single data point.
onehot : ndarray, shape (n_samples, n_clusters)
Each row has a 1 indicating cluster membership, other entries are 0.
cov_type : {'full', 'tied', 'diag', 'spherical'}
Covariance type for Gaussian mixture model
"""
n = X.shape[0]
weights, means, covariances = _estimate_gaussian_parameters(
X, onehot, 1e-06, cov_type)
weights /= n
precisions_cholesky_ = _compute_precision_cholesky(covariances, cov_type)
if cov_type == "tied":
c = precisions_cholesky_
precisions = np.dot(c, c.T)
elif cov_type == "diag":
precisions = precisions_cholesky_
else:
precisions = [np.dot(c, c.T) for c in precisions_cholesky_]
return weights, means, precisions
def add_new_component(self,
x_proj,
q_abnormal_thres=0.95,
acculumated_X_train_proj=None):
##########################################################################################
# sub-scenario: create a new component for the new x
# self.novelty_thres < _y_score < self.abnormal_thres
# x is predicted as a novelty datapoint (but still is a normal datapoint), so we create a new
# component and update GMM.
self.n_components += 1
# compute the mean and covariance of the new components
# For the mean, we use the x value as the mean of the new component
# (because the new component only has one point (i.e., x)), and append it to the previous means.
new_mean = x_proj
new_covar = self.generate_new_covariance(x_proj, self.means_,
self.covariances_)
self.means_ = np.concatenate([self.means_, new_mean], axis=0)
_, dim = new_mean.shape
if self.covariance_type == 'diag':
self.covariances_ = np.concatenate(
[self.covariances_,
new_covar.reshape(1, dim)], axis=0)
else:
self.covariances_ = np.concatenate(
[self.covariances_,
new_covar.reshape(1, dim, dim)], axis=0)
# print(f'new_model.params: {self.get_params()}')
n = acculumated_X_train_proj.shape[0]
self.weights_ = np.asarray([n / (n + 1) * v for v in self.weights_])
self.weights_ = np.concatenate(
[self.weights_, np.ones((1, )) * (1 / (n + 1))], axis=0)
self.sum_resp = np.concatenate([self.sum_resp, np.ones((1, ))], axis=0)
# f_(k+1): the new component of GMM
n_feats = self.n_components
# get log probabliity of acculumated data to update the threshold.
diff = (acculumated_X_train_proj -
new_mean).T # X and mu should be column vectors
if self.covariance_type == 'diag':
log_dist = -0.5 * diff.T**2 @ (1 / new_covar)
log_det = np.product(new_covar)
else:
log_dist = np.diag(
-0.5 *
np.matmul(np.matmul(diff.T, np.linalg.inv(new_covar)), diff))
log_det = np.log(np.linalg.det(new_covar))
log_det = 1e-6 if np.isnan(log_det) or np.isinf(log_det) else log_det
f_k_1 = -.5 * (n_feats * np.log(2 * np.pi) + log_det) + log_dist
# update self.self.precisions_cholesky_,
self.precisions_cholesky_ = _compute_precision_cholesky(
self.covariances_, self.covariance_type)
self.y_score = n / (n + 1) * self.decision_function(
acculumated_X_train_proj) + 1 / (n + 1) * f_k_1
self.abnormal_thres = np.quantile(
self.y_score, q=q_abnormal_thres) # abnormal threshold
def train(self, data):
if self.initialized:
ff_tmp = self.params['forgetting_factor'].get_value()
self.short_term_model = IGMM(self.params['init_components'])
self.short_term_model.get_best_gmm(data,lims=[1, self.params['max_step_components']])
# lims=[self.params['init_components'], self.params['max_step_components']])
weights_st = self.short_term_model.weights_
weights_st = ff_tmp * weights_st
self.short_term_model.weights_ = weights_st
#print(ff_tmp)
weights_lt = self.weights_
weights_lt = (self.weights_.sum() - ff_tmp) * weights_lt # Regularization to keep sum(w)=1.0
self.weights_ = weights_lt
gmm_new = copy.deepcopy(self.short_term_model)
gmm_new = self.merge_similar_gaussians_in_gmm_minim(gmm_new)
self.mergeGMM(gmm_new)
self.weights_=self.weights_/sum(self.weights_) #Regularization
else:
#self.get_best_gmm(data, lims=[self.params['init_components'], self.params['max_step_components']])
self.get_best_gmm(data, lims=[self.params['init_components'],self.params['init_components']])
self.short_term_model = GMM(self.n_components)
self.initialized = True
self.precisions_cholesky_ = _compute_precision_cholesky(self.covariances_, "full")
if self.params['infer_fixed']:
y_dims = self.params['y_dims']
x_dims = self.params['x_dims']
SIGMA_YY_inv = np.zeros((self.n_components,len(y_dims),len(y_dims)))
SIGMA_XY = np.zeros((self.n_components,len(x_dims),len(y_dims)))
for k, (Mu, Sigma) in enumerate(zip(self.means_, self.covariances_)):
Sigma_yy = Sigma[:, y_dims]
Sigma_yy = Sigma_yy[y_dims, :]
Sigma_xy = Sigma[x_dims, :]
Sigma_xy = Sigma_xy[:, y_dims]
Sigma_yy_inv = linalg.inv(Sigma_yy)
SIGMA_YY_inv[k,:,:] = Sigma_yy_inv
SIGMA_XY[k,:, :] = Sigma_xy
self.SIGMA_YY_inv = SIGMA_YY_inv
self.SIGMA_XY = SIGMA_XY
def _set_pX(self):
# probability density function of X
self.pX = sklearn.mixture.GaussianMixture(
n_components=self.n_mix, covariance_type="full"
)
self.pX.weights_ = self.w
self.pX.means_ = self.meanX
self.pX.covariances_ = self.covXX
# following function is required to estimate porsterior
self.pX.precisions_cholesky_ = _compute_precision_cholesky(self.covXX, "full")
return
def test_suffstat_sk_full():
# compare the precision matrix compute from the
# EmpiricalCovariance.covariance fitted on X*sqrt(resp)
# with _sufficient_sk_full, n_components=1
rng = np.random.RandomState(0)
n_samples, n_features = 500, 2
# special case 1, assuming data is "centered"
X = rng.rand(n_samples, n_features)
resp = rng.rand(n_samples, 1)
X_resp = np.sqrt(resp) * X
nk = np.array([n_samples])
xk = np.zeros((1, n_features))
covars_pred = _estimate_gaussian_covariances_full(resp, X, nk, xk, 0)
ecov = EmpiricalCovariance(assume_centered=True)
ecov.fit(X_resp)
assert_almost_equal(ecov.error_norm(covars_pred[0], norm='frobenius'), 0)
assert_almost_equal(ecov.error_norm(covars_pred[0], norm='spectral'), 0)
# check the precision computation
precs_chol_pred = _compute_precision_cholesky(covars_pred, 'full')
precs_pred = np.array([np.dot(prec, prec.T) for prec in precs_chol_pred])
precs_est = np.array([linalg.inv(cov) for cov in covars_pred])
assert_array_almost_equal(precs_est, precs_pred)
# special case 2, assuming resp are all ones
resp = np.ones((n_samples, 1))
nk = np.array([n_samples])
xk = X.mean(axis=0).reshape((1, -1))
covars_pred = _estimate_gaussian_covariances_full(resp, X, nk, xk, 0)
ecov = EmpiricalCovariance(assume_centered=False)
ecov.fit(X)
assert_almost_equal(ecov.error_norm(covars_pred[0], norm='frobenius'), 0)
assert_almost_equal(ecov.error_norm(covars_pred[0], norm='spectral'), 0)
# check the precision computation
precs_chol_pred = _compute_precision_cholesky(covars_pred, 'full')
precs_pred = np.array([np.dot(prec, prec.T) for prec in precs_chol_pred])
precs_est = np.array([linalg.inv(cov) for cov in covars_pred])
assert_array_almost_equal(precs_est, precs_pred)
def jsd_diss(self, w1, mu1, cov1, w2, mu2, cov2):
"""
Calculates Jensen-Shannon divergence of two gmm's
:param gmm_p: mixture.GaussianMixture
:param gmm_q: mixture.GaussianMixture
:param sample_count: number of monte carlo samples to use
:return: Jensen-Shannon divergence
"""
gmm_p = GaussianMixture(n_components=n_components,
covariance_type="full")
gmm_p.weights_ = w1
gmm_p.covariances_ = cov1
gmm_p.means_ = mu1
gmm_p.n_components = 1
gmm_p.precisions_cholesky_ = _compute_precision_cholesky(cov1, "full")
gmm_q = GaussianMixture(n_components=n_components,
covariance_type="full")
gmm_q.weights_ = w2
gmm_q.covariances_ = cov2
gmm_q.means_ = mu2
gmm_q.n_components = 1
gmm_q.precisions_cholesky_ = _compute_precision_cholesky(cov2, "full")
X = gmm_p.sample(sample_count)[0]
log_p_X = gmm_p.score_samples(X)
log_q_X = gmm_q.score_samples(X)
log_mix_X = np.logaddexp(log_p_X, log_q_X)
Y = gmm_q.sample(sample_count)[0]
log_p_Y = gmm_p.score_samples(Y)
log_q_Y = gmm_q.score_samples(Y)
log_mix_Y = np.logaddexp(log_p_Y, log_q_Y)
# black magic?
return (log_p_X.mean() -
(log_mix_X.mean() - np.log(2)) + log_q_Y.mean() -
(log_mix_Y.mean() - np.log(2))) / 2
def _m_step(self, X, log_resp):
"""M step.
Parameters
----------
X : array-like, shape (n_samples, n_features)
log_resp : array-like, shape (n_samples, n_components)
Logarithm of the posterior probabilities (or responsibilities) of
the point of each sample in X.
"""
resp = (self.Xweights * np.exp(log_resp).T).T
self.weights_, self.means_, self.covariances_ = (
_estimate_gaussian_parameters(X, resp, self.reg_covar,
self.covariance_type))
self.weights_ /= np.sum(self.Xweights)
self.precisions_cholesky_ = _compute_precision_cholesky(
self.covariances_, self.covariance_type)
def test_compute_log_det_cholesky():
n_features = 2
rand_data = RandomData(np.random.RandomState(0))
for covar_type in COVARIANCE_TYPE:
covariance = rand_data.covariances[covar_type]
if covar_type == 'full':
predected_det = np.array([linalg.det(cov) for cov in covariance])
elif covar_type == 'tied':
predected_det = linalg.det(covariance)
elif covar_type == 'diag':
predected_det = np.array([np.prod(cov) for cov in covariance])
elif covar_type == 'spherical':
predected_det = covariance ** n_features
# We compute the cholesky decomposition of the covariance matrix
expected_det = _compute_log_det_cholesky(_compute_precision_cholesky(
covariance, covar_type), covar_type, n_features=n_features)
assert_array_almost_equal(expected_det, - .5 * np.log(predected_det))
def test_gaussian_suffstat_sk_spherical():
# computing spherical covariance equals to the variance of one-dimension
# data after flattening, n_components=1
rng = np.random.RandomState(0)
n_samples, n_features = 500, 2
X = rng.rand(n_samples, n_features)
X = X - X.mean()
resp = np.ones((n_samples, 1))
nk = np.array([n_samples])
xk = X.mean()
covars_pred_spherical = _estimate_gaussian_covariances_spherical(resp, X, nk, xk, 0)
covars_pred_spherical2 = np.dot(X.flatten().T, X.flatten()) / (
n_features * n_samples
)
assert_almost_equal(covars_pred_spherical, covars_pred_spherical2)
# check the precision computation
precs_chol_pred = _compute_precision_cholesky(covars_pred_spherical, "spherical")
assert_almost_equal(covars_pred_spherical, 1.0 / precs_chol_pred**2)
def _set_gmm(self):
"""
Make a GMM object for sampling
"""
from sklearn.mixture._gaussian_mixture import (
_compute_precision_cholesky)
# these numbers are not used because we set the means, etc by hand
ngauss = self.weights.size
gmm = self._make_gmm(ngauss)
gmm.means_ = self.means.copy()
gmm.covariances_ = self.covars.copy()
gmm.weights_ = self.weights.copy()
gmm.precisions_cholesky_ = _compute_precision_cholesky(
self.covars,
"full",
)
self._gmm = gmm
def get_best_gmm(self, data, lims=[1, 10]):
lowest_bic = np.infty
bic = []
aic = []
# minim = False
# minim_flag = 2
n_components_range = range(lims[0], lims[1] + 1, 1)
for n_components in n_components_range:
# Fit a mixture of Gaussians with EM, beware for cases where te model is not found in any case
gmm = GMM(n_components=n_components,
covariance_type='full')
gmm.fit(data)
bic.append(gmm.bic(data))
aic.append(gmm.aic(data))
if bic[-1] < lowest_bic:
lowest_bic = bic[-1]
best_gmm = n_components
try:
if (bic[-1] > bic[-2] > bic[-3] and
bic[-3] < bic[-4] < bic[-5]):
best_gmm = n_components - 2
break
except IndexError:
pass
# if best_gmm <= 6: # The derivative does not make sense here #THS MUST BE CHECKED
best_gmm = np.array(bic).argmin() + lims[0]
gmm = GMM(n_components=best_gmm,
covariance_type='full')
gmm.fit(data)
self.weights_ = gmm.weights_
self.covariances_ = gmm.covariances_ # self.covariances_ = gmm._get_covars()
self.means_ = gmm.means_
self.n_components = gmm.n_components
self.precisions_cholesky_ = _compute_precision_cholesky(gmm.covariances_, "full")
def get_3d_grid_gmm(subdivisions=[5,5,5], variance=0.04):
"""
Compute the weight, mean and covariance of a gmm placed on a 3D grid
:param subdivisions: 2 element list of number of subdivisions of the 3D space in each axes to form the grid
:param variance: scalar for spherical gmm.p
:return gmm: gmm: instance of sklearn GaussianMixture (GMM) object Gauassian mixture model
"""
# n_gaussians = reduce(lambda x, y: x*y,subdivisions)
n_gaussians = np.prod(np.array(subdivisions))
step = [1.0/(subdivisions[0]), 1.0/(subdivisions[1]), 1.0/(subdivisions[2])]
means = np.mgrid[ step[0]-1: 1.0-step[0]: complex(0, subdivisions[0]),
step[1]-1: 1.0-step[1]: complex(0, subdivisions[1]),
step[2]-1: 1.0-step[2]: complex(0, subdivisions[2])]
means = np.reshape(means, [3, -1]).T
covariances = variance*np.ones_like(means)
weights = (1.0/n_gaussians)*np.ones(n_gaussians)
gmm = GaussianMixture(n_components=n_gaussians, covariance_type='diag')
gmm.weights_ = weights
gmm.covariances_ = covariances
gmm.means_ = means
gmm.precisions_cholesky_ = _compute_precision_cholesky(covariances, 'diag')
return gmm
def test_suffstat_sk_diag():
# test against 'full' case
rng = np.random.RandomState(0)
n_samples, n_features, n_components = 500, 2, 2
resp = rng.rand(n_samples, n_components)
resp = resp / resp.sum(axis=1)[:, np.newaxis]
X = rng.rand(n_samples, n_features)
nk = resp.sum(axis=0)
xk = np.dot(resp.T, X) / nk[:, np.newaxis]
covars_pred_full = _estimate_gaussian_covariances_full(resp, X, nk, xk, 0)
covars_pred_diag = _estimate_gaussian_covariances_diag(resp, X, nk, xk, 0)
ecov = EmpiricalCovariance()
for (cov_full, cov_diag) in zip(covars_pred_full, covars_pred_diag):
ecov.covariance_ = np.diag(np.diag(cov_full))
cov_diag = np.diag(cov_diag)
assert_almost_equal(ecov.error_norm(cov_diag, norm='frobenius'), 0)
assert_almost_equal(ecov.error_norm(cov_diag, norm='spectral'), 0)
# check the precision computation
precs_chol_pred = _compute_precision_cholesky(covars_pred_diag, 'diag')
assert_almost_equal(covars_pred_diag, 1. / precs_chol_pred**2)
def get_means_resp(X,log_resp, cov):
_, means_, covariances_ = _gm._estimate_gaussian_parameters(X, np.exp(log_resp), 1e-6, cov)
precisions_cholesky_ = _gm._compute_precision_cholesky( covariances_, cov)
log_resp = _gm._estimate_log_gaussian_prob( X, means_, precisions_cholesky_,cov)
return means_, log_resp
cov_init_grav = np.array([gmmref.covariances_[:, 0]]).reshape((3, 1, 1))
clfgrav = utils.pgi_utils.GaussianMixture(
n_components=3,
means_init=means_init_grav,
precisions_init=cov_init_grav,
n_init=1,
max_iter=2,
tol=np.inf,
)
# random fit, we set values after.
clfgrav.fit(np.random.randn(10, 1))
clfgrav.means_ = means_init_grav
clfgrav.covariances_ = cov_init_grav
from sklearn.mixture._gaussian_mixture import _compute_precision_cholesky
clfgrav.precisions_cholesky_ = _compute_precision_cholesky(
clfgrav.covariances_, clfgrav.covariance_type)
clfgrav.weights_ = gmmref.weights_
testXplot_grav = np.linspace(-1.2, 0.1, 1000)[:, np.newaxis]
score_grav = clfgrav.score_samples(testXplot_grav)
ax2.plot(
testXplot_grav,
np.exp(score_grav),
linewidth=3.0,
label="1D Probability Density Distribution",
c="k",
)
ax2.set_ylim([0.0, 2])
ax2.legend(fontsize=ticksize)
# create the 1D GMM profile for mag. susc.
means_init_mag = gmmref.means_[:, 1].reshape(3, 1)
def fit(self, data):
if (len(data) <= 20):
return 0
best_gmm = self.trainBestModel(data)
if (self.initialized == False):
self.weights_ = best_gmm.weights_
self.covariances_ = best_gmm.covariances_ # self.covariances_ = gmm._get_covars()
self.means_ = best_gmm.means_
self.n_components = best_gmm.n_components
self.precisions_cholesky_ = _compute_precision_cholesky(
best_gmm.covariances_, "full")
self.initialized = True
logging.debug(
f'TRAIN AWAL component : {best_gmm.n_components} \t W: {best_gmm.weights_} '
)
else:
w_all = np.concatenate((self.weights_, best_gmm.weights_),
axis=None)
mu_all = np.concatenate((self.means_, best_gmm.means_), axis=0)
cov_all = np.concatenate(
(self.covariances_, best_gmm.covariances_), axis=0)
n_components_range = range(
self.n_components + best_gmm.n_components,
self.n_components - 1, -1)
#logging.debug(f'Search from:{n_components_range}')
bicreduced = []
lowest_bic = np.infty
jumlahSample = 5 * len(data)
currentSample = self.sample(2 * jumlahSample)[0]
dataxx = np.concatenate((currentSample, data), axis=0)
for n_components in n_components_range:
#print(n_components)
w, m, c = self.mixture_reduction(w_all,
mu_all,
cov_all,
n_components,
isomorphic=True,
verbose=False,
optimization=False)
gmm_p = GaussianMixture(n_components=n_components,
covariance_type="full")
gmm_p.weights_ = w
gmm_p.covariances_ = c
gmm_p.means_ = m
gmm_p.precisions_cholesky_ = _compute_precision_cholesky(
c, "full")
bic_ = gmm_p.bic(dataxx)
bicreduced.append(bic_)
#print('REDUCD BIC components {0} = {1}'.format(n_components, bic_))
if bic_ < lowest_bic:
lowest_bic = bic_
best_gmm = gmm_p
logging.debug(
f'N_component Awal: {self.n_components} \t Drift Comp: {best_gmm.n_components} '
)
self.weights_ = best_gmm.weights_ / np.sum(best_gmm.weights_)
self.means_ = best_gmm.means_
self.covariances_ = best_gmm.covariances_
self.n_components = best_gmm.n_components
#print("W awal:", self.weights_)
#Compute the Cholesky decomposition of the precisions.
self.prune()
#print("W Prune:", self.weights_)
self.precisions_cholesky_ = _compute_precision_cholesky(
self.covariances_, self.covariance_type)
