def test_generate_data(self):
if self.comm.Get_rank() == 0:
self.mu = [np.repeat(np.double(k), self.d) for k in range(self.K)]
r = np.zeros(self.d)
r[0] = 3.
r[1] = -1
self.sigma = [spl.toeplitz(r) for k in range(self.K)]
self.data_sigma = [[
invwishartrand(5, self.sigma[k]) for k in range(self.K)
] for n in range(self.n)] # @UnusedVariable
self.data_mu = [[
self.mu[k] + np.random.randn(self.d) * 0.2
for k in range(self.K)
] for n in range(self.n)] # @UnusedVariable
self.Y = []
for i in range(self.n):
mix = mixture(K=self.K)
mix.mu = self.data_mu[i]
mix.sigma = self.data_sigma[i]
mix.p = np.ones(self.K) / self.K
mix.d = self.d
self.Y.append(mix.simulate_data(self.n_obs))
else:
self.Y = None
self.hGMM.set_data(self.Y)
self.hGMM.set_prior_param0()
def simulate_data(nCells = 5*10**4, nPersons = 40, seed = 123456, ratio_P = [1., 1., 0.8, 0.1]): """ Simulates the data following the instruction presented in the article """ if seed != None: npr.seed(seed) nClass = 4 dim = 3 P = [0.49, 0.3, 0.2 , 0.01 ] Thetas = [np.array([0.,0, 0]), np.array([0, -2, 1]), np.array([1., 2, 0]), np.array([-2,2,1.5])] Z_Sigma = [np.array([[1.27, 0.25, 0],[0.25, 0.27, -0.001],[0., -0.001, 0.001]]), np.array([[0.06, 0.04, -0.03],[0.04, 0.05, 0],[-0.03, 0., 0.09]]), np.array([[0.44, 0.08, 0.08],[0.08, 0.16, 0],[0.08, 0., 0.16]]), 0.01*np.eye(3)] Sigmas = [0.1*np.eye(3), 0.1*spl.toeplitz([2.,0.5,0]),0.1* spl.toeplitz([2.,-0.5,1]), 0.1*spl.toeplitz([1.,.3,.3]) ] act_Class = np.zeros((nPersons,4)) for i in range(nClass): act_Class[:np.ceil(nPersons*ratio_P[i]),i] = 1. Y = [] nu = 100 mus = [] for i in range(nPersons): mix_obj = GMM.mixture(K = np.int(np.sum(act_Class[i, :]))) theta_temp = [] sigma_temp = [] for j in range(nClass): if act_Class[i, j] == 1: theta_temp.append(Thetas[j] + npr.multivariate_normal(np.zeros(3), Z_Sigma[j])) sigma_temp.append(wishart.invwishartrand(nu, (nu - dim - 1)* Sigmas[j])) else: theta_temp.append(np.ones(dim)*np.NAN) sigma_temp.append(np.ones((dim,dim))*np.NAN) theta_temp_ = [ theta_temp[aC] for aC in np.where(act_Class[i, :] == 1)[0]] sigma_temp_ = [ sigma_temp[aC] for aC in np.where(act_Class[i, :] == 1)[0]] mix_obj.mu = theta_temp_ mus.append(theta_temp) mix_obj.sigma = sigma_temp_ p_ = np.array([ (0.2*np.random.rand()+0.9) * P[aC] for aC in np.where(act_Class[i, :] == 1)[0]] ) p_ /= np.sum(p_) mix_obj.p = p_ mix_obj.d = dim Y.append(mix_obj.simulate_data(nCells)) mus = np.array(mus) return Y, act_Class, mus.T, Thetas, Sigmas, P
def test_simulate(self):
self.startup2()
row = np.zeros(self.d)
row[0] = 5.
row[1] = -1
row[2] = 2.
Sigma = spl.toeplitz(row)
Y = []
for i in range(self.sim): # @UnusedVariable
Y.append(invwishartrand(10, Sigma))
self.dist.set_data(Y)
self.dist.sample()
def test_simulate(self): self.startup2() row = np.zeros(self.d) row[0] = 5. row[1] = -1 row[2] = 2. Sigma = spl.toeplitz(row) Y = [] for i in range(self.sim): # @UnusedVariable Y.append( invwishartrand(10, Sigma)) self.dist.set_data(Y) self.dist.sample()
def sample_data(self): self.data = [] r = np.zeros(self.d) nu =10. r[0] = 5. r[1] = -0.5 r[2] = 1. self.Sigma = spl.toeplitz(r)/nu for i in range(self.sim): # @UnusedVariable self.data.append( invwishartrand(nu, self.Sigma)) self.dist.set_data(self.data) Q = np.zeros((self.d, self.d)) for i in range(self.sim): Q += self.data[i]
def setdata(self): self.mu = [np.repeat(np.double(k),self.d) for k in range(self.K)] self.r1 = np.zeros(self.d) self.r1[0] = 3. self.r1[1] = -1 self.r1[2] = 1 self.r1[2] = .5 self.r2 = np.zeros(self.d) self.r2[0] = 2. self.r2[1] = -1.5 self.r2[2] = 0 self.r2[2] = 0. self.sigma =[np.eye(self.d), spl.toeplitz(self.r1), spl.toeplitz(self.r1), 0.5*np.eye(self.d)] self.p = np.array([1.,1.,1.,0.1]) self.p2 = np.array([1.,1.,1.]) self.p /= np.sum(self.p) self.p2 /= np.sum(self.p2) self.data_sigma = [ [invwishartrand(5, self.sigma[k]) for k in range(self.K)] for n in range(self.n_y)] # @UnusedVariable self.data_mu = [ [self.mu[k]+ np.random.randn(self.d)*0.2 for k in range(self.K)] for n in range(self.n_y)] # @UnusedVariable self.Y = [] for i in range(self.n_y): if i < 17: mix = mixture(K = self.K) mix.mu = self.data_mu[i] mix.sigma = self.data_sigma[i] mix.p = self.p else: mix = mixture(K = self.K-1) mix.mu = self.data_mu[i][:-1] mix.sigma = self.data_sigma[i][:-1] mix.p = self.p2 mix.d = self.d self.Y.append(mix.simulate_data(self.n_obs)) self.hGMM.set_data(self.Y) self.hGMM.set_prior_param0() self.hGMM.set_p_labelswitch(0.5) self.hGMM.set_p_activation([0,0])
def setdata(self): if MPI.COMM_WORLD.Get_rank() == 0: # @UndefinedVariable self.mu = [np.repeat(np.double(k),self.d) for k in range(self.K)] self.r1 = np.zeros(self.d) self.r1[0] = 3. self.r1[1] = -1 self.r1[2] = 1 self.r1[2] = .5 self.r2 = np.zeros(self.d) self.r2[0] = 2. self.r2[1] = -1.5 self.r2[2] = 0 self.r2[2] = 0. self.sigma =[np.eye(self.d), spl.toeplitz(self.r1), spl.toeplitz(self.r1), 0.5*np.eye(self.d)] self.p = np.array([1.,1.,1.,0.1]) self.p2 = np.array([1.,1.,1.]) self.p /= np.sum(self.p) self.p2 /= np.sum(self.p2) self.data_sigma = [ [invwishartrand(5, self.sigma[k]) for k in range(self.K)] for n in range(self.n_y)] # @UnusedVariable self.data_mu = [ [self.mu[k]+ np.random.randn(self.d)*0.2 for k in range(self.K)] for n in range(self.n_y)] # @UnusedVariable self.Y = [] for i in range(self.n_y): if i < 17: mix = mixture(K = self.K) mix.mu = self.data_mu[i] mix.sigma = self.data_sigma[i] mix.p = self.p else: mix = mixture(K = self.K-1) mix.mu = self.data_mu[i][:-1] mix.sigma = self.data_sigma[i][:-1] mix.p = self.p2 mix.d = self.d self.Y.append(mix.simulate_data(self.n_obs)) else: self.Y = None
def generate_data(n_obs, K):
d = 4
n_y = 20
mu = [np.repeat(np.double(k), d) for k in range(K)]
r1 = np.zeros(d)
r1[0] = 3.
r1[1] = -1
r1[2] = 1
r1[2] = .5
r2 = np.zeros(d)
r2[0] = 2.
r2[1] = -1.5
r2[2] = 0
r2[2] = 0.
sigma = [np.eye(d), spl.toeplitz(r1), spl.toeplitz(r1), 0.5 * np.eye(d)]
p = np.array([1., 1., 1., 0.1])
p2 = np.array([1., 1., 1.])
p /= np.sum(p)
p2 /= np.sum(p2)
data_sigma = [[invwishartrand(10 + d, sigma[k]) for k in range(K)]
for n in range(n_y)] # @UnusedVariable
data_mu = [[mu[k] + np.random.randn(d) * 0.005 for k in range(K)]
for n in range(n_y)] # @UnusedVariable
Y = []
for i in range(n_y):
if i < 17:
mix = mixture(K=K)
mix.mu = data_mu[i]
mix.sigma = data_sigma[i]
mix.p = p
else:
mix = mixture(K=K - 1)
mix.mu = data_mu[i][:-1]
mix.sigma = data_sigma[i][:-1]
mix.p = p2
mix.d = d
Y.append(mix.simulate_data(n_obs))
return Y, data_sigma, data_mu
def generate_data( n_obs, K): d = 4 n_y = 20 mu = [np.repeat(np.double(k),d) for k in range(K)] r1 = np.zeros(d) r1[0] = 3. r1[1] = -1 r1[2] = 1 r1[2] = .5 r2 = np.zeros(d) r2[0] = 2. r2[1] = -1.5 r2[2] = 0 r2[2] = 0. sigma =[np.eye(d), spl.toeplitz(r1), spl.toeplitz(r1), 0.5*np.eye(d)] p = np.array([1.,1.,1.,0.1]) p2 = np.array([1.,1.,1.]) p /= np.sum(p) p2 /= np.sum(p2) data_sigma = [ [invwishartrand(10+d, sigma[k]) for k in range(K)] for n in range(n_y)] # @UnusedVariable data_mu = [ [mu[k]+ np.random.randn(d)*0.005 for k in range(K)] for n in range(n_y)] # @UnusedVariable Y = [] for i in range(n_y): if i < 17: mix = mixture(K = K) mix.mu = data_mu[i] mix.sigma = data_sigma[i] mix.p = p else: mix = mixture(K = K-1) mix.mu = data_mu[i][:-1] mix.sigma = data_sigma[i][:-1] mix.p = p2 mix.d = d Y.append(mix.simulate_data(n_obs)) return Y, data_sigma, data_mu
def test_generate_data(self): if self.comm.Get_rank() == 0: self.mu = [np.repeat(np.double(k),self.d) for k in range(self.K)] r = np.zeros(self.d) r[0] = 3. r[1] = -1 self.sigma = [spl.toeplitz(r) for k in range(self.K)] self.data_sigma = [ [invwishartrand(5, self.sigma[k]) for k in range(self.K)] for n in range(self.n)] # @UnusedVariable self.data_mu = [ [self.mu[k]+ np.random.randn(self.d)*0.2 for k in range(self.K)] for n in range(self.n)] # @UnusedVariable self.Y = [] for i in range(self.n): mix = mixture(K = self.K) mix.mu = self.data_mu[i] mix.sigma = self.data_sigma[i] mix.p = np.ones(self.K) / self.K mix.d = self.d self.Y.append(mix.simulate_data(self.n_obs)) else: self.Y = None self.hGMM.set_data(self.Y) self.hGMM.set_prior_param0()
def simulate_data_(thetas,
sigma_theta,
sigmas,
weights,
nu=100,
ratio_act=None,
n_cells=5 * 10**4,
n_persons=40,
seed=None,
silent=True):
"""
simulating data given:
*thetas* list of latent means
*sigma_theta* variation between the means
*sigmas* list of latent covariances
*weights* list of probabilites
*nu* inverse wishart parameter
*ratio_act* probabilility that the cluster is active at a person
*n_cells* number of cells at a person
*n_persons* number of persons
*seed* random number generator
"""
if seed is None:
npr.seed(seed)
K = len(weights)
dim = thetas[0].shape[0]
if ratio_act is None:
ratio_act = np.ones(K)
act_class = np.zeros((n_persons, K))
for i in range(K):
act_class[:np.int(np.ceil(n_persons * ratio_act[i])), i] = 1.
Y = []
x = []
nu = 100
mus = []
for i in range(n_persons):
if not silent:
print("setting up person_{i}: ".format(i=i), end='')
sys.stdout.flush()
mix_obj = GMM.mixture(K=np.int(np.sum(act_class[i, :])))
theta_temp = []
sigma_temp = []
for j in range(K):
if act_class[i, j] == 1:
theta_temp.append(thetas[j] +
util.rmvn(np.zeros((dim,
1)), sigma_theta[j]))
sigma_temp.append(
wishart.invwishartrand(nu, (nu - dim - 1) * sigmas[j]))
else:
theta_temp.append(np.ones(dim) * np.NAN)
sigma_temp.append(np.ones((dim, dim)) * np.NAN)
theta_temp_ = [
theta_temp[aC] for aC in np.where(act_class[i, :] == 1)[0]
]
sigma_temp_ = [
sigma_temp[aC] for aC in np.where(act_class[i, :] == 1)[0]
]
mix_obj.mu = theta_temp_
mus.append(theta_temp)
mix_obj.sigma = sigma_temp_
p_ = np.array([(0.2 * np.random.rand() + 0.9) * weights[aC]
for aC in np.where(act_class[i, :] == 1)[0]])
p_ /= np.sum(p_)
mix_obj.p = p_
mix_obj.d = dim
#Y_, x_ = mix_obj.simulate_data2(np.int(np.floor(0.99*n_cells)))
Y_, x_ = mix_obj.simulate_data2(n_cells)
noise_variance = np.eye(mix_obj.d)
np.fill_diagonal(noise_variance, np.var(Y_, 0))
#Y_noise = npr.multivariate_normal(np.mean(Y_,0), noise_variance, size = np.int(np.ceil(0.01*n_cells)))
#Y_ = np.vstack((Y_,Y_noise))
#np.random.shuffle(Y_)
Y.append(Y_)
x.append(x_)
if not silent:
print("done")
sys.stdout.flush()
mus = np.array(mus)
return Y, act_class, mus.T, x
def simulate_data_( thetas, sigma_theta, sigmas, weights, nu = 100, ratio_act = None, n_cells = 5*10**4, n_persons = 40,
seed = None, silent = True):
"""
simulating data given:
*thetas* list of latent means
*sigma_theta* variation between the means
*sigmas* list of latent covariances
*weights* list of probabilites
*nu* inverse wishart parameter
*ratio_act* probabilility that the cluster is active at a person
*n_cells* number of cells at a person
*n_persons* number of persons
*seed* random number generator
"""
if seed is None:
npr.seed(seed)
K = len(weights)
dim = thetas[0].shape[0]
if ratio_act is None:
ratio_act = np.ones(K )
act_class = np.zeros((n_persons, K))
for i in range(K):
act_class[:np.int(np.ceil(n_persons * ratio_act[i])), i] = 1.
Y = []
x = []
nu = 100
mus = []
for i in range(n_persons):
if not silent:
print("setting up person_{i}: ".format(i = i),end = '')
sys.stdout.flush()
mix_obj = GMM.mixture(K = np.int(np.sum(act_class[i, :])))
theta_temp = []
sigma_temp = []
for j in range(K):
if act_class[i, j] == 1:
theta_temp.append(thetas[j] + util.rmvn( np.zeros((dim, 1)), sigma_theta[j] ))
sigma_temp.append(wishart.invwishartrand(nu, (nu - dim - 1) * sigmas[j]))
else:
theta_temp.append(np.ones(dim) * np.NAN)
sigma_temp.append(np.ones((dim,dim)) * np.NAN)
theta_temp_ = [ theta_temp[aC] for aC in np.where(act_class[i, :] == 1)[0]]
sigma_temp_ = [ sigma_temp[aC] for aC in np.where(act_class[i, :] == 1)[0]]
mix_obj.mu = theta_temp_
mus.append(theta_temp)
mix_obj.sigma = sigma_temp_
p_ = np.array([ (0.2*np.random.rand()+0.9) * weights[aC] for aC in np.where(act_class[i, :] == 1)[0]] )
p_ /= np.sum(p_)
mix_obj.p = p_
mix_obj.d = dim
#Y_, x_ = mix_obj.simulate_data2(np.int(np.floor(0.99*n_cells)))
Y_, x_ = mix_obj.simulate_data2(n_cells)
noise_variance = np.eye(mix_obj.d)
np.fill_diagonal(noise_variance, np.var(Y_,0))
#Y_noise = npr.multivariate_normal(np.mean(Y_,0), noise_variance, size = np.int(np.ceil(0.01*n_cells)))
#Y_ = np.vstack((Y_,Y_noise))
#np.random.shuffle(Y_)
Y.append(Y_)
x.append(x_)
if not silent:
print("done")
sys.stdout.flush()
mus = np.array(mus)
return Y, act_class, mus.T, x
def simulate_data(nCells=5 * 10**4,
nPersons=40,
seed=123456,
ratio_P=[1., 1., 0.8, 0.1]):
"""
Simulates the data following the instruction presented in the article
"""
if seed != None:
npr.seed(seed)
nClass = 4
dim = 3
P = [0.49, 0.3, 0.2, 0.01]
Thetas = [
np.array([0., 0, 0]),
np.array([0, -2, 1]),
np.array([1., 2, 0]),
np.array([-2, 2, 1.5])
]
Z_Sigma = [
np.array([[1.27, 0.25, 0], [0.25, 0.27, -0.001], [0., -0.001, 0.001]]),
np.array([[0.06, 0.04, -0.03], [0.04, 0.05, 0], [-0.03, 0., 0.09]]),
np.array([[0.44, 0.08, 0.08], [0.08, 0.16, 0], [0.08, 0., 0.16]]),
0.01 * np.eye(3)
]
Sigmas = [
0.1 * np.eye(3), 0.1 * spl.toeplitz([2., 0.5, 0]),
0.1 * spl.toeplitz([2., -0.5, 1]), 0.1 * spl.toeplitz([1., .3, .3])
]
act_Class = np.zeros((nPersons, 4))
for i in range(nClass):
act_Class[:np.ceil(nPersons * ratio_P[i]), i] = 1.
Y = []
nu = 100
mus = []
for i in range(nPersons):
mix_obj = GMM.mixture(K=np.int(np.sum(act_Class[i, :])))
theta_temp = []
sigma_temp = []
for j in range(nClass):
if act_Class[i, j] == 1:
theta_temp.append(
Thetas[j] +
npr.multivariate_normal(np.zeros(3), Z_Sigma[j]))
sigma_temp.append(
wishart.invwishartrand(nu, (nu - dim - 1) * Sigmas[j]))
else:
theta_temp.append(np.ones(dim) * np.NAN)
sigma_temp.append(np.ones((dim, dim)) * np.NAN)
theta_temp_ = [
theta_temp[aC] for aC in np.where(act_Class[i, :] == 1)[0]
]
sigma_temp_ = [
sigma_temp[aC] for aC in np.where(act_Class[i, :] == 1)[0]
]
mix_obj.mu = theta_temp_
mus.append(theta_temp)
mix_obj.sigma = sigma_temp_
p_ = np.array([(0.2 * np.random.rand() + 0.9) * P[aC]
for aC in np.where(act_Class[i, :] == 1)[0]])
p_ /= np.sum(p_)
mix_obj.p = p_
mix_obj.d = dim
Y.append(mix_obj.simulate_data(nCells))
mus = np.array(mus)
return Y, act_Class, mus.T, Thetas, Sigmas, P