def setUp(self):
np.random.seed(1)
# define phenotype
N = 200
P = 2
self.Y = sp.randn(N, P)
# define fixed effects
self.F = []
self.A = []
self.F.append(1. * (sp.rand(N, 2) < 0.5))
self.A.append(sp.eye(P))
# define row caoriance
f = 10
X = 1. * (sp.rand(N, f) < 0.2)
self.R = covar_rescale(sp.dot(X, X.T))
self.R += 1e-4 * sp.eye(N)
# define col covariances
self.Cg = FreeFormCov(P)
self.Cn = FreeFormCov(P)
self.Cg.setCovariance(0.5 * sp.cov(self.Y.T))
self.Cn.setCovariance(0.5 * sp.cov(self.Y.T))
# define gp
self.gp = GP2KronSum(Y=self.Y,
F=self.F,
A=self.A,
Cg=self.Cg,
Cn=self.Cn,
R=self.R)
def setUp(self):
np.random.seed(1)
dim_r = 10
dim_c = 3
X = sp.rand(dim_r, dim_r)
R = covar_rescale(sp.dot(X,X.T))
C = FreeFormCov(dim_c)
self._cov = KronCov(C, R)
self._Iok = sp.randn(self._cov.dim)<0.9
def gen_data(N=100, P=4):
f = 20
G = 1.*(sp.rand(N, f)<0.2)
X = 1.*(sp.rand(N, f)<0.2)
R = covar_rescale(sp.dot(X,X.T))
R+= 1e-4 * sp.eye(N)
S, U = la.eigh(R)
Y = sp.randn(N, P)
return Y, S, U, G
def gen_data(N=100, P=4):
f = 20
G = 1. * (sp.rand(N, f) < 0.2)
X = 1. * (sp.rand(N, f) < 0.2)
R = covar_rescale(sp.dot(X, X.T))
R += 1e-4 * sp.eye(N)
S, U = la.eigh(R)
Y = sp.randn(N, P)
return Y, S, U, G
def setUp(self):
sp.random.seed(1)
# define row caoriance
dim_r = 4
X = sp.rand(dim_r, dim_r)
self.R = covar_rescale(sp.dot(X,X.T))
# define col covariances
dim_c = 2
Cg = FreeFormCov(dim_c)
Cn = FreeFormCov(dim_c)
self.C = Cov2KronSum(Cg = Cg, Cn = Cn, R = self.R)
self.name = 'cov2kronSum'
self.C.setRandomParams()
def setUp(self):
sp.random.seed(1)
# define row caoriance
dim_r = 4
X = sp.rand(dim_r, dim_r)
self.R = covar_rescale(sp.dot(X,X.T))
# define col covariances
dim_c = 2
Cg = FreeFormCov(dim_c)
Cn = FreeFormCov(dim_c)
self.C = Cov2KronSum(Cg = Cg, Cn = Cn, R = self.R)
self.name = 'cov2kronSum'
self.C.setRandomParams()
def setUp(self):
sp.random.seed(1)
# define row caoriance
dim_r = 4
rank_r = 2
G = sp.rand(dim_r, rank_r)
X = sp.rand(dim_r, dim_r)
R = covar_rescale(sp.dot(X, X.T))
# define col covariances
dim_c = 2
Cg = FreeFormCov(dim_c)
Cn = FreeFormCov(dim_c)
self.C = Cov3KronSumLR(Cn=Cn, Cg=Cg, R=R, G=G, rank=1)
self.name = 'cov3kronSumLR'
self.C.setRandomParams()
def setUp(self):
sp.random.seed(1)
# define row caoriance
dim_r = 4
rank_r = 2
G = sp.rand(dim_r, rank_r)
X = sp.rand(dim_r, dim_r)
R = covar_rescale(sp.dot(X,X.T))
# define col covariances
dim_c = 2
Cg = FreeFormCov(dim_c)
Cn = FreeFormCov(dim_c)
self.C = Cov3KronSumLR(Cn = Cn, Cg = Cg, R = R, G = G, rank = 1)
self.name = 'cov3kronSumLR'
self.C.setRandomParams()
def setUp(self):
np.random.seed(1)
# define phenotype
N = 10
P = 3
Y = sp.randn(N,P)
# pheno with missing data
Ym = Y.copy()
Im = sp.rand(N, P)<0.2
Ym[Im] = sp.nan
# define fixed effects
F = []; A = []
F.append(1.*(sp.rand(N,2)<0.5))
A.append(sp.eye(P))
mean = MeanKronSum(Y, F=F, A=A)
mean_m = MeanKronSum(Ym, F=F, A=A)
# define row caoriance
f = 10
X = 1.*(sp.rand(N, f)<0.2)
R = covar_rescale(sp.dot(X,X.T))
R+= 1e-4 * sp.eye(N)
# define col covariances
Cg = FreeFormCov(P)
Cn = FreeFormCov(P)
Cg.setRandomParams()
Cn.setRandomParams()
# define covariance matrices
covar1 = KronCov(Cg, R)
covar2 = KronCov(Cn, sp.eye(N))
covar = SumCov(covar1,covar2)
# define covariance matrice with missing data
Iok = (~Im).reshape(N * P, order='F')
covar1_m = KronCov(copy.copy(Cg), R, Iok=Iok)
covar2_m = KronCov(copy.copy(Cn), sp.eye(N), Iok=Iok)
covar_m = SumCov(covar1_m,covar2_m)
# define gp
self._gp = GP(covar=covar, mean=mean)
self._gpm = GP(covar=covar_m, mean=mean_m)
self._gp2ks = GP2KronSum(Y=Y, F=F, A=A, Cg=Cg, Cn=Cn, R=R)
def test_too_expensive_exceptions(self):
dim_r = 100
rank_r = 2
G = sp.rand(dim_r, rank_r)
X = sp.rand(dim_r, dim_r)
R = covar_rescale(sp.dot(X,X.T))
dim_c = 5001
Cg = FreeFormCov(dim_c)
Cn = FreeFormCov(dim_c)
C = Cov3KronSumLR(Cn = Cn, Cg = Cg, R = R, G = G, rank = 1)
with self.assertRaises(TooExpensiveOperationError):
C.L()
with self.assertRaises(TooExpensiveOperationError):
C.K()
with self.assertRaises(TooExpensiveOperationError):
C.K_grad_i(0)
def test_too_expensive_exceptions(self):
dim_r = 100
rank_r = 2
G = sp.rand(dim_r, rank_r)
X = sp.rand(dim_r, dim_r)
R = covar_rescale(sp.dot(X, X.T))
dim_c = 5001
Cg = FreeFormCov(dim_c)
Cn = FreeFormCov(dim_c)
C = Cov3KronSumLR(Cn=Cn, Cg=Cg, R=R, G=G, rank=1)
with self.assertRaises(TooExpensiveOperationError):
C.L()
with self.assertRaises(TooExpensiveOperationError):
C.K()
with self.assertRaises(TooExpensiveOperationError):
C.K_grad_i(0)
def setUp(self):
np.random.seed(1)
# define phenotype
N = 200
P = 2
Y = sp.randn(N, P)
# define row caoriance
f = 10
G = 1. * (sp.rand(N, f) < 0.2)
X = 1. * (sp.rand(N, f) < 0.2)
R = covar_rescale(sp.dot(X, X.T))
R += 1e-4 * sp.eye(N)
# define col covariances
Cg = FreeFormCov(P)
self._Cg = Cg
Cn = FreeFormCov(P)
Cg.setCovariance(0.5 * sp.cov(Y.T))
Cn.setCovariance(0.5 * sp.cov(Y.T))
# define gp
self.gp = GP3KronSumLR(Y=Y, Cg=Cg, Cn=Cn, R=R, G=G, rank=1)
def setUp(self):
np.random.seed(1)
# define phenotype
N = 200
P = 2
Y = sp.randn(N,P)
# define row caoriance
f = 10
G = 1.*(sp.rand(N, f)<0.2)
X = 1.*(sp.rand(N, f)<0.2)
R = covar_rescale(sp.dot(X,X.T))
R+= 1e-4 * sp.eye(N)
# define col covariances
Cg = FreeFormCov(P)
self._Cg = Cg
Cn = FreeFormCov(P)
Cg.setCovariance(0.5 * sp.cov(Y.T))
Cn.setCovariance(0.5 * sp.cov(Y.T))
# define gp
self.gp = GP3KronSumLR(Y = Y, Cg = Cg, Cn = Cn, R = R, G = G, rank = 1)
def setUp(self):
np.random.seed(1)
# define phenotype
N = 200
P = 2
self.Y = sp.randn(N, P)
# define fixed effects
self.F = []; self.A = []
self.F.append(1.*(sp.rand(N,2)<0.5))
self.A.append(sp.eye(P))
# define row caoriance
f = 10
X = 1.*(sp.rand(N, f)<0.2)
self.R = covar_rescale(sp.dot(X,X.T))
self.R += 1e-4 * sp.eye(N)
# define col covariances
self.Cg = FreeFormCov(P)
self.Cn = FreeFormCov(P)
self.Cg.setCovariance(0.5 * sp.cov(self.Y.T))
self.Cn.setCovariance(0.5 * sp.cov(self.Y.T))
# define gp
self.gp = GP2KronSum(Y=self.Y, F=self.F, A=self.A, Cg=self.Cg,
Cn=self.Cn, R=self.R)
from limix.core.gp import GP3KronSumLR
from limix.core.gp import GP
from limix.utils.preprocess import covar_rescale
import time
import copy
import pdb
if __name__ == '__main__':
# define region and bg terms
N = 500
f = 10
P = 3
G = 1. * (sp.rand(N, f) < 0.2)
X = 1. * (sp.rand(N, f) < 0.2)
R = covar_rescale(sp.dot(X, X.T))
R += 1e-4 * sp.eye(N)
# define col covariances
Cg = FreeFormCov(P)
Cn = FreeFormCov(P)
Cg.setRandomParams()
Cn.setRandomParams()
# define pheno
Y = sp.randn(N, P)
# define GP and optimize
gp = GP3KronSumLR(Y=Y, Cg=Cg, Cn=Cn, R=R, G=G, rank=1)
gp.optimize()
params0 = {'covar': sp.concatenate([params, params])}
else:
if self.P == 1:
params_cn = sp.array([1.])
else:
cov = sp.cov(self.Y.T) + 1e-4 * sp.eye(self.P)
chol = la.cholesky(cov, lower=True)
params_cn = chol[sp.tril_indices(self.P)]
params0 = {'covar': params_cn}
return params0
if __name__ == '__main__':
from limix.utils.preprocess import covar_rescale
sp.random.seed(0)
#generate phenotype
n = 1000
p = 4
f = 10
Y = sp.randn(n, p)
X = sp.randn(n, f)
G = sp.randn(n, f)
R = sp.dot(X, X.T)
R = covar_rescale(R)
mts = mtset(Y, R=R)
nullMTInfo = mts.fitNull(cache=False)
mts.optimize(G)
r+= sp.sum(sp.log(self.Cn.S())) * self.dim_r
r+= 2 * sp.log(sp.diag(self.H_chol())).sum()
return r
@cached(['col_cov', 'row_cov', 'G', 'logdet_grad_i'])
def logdet_grad_i(self, i):
r = (self.d() * self.diag_Ctilde_o_Sr(i)).sum()
r-= (self.H_inv() * self.Kbar(i)).sum()
return r
if __name__ == '__main__':
from limix.core.covar import FreeFormCov
from limix.utils.preprocess import covar_rescale
# define row caoriance
dim_r = 10
X = sp.rand(dim_r, dim_r)
R = covar_rescale(sp.dot(X,X.T))
# define col covariances
dim_c = 3
Cg = FreeFormCov(dim_c)
Cn = FreeFormCov(dim_c)
# cov = Cov3KronSum(Cg = Cg, Cn = Cn, R = R)
# cov.setRandomParams()
#
# print cov.K()
# print cov.K_grad_i(0)
def standardize_cov(K):
from limix.utils.preprocess import covar_rescale
return covar_rescale(K)
params0 = {'covar': sp.concatenate([params, params])}
else:
if self.P==1:
params_cn = sp.array([1.])
else:
cov = sp.cov(self.Y.T) + 1e-4*sp.eye(self.P)
chol = la.cholesky(cov, lower=True)
params_cn = chol[sp.tril_indices(self.P)]
params0 = {'covar': params_cn}
return params0
if __name__=='__main__':
from limix.utils.preprocess import covar_rescale
sp.random.seed(0)
#generate phenotype
n = 1000
p = 4
f = 10
Y = sp.randn(n, p)
X = sp.randn(n, f)
G = sp.randn(n, f)
R = sp.dot(X, X.T)
R = covar_rescale(R)
mts = mtset(Y, R=R)
nullMTInfo = mts.fitNull(cache=False)
mts.optimize(G)
def VD(self, DGE, IGE, IEE, cageEffect):
""" defines covariance for variance decomposition."""
#defines mean
mean = lin_mean(self.pheno,self.covs)
#define cagemate assignment - required for SGE, SEE, and cage effects. Z is N focal x N_cm and has 0s in cells Z_i,i (i.e. an animal is not its own cage mate)
same_cage = 1. * (self.cage==self.cage_cm)
diff_inds = 1. * (self.sampleID[:,sp.newaxis]!=self.sampleID_cm)
Z = same_cage * diff_inds
#define the overall genetic covariance matrix
if DGE or IGE:
#scales kinship (DGE component) to sample variance 1
sf_K = covar_rescaling_factor(self.kinship)
self.kinship *= sf_K
#now create and scale SGE and DGE/SGE covariance components
if IGE:
#first SGE component: ZKcmZ' in this code (ZKZ' in paper)
_ZKcmZ = sp.dot(Z,sp.dot(self.kinship_cm,Z.T))
sf_ZKcmZ = covar_rescaling_factor(_ZKcmZ)
self.kinship_cm *= sf_ZKcmZ
#second DGE/SGE covariance:
self.kinship_cross *= sp.sqrt(sf_K * sf_ZKcmZ)
if DGE and not IGE:
self._genoCov = FixedCov(self.kinship)
elif IGE and not DGE:
self._genoCov = FixedCov(_ZKcmZ)
elif DGE and IGE:
self._genoCov = DirIndirCov(self.kinship,Z,kinship_cm=self.kinship_cm,kinship_cross=self.kinship_cross)
else:
self._genoCov = None
#define the overall environmental covariance matrix
#there is always DEE
#env naturally has sample variance 1 so no need to scale it
if IEE:
#_ZZ = ZIcmZ'
_ZZ = sp.dot(Z,Z.T)
sf_ZZ = covar_rescaling_factor(_ZZ)
self.env_cm *= sf_ZZ
self.env_cross *= sp.sqrt(1 * sf_ZZ)
self._envCov = DirIndirCov(self.env,Z,kinship_cm=self.env_cm,kinship_cross=self.env_cross)
else:
self._envCov = FixedCov(self.env)
##define cage effect covariance matrix
if cageEffect:
N = self.pheno.shape[0]
uCage = sp.unique(self.cage)
#W, the cage design matrix, is N x n_cages (where N is number of focal animals)
W = sp.zeros((N,uCage.shape[0]))
for cv_i, cv in enumerate(uCage):
W[:,cv_i] = 1.*(self.cage[:,0]==cv)
#WW, the cage effect covariance matrix, is N x N and has 1s in cells WW_i,i
WW = sp.dot(W,W.T)
#this is equivalent to getting covar_rescaling_factor first and then multiplying, as done for other matrices above
WW = covar_rescale(WW)
self._cageCov = FixedCov(WW)
else:
self._cageCov = None
# define overall covariance matrix as sum of genetic, environmental and cage covariance matrices
if self._genoCov is None:
if self._cageCov is None:
self.covar = SumCov(self._envCov)
else:
self.covar = SumCov(self._envCov,self._cageCov)
else:
if self._cageCov is None:
self.covar = SumCov(self._genoCov,self._envCov)
else:
self.covar = SumCov(self._genoCov,self._envCov,self._cageCov)
## define gp
self._gp = GP(covar=self.covar,mean=mean)
