def setUp(self): np.random.seed(1) # define phenotype N = 200 P = 2 self.Y = sp.randn(N, P) # define fixed effects self.F = [] self.A = [] self.F.append(1. * (sp.rand(N, 2) < 0.5)) self.A.append(sp.eye(P)) # define row caoriance f = 10 X = 1. * (sp.rand(N, f) < 0.2) self.R = covar_rescale(sp.dot(X, X.T)) self.R += 1e-4 * sp.eye(N) # define col covariances self.Cg = FreeFormCov(P) self.Cn = FreeFormCov(P) self.Cg.setCovariance(0.5 * sp.cov(self.Y.T)) self.Cn.setCovariance(0.5 * sp.cov(self.Y.T)) # define gp self.gp = GP2KronSum(Y=self.Y, F=self.F, A=self.A, Cg=self.Cg, Cn=self.Cn, R=self.R)
def setUp(self): np.random.seed(1) dim_r = 10 dim_c = 3 X = sp.rand(dim_r, dim_r) R = covar_rescale(sp.dot(X,X.T)) C = FreeFormCov(dim_c) self._cov = KronCov(C, R) self._Iok = sp.randn(self._cov.dim)<0.9
def gen_data(N=100, P=4): f = 20 G = 1.*(sp.rand(N, f)<0.2) X = 1.*(sp.rand(N, f)<0.2) R = covar_rescale(sp.dot(X,X.T)) R+= 1e-4 * sp.eye(N) S, U = la.eigh(R) Y = sp.randn(N, P) return Y, S, U, G
def gen_data(N=100, P=4): f = 20 G = 1. * (sp.rand(N, f) < 0.2) X = 1. * (sp.rand(N, f) < 0.2) R = covar_rescale(sp.dot(X, X.T)) R += 1e-4 * sp.eye(N) S, U = la.eigh(R) Y = sp.randn(N, P) return Y, S, U, G
def setUp(self): sp.random.seed(1) # define row caoriance dim_r = 4 X = sp.rand(dim_r, dim_r) self.R = covar_rescale(sp.dot(X,X.T)) # define col covariances dim_c = 2 Cg = FreeFormCov(dim_c) Cn = FreeFormCov(dim_c) self.C = Cov2KronSum(Cg = Cg, Cn = Cn, R = self.R) self.name = 'cov2kronSum' self.C.setRandomParams()
def setUp(self): sp.random.seed(1) # define row caoriance dim_r = 4 rank_r = 2 G = sp.rand(dim_r, rank_r) X = sp.rand(dim_r, dim_r) R = covar_rescale(sp.dot(X, X.T)) # define col covariances dim_c = 2 Cg = FreeFormCov(dim_c) Cn = FreeFormCov(dim_c) self.C = Cov3KronSumLR(Cn=Cn, Cg=Cg, R=R, G=G, rank=1) self.name = 'cov3kronSumLR' self.C.setRandomParams()
def setUp(self): sp.random.seed(1) # define row caoriance dim_r = 4 rank_r = 2 G = sp.rand(dim_r, rank_r) X = sp.rand(dim_r, dim_r) R = covar_rescale(sp.dot(X,X.T)) # define col covariances dim_c = 2 Cg = FreeFormCov(dim_c) Cn = FreeFormCov(dim_c) self.C = Cov3KronSumLR(Cn = Cn, Cg = Cg, R = R, G = G, rank = 1) self.name = 'cov3kronSumLR' self.C.setRandomParams()
def setUp(self): np.random.seed(1) # define phenotype N = 10 P = 3 Y = sp.randn(N,P) # pheno with missing data Ym = Y.copy() Im = sp.rand(N, P)<0.2 Ym[Im] = sp.nan # define fixed effects F = []; A = [] F.append(1.*(sp.rand(N,2)<0.5)) A.append(sp.eye(P)) mean = MeanKronSum(Y, F=F, A=A) mean_m = MeanKronSum(Ym, F=F, A=A) # define row caoriance f = 10 X = 1.*(sp.rand(N, f)<0.2) R = covar_rescale(sp.dot(X,X.T)) R+= 1e-4 * sp.eye(N) # define col covariances Cg = FreeFormCov(P) Cn = FreeFormCov(P) Cg.setRandomParams() Cn.setRandomParams() # define covariance matrices covar1 = KronCov(Cg, R) covar2 = KronCov(Cn, sp.eye(N)) covar = SumCov(covar1,covar2) # define covariance matrice with missing data Iok = (~Im).reshape(N * P, order='F') covar1_m = KronCov(copy.copy(Cg), R, Iok=Iok) covar2_m = KronCov(copy.copy(Cn), sp.eye(N), Iok=Iok) covar_m = SumCov(covar1_m,covar2_m) # define gp self._gp = GP(covar=covar, mean=mean) self._gpm = GP(covar=covar_m, mean=mean_m) self._gp2ks = GP2KronSum(Y=Y, F=F, A=A, Cg=Cg, Cn=Cn, R=R)
def test_too_expensive_exceptions(self): dim_r = 100 rank_r = 2 G = sp.rand(dim_r, rank_r) X = sp.rand(dim_r, dim_r) R = covar_rescale(sp.dot(X,X.T)) dim_c = 5001 Cg = FreeFormCov(dim_c) Cn = FreeFormCov(dim_c) C = Cov3KronSumLR(Cn = Cn, Cg = Cg, R = R, G = G, rank = 1) with self.assertRaises(TooExpensiveOperationError): C.L() with self.assertRaises(TooExpensiveOperationError): C.K() with self.assertRaises(TooExpensiveOperationError): C.K_grad_i(0)
def test_too_expensive_exceptions(self): dim_r = 100 rank_r = 2 G = sp.rand(dim_r, rank_r) X = sp.rand(dim_r, dim_r) R = covar_rescale(sp.dot(X, X.T)) dim_c = 5001 Cg = FreeFormCov(dim_c) Cn = FreeFormCov(dim_c) C = Cov3KronSumLR(Cn=Cn, Cg=Cg, R=R, G=G, rank=1) with self.assertRaises(TooExpensiveOperationError): C.L() with self.assertRaises(TooExpensiveOperationError): C.K() with self.assertRaises(TooExpensiveOperationError): C.K_grad_i(0)
def setUp(self): np.random.seed(1) # define phenotype N = 200 P = 2 Y = sp.randn(N, P) # define row caoriance f = 10 G = 1. * (sp.rand(N, f) < 0.2) X = 1. * (sp.rand(N, f) < 0.2) R = covar_rescale(sp.dot(X, X.T)) R += 1e-4 * sp.eye(N) # define col covariances Cg = FreeFormCov(P) self._Cg = Cg Cn = FreeFormCov(P) Cg.setCovariance(0.5 * sp.cov(Y.T)) Cn.setCovariance(0.5 * sp.cov(Y.T)) # define gp self.gp = GP3KronSumLR(Y=Y, Cg=Cg, Cn=Cn, R=R, G=G, rank=1)
def setUp(self): np.random.seed(1) # define phenotype N = 200 P = 2 Y = sp.randn(N,P) # define row caoriance f = 10 G = 1.*(sp.rand(N, f)<0.2) X = 1.*(sp.rand(N, f)<0.2) R = covar_rescale(sp.dot(X,X.T)) R+= 1e-4 * sp.eye(N) # define col covariances Cg = FreeFormCov(P) self._Cg = Cg Cn = FreeFormCov(P) Cg.setCovariance(0.5 * sp.cov(Y.T)) Cn.setCovariance(0.5 * sp.cov(Y.T)) # define gp self.gp = GP3KronSumLR(Y = Y, Cg = Cg, Cn = Cn, R = R, G = G, rank = 1)
def setUp(self): np.random.seed(1) # define phenotype N = 200 P = 2 self.Y = sp.randn(N, P) # define fixed effects self.F = []; self.A = [] self.F.append(1.*(sp.rand(N,2)<0.5)) self.A.append(sp.eye(P)) # define row caoriance f = 10 X = 1.*(sp.rand(N, f)<0.2) self.R = covar_rescale(sp.dot(X,X.T)) self.R += 1e-4 * sp.eye(N) # define col covariances self.Cg = FreeFormCov(P) self.Cn = FreeFormCov(P) self.Cg.setCovariance(0.5 * sp.cov(self.Y.T)) self.Cn.setCovariance(0.5 * sp.cov(self.Y.T)) # define gp self.gp = GP2KronSum(Y=self.Y, F=self.F, A=self.A, Cg=self.Cg, Cn=self.Cn, R=self.R)
from limix.core.gp import GP3KronSumLR from limix.core.gp import GP from limix.utils.preprocess import covar_rescale import time import copy import pdb if __name__ == '__main__': # define region and bg terms N = 500 f = 10 P = 3 G = 1. * (sp.rand(N, f) < 0.2) X = 1. * (sp.rand(N, f) < 0.2) R = covar_rescale(sp.dot(X, X.T)) R += 1e-4 * sp.eye(N) # define col covariances Cg = FreeFormCov(P) Cn = FreeFormCov(P) Cg.setRandomParams() Cn.setRandomParams() # define pheno Y = sp.randn(N, P) # define GP and optimize gp = GP3KronSumLR(Y=Y, Cg=Cg, Cn=Cn, R=R, G=G, rank=1) gp.optimize()
params0 = {'covar': sp.concatenate([params, params])} else: if self.P == 1: params_cn = sp.array([1.]) else: cov = sp.cov(self.Y.T) + 1e-4 * sp.eye(self.P) chol = la.cholesky(cov, lower=True) params_cn = chol[sp.tril_indices(self.P)] params0 = {'covar': params_cn} return params0 if __name__ == '__main__': from limix.utils.preprocess import covar_rescale sp.random.seed(0) #generate phenotype n = 1000 p = 4 f = 10 Y = sp.randn(n, p) X = sp.randn(n, f) G = sp.randn(n, f) R = sp.dot(X, X.T) R = covar_rescale(R) mts = mtset(Y, R=R) nullMTInfo = mts.fitNull(cache=False) mts.optimize(G)
r+= sp.sum(sp.log(self.Cn.S())) * self.dim_r r+= 2 * sp.log(sp.diag(self.H_chol())).sum() return r @cached(['col_cov', 'row_cov', 'G', 'logdet_grad_i']) def logdet_grad_i(self, i): r = (self.d() * self.diag_Ctilde_o_Sr(i)).sum() r-= (self.H_inv() * self.Kbar(i)).sum() return r if __name__ == '__main__': from limix.core.covar import FreeFormCov from limix.utils.preprocess import covar_rescale # define row caoriance dim_r = 10 X = sp.rand(dim_r, dim_r) R = covar_rescale(sp.dot(X,X.T)) # define col covariances dim_c = 3 Cg = FreeFormCov(dim_c) Cn = FreeFormCov(dim_c) # cov = Cov3KronSum(Cg = Cg, Cn = Cn, R = R) # cov.setRandomParams() # # print cov.K() # print cov.K_grad_i(0)
def standardize_cov(K): from limix.utils.preprocess import covar_rescale return covar_rescale(K)
params0 = {'covar': sp.concatenate([params, params])} else: if self.P==1: params_cn = sp.array([1.]) else: cov = sp.cov(self.Y.T) + 1e-4*sp.eye(self.P) chol = la.cholesky(cov, lower=True) params_cn = chol[sp.tril_indices(self.P)] params0 = {'covar': params_cn} return params0 if __name__=='__main__': from limix.utils.preprocess import covar_rescale sp.random.seed(0) #generate phenotype n = 1000 p = 4 f = 10 Y = sp.randn(n, p) X = sp.randn(n, f) G = sp.randn(n, f) R = sp.dot(X, X.T) R = covar_rescale(R) mts = mtset(Y, R=R) nullMTInfo = mts.fitNull(cache=False) mts.optimize(G)
def VD(self, DGE, IGE, IEE, cageEffect): """ defines covariance for variance decomposition.""" #defines mean mean = lin_mean(self.pheno,self.covs) #define cagemate assignment - required for SGE, SEE, and cage effects. Z is N focal x N_cm and has 0s in cells Z_i,i (i.e. an animal is not its own cage mate) same_cage = 1. * (self.cage==self.cage_cm) diff_inds = 1. * (self.sampleID[:,sp.newaxis]!=self.sampleID_cm) Z = same_cage * diff_inds #define the overall genetic covariance matrix if DGE or IGE: #scales kinship (DGE component) to sample variance 1 sf_K = covar_rescaling_factor(self.kinship) self.kinship *= sf_K #now create and scale SGE and DGE/SGE covariance components if IGE: #first SGE component: ZKcmZ' in this code (ZKZ' in paper) _ZKcmZ = sp.dot(Z,sp.dot(self.kinship_cm,Z.T)) sf_ZKcmZ = covar_rescaling_factor(_ZKcmZ) self.kinship_cm *= sf_ZKcmZ #second DGE/SGE covariance: self.kinship_cross *= sp.sqrt(sf_K * sf_ZKcmZ) if DGE and not IGE: self._genoCov = FixedCov(self.kinship) elif IGE and not DGE: self._genoCov = FixedCov(_ZKcmZ) elif DGE and IGE: self._genoCov = DirIndirCov(self.kinship,Z,kinship_cm=self.kinship_cm,kinship_cross=self.kinship_cross) else: self._genoCov = None #define the overall environmental covariance matrix #there is always DEE #env naturally has sample variance 1 so no need to scale it if IEE: #_ZZ = ZIcmZ' _ZZ = sp.dot(Z,Z.T) sf_ZZ = covar_rescaling_factor(_ZZ) self.env_cm *= sf_ZZ self.env_cross *= sp.sqrt(1 * sf_ZZ) self._envCov = DirIndirCov(self.env,Z,kinship_cm=self.env_cm,kinship_cross=self.env_cross) else: self._envCov = FixedCov(self.env) ##define cage effect covariance matrix if cageEffect: N = self.pheno.shape[0] uCage = sp.unique(self.cage) #W, the cage design matrix, is N x n_cages (where N is number of focal animals) W = sp.zeros((N,uCage.shape[0])) for cv_i, cv in enumerate(uCage): W[:,cv_i] = 1.*(self.cage[:,0]==cv) #WW, the cage effect covariance matrix, is N x N and has 1s in cells WW_i,i WW = sp.dot(W,W.T) #this is equivalent to getting covar_rescaling_factor first and then multiplying, as done for other matrices above WW = covar_rescale(WW) self._cageCov = FixedCov(WW) else: self._cageCov = None # define overall covariance matrix as sum of genetic, environmental and cage covariance matrices if self._genoCov is None: if self._cageCov is None: self.covar = SumCov(self._envCov) else: self.covar = SumCov(self._envCov,self._cageCov) else: if self._cageCov is None: self.covar = SumCov(self._genoCov,self._envCov) else: self.covar = SumCov(self._genoCov,self._envCov,self._cageCov) ## define gp self._gp = GP(covar=self.covar,mean=mean)