def test_lmm2(self): """another test, establishing an lmm-equivalent by a design matrix choice""" for dn in self.datasets: D = data.load(os.path.join(self.dir_name,dn)) #construct Kronecker LMM model which has the special case of standard LMM #covar1: genotype matrix N = D['K'].shape[0] P = 3 K1r = D['K'] #K1c = SP.zeros([2,2]) #K1c[0,0] = 1 K1c = SP.eye(P) K2r = SP.eye(N) K2c = SP.eye(P) #A = SP.zeros([1,2]) #A[0,0] =1 A = SP.eye(P) Acov = SP.eye(P) Xcov = D['Cov'][:,SP.newaxis] X = D['X'] Y = D['Y'][:,SP.newaxis] Y = SP.tile(Y,(1,P)) lmm = limix.CKroneckerLMM() lmm.setK1r(K1r) lmm.setK1c(K1c) lmm.setK2r(K2r) lmm.setK2c(K2c) lmm.setSNPs(X) #add covariates lmm.addCovariates(Xcov,Acov) #add SNP design lmm.setSNPcoldesign(A) lmm.setPheno(Y) lmm.setNumIntervalsAlt(0) lmm.setNumIntervals0(100) lmm.process() #get p-values with P-dof: pv_Pdof = lmm.getPv().ravel() #transform in P-values with a single DOF: import scipy.stats as st lrt = st.chi2.isf(pv_Pdof,P)/P pv = st.chi2.sf(lrt,1) #compare with single DOF P-values: D2= ((SP.log10(pv)-SP.log10(D['pv']))**2) RV = SP.sqrt(D2.mean()) #print "\n" #print pv[0:10] #print D['pv'][0:10] #print RV #pdb.set_trace() self.assertTrue(RV<1E-6)
def test_lmm(self): """basic test, comparing pv to a standard LMM equivalent""" for dn in self.datasets: D = data.load(os.path.join(self.dir_name,dn)) #construct Kronecker LMM model which has the special case of standard LMM #covar1: genotype matrix K1r = D['K'] K1c = SP.eye(1) K2r = SP.eye(D['K'].shape[0]) K2c = SP.eye(1) A = SP.eye(1) Acov = SP.eye(1) Xcov = D['Cov'][:,SP.newaxis] X = D['X'] Y = D['Y'][:,SP.newaxis] lmm = limix.CKroneckerLMM() lmm.setK1r(K1r) lmm.setK1c(K1c) lmm.setK2r(K2r) lmm.setK2c(K2c) lmm.setSNPs(X) #add covariates lmm.addCovariates(Xcov,Acov) #add SNP design lmm.setSNPcoldesign(A) lmm.setPheno(Y) lmm.setNumIntervalsAlt(0) lmm.setNumIntervals0(100) lmm.process() pv = lmm.getPv().ravel() D2= ((SP.log10(pv)-SP.log10(D['pv']))**2) RV = SP.sqrt(D2.mean()) #print "\n" #print pv[0:10] #print D['pv'][0:10] #print RV #pdb.set_trace() self.assertTrue(RV<1E-6)
def simple_interaction_kronecker_deprecated(snps, phenos, covs=None, Acovs=None, Asnps1=None, Asnps0=None, K1r=None, K1c=None, K2r=None, K2c=None, covar_type='lowrank_diag', rank=1, searchDelta=False): """ I-variate fixed effects interaction test for phenotype specific SNP effects. (Runs multiple likelihood ratio tests and computes the P-values in python from the likelihood ratios) Args: snps: [N x S] SP.array of S SNPs for N individuals (test SNPs) phenos: [N x P] SP.array of P phenotypes for N individuals covs: list of SP.arrays holding covariates. Each covs[i] has one corresponding Acovs[i] Acovs: list of SP.arrays holding the phenotype design matrices for covariates. Each covs[i] has one corresponding Acovs[i]. Asnps1: list of SP.arrays of I interaction variables to be tested for N individuals. Note that it is assumed that Asnps0 is already included. If not provided, the alternative model will be the independent model Asnps0: single SP.array of I0 interaction variables to be included in the background model when testing for interaction with Inters K1r: [N x N] SP.array of LMM-covariance/kinship koefficients (optional) If not provided, then linear regression analysis is performed K1c: [P x P] SP.array of LMM-covariance/kinship koefficients (optional) If not provided, then linear regression analysis is performed K2r: [N x N] SP.array of LMM-covariance/kinship koefficients (optional) If not provided, then linear regression analysis is performed K2c: [P x P] SP.array of LMM-covariance/kinship koefficients (optional) If not provided, then linear regression analysis is performed covar_type: type of covaraince to use. Default 'freeform'. possible values are 'freeform': free form optimization, 'fixed': use a fixed matrix specified in covar_K0, 'diag': optimize a diagonal matrix, 'lowrank': optimize a low rank matrix. The rank of the lowrank part is specified in the variable rank, 'lowrank_id': optimize a low rank matrix plus the weight of a constant diagonal matrix. The rank of the lowrank part is specified in the variable rank, 'lowrank_diag': optimize a low rank matrix plus a free diagonal matrix. The rank of the lowrank part is specified in the variable rank, 'block': optimize the weight of a constant P x P block matrix of ones, 'block_id': optimize the weight of a constant P x P block matrix of ones plus the weight of a constant diagonal matrix, 'block_diag': optimize the weight of a constant P x P block matrix of ones plus a free diagonal matrix, rank: rank of a possible lowrank component (default 1) searchDelta: Boolean indicator if delta is optimized during SNP testing (default False) Returns: pv: P-values of the interaction test lrt0: log likelihood ratio statistics of the null model pv0: P-values of the null model lrt: log likelihood ratio statistics of the interaction test lrtAlt: log likelihood ratio statistics of the alternative model pvAlt: P-values of the alternative model """ S = snps.shape[1] #0. checks N = phenos.shape[0] P = phenos.shape[1] if K1r == None: K1r = SP.dot(snps, snps.T) else: assert K1r.shape[0] == N, 'K1r: dimensions dismatch' assert K1r.shape[1] == N, 'K1r: dimensions dismatch' if K2r == None: K2r = SP.eye(N) else: assert K2r.shape[0] == N, 'K2r: dimensions dismatch' assert K2r.shape[1] == N, 'K2r: dimensions dismatch' covs, Acovs = updateKronCovs(covs, Acovs, N, P) #Asnps can be several designs if (Asnps0 is None): Asnps0 = [SP.ones([1, P])] if Asnps1 is None: Asnps1 = [SP.eye([P])] if (type(Asnps0) != list): Asnps0 = [Asnps0] if (type(Asnps1) != list): Asnps1 = [Asnps1] assert (len(Asnps0) == 1) and ( len(Asnps1) > 0), "need at least one Snp design matrix for null and alt model" #one row per column design matrix pv = SP.zeros((len(Asnps1), snps.shape[1])) lrt = SP.zeros((len(Asnps1), snps.shape[1])) pvAlt = SP.zeros((len(Asnps1), snps.shape[1])) lrtAlt = SP.zeros((len(Asnps1), snps.shape[1])) #1. run GP model to infer suitable covariance structure if K1c == None or K2c == None: vc = estimateKronCovariances(phenos=phenos, K1r=K1r, K2r=K2r, K1c=K1c, K2c=K2c, covs=covs, Acovs=Acovs, covar_type=covar_type, rank=rank) K1c = vc.getEstTraitCovar(0) K2c = vc.getEstTraitCovar(1) else: assert K1c.shape[0] == P, 'K1c: dimensions dismatch' assert K1c.shape[1] == P, 'K1c: dimensions dismatch' assert K2c.shape[0] == P, 'K2c: dimensions dismatch' assert K2c.shape[1] == P, 'K2c: dimensions dismatch' #2. run kroneckerLMM for null model lmm = limix.CKroneckerLMM() lmm.setK1r(K1r) lmm.setK1c(K1c) lmm.setK2r(K2r) lmm.setK2c(K2c) lmm.setSNPs(snps) #add covariates for ic in xrange(len(Acovs)): lmm.addCovariates(covs[ic], Acovs[ic]) lmm.setPheno(phenos) if searchDelta: lmm.setNumIntervalsAlt(100) else: lmm.setNumIntervalsAlt(0) lmm.setNumIntervals0(100) #add SNP design lmm.setSNPcoldesign(Asnps0[0]) lmm.process() dof0 = Asnps0[0].shape[0] pv0 = lmm.getPv() lrt0 = ST.chi2.isf(pv0, dof0) for iA in xrange(len(Asnps1)): dof1 = Asnps1[iA].shape[0] dof = dof1 - dof0 lmm.setSNPcoldesign(Asnps1[iA]) lmm.process() pvAlt[iA, :] = lmm.getPv()[0] lrtAlt[iA, :] = ST.chi2.isf(pvAlt[iA, :], dof1) lrt[iA, :] = lrtAlt[iA, :] - lrt0[ 0] # Don't need the likelihood ratios, as null model is the same between the two models pv[iA, :] = ST.chi2.sf(lrt[iA, :], dof) return pv, lrt0, pv0, lrt, lrtAlt, pvAlt
def kronecker_lmm(snps, phenos, covs=None, Acovs=None, Asnps=None, K1r=None, K1c=None, K2r=None, K2c=None, covar_type='lowrank_diag', rank=1, NumIntervalsDelta0=100, NumIntervalsDeltaAlt=0, searchDelta=False): """ simple wrapper for kroneckerLMM code Args: snps: [N x S] SP.array of S SNPs for N individuals (test SNPs) phenos: [N x P] SP.array of P phenotypes for N individuals covs: list of SP.arrays holding covariates. Each covs[i] has one corresponding Acovs[i] Acovs: list of SP.arrays holding the phenotype design matrices for covariates. Each covs[i] has one corresponding Acovs[i]. Asnps: single SP.array of I0 interaction variables to be included in the background model when testing for interaction with Inters If not provided, the alternative model will be the independent model K1r: [N x N] SP.array of LMM-covariance/kinship koefficients (optional) If not provided, then linear regression analysis is performed K1c: [P x P] SP.array of LMM-covariance/kinship koefficients (optional) If not provided, then linear regression analysis is performed K2r: [N x N] SP.array of LMM-covariance/kinship koefficients (optional) If not provided, then linear regression analysis is performed K2c: [P x P] SP.array of LMM-covariance/kinship koefficients (optional) If not provided, then linear regression analysis is performed covar_type: type of covaraince to use. Default 'freeform'. possible values are 'freeform': free form optimization, 'fixed': use a fixed matrix specified in covar_K0, 'diag': optimize a diagonal matrix, 'lowrank': optimize a low rank matrix. The rank of the lowrank part is specified in the variable rank, 'lowrank_id': optimize a low rank matrix plus the weight of a constant diagonal matrix. The rank of the lowrank part is specified in the variable rank, 'lowrank_diag': optimize a low rank matrix plus a free diagonal matrix. The rank of the lowrank part is specified in the variable rank, 'block': optimize the weight of a constant P x P block matrix of ones, 'block_id': optimize the weight of a constant P x P block matrix of ones plus the weight of a constant diagonal matrix, 'block_diag': optimize the weight of a constant P x P block matrix of ones plus a free diagonal matrix, rank: rank of a possible lowrank component (default 1) NumIntervalsDelta0: number of steps for delta optimization on the null model (100) NumIntervalsDeltaAlt:number of steps for delta optimization on the alt. model (0 - no optimization) searchDelta: Boolean indicator if delta is optimized during SNP testing (default False) Returns: CKroneckerLMM object P-values for all SNPs from liklelihood ratio test """ #0. checks N = phenos.shape[0] P = phenos.shape[1] if K1r == None: K1r = SP.dot(snps, snps.T) else: assert K1r.shape[0] == N, 'K1r: dimensions dismatch' assert K1r.shape[1] == N, 'K1r: dimensions dismatch' if K2r == None: K2r = SP.eye(N) else: assert K2r.shape[0] == N, 'K2r: dimensions dismatch' assert K2r.shape[1] == N, 'K2r: dimensions dismatch' covs, Acovs = updateKronCovs(covs, Acovs, N, P) #Asnps can be several designs if Asnps is None: Asnps = [SP.ones([1, P])] if (type(Asnps) != list): Asnps = [Asnps] assert len(Asnps) > 0, "need at least one Snp design matrix" #one row per column design matrix pv = SP.zeros((len(Asnps), snps.shape[1])) #1. run GP model to infer suitable covariance structure if K1c == None or K2c == None: vc = estimateKronCovariances(phenos=phenos, K1r=K1r, K2r=K2r, K1c=K1c, K2c=K2c, covs=covs, Acovs=Acovs, covar_type=covar_type, rank=rank) K1c = vc.getEstTraitCovar(0) K2c = vc.getEstTraitCovar(1) else: assert K1c.shape[0] == P, 'K1c: dimensions dismatch' assert K1c.shape[1] == P, 'K1c: dimensions dismatch' assert K2c.shape[0] == P, 'K2c: dimensions dismatch' assert K2c.shape[1] == P, 'K2c: dimensions dismatch' #2. run kroneckerLMM lmm = limix.CKroneckerLMM() lmm.setK1r(K1r) lmm.setK1c(K1c) lmm.setK2r(K2r) lmm.setK2c(K2c) lmm.setSNPs(snps) #add covariates for ic in xrange(len(Acovs)): lmm.addCovariates(covs[ic], Acovs[ic]) lmm.setPheno(phenos) #delta serch on alt. model? if searchDelta: lmm.setNumIntervalsAlt(NumIntervalsDeltaAlt) else: lmm.setNumIntervalsAlt(0) lmm.setNumIntervals0(NumIntervalsDelta0) for iA in xrange(len(Asnps)): #add SNP design lmm.setSNPcoldesign(Asnps[iA]) lmm.process() pv[iA, :] = lmm.getPv()[0] return lmm, pv
def simple_interaction_kronecker(snps, phenos, covs=None, Acovs=None, Asnps1=None, Asnps0=None, K1r=None, K1c=None, K2r=None, K2c=None, covar_type='lowrank_diag', rank=1, NumIntervalsDelta0=100, NumIntervalsDeltaAlt=0, searchDelta=False): """ I-variate fixed effects interaction test for phenotype specific SNP effects Args: snps: [N x S] SP.array of S SNPs for N individuals (test SNPs) phenos: [N x P] SP.array of P phenotypes for N individuals covs: list of SP.arrays holding covariates. Each covs[i] has one corresponding Acovs[i] Acovs: list of SP.arrays holding the phenotype design matrices for covariates. Each covs[i] has one corresponding Acovs[i]. Asnps1: list of SP.arrays of I interaction variables to be tested for N individuals. Note that it is assumed that Asnps0 is already included. If not provided, the alternative model will be the independent model Asnps0: single SP.array of I0 interaction variables to be included in the background model when testing for interaction with Inters K1r: [N x N] SP.array of LMM-covariance/kinship koefficients (optional) If not provided, then linear regression analysis is performed K1c: [P x P] SP.array of LMM-covariance/kinship koefficients (optional) If not provided, then linear regression analysis is performed K2r: [N x N] SP.array of LMM-covariance/kinship koefficients (optional) If not provided, then linear regression analysis is performed K2c: [P x P] SP.array of LMM-covariance/kinship koefficients (optional) If not provided, then linear regression analysis is performed covar_type: type of covaraince to use. Default 'freeform'. possible values are 'freeform': free form optimization, 'fixed': use a fixed matrix specified in covar_K0, 'diag': optimize a diagonal matrix, 'lowrank': optimize a low rank matrix. The rank of the lowrank part is specified in the variable rank, 'lowrank_id': optimize a low rank matrix plus the weight of a constant diagonal matrix. The rank of the lowrank part is specified in the variable rank, 'lowrank_diag': optimize a low rank matrix plus a free diagonal matrix. The rank of the lowrank part is specified in the variable rank, 'block': optimize the weight of a constant P x P block matrix of ones, 'block_id': optimize the weight of a constant P x P block matrix of ones plus the weight of a constant diagonal matrix, 'block_diag': optimize the weight of a constant P x P block matrix of ones plus a free diagonal matrix, rank: rank of a possible lowrank component (default 1) NumIntervalsDelta0: number of steps for delta optimization on the null model (100) NumIntervalsDeltaAlt:number of steps for delta optimization on the alt. model (0 - no optimization) searchDelta: Carry out delta optimization on the alternative model? if yes We use NumIntervalsDeltaAlt steps Returns: pv: P-values of the interaction test pv0: P-values of the null model pvAlt: P-values of the alternative model """ S = snps.shape[1] #0. checks N = phenos.shape[0] P = phenos.shape[1] if K1r == None: K1r = SP.dot(snps, snps.T) else: assert K1r.shape[0] == N, 'K1r: dimensions dismatch' assert K1r.shape[1] == N, 'K1r: dimensions dismatch' if K2r == None: K2r = SP.eye(N) else: assert K2r.shape[0] == N, 'K2r: dimensions dismatch' assert K2r.shape[1] == N, 'K2r: dimensions dismatch' covs, Acovs = updateKronCovs(covs, Acovs, N, P) #Asnps can be several designs if (Asnps0 is None): Asnps0 = [SP.ones([1, P])] if Asnps1 is None: Asnps1 = [SP.eye([P])] if (type(Asnps0) != list): Asnps0 = [Asnps0] if (type(Asnps1) != list): Asnps1 = [Asnps1] assert (len(Asnps0) == 1) and ( len(Asnps1) > 0), "need at least one Snp design matrix for null and alt model" #one row per column design matrix pv = SP.zeros((len(Asnps1), snps.shape[1])) lrt = SP.zeros((len(Asnps1), snps.shape[1])) pvAlt = SP.zeros((len(Asnps1), snps.shape[1])) lrtAlt = SP.zeros((len(Asnps1), snps.shape[1])) #1. run GP model to infer suitable covariance structure if K1c == None or K2c == None: vc = estimateKronCovariances(phenos=phenos, K1r=K1r, K2r=K2r, K1c=K1c, K2c=K2c, covs=covs, Acovs=Acovs, covar_type=covar_type, rank=rank) K1c = vc.getEstTraitCovar(0) K2c = vc.getEstTraitCovar(1) else: assert K1c.shape[0] == P, 'K1c: dimensions dismatch' assert K1c.shape[1] == P, 'K1c: dimensions dismatch' assert K2c.shape[0] == P, 'K2c: dimensions dismatch' assert K2c.shape[1] == P, 'K2c: dimensions dismatch' #2. run kroneckerLMM for null model lmm = limix.CKroneckerLMM() lmm.setK1r(K1r) lmm.setK1c(K1c) lmm.setK2r(K2r) lmm.setK2c(K2c) lmm.setSNPs(snps) #add covariates for ic in xrange(len(Acovs)): lmm.addCovariates(covs[ic], Acovs[ic]) lmm.setPheno(phenos) #delta serch on alt. model? if searchDelta: lmm.setNumIntervalsAlt(NumIntervalsDeltaAlt) lmm.setNumIntervals0_inter(NumIntervalsDeltaAlt) else: lmm.setNumIntervalsAlt(0) lmm.setNumIntervals0_inter(0) lmm.setNumIntervals0(NumIntervalsDelta0) #add SNP design lmm.setSNPcoldesign0_inter(Asnps0[0]) for iA in xrange(len(Asnps1)): lmm.setSNPcoldesign(Asnps1[iA]) lmm.process() pvAlt[iA, :] = lmm.getPv()[0] pv[iA, :] = lmm.getPv()[1] pv0 = lmm.getPv()[2] return pv, pv0, pvAlt