def setUp(self):
#check: do we have a csv File?
self.dir_name = os.path.dirname(__file__)
self.dataset = os.path.join(self.dir_name,'varDecomp')
if (not os.path.exists(self.dataset)) or 'recalc' in sys.argv:
if not os.path.exists(self.dataset):
os.makedirs(self.dataset)
SP.random.seed(1)
self.N = 200
self.S = 1000
self.P = 2
self.D = {}
self.genGeno()
self.genPheno()
self.generate = True
else:
self.generate=False
#self.D = data.load(os.path.join(self.dir_name,self.dataset))
self.D = data.load(self.dataset)
self.N = self.D['X'].shape[0]
self.S = self.D['X'].shape[1]
self.P = self.D['Y'].shape[1]
self.Kg = SP.dot(self.D['X'],self.D['X'].T)
self.Kg = self.Kg/self.Kg.diagonal().mean()
self.vc = VAR.VarianceDecomposition(self.D['Y'])
self.vc.addRandomEffect(self.Kg,jitter=0)
self.vc.addRandomEffect(is_noise=True,jitter=0)
self.vc.addFixedEffect()
def test_permutation(self):
#test permutation function
for dn in self.datasets:
D = data.load(os.path.join(self.dir_name,dn))
perm = SP.random.permutation(D['X'].shape[0])
#1. set permuattion
lmm = dlimix.CLMM()
lmm.setK(D['K'])
lmm.setSNPs(D['X'])
lmm.setCovs(D['Cov'])
lmm.setPheno(D['Y'])
if 1:
#pdb.set_trace()
perm = SP.array(perm,dtype='int32')#Windows needs int32 as long -> fix interface to accept int64 types
lmm.setPermutation(perm)
lmm.process()
pv_perm1 = lmm.getPv().ravel()
#2. do by hand
lmm = dlimix.CLMM()
lmm.setK(D['K'])
lmm.setSNPs(D['X'][perm])
lmm.setCovs(D['Cov'])
lmm.setPheno(D['Y'])
lmm.process()
pv_perm2 = lmm.getPv().ravel()
D2 = (SP.log10(pv_perm1)-SP.log10(pv_perm2))**2
RV = SP.sqrt(D2.mean())
self.assertTrue(RV<1E-6)
def test_permutation(self):
#test permutation function
for dn in self.datasets:
D = data.load(os.path.join(self.dir_name, dn))
perm = SP.random.permutation(D['X'].shape[0])
#1. set permuattion
lmm = dlimix.CLMM()
lmm.setK(D['K'])
lmm.setSNPs(D['X'])
lmm.setCovs(D['Cov'])
lmm.setPheno(D['Y'])
if 1:
#pdb.set_trace()
perm = SP.array(
perm, dtype='int32'
) #Windows needs int32 as long -> fix interface to accept int64 types
lmm.setPermutation(perm)
lmm.process()
pv_perm1 = lmm.getPv().ravel()
#2. do by hand
lmm = dlimix.CLMM()
lmm.setK(D['K'])
lmm.setSNPs(D['X'][perm])
lmm.setCovs(D['Cov'])
lmm.setPheno(D['Y'])
lmm.process()
pv_perm2 = lmm.getPv().ravel()
D2 = (SP.log10(pv_perm1) - SP.log10(pv_perm2))**2
RV = SP.sqrt(D2.mean())
self.assertTrue(RV < 1E-6)
def test_lmm2(self):
"""another test, establishing an lmm-equivalent by a design matrix choice"""
for dn in self.datasets:
D = data.load(os.path.join(self.dir_name, dn))
#construct Kronecker LMM model which has the special case of standard LMM
#covar1: genotype matrix
N = D['K'].shape[0]
P = 3
K1r = D['K']
#K1c = SP.zeros([2,2])
#K1c[0,0] = 1
K1c = SP.eye(P)
K2r = SP.eye(N)
K2c = SP.eye(P)
#A = SP.zeros([1,2])
#A[0,0] =1
A = SP.eye(P)
Acov = SP.eye(P)
Xcov = D['Cov'][:, SP.newaxis]
X = D['X']
Y = D['Y'][:, SP.newaxis]
Y = SP.tile(Y, (1, P))
lmm = dlimix.CKroneckerLMM()
lmm.setK1r(K1r)
lmm.setK1c(K1c)
lmm.setK2r(K2r)
lmm.setK2c(K2c)
lmm.setSNPs(X)
#add covariates
lmm.addCovariates(Xcov, Acov)
#add SNP design
lmm.setSNPcoldesign(A)
lmm.setPheno(Y)
lmm.setNumIntervalsAlt(0)
lmm.setNumIntervals0(100)
lmm.process()
#get p-values with P-dof:
pv_Pdof = lmm.getPv().ravel()
#transform in P-values with a single DOF:
import scipy.stats as st
lrt = st.chi2.isf(pv_Pdof, P) / P
pv = st.chi2.sf(lrt, 1)
#compare with single DOF P-values:
D2 = ((SP.log10(pv) - SP.log10(D['pv']))**2)
RV = SP.sqrt(D2.mean())
#print "\n"
#print pv[0:10]
#print D['pv'][0:10]
#print RV
#pdb.set_trace()
self.assertTrue(RV < 1E-6)
def test_lmm2(self):
"""another test, establishing an lmm-equivalent by a design matrix choice"""
for dn in self.datasets:
D = data.load(os.path.join(self.dir_name,dn))
#construct Kronecker LMM model which has the special case of standard LMM
#covar1: genotype matrix
N = D['K'].shape[0]
P = 3
K1r = D['K']
#K1c = SP.zeros([2,2])
#K1c[0,0] = 1
K1c = SP.eye(P)
K2r = SP.eye(N)
K2c = SP.eye(P)
#A = SP.zeros([1,2])
#A[0,0] =1
A = SP.eye(P)
Acov = SP.eye(P)
Xcov = D['Cov'][:,SP.newaxis]
X = D['X']
Y = D['Y'][:,SP.newaxis]
Y = SP.tile(Y,(1,P))
lmm = dlimix.CKroneckerLMM()
lmm.setK1r(K1r)
lmm.setK1c(K1c)
lmm.setK2r(K2r)
lmm.setK2c(K2c)
lmm.setSNPs(X)
#add covariates
lmm.addCovariates(Xcov,Acov)
#add SNP design
lmm.setSNPcoldesign(A)
lmm.setPheno(Y)
lmm.setNumIntervalsAlt(0)
lmm.setNumIntervals0(100)
lmm.process()
#get p-values with P-dof:
pv_Pdof = lmm.getPv().ravel()
#transform in P-values with a single DOF:
import scipy.stats as st
lrt = st.chi2.isf(pv_Pdof,P)/P
pv = st.chi2.sf(lrt,1)
#compare with single DOF P-values:
D2= ((SP.log10(pv)-SP.log10(D['pv']))**2)
RV = SP.sqrt(D2.mean())
#print "\n"
#print pv[0:10]
#print D['pv'][0:10]
#print RV
#pdb.set_trace()
self.assertTrue(RV<1E-6)
def test_exceptions(self):
D = data.load(os.path.join(self.dir_name,self.datasets[0]))
lmm = dlimix.CLMM()
N = 100
K = SP.eye(N)
X = SP.randn(N,100)
Y = SP.randn(N+1,1)
Cov = SP.randn(N,1)
lmm.setK(K)
lmm.setSNPs(X)
lmm.setCovs(Cov)
lmm.setPheno(Y)
try:
lmm.process()
except Exception,e:
self.assertTrue(1==1)
pass
def test_exceptions(self):
D = data.load(os.path.join(self.dir_name, self.datasets[0]))
lmm = dlimix.CLMM()
N = 100
K = SP.eye(N)
X = SP.randn(N, 100)
Y = SP.randn(N + 1, 1)
Cov = SP.randn(N, 1)
lmm.setK(K)
lmm.setSNPs(X)
lmm.setCovs(Cov)
lmm.setPheno(Y)
try:
lmm.process()
except Exception as e:
self.assertTrue(1 == 1)
pass
def test_lmm1(self):
"""basic test, comapring pv"""
for dn in self.datasets:
D = data.load(os.path.join(self.dir_name, dn))
lmm = dlimix.CLMM()
lmm.setK(D['K'])
lmm.setSNPs(D['X'])
lmm.setCovs(D['Cov'])
lmm.setPheno(D['Y'])
lmm.process()
pv = lmm.getPv().ravel()
BetaSte = lmm.getBetaSNPste().ravel()
Beta = lmm.getBetaSNP()
D2pv = ((SP.log10(pv) - SP.log10(D['pv']))**2)
# D2Beta= (Beta-D['Beta'])**2
# D2BetaSte = (BetaSte-D['BetaSte'])**2
RV = SP.sqrt(D2pv.mean()) < 1E-6
# RV = RV & (D2Beta.mean()<1E-6)
# RV = RV & (D2BetaSte.mean()<1E-6)
self.assertTrue(RV)
def test_lmm1(self):
"""basic test, comapring pv"""
for dn in self.datasets:
D = data.load(os.path.join(self.dir_name,dn))
lmm = dlimix.CLMM()
lmm.setK(D['K'])
lmm.setSNPs(D['X'])
lmm.setCovs(D['Cov'])
lmm.setPheno(D['Y'])
lmm.process()
pv = lmm.getPv().ravel()
BetaSte = lmm.getBetaSNPste().ravel()
Beta = lmm.getBetaSNP()
D2pv= ((SP.log10(pv)-SP.log10(D['pv']))**2)
# D2Beta= (Beta-D['Beta'])**2
# D2BetaSte = (BetaSte-D['BetaSte'])**2
RV = SP.sqrt(D2pv.mean())<1E-6
# RV = RV & (D2Beta.mean()<1E-6)
# RV = RV & (D2BetaSte.mean()<1E-6)
self.assertTrue(RV)
def test_lmm(self):
"""basic test, comparing pv to a standard LMM equivalent"""
for dn in self.datasets:
D = data.load(os.path.join(self.dir_name,dn))
#construct Kronecker LMM model which has the special case of standard LMM
#covar1: genotype matrix
K1r = D['K']
K1c = SP.eye(1)
K2r = SP.eye(D['K'].shape[0])
K2c = SP.eye(1)
A = SP.eye(1)
Acov = SP.eye(1)
Xcov = D['Cov'][:,SP.newaxis]
X = D['X']
Y = D['Y'][:,SP.newaxis]
lmm = dlimix.CKroneckerLMM()
lmm.setK1r(K1r)
lmm.setK1c(K1c)
lmm.setK2r(K2r)
lmm.setK2c(K2c)
lmm.setSNPs(X)
#add covariates
lmm.addCovariates(Xcov,Acov)
#add SNP design
lmm.setSNPcoldesign(A)
lmm.setPheno(Y)
lmm.setNumIntervalsAlt(0)
lmm.setNumIntervals0(100)
lmm.process()
pv = lmm.getPv().ravel()
D2= ((SP.log10(pv)-SP.log10(D['pv']))**2)
RV = SP.sqrt(D2.mean())
#print "\n"
#print pv[0:10]
#print D['pv'][0:10]
#print RV
#pdb.set_trace()
self.assertTrue(RV<1E-6)
def test_lmm(self):
"""basic test, comparing pv to a standard LMM equivalent"""
for dn in self.datasets:
D = data.load(os.path.join(self.dir_name, dn))
#construct Kronecker LMM model which has the special case of standard LMM
#covar1: genotype matrix
K1r = D['K']
K1c = SP.eye(1)
K2r = SP.eye(D['K'].shape[0])
K2c = SP.eye(1)
A = SP.eye(1)
Acov = SP.eye(1)
Xcov = D['Cov'][:, SP.newaxis]
X = D['X']
Y = D['Y'][:, SP.newaxis]
lmm = dlimix.CKroneckerLMM()
lmm.setK1r(K1r)
lmm.setK1c(K1c)
lmm.setK2r(K2r)
lmm.setK2c(K2c)
lmm.setSNPs(X)
#add covariates
lmm.addCovariates(Xcov, Acov)
#add SNP design
lmm.setSNPcoldesign(A)
lmm.setPheno(Y)
lmm.setNumIntervalsAlt(0)
lmm.setNumIntervals0(100)
lmm.process()
pv = lmm.getPv().ravel()
D2 = ((SP.log10(pv) - SP.log10(D['pv']))**2)
RV = SP.sqrt(D2.mean())
#print "\n"
#print pv[0:10]
#print D['pv'][0:10]
#print RV
#pdb.set_trace()
self.assertTrue(RV < 1E-6)
def test_lmm(self):
"""basic main effec tests"""
for dn in self.datasets:
D = data.load(os.path.join(self.dir_name,dn))
N = D['X'].shape[0]
inter0 = SP.zeros([N,0])#fixed verion: all 0 feature did not work: #inter0 = SP.zeros([N,1])N-by-0 matrix instead of N-by-1 works
inter1 = SP.ones([N,1])
lmm = dlimix.CInteractLMM()
lmm.setK(D['K'])
lmm.setSNPs(D['X'])
lmm.setCovs(D['Cov'])
lmm.setPheno(D['Y'])
#inter0[:]=1
#inter1[0:N/2]=-1
lmm.setInter0(inter0)
lmm.setInter(inter1)
lmm.process()
pv = lmm.getPv().ravel()
D2= SP.sqrt( ((SP.log10(pv)-SP.log10(D['pv']))**2).mean())
self.assertTrue(D2<1E-6)
def test_permutation(self):
#test permutation function
for dn in self.datasets:
D = data.load(os.path.join(self.dir_name, dn))
perm = SP.random.permutation(D['X'].shape[0])
N = D['X'].shape[0]
inter0 = SP.zeros(
[N, 0]
) #fix, as old version with all zero feature does not work#N-by-0 matrix instead of N-by-1 works
inter1 = SP.ones([N, 1])
#1. set permuattion
lmm = dlimix.CInteractLMM()
lmm.setInter0(inter0)
lmm.setInter(inter1)
lmm.setK(D['K'])
lmm.setSNPs(D['X'])
lmm.setCovs(D['Cov'])
lmm.setPheno(D['Y'])
if 1:
#pdb.set_trace()
perm = SP.array(
perm, dtype='int32'
) #Windows needs int32 as long -> fix interface to accept int64 types
lmm.setPermutation(perm)
lmm.process()
pv_perm1 = lmm.getPv().ravel()
#2. do by hand
lmm = dlimix.CInteractLMM()
lmm.setInter0(inter0)
lmm.setInter(inter1)
lmm.setK(D['K'])
lmm.setSNPs(D['X'][perm])
lmm.setCovs(D['Cov'])
lmm.setPheno(D['Y'])
lmm.process()
pv_perm2 = lmm.getPv().ravel()
D2 = (SP.log10(pv_perm1) - SP.log10(pv_perm2))**2
RV = SP.sqrt(D2.mean())
self.assertTrue(RV < 1E-6)
def setUp(self):
#check: do we have a csv File?
self.dir_name = os.path.dirname(os.path.realpath(__file__))
self.dataset = os.path.join(self.dir_name,'lmmlasso')
if (not os.path.exists(self.dataset)) or 'recalc' in sys.argv:
if not os.path.exists(self.dataset):
os.makedirs(self.dataset)
SP.random.seed(1)
self.N = 500
self.S = 100
self.D = {}
self.genGeno()
self.genKernel()
self.genPheno()
self.generate = True
else:
self.generate=False
self.D = data.load(self.dataset)
self.N = self.D['X'].shape[0]
self.S = self.D['X'].shape[1]
self.lmmlasso = lmmlasso.LmmLasso()
def test_lmm(self):
"""basic main effec tests"""
for dn in self.datasets:
D = data.load(os.path.join(self.dir_name, dn))
N = D['X'].shape[0]
inter0 = SP.zeros(
[N, 0]
) #fixed verion: all 0 feature did not work: #inter0 = SP.zeros([N,1])N-by-0 matrix instead of N-by-1 works
inter1 = SP.ones([N, 1])
lmm = dlimix.CInteractLMM()
lmm.setK(D['K'])
lmm.setSNPs(D['X'])
lmm.setCovs(D['Cov'])
lmm.setPheno(D['Y'])
#inter0[:]=1
#inter1[0:N/2]=-1
lmm.setInter0(inter0)
lmm.setInter(inter1)
lmm.process()
pv = lmm.getPv().ravel()
D2 = SP.sqrt(((SP.log10(pv) - SP.log10(D['pv']))**2).mean())
self.assertTrue(D2 < 1E-6)
def test_lmm1(self):
"""basic test, comapring pv"""
for dn in self.datasets:
D = data.load(os.path.join(self.dir_name, dn))
#make vllarg X. This needs to be changed later
NL = 1000
self.NL = NL
X = SP.tile(D['X'], (1, self.NL))
lmm = dlimix.CLMM()
lmm.setK(D['K'])
lmm.setSNPs(X)
lmm.setCovs(D['Cov'])
lmm.setPheno(D['Y'])
lmm.process()
pv = lmm.getPv().ravel()
BetaSte = lmm.getBetaSNPste().ravel()
Beta = lmm.getBetaSNP()
D2pv = (SP.log10(pv) - SP.log10(SP.tile(D['pv'], self.NL))**2)
# D2Beta= (Beta-SP.tile(D['Beta'],self.NL))**2
# D2BetaSte = (BetaSte-SP.tile(D['BetaSte'],self.NL))**2
RV = SP.sqrt(D2pv.mean()) < 1E-6
# RV = RV & (D2Beta.mean()<1E-6)
# RV = RV & (D2BetaSte.mean()<1E-6)
self.assertTrue(RV)
def test_permutation(self):
#test permutation function
for dn in self.datasets:
D = data.load(os.path.join(self.dir_name,dn))
perm = SP.random.permutation(D['X'].shape[0])
N = D['X'].shape[0]
inter0 = SP.zeros([N,0])#fix, as old version with all zero feature does not work#N-by-0 matrix instead of N-by-1 works
inter1 = SP.ones([N,1])
#1. set permuattion
lmm = dlimix.CInteractLMM()
lmm.setInter0(inter0)
lmm.setInter(inter1)
lmm.setK(D['K'])
lmm.setSNPs(D['X'])
lmm.setCovs(D['Cov'])
lmm.setPheno(D['Y'])
if 1:
#pdb.set_trace()
perm = SP.array(perm,dtype='int32')#Windows needs int32 as long -> fix interface to accept int64 types
lmm.setPermutation(perm)
lmm.process()
pv_perm1 = lmm.getPv().ravel()
#2. do by hand
lmm = dlimix.CInteractLMM()
lmm.setInter0(inter0)
lmm.setInter(inter1)
lmm.setK(D['K'])
lmm.setSNPs(D['X'][perm])
lmm.setCovs(D['Cov'])
lmm.setPheno(D['Y'])
lmm.process()
pv_perm2 = lmm.getPv().ravel()
D2 = (SP.log10(pv_perm1)-SP.log10(pv_perm2))**2
RV = SP.sqrt(D2.mean())
self.assertTrue(RV<1E-6)
def test_lmm1(self):
"""basic test, comapring pv"""
for dn in self.datasets:
D = data.load(os.path.join(self.dir_name,dn))
#make vllarg X. This needs to be changed later
NL = 1000
self.NL = NL
X = SP.tile(D['X'],(1,self.NL))
lmm = dlimix.CLMM()
lmm.setK(D['K'])
lmm.setSNPs(X)
lmm.setCovs(D['Cov'])
lmm.setPheno(D['Y'])
lmm.process()
pv = lmm.getPv().ravel()
BetaSte = lmm.getBetaSNPste().ravel()
Beta = lmm.getBetaSNP()
D2pv= (SP.log10(pv)-SP.log10(SP.tile(D['pv'],self.NL))**2)
# D2Beta= (Beta-SP.tile(D['Beta'],self.NL))**2
# D2BetaSte = (BetaSte-SP.tile(D['BetaSte'],self.NL))**2
RV = SP.sqrt(D2pv.mean())<1E-6
# RV = RV & (D2Beta.mean()<1E-6)
# RV = RV & (D2BetaSte.mean()<1E-6)
self.assertTrue(RV)