def fitPairwiseModel(Y,XX=None,S_XX=None,U_XX=None,verbose=False):
N,P = Y.shape
""" initilizes parameters """
RV = fitSingleTraitModel(Y,XX=XX,S_XX=S_XX,U_XX=U_XX,verbose=verbose)
Cg = covariance.freeform(2)
Cn = covariance.freeform(2)
gp = gp2kronSum(mean(Y[:,0:2]),Cg,Cn,XX=XX,S_XX=S_XX,U_XX=U_XX)
conv2 = SP.ones((P,P),dtype=bool)
rho_g = SP.ones((P,P))
rho_n = SP.ones((P,P))
for p1 in range(P):
for p2 in range(p1):
if verbose:
print '.. fitting correlation (%d,%d)'%(p1,p2)
gp.setY(Y[:,[p1,p2]])
Cg_params0 = SP.array([SP.sqrt(RV['varST'][p1,0]),1e-6*SP.randn(),SP.sqrt(RV['varST'][p2,0])])
Cn_params0 = SP.array([SP.sqrt(RV['varST'][p1,1]),1e-6*SP.randn(),SP.sqrt(RV['varST'][p2,1])])
params0 = {'Cg':Cg_params0,'Cn':Cn_params0}
conv2[p1,p2],info = OPT.opt_hyper(gp,params0,factr=1e3)
rho_g[p1,p2] = Cg.K()[0,1]/SP.sqrt(Cg.K().diagonal().prod())
rho_n[p1,p2] = Cn.K()[0,1]/SP.sqrt(Cn.K().diagonal().prod())
conv2[p2,p1] = conv2[p1,p2]; rho_g[p2,p1] = rho_g[p1,p2]; rho_n[p2,p1] = rho_n[p1,p2]
RV['Cg0'] = rho_g*SP.dot(SP.sqrt(RV['varST'][:,0:1]),SP.sqrt(RV['varST'][:,0:1].T))
RV['Cn0'] = rho_n*SP.dot(SP.sqrt(RV['varST'][:,1:2]),SP.sqrt(RV['varST'][:,1:2].T))
RV['conv2'] = conv2
#3. regularizes covariance matrices
offset_g = abs(SP.minimum(LA.eigh(RV['Cg0'])[0].min(),0))+1e-4
offset_n = abs(SP.minimum(LA.eigh(RV['Cn0'])[0].min(),0))+1e-4
RV['Cg0_reg'] = RV['Cg0']+offset_g*SP.eye(P)
RV['Cn0_reg'] = RV['Cn0']+offset_n*SP.eye(P)
RV['params0_Cg']=LA.cholesky(RV['Cg0_reg'])[SP.tril_indices(P)]
RV['params0_Cn']=LA.cholesky(RV['Cn0_reg'])[SP.tril_indices(P)]
return RV
def fitPairwiseModel(Y, XX=None, S_XX=None, U_XX=None, verbose=False):
N, P = Y.shape
""" initilizes parameters """
RV = fitSingleTraitModel(Y, XX=XX, S_XX=S_XX, U_XX=U_XX, verbose=verbose)
Cg = covariance.freeform(2)
Cn = covariance.freeform(2)
gp = gp2kronSum(mean(Y[:, 0:2]), Cg, Cn, XX=XX, S_XX=S_XX, U_XX=U_XX)
conv2 = SP.ones((P, P), dtype=bool)
rho_g = SP.ones((P, P))
rho_n = SP.ones((P, P))
for p1 in range(P):
for p2 in range(p1):
if verbose:
print '.. fitting correlation (%d,%d)' % (p1, p2)
gp.setY(Y[:, [p1, p2]])
Cg_params0 = SP.array([
SP.sqrt(RV['varST'][p1, 0]), 1e-6 * SP.randn(),
SP.sqrt(RV['varST'][p2, 0])
])
Cn_params0 = SP.array([
SP.sqrt(RV['varST'][p1, 1]), 1e-6 * SP.randn(),
SP.sqrt(RV['varST'][p2, 1])
])
params0 = {'Cg': Cg_params0, 'Cn': Cn_params0}
conv2[p1, p2], info = OPT.opt_hyper(gp, params0, factr=1e3)
rho_g[p1, p2] = Cg.K()[0, 1] / SP.sqrt(Cg.K().diagonal().prod())
rho_n[p1, p2] = Cn.K()[0, 1] / SP.sqrt(Cn.K().diagonal().prod())
conv2[p2, p1] = conv2[p1, p2]
rho_g[p2, p1] = rho_g[p1, p2]
rho_n[p2, p1] = rho_n[p1, p2]
RV['Cg0'] = rho_g * SP.dot(SP.sqrt(RV['varST'][:, 0:1]),
SP.sqrt(RV['varST'][:, 0:1].T))
RV['Cn0'] = rho_n * SP.dot(SP.sqrt(RV['varST'][:, 1:2]),
SP.sqrt(RV['varST'][:, 1:2].T))
RV['conv2'] = conv2
#3. regularizes covariance matrices
offset_g = abs(SP.minimum(LA.eigh(RV['Cg0'])[0].min(), 0)) + 1e-4
offset_n = abs(SP.minimum(LA.eigh(RV['Cn0'])[0].min(), 0)) + 1e-4
RV['Cg0_reg'] = RV['Cg0'] + offset_g * SP.eye(P)
RV['Cn0_reg'] = RV['Cn0'] + offset_n * SP.eye(P)
RV['params0_Cg'] = LA.cholesky(RV['Cg0_reg'])[SP.tril_indices(P)]
RV['params0_Cn'] = LA.cholesky(RV['Cn0_reg'])[SP.tril_indices(P)]
return RV
def _buildTraitCovar(self,trait_covar_type,rank=1):
"""
Internal functions that builds the trait covariance matrix using the LIMIX framework
Args:
trait_covar_type: type of covaraince to use in {freeform,lowrank_id,lowrank_diag}
rank: rank of a possible lowrank component (default 1)
Returns:
LIMIX::PCovarianceFunction for Trait covariance matrix
"""
assert trait_covar_type in 'freeform', '%s not supported yet'%trait_covar_type
cov = covariance.freeform(self.P)
return cov
def _buildTraitCovar(self, trait_covar_type, rank=1):
"""
Internal functions that builds the trait covariance matrix using the LIMIX framework
Args:
trait_covar_type: type of covaraince to use in {freeform,lowrank_id,lowrank_diag}
rank: rank of a possible lowrank component (default 1)
Returns:
LIMIX::PCovarianceFunction for Trait covariance matrix
"""
assert trait_covar_type in 'freeform', '%s not supported yet' % trait_covar_type
cov = covariance.freeform(self.P)
return cov
100, 150, 200, 300, 500, 800, 1200, 1600, 2000, 3000, 4000, 5000,
6000, 8000, 10000, 12000, 14000, 16000, 20000, 24000, 32000, 40000
])
n_rips = 5
t = sp.zeros((Ns.shape[0], n_rips))
t0 = sp.zeros((Ns.shape[0], n_rips))
r = sp.zeros((Ns.shape[0], n_rips))
for ni, n in enumerate(Ns):
for ri in range(n_rips):
print '.. %d individuals - rip %d' % (n, ri)
print ' .. generating data'
Y, F, G = gen_data(N=n, P=P)
# define GPs
gp = GP2KronSumLR(Y=Y, F=F, A=sp.eye(P), Cn=Cn, G=G)
gp0 = gp2ks0(Y, covariance.freeform(P), F=F, rank=1)
gp0.set_Xr(G)
if 1:
gp.covar.setRandomParams()
else:
n_params = gp.covar.Cr.getNumberParams()
n_params += gp.covar.Cn.getNumberParams()
params1 = {'covar': sp.randn(n_params)}
gp.setParams(params1)
params = {}
params['Cr'] = gp.covar.Cr.getParams().copy()
params['Cn'] = gp.covar.Cn.getParams().copy()
gp0.setParams(params)
print ' .. optimization'
if not os.path.exists(out_file) or 'recalc' in sys.argv:
Ns = sp.array([100,150,200,300,500,800,1200,1600,2000,3000,4000,5000])
n_rips = 5
t = sp.zeros((Ns.shape[0], n_rips))
t0 = sp.zeros((Ns.shape[0], n_rips))
r = sp.zeros((Ns.shape[0], n_rips))
for ni, n in enumerate(Ns):
for ri in range(n_rips):
print '.. %d individuals - rip %d' % (n, ri)
print ' .. generating data'
Y, S, U, G = gen_data(N=n, P=P)
# define GPs
gp = GP3KronSumLR(Y = Y, Cg = Cg, Cn = Cn, S_R = S, U_R = U, G = G, rank = 1)
gp0 = gp3ks0(mean(Y), covariance.freeform(P), covariance.freeform(P), S_XX=S, U_XX=U, rank=1)
gp0.set_Xr(G)
gp._reset_profiler()
if 1:
gp.covar.setRandomParams()
else:
n_params = gp.covar.Cr.getNumberParams()
n_params+= gp.covar.Cg.getNumberParams()
n_params+= gp.covar.Cn.getNumberParams()
params1 = {'covar': sp.randn(n_params)}
gp.setParams(params1)
params = {}
params['Cr'] = gp.covar.Cr.getParams().copy()
params['Cg'] = gp.covar.Cg.getParams().copy()
params['Cn'] = gp.covar.Cn.getParams().copy()
import time
import copy
import pdb
import h5py
import os
from limix.utils.util_functions import smartDumpDictHdf5
if __name__=='__main__':
P = 4
pdb.set_trace()
# define col covariances
C = FreeFormCov(P, jitter=0)
C0 = covariance.freeform(P)
t1 = 0
t0 = 0
for ti in range(1000):
C.setRandomParams()
C0.setParams(C.getParams())
for i in range(int(0.5*P*(P+1))):
_t0 = time.time()
C0.Kgrad_param(i)
_t1 = time.time()
C.K_grad_i(i)
_t2 = time.time()
t0 += _t1-_t0
if not os.path.exists(out_file) or 'recalc' in sys.argv:
Ns = sp.array([100,150,200,300,500,800,1200,1600,2000,3000,4000,5000,6000,8000,10000,12000,14000,16000,20000,24000,32000,40000])
n_rips = 5
t = sp.zeros((Ns.shape[0], n_rips))
t0 = sp.zeros((Ns.shape[0], n_rips))
r = sp.zeros((Ns.shape[0], n_rips))
for ni, n in enumerate(Ns):
for ri in range(n_rips):
print '.. %d individuals - rip %d' % (n, ri)
print ' .. generating data'
Y, F, G = gen_data(N=n, P=P)
# define GPs
gp = GP2KronSumLR(Y=Y, F=F, A=sp.eye(P), Cn=Cn, G=G)
gp0 = gp2ks0(Y, covariance.freeform(P), F=F, rank=1)
gp0.set_Xr(G)
if 1:
gp.covar.setRandomParams()
else:
n_params = gp.covar.Cr.getNumberParams()
n_params+= gp.covar.Cn.getNumberParams()
params1 = {'covar': sp.randn(n_params)}
gp.setParams(params1)
params = {}
params['Cr'] = gp.covar.Cr.getParams().copy()
params['Cn'] = gp.covar.Cn.getParams().copy()
gp0.setParams(params)
print ' .. optimization'
def test_lmm_lr_speed(G,y,Z,Kbg,Covs=None,S=None,U=None):
"""
low-rank lmm
input:
G : genotypes
y : phenotype
Z : features of low-rank matrix
Kbg : background covariance matrix
Covs : fixed effect covariates
using mtset implementation
"""
m = mean(y)
one = np.ones((1,1))
if Z.shape[1] > G.shape[0]:
return test_lmm_lr(G, y, Z, Kbg, Covs=Covs)
if Covs is not None:
m.addFixedEffect(Covs)
nCovs = Covs.shape[1]
Cg = covariance.freeform(1)
Cn = covariance.freeform(1)
Z/=np.sqrt(Z.shape[1])
gp = gp3kronSum(m,Cg,Cn,XX=Kbg,Xr=Z,S_XX=S,U_XX=U)
params_rnd = {}
params_rnd['Cg'] = 1e-4*np.random.randn(1)
params_rnd['Cn'] = 1e-4*np.random.randn(1)
params_rnd['Cr'] = 1e-4*np.random.randn(1)
if Covs is not None:
params_rnd['mean'] = 1e-6*np.random.randn(nCovs)
conv,info = OPT.opt_hyper(gp,params_rnd)
LML0 = gp.LML()
params0 = gp.getParams()
params_rnd = params0.copy()
if Covs is not None:
mean0 = params0['mean']
params_rnd['mean'] = 1e-6*np.random.randn(nCovs+1)
params_rnd['mean'][:nCovs] = mean0
else:
params_rnd['mean'] = 1e-6*np.random.randn(1)
F = G.shape[1]
LML = np.zeros(F)
beta = np.zeros(F)
for f in xrange(F):
m.clearFixedEffect()
if Covs is not None: m.addFixedEffect(Covs)
m.addFixedEffect(G[:,[f]])
conv,info = OPT.opt_hyper(gp, params_rnd)
beta[f] = m.getParams()[-1]
LML[f] = gp.LML()
LRT = 2*(LML0-LML)
pv = stats.chi2.sf(LRT,1)
return pv, beta
Ns = sp.array(
[100, 150, 200, 300, 500, 800, 1200, 1600, 2000, 3000, 4000, 5000])
n_rips = 5
t = sp.zeros((Ns.shape[0], n_rips))
t0 = sp.zeros((Ns.shape[0], n_rips))
r = sp.zeros((Ns.shape[0], n_rips))
for ni, n in enumerate(Ns):
for ri in range(n_rips):
print '.. %d individuals - rip %d' % (n, ri)
print ' .. generating data'
Y, S, U, G = gen_data(N=n, P=P)
# define GPs
gp = GP3KronSumLR(Y=Y, Cg=Cg, Cn=Cn, S_R=S, U_R=U, G=G, rank=1)
gp0 = gp3ks0(mean(Y),
covariance.freeform(P),
covariance.freeform(P),
S_XX=S,
U_XX=U,
rank=1)
gp0.set_Xr(G)
gp._reset_profiler()
if 1:
gp.covar.setRandomParams()
else:
n_params = gp.covar.Cr.getNumberParams()
n_params += gp.covar.Cg.getNumberParams()
n_params += gp.covar.Cn.getNumberParams()
params1 = {'covar': sp.randn(n_params)}
gp.setParams(params1)
n_rips = 5
t = sp.zeros((Ps.shape[0], n_rips))
t0 = sp.zeros((Ps.shape[0], n_rips))
r = sp.zeros((Ps.shape[0], n_rips))
for pi, p in enumerate(Ps):
for ri in range(n_rips):
print '.. %d traits - rip %d' % (p, ri)
print ' .. generating data'
Y, S, U, G = gen_data(N=N, P=p)
Cg = FreeFormCov(p, jitter=0)
Cn = FreeFormCov(p)
# define GPs
gp = GP3KronSumLR(Y=Y, Cg=Cg, Cn=Cn, S_R=S, U_R=U, G=G, rank=1)
gp0 = gp3ks0(mean(Y),
covariance.freeform(p),
covariance.freeform(p),
S_XX=S,
U_XX=U,
rank=1)
gp0.set_Xr(G)
gp._reset_profiler()
if 1:
gp.covar.setRandomParams()
else:
n_params = gp.covar.Cr.getNumberParams()
n_params += gp.covar.Cg.getNumberParams()
n_params += gp.covar.Cn.getNumberParams()
params1 = {'covar': sp.randn(n_params)}
gp.setParams(params1)
