def fitPairwiseModel(Y,XX=None,S_XX=None,U_XX=None,verbose=False): N,P = Y.shape """ initilizes parameters """ RV = fitSingleTraitModel(Y,XX=XX,S_XX=S_XX,U_XX=U_XX,verbose=verbose) Cg = covariance.freeform(2) Cn = covariance.freeform(2) gp = gp2kronSum(mean(Y[:,0:2]),Cg,Cn,XX=XX,S_XX=S_XX,U_XX=U_XX) conv2 = SP.ones((P,P),dtype=bool) rho_g = SP.ones((P,P)) rho_n = SP.ones((P,P)) for p1 in range(P): for p2 in range(p1): if verbose: print '.. fitting correlation (%d,%d)'%(p1,p2) gp.setY(Y[:,[p1,p2]]) Cg_params0 = SP.array([SP.sqrt(RV['varST'][p1,0]),1e-6*SP.randn(),SP.sqrt(RV['varST'][p2,0])]) Cn_params0 = SP.array([SP.sqrt(RV['varST'][p1,1]),1e-6*SP.randn(),SP.sqrt(RV['varST'][p2,1])]) params0 = {'Cg':Cg_params0,'Cn':Cn_params0} conv2[p1,p2],info = OPT.opt_hyper(gp,params0,factr=1e3) rho_g[p1,p2] = Cg.K()[0,1]/SP.sqrt(Cg.K().diagonal().prod()) rho_n[p1,p2] = Cn.K()[0,1]/SP.sqrt(Cn.K().diagonal().prod()) conv2[p2,p1] = conv2[p1,p2]; rho_g[p2,p1] = rho_g[p1,p2]; rho_n[p2,p1] = rho_n[p1,p2] RV['Cg0'] = rho_g*SP.dot(SP.sqrt(RV['varST'][:,0:1]),SP.sqrt(RV['varST'][:,0:1].T)) RV['Cn0'] = rho_n*SP.dot(SP.sqrt(RV['varST'][:,1:2]),SP.sqrt(RV['varST'][:,1:2].T)) RV['conv2'] = conv2 #3. regularizes covariance matrices offset_g = abs(SP.minimum(LA.eigh(RV['Cg0'])[0].min(),0))+1e-4 offset_n = abs(SP.minimum(LA.eigh(RV['Cn0'])[0].min(),0))+1e-4 RV['Cg0_reg'] = RV['Cg0']+offset_g*SP.eye(P) RV['Cn0_reg'] = RV['Cn0']+offset_n*SP.eye(P) RV['params0_Cg']=LA.cholesky(RV['Cg0_reg'])[SP.tril_indices(P)] RV['params0_Cn']=LA.cholesky(RV['Cn0_reg'])[SP.tril_indices(P)] return RV
def fitPairwiseModel(Y, XX=None, S_XX=None, U_XX=None, verbose=False): N, P = Y.shape """ initilizes parameters """ RV = fitSingleTraitModel(Y, XX=XX, S_XX=S_XX, U_XX=U_XX, verbose=verbose) Cg = covariance.freeform(2) Cn = covariance.freeform(2) gp = gp2kronSum(mean(Y[:, 0:2]), Cg, Cn, XX=XX, S_XX=S_XX, U_XX=U_XX) conv2 = SP.ones((P, P), dtype=bool) rho_g = SP.ones((P, P)) rho_n = SP.ones((P, P)) for p1 in range(P): for p2 in range(p1): if verbose: print '.. fitting correlation (%d,%d)' % (p1, p2) gp.setY(Y[:, [p1, p2]]) Cg_params0 = SP.array([ SP.sqrt(RV['varST'][p1, 0]), 1e-6 * SP.randn(), SP.sqrt(RV['varST'][p2, 0]) ]) Cn_params0 = SP.array([ SP.sqrt(RV['varST'][p1, 1]), 1e-6 * SP.randn(), SP.sqrt(RV['varST'][p2, 1]) ]) params0 = {'Cg': Cg_params0, 'Cn': Cn_params0} conv2[p1, p2], info = OPT.opt_hyper(gp, params0, factr=1e3) rho_g[p1, p2] = Cg.K()[0, 1] / SP.sqrt(Cg.K().diagonal().prod()) rho_n[p1, p2] = Cn.K()[0, 1] / SP.sqrt(Cn.K().diagonal().prod()) conv2[p2, p1] = conv2[p1, p2] rho_g[p2, p1] = rho_g[p1, p2] rho_n[p2, p1] = rho_n[p1, p2] RV['Cg0'] = rho_g * SP.dot(SP.sqrt(RV['varST'][:, 0:1]), SP.sqrt(RV['varST'][:, 0:1].T)) RV['Cn0'] = rho_n * SP.dot(SP.sqrt(RV['varST'][:, 1:2]), SP.sqrt(RV['varST'][:, 1:2].T)) RV['conv2'] = conv2 #3. regularizes covariance matrices offset_g = abs(SP.minimum(LA.eigh(RV['Cg0'])[0].min(), 0)) + 1e-4 offset_n = abs(SP.minimum(LA.eigh(RV['Cn0'])[0].min(), 0)) + 1e-4 RV['Cg0_reg'] = RV['Cg0'] + offset_g * SP.eye(P) RV['Cn0_reg'] = RV['Cn0'] + offset_n * SP.eye(P) RV['params0_Cg'] = LA.cholesky(RV['Cg0_reg'])[SP.tril_indices(P)] RV['params0_Cn'] = LA.cholesky(RV['Cn0_reg'])[SP.tril_indices(P)] return RV
def _buildTraitCovar(self,trait_covar_type,rank=1): """ Internal functions that builds the trait covariance matrix using the LIMIX framework Args: trait_covar_type: type of covaraince to use in {freeform,lowrank_id,lowrank_diag} rank: rank of a possible lowrank component (default 1) Returns: LIMIX::PCovarianceFunction for Trait covariance matrix """ assert trait_covar_type in 'freeform', '%s not supported yet'%trait_covar_type cov = covariance.freeform(self.P) return cov
def _buildTraitCovar(self, trait_covar_type, rank=1): """ Internal functions that builds the trait covariance matrix using the LIMIX framework Args: trait_covar_type: type of covaraince to use in {freeform,lowrank_id,lowrank_diag} rank: rank of a possible lowrank component (default 1) Returns: LIMIX::PCovarianceFunction for Trait covariance matrix """ assert trait_covar_type in 'freeform', '%s not supported yet' % trait_covar_type cov = covariance.freeform(self.P) return cov
100, 150, 200, 300, 500, 800, 1200, 1600, 2000, 3000, 4000, 5000, 6000, 8000, 10000, 12000, 14000, 16000, 20000, 24000, 32000, 40000 ]) n_rips = 5 t = sp.zeros((Ns.shape[0], n_rips)) t0 = sp.zeros((Ns.shape[0], n_rips)) r = sp.zeros((Ns.shape[0], n_rips)) for ni, n in enumerate(Ns): for ri in range(n_rips): print '.. %d individuals - rip %d' % (n, ri) print ' .. generating data' Y, F, G = gen_data(N=n, P=P) # define GPs gp = GP2KronSumLR(Y=Y, F=F, A=sp.eye(P), Cn=Cn, G=G) gp0 = gp2ks0(Y, covariance.freeform(P), F=F, rank=1) gp0.set_Xr(G) if 1: gp.covar.setRandomParams() else: n_params = gp.covar.Cr.getNumberParams() n_params += gp.covar.Cn.getNumberParams() params1 = {'covar': sp.randn(n_params)} gp.setParams(params1) params = {} params['Cr'] = gp.covar.Cr.getParams().copy() params['Cn'] = gp.covar.Cn.getParams().copy() gp0.setParams(params) print ' .. optimization'
if not os.path.exists(out_file) or 'recalc' in sys.argv: Ns = sp.array([100,150,200,300,500,800,1200,1600,2000,3000,4000,5000]) n_rips = 5 t = sp.zeros((Ns.shape[0], n_rips)) t0 = sp.zeros((Ns.shape[0], n_rips)) r = sp.zeros((Ns.shape[0], n_rips)) for ni, n in enumerate(Ns): for ri in range(n_rips): print '.. %d individuals - rip %d' % (n, ri) print ' .. generating data' Y, S, U, G = gen_data(N=n, P=P) # define GPs gp = GP3KronSumLR(Y = Y, Cg = Cg, Cn = Cn, S_R = S, U_R = U, G = G, rank = 1) gp0 = gp3ks0(mean(Y), covariance.freeform(P), covariance.freeform(P), S_XX=S, U_XX=U, rank=1) gp0.set_Xr(G) gp._reset_profiler() if 1: gp.covar.setRandomParams() else: n_params = gp.covar.Cr.getNumberParams() n_params+= gp.covar.Cg.getNumberParams() n_params+= gp.covar.Cn.getNumberParams() params1 = {'covar': sp.randn(n_params)} gp.setParams(params1) params = {} params['Cr'] = gp.covar.Cr.getParams().copy() params['Cg'] = gp.covar.Cg.getParams().copy() params['Cn'] = gp.covar.Cn.getParams().copy()
import time import copy import pdb import h5py import os from limix.utils.util_functions import smartDumpDictHdf5 if __name__=='__main__': P = 4 pdb.set_trace() # define col covariances C = FreeFormCov(P, jitter=0) C0 = covariance.freeform(P) t1 = 0 t0 = 0 for ti in range(1000): C.setRandomParams() C0.setParams(C.getParams()) for i in range(int(0.5*P*(P+1))): _t0 = time.time() C0.Kgrad_param(i) _t1 = time.time() C.K_grad_i(i) _t2 = time.time() t0 += _t1-_t0
if not os.path.exists(out_file) or 'recalc' in sys.argv: Ns = sp.array([100,150,200,300,500,800,1200,1600,2000,3000,4000,5000,6000,8000,10000,12000,14000,16000,20000,24000,32000,40000]) n_rips = 5 t = sp.zeros((Ns.shape[0], n_rips)) t0 = sp.zeros((Ns.shape[0], n_rips)) r = sp.zeros((Ns.shape[0], n_rips)) for ni, n in enumerate(Ns): for ri in range(n_rips): print '.. %d individuals - rip %d' % (n, ri) print ' .. generating data' Y, F, G = gen_data(N=n, P=P) # define GPs gp = GP2KronSumLR(Y=Y, F=F, A=sp.eye(P), Cn=Cn, G=G) gp0 = gp2ks0(Y, covariance.freeform(P), F=F, rank=1) gp0.set_Xr(G) if 1: gp.covar.setRandomParams() else: n_params = gp.covar.Cr.getNumberParams() n_params+= gp.covar.Cn.getNumberParams() params1 = {'covar': sp.randn(n_params)} gp.setParams(params1) params = {} params['Cr'] = gp.covar.Cr.getParams().copy() params['Cn'] = gp.covar.Cn.getParams().copy() gp0.setParams(params) print ' .. optimization'
def test_lmm_lr_speed(G,y,Z,Kbg,Covs=None,S=None,U=None): """ low-rank lmm input: G : genotypes y : phenotype Z : features of low-rank matrix Kbg : background covariance matrix Covs : fixed effect covariates using mtset implementation """ m = mean(y) one = np.ones((1,1)) if Z.shape[1] > G.shape[0]: return test_lmm_lr(G, y, Z, Kbg, Covs=Covs) if Covs is not None: m.addFixedEffect(Covs) nCovs = Covs.shape[1] Cg = covariance.freeform(1) Cn = covariance.freeform(1) Z/=np.sqrt(Z.shape[1]) gp = gp3kronSum(m,Cg,Cn,XX=Kbg,Xr=Z,S_XX=S,U_XX=U) params_rnd = {} params_rnd['Cg'] = 1e-4*np.random.randn(1) params_rnd['Cn'] = 1e-4*np.random.randn(1) params_rnd['Cr'] = 1e-4*np.random.randn(1) if Covs is not None: params_rnd['mean'] = 1e-6*np.random.randn(nCovs) conv,info = OPT.opt_hyper(gp,params_rnd) LML0 = gp.LML() params0 = gp.getParams() params_rnd = params0.copy() if Covs is not None: mean0 = params0['mean'] params_rnd['mean'] = 1e-6*np.random.randn(nCovs+1) params_rnd['mean'][:nCovs] = mean0 else: params_rnd['mean'] = 1e-6*np.random.randn(1) F = G.shape[1] LML = np.zeros(F) beta = np.zeros(F) for f in xrange(F): m.clearFixedEffect() if Covs is not None: m.addFixedEffect(Covs) m.addFixedEffect(G[:,[f]]) conv,info = OPT.opt_hyper(gp, params_rnd) beta[f] = m.getParams()[-1] LML[f] = gp.LML() LRT = 2*(LML0-LML) pv = stats.chi2.sf(LRT,1) return pv, beta
Ns = sp.array( [100, 150, 200, 300, 500, 800, 1200, 1600, 2000, 3000, 4000, 5000]) n_rips = 5 t = sp.zeros((Ns.shape[0], n_rips)) t0 = sp.zeros((Ns.shape[0], n_rips)) r = sp.zeros((Ns.shape[0], n_rips)) for ni, n in enumerate(Ns): for ri in range(n_rips): print '.. %d individuals - rip %d' % (n, ri) print ' .. generating data' Y, S, U, G = gen_data(N=n, P=P) # define GPs gp = GP3KronSumLR(Y=Y, Cg=Cg, Cn=Cn, S_R=S, U_R=U, G=G, rank=1) gp0 = gp3ks0(mean(Y), covariance.freeform(P), covariance.freeform(P), S_XX=S, U_XX=U, rank=1) gp0.set_Xr(G) gp._reset_profiler() if 1: gp.covar.setRandomParams() else: n_params = gp.covar.Cr.getNumberParams() n_params += gp.covar.Cg.getNumberParams() n_params += gp.covar.Cn.getNumberParams() params1 = {'covar': sp.randn(n_params)} gp.setParams(params1)
n_rips = 5 t = sp.zeros((Ps.shape[0], n_rips)) t0 = sp.zeros((Ps.shape[0], n_rips)) r = sp.zeros((Ps.shape[0], n_rips)) for pi, p in enumerate(Ps): for ri in range(n_rips): print '.. %d traits - rip %d' % (p, ri) print ' .. generating data' Y, S, U, G = gen_data(N=N, P=p) Cg = FreeFormCov(p, jitter=0) Cn = FreeFormCov(p) # define GPs gp = GP3KronSumLR(Y=Y, Cg=Cg, Cn=Cn, S_R=S, U_R=U, G=G, rank=1) gp0 = gp3ks0(mean(Y), covariance.freeform(p), covariance.freeform(p), S_XX=S, U_XX=U, rank=1) gp0.set_Xr(G) gp._reset_profiler() if 1: gp.covar.setRandomParams() else: n_params = gp.covar.Cr.getNumberParams() n_params += gp.covar.Cg.getNumberParams() n_params += gp.covar.Cn.getNumberParams() params1 = {'covar': sp.randn(n_params)} gp.setParams(params1)