def _initMean(self, Y, F=None, tol=1e-6): """ initialize the mean term Args: F: sample design of the fixed effect """ if F is not None: R = LA.qr(F, mode='r')[0][:F.shape[1], :] I = (abs(R.diagonal()) > tol) if SP.any(~I): warnings.warn( 'cols ' + str(SP.where(~I)[0]) + ' have been removed because linearly dependent on the others' ) self.F = F[:, I] else: self.F = None #dimensions self.N, self.P = Y.shape #get F and Y self.Y = Y # build mean self.mean = mean(Y) if F is not None: A = SP.eye(self.P) self.mean.addFixedEffect(F=self.F, A=A)
def fitPairwiseModel(Y,XX=None,S_XX=None,U_XX=None,verbose=False): N,P = Y.shape """ initilizes parameters """ RV = fitSingleTraitModel(Y,XX=XX,S_XX=S_XX,U_XX=U_XX,verbose=verbose) Cg = covariance.freeform(2) Cn = covariance.freeform(2) gp = gp2kronSum(mean(Y[:,0:2]),Cg,Cn,XX=XX,S_XX=S_XX,U_XX=U_XX) conv2 = SP.ones((P,P),dtype=bool) rho_g = SP.ones((P,P)) rho_n = SP.ones((P,P)) for p1 in range(P): for p2 in range(p1): if verbose: print '.. fitting correlation (%d,%d)'%(p1,p2) gp.setY(Y[:,[p1,p2]]) Cg_params0 = SP.array([SP.sqrt(RV['varST'][p1,0]),1e-6*SP.randn(),SP.sqrt(RV['varST'][p2,0])]) Cn_params0 = SP.array([SP.sqrt(RV['varST'][p1,1]),1e-6*SP.randn(),SP.sqrt(RV['varST'][p2,1])]) params0 = {'Cg':Cg_params0,'Cn':Cn_params0} conv2[p1,p2],info = OPT.opt_hyper(gp,params0,factr=1e3) rho_g[p1,p2] = Cg.K()[0,1]/SP.sqrt(Cg.K().diagonal().prod()) rho_n[p1,p2] = Cn.K()[0,1]/SP.sqrt(Cn.K().diagonal().prod()) conv2[p2,p1] = conv2[p1,p2]; rho_g[p2,p1] = rho_g[p1,p2]; rho_n[p2,p1] = rho_n[p1,p2] RV['Cg0'] = rho_g*SP.dot(SP.sqrt(RV['varST'][:,0:1]),SP.sqrt(RV['varST'][:,0:1].T)) RV['Cn0'] = rho_n*SP.dot(SP.sqrt(RV['varST'][:,1:2]),SP.sqrt(RV['varST'][:,1:2].T)) RV['conv2'] = conv2 #3. regularizes covariance matrices offset_g = abs(SP.minimum(LA.eigh(RV['Cg0'])[0].min(),0))+1e-4 offset_n = abs(SP.minimum(LA.eigh(RV['Cn0'])[0].min(),0))+1e-4 RV['Cg0_reg'] = RV['Cg0']+offset_g*SP.eye(P) RV['Cn0_reg'] = RV['Cn0']+offset_n*SP.eye(P) RV['params0_Cg']=LA.cholesky(RV['Cg0_reg'])[SP.tril_indices(P)] RV['params0_Cn']=LA.cholesky(RV['Cn0_reg'])[SP.tril_indices(P)] return RV
def fitPairwiseModel(Y, XX=None, S_XX=None, U_XX=None, verbose=False): N, P = Y.shape """ initilizes parameters """ RV = fitSingleTraitModel(Y, XX=XX, S_XX=S_XX, U_XX=U_XX, verbose=verbose) Cg = covariance.freeform(2) Cn = covariance.freeform(2) gp = gp2kronSum(mean(Y[:, 0:2]), Cg, Cn, XX=XX, S_XX=S_XX, U_XX=U_XX) conv2 = SP.ones((P, P), dtype=bool) rho_g = SP.ones((P, P)) rho_n = SP.ones((P, P)) for p1 in range(P): for p2 in range(p1): if verbose: print '.. fitting correlation (%d,%d)' % (p1, p2) gp.setY(Y[:, [p1, p2]]) Cg_params0 = SP.array([ SP.sqrt(RV['varST'][p1, 0]), 1e-6 * SP.randn(), SP.sqrt(RV['varST'][p2, 0]) ]) Cn_params0 = SP.array([ SP.sqrt(RV['varST'][p1, 1]), 1e-6 * SP.randn(), SP.sqrt(RV['varST'][p2, 1]) ]) params0 = {'Cg': Cg_params0, 'Cn': Cn_params0} conv2[p1, p2], info = OPT.opt_hyper(gp, params0, factr=1e3) rho_g[p1, p2] = Cg.K()[0, 1] / SP.sqrt(Cg.K().diagonal().prod()) rho_n[p1, p2] = Cn.K()[0, 1] / SP.sqrt(Cn.K().diagonal().prod()) conv2[p2, p1] = conv2[p1, p2] rho_g[p2, p1] = rho_g[p1, p2] rho_n[p2, p1] = rho_n[p1, p2] RV['Cg0'] = rho_g * SP.dot(SP.sqrt(RV['varST'][:, 0:1]), SP.sqrt(RV['varST'][:, 0:1].T)) RV['Cn0'] = rho_n * SP.dot(SP.sqrt(RV['varST'][:, 1:2]), SP.sqrt(RV['varST'][:, 1:2].T)) RV['conv2'] = conv2 #3. regularizes covariance matrices offset_g = abs(SP.minimum(LA.eigh(RV['Cg0'])[0].min(), 0)) + 1e-4 offset_n = abs(SP.minimum(LA.eigh(RV['Cn0'])[0].min(), 0)) + 1e-4 RV['Cg0_reg'] = RV['Cg0'] + offset_g * SP.eye(P) RV['Cn0_reg'] = RV['Cn0'] + offset_n * SP.eye(P) RV['params0_Cg'] = LA.cholesky(RV['Cg0_reg'])[SP.tril_indices(P)] RV['params0_Cn'] = LA.cholesky(RV['Cn0_reg'])[SP.tril_indices(P)] return RV
def fitSingleTraitModel(Y,XX=None,S_XX=None,U_XX=None,verbose=False): """ fit single trait model """ N,P = Y.shape RV = {} Cg = covariance.lowrank(1) Cn = covariance.lowrank(1) gp = gp2kronSum(mean(Y[:,0:1]),Cg,Cn,XX=XX,S_XX=S_XX,U_XX=U_XX) params0 = {'Cg':SP.sqrt(0.5)*SP.ones(1),'Cn':SP.sqrt(0.5)*SP.ones(1)} var = SP.zeros((P,2)) conv1 = SP.zeros(P,dtype=bool) for p in range(P): if verbose: print '.. fitting variance trait %d'%p gp.setY(Y[:,p:p+1]) conv1[p],info = OPT.opt_hyper(gp,params0,factr=1e3) var[p,0] = Cg.K()[0,0] var[p,1] = Cn.K()[0,0] RV['conv1'] = conv1 RV['varST'] = var return RV
def _initMean(self,Y,F=None,tol=1e-6): """ initialize the mean term Args: F: sample design of the fixed effect """ if F is not None: R = LA.qr(F,mode='r')[0][:F.shape[1],:] I = (abs(R.diagonal())>tol) if SP.any(~I): warnings.warn('cols '+str(SP.where(~I)[0])+' have been removed because linearly dependent on the others') self.F = F[:,I] else: self.F = None #dimensions self.N,self.P = Y.shape #get F and Y self.Y=Y # build mean self.mean = mean(Y) if F is not None: A = SP.eye(self.P) self.mean.addFixedEffect(F=self.F,A=A)
def fitSingleTraitModel(Y, XX=None, S_XX=None, U_XX=None, verbose=False): """ fit single trait model """ N, P = Y.shape RV = {} Cg = covariance.lowrank(1) Cn = covariance.lowrank(1) gp = gp2kronSum(mean(Y[:, 0:1]), Cg, Cn, XX=XX, S_XX=S_XX, U_XX=U_XX) params0 = { 'Cg': SP.sqrt(0.5) * SP.ones(1), 'Cn': SP.sqrt(0.5) * SP.ones(1) } var = SP.zeros((P, 2)) conv1 = SP.zeros(P, dtype=bool) for p in range(P): if verbose: print '.. fitting variance trait %d' % p gp.setY(Y[:, p:p + 1]) conv1[p], info = OPT.opt_hyper(gp, params0, factr=1e3) var[p, 0] = Cg.K()[0, 0] var[p, 1] = Cn.K()[0, 0] RV['conv1'] = conv1 RV['varST'] = var return RV
Xr*= SP.sqrt(N/(Xr**2).sum()) # define mean term mu = mean(Y) # add first fixed effect F = 1.*(SP.rand(N,2)<0.2); A = SP.eye(P) # add second fixed effect F2 = 1.*(SP.rand(N,3)<0.2); A2 = SP.ones((1,P)) mu.addFixedEffect(F=F,A=A) mu.addFixedEffect(F=F2,A=A2) if mtSet_present: mu1 = MEAN.mean(Y) mu1.addFixedEffect(F=F,A=A) mu1.addFixedEffect(F=F2,A=A2) # define covariance matrices Cg = freeform(P) Cn = freeform(P) Cg1 = freeform(P) Cn1 = freeform(P) ipdb.set_trace() if 1: # compare with mtSet implementation params = {}
if not os.path.exists(out_file) or 'recalc' in sys.argv: Ns = sp.array([100,150,200,300,500,800,1200,1600,2000,3000,4000,5000]) n_rips = 5 t = sp.zeros((Ns.shape[0], n_rips)) t0 = sp.zeros((Ns.shape[0], n_rips)) r = sp.zeros((Ns.shape[0], n_rips)) for ni, n in enumerate(Ns): for ri in range(n_rips): print '.. %d individuals - rip %d' % (n, ri) print ' .. generating data' Y, S, U, G = gen_data(N=n, P=P) # define GPs gp = GP3KronSumLR(Y = Y, Cg = Cg, Cn = Cn, S_R = S, U_R = U, G = G, rank = 1) gp0 = gp3ks0(mean(Y), covariance.freeform(P), covariance.freeform(P), S_XX=S, U_XX=U, rank=1) gp0.set_Xr(G) gp._reset_profiler() if 1: gp.covar.setRandomParams() else: n_params = gp.covar.Cr.getNumberParams() n_params+= gp.covar.Cg.getNumberParams() n_params+= gp.covar.Cn.getNumberParams() params1 = {'covar': sp.randn(n_params)} gp.setParams(params1) params = {} params['Cr'] = gp.covar.Cr.getParams().copy() params['Cg'] = gp.covar.Cg.getParams().copy() params['Cn'] = gp.covar.Cn.getParams().copy()
# generate data h2 = 0.3 N = 2000; P = 4; S = 1000 X = 1.*(SP.rand(N,S)<0.2) beta = SP.randn(S,P) Yg = SP.dot(X,beta); Yg*=SP.sqrt(h2/Yg.var(0).mean()) Yn = SP.randn(N,P); Yn*=SP.sqrt((1-h2)/Yn.var(0).mean()) Y = Yg+Yn; Y-=Y.mean(0); Y/=Y.std(0) XX = SP.dot(X,X.T) XX/= XX.diagonal().mean() Xr = 1.*(SP.rand(N,10)<0.2) Xr*= SP.sqrt(N/(Xr**2).sum()) # define mean term mean = MEAN.mean(Y) # add first fixed effect F = 1.*(SP.rand(N,2)<0.2); A = SP.eye(P) mean.addFixedEffect(F=F,A=A) # add first fixed effect F = 1.*(SP.rand(N,3)<0.2); A = SP.ones((1,P)) mean.addFixedEffect(F=F,A=A) # define covariance matrices Cg = limix.CFreeFormCF(P) Cn = limix.CFreeFormCF(P) if 0: # generate parameters params = {} params['Cg'] = SP.randn(int(0.5*P*(P+1)))
if not os.path.exists(out_file) or 'recalc' in sys.argv: Ns = sp.array( [100, 150, 200, 300, 500, 800, 1200, 1600, 2000, 3000, 4000, 5000]) n_rips = 5 t = sp.zeros((Ns.shape[0], n_rips)) t0 = sp.zeros((Ns.shape[0], n_rips)) r = sp.zeros((Ns.shape[0], n_rips)) for ni, n in enumerate(Ns): for ri in range(n_rips): print '.. %d individuals - rip %d' % (n, ri) print ' .. generating data' Y, S, U, G = gen_data(N=n, P=P) # define GPs gp = GP3KronSumLR(Y=Y, Cg=Cg, Cn=Cn, S_R=S, U_R=U, G=G, rank=1) gp0 = gp3ks0(mean(Y), covariance.freeform(P), covariance.freeform(P), S_XX=S, U_XX=U, rank=1) gp0.set_Xr(G) gp._reset_profiler() if 1: gp.covar.setRandomParams() else: n_params = gp.covar.Cr.getNumberParams() n_params += gp.covar.Cg.getNumberParams() n_params += gp.covar.Cn.getNumberParams() params1 = {'covar': sp.randn(n_params)}