def test_gpkronprod(self): # initialize covar_c = linear.LinearCF(n_dimensions=self.n_latent) covar_r = linear.LinearCF(n_dimensions=self.n_dimensions) X0_c = SP.random.randn(self.n_tasks,self.n_latent) lik = likelihood_base.GaussIsoLik() gp = gp_kronprod.KronProdGP(covar_c=covar_c, covar_r=covar_r, likelihood=lik) gp.setData(Y=self.Ykronprod['train'],X_r=self.X['train']) hyperparams = {'lik':SP.array([0.5]), 'X_c':X0_c, 'covar_r':SP.array([0.5]), 'covar_c':SP.array([0.5]), 'X_r':self.X['train']} # check predictions, likelihood and gradients gp.predict(hyperparams,Xstar_r=self.X['test'],debugging=True) gp._LML_covar(hyperparams,debugging=True) gp._LMLgrad_covar(hyperparams,debugging=True) gp._LMLgrad_lik(hyperparams,debugging=True) gp._LMLgrad_x(hyperparams,debugging=True) # optimize hyperparams = {'lik':SP.array([0.5]), 'X_c':X0_c, 'covar_r':SP.array([0.5]), 'covar_c':SP.array([0.5])} opts = {'gradcheck':True} hyperparams_opt,lml_opt = optimize_base.opt_hyper(gp,hyperparams,opts=opts) Kest = covar_c.K(hyperparams_opt['covar_c']) # check predictions, likelihood and gradients gp._invalidate_cache() # otherwise debugging parameters are not up to date! gp.predict(hyperparams_opt,debugging=True,Xstar_r=self.X['test']) gp._LML_covar(hyperparams_opt,debugging=True) gp._LMLgrad_covar(hyperparams_opt,debugging=True) gp._LMLgrad_lik(hyperparams_opt,debugging=True) gp._LMLgrad_x(hyperparams_opt,debugging=True)
def test_gplvm(self): covar = linear.LinearCF(n_dimensions=self.n_latent) lik = likelihood_base.GaussIsoLik() prior = priors.GaussianPrior(key='X', theta=SP.array([1.])) gp = gplvm.GPLVM(covar=covar, likelihood=lik, prior=prior) X0 = SP.random.randn(self.n_tasks, self.n_latent) X0 = self.Xlatent covar.X = X0 gp.setData(Y=self.Ylatent) # gradient with respect to X hyperparams = { 'covar': SP.array([0.5]), 'lik': SP.array([0.5]), 'X': X0 } LML = gp.LML(hyperparams) LMLgrad = gp.LMLgrad(hyperparams) LMLgrad_x = SP.zeros((self.n_tasks, self.n_latent)) W = gp.get_covariances(hyperparams)['W'] for d in xrange(self.n_latent): for n in xrange(self.n_tasks): Knd_grad = covar.Kgrad_x(hyperparams['covar'], d, n) LMLgrad_x[n, d] = 0.5 * (W * Knd_grad).sum() LMLgrad_x += prior.LMLgrad(hyperparams)['X'] assert SP.allclose( LMLgrad['X'], LMLgrad_x), 'ouch, gradient with respect to X is wrong' # optimize opts = {'gradcheck': True} hyperparams_opt, lml_opt = optimize_base.opt_hyper(gp, hyperparams, opts=opts) Ktrue = SP.dot(self.Xlatent, self.Xlatent.T) covar.X = hyperparams_opt['X'] Kest = covar.K(hyperparams_opt['covar']) # gradient with respect to X LML = gp.LML(hyperparams_opt) LMLgrad = gp.LMLgrad(hyperparams_opt) LMLgrad_x = SP.zeros((self.n_tasks, self.n_latent)) W = gp.get_covariances(hyperparams_opt)['W'] for d in xrange(self.n_latent): for n in xrange(self.n_tasks): Knd_grad = covar.Kgrad_x(hyperparams_opt['covar'], d, n) LMLgrad_x[n, d] = 0.5 * (W * Knd_grad).sum() LMLgrad_x += prior.LMLgrad(hyperparams_opt)['X'] assert SP.allclose( LMLgrad['X'], LMLgrad_x), 'ouch, gradient with respect to X is wrong'
def run_optimizer(method, gp, opts, Y, X_r, Icv, cv_idx, X_o=None): if 'min_iter' in opts: min_iter = opts['min_iter'] else: min_iter = 10 if 'max_iter' in opts: max_iter = opts['max_iter'] else: max_iter = 100 # initialize LG.info('Optimize %s' % method) converged = False lmltest_global = SP.inf hyperparams_global = None Ypred_global = None r2_global = -SP.inf # hold nfolds of the data out Itrain = Icv != cv_idx Itest = Icv == cv_idx i = 1 while True: LG.info('Iteration: %d' % i) converged = False # stop, if maximum number of iterations is reached if i > max_iter: break # set data if X_o == None: gp.setData(Y=Y[Itrain], X_r=X_r[Itrain]) else: gp.setData(Y=Y[Itrain], X_r=X_r[Itrain], X_o=X_o[Itrain]) hyperparams, Ifilter, bounds = initialize.init(method, Y[Itrain].T, X_r[Itrain], opts) try: [hyperparams_opt, lmltrain] = opt.opt_hyper(gp, hyperparams, opts=opts, Ifilter=Ifilter, bounds=bounds) # gradient need not to be 0, because we have bounds on the hyperparameters... gradient = SP.array([ LA.norm(x) for x in gp.LMLgrad(hyperparams_opt).values() ]).mean() LG.info('LMLtrain: %.3f' % gp.LML(hyperparams_opt)) LG.info('Gradient: %.3f' % (gradient)) converged = True except AssertionError, error: print 'Assertion Error: %s' % error continue except:
def test_gpkronprod(self): # initialize covar_c = linear.LinearCF(n_dimensions=self.n_latent) covar_r = linear.LinearCF(n_dimensions=self.n_dimensions) X0_c = SP.random.randn(self.n_tasks, self.n_latent) lik = likelihood_base.GaussIsoLik() gp = gp_kronprod.KronProdGP(covar_c=covar_c, covar_r=covar_r, likelihood=lik) gp.setData(Y=self.Ykronprod['train'], X_r=self.X['train']) hyperparams = { 'lik': SP.array([0.5]), 'X_c': X0_c, 'covar_r': SP.array([0.5]), 'covar_c': SP.array([0.5]), 'X_r': self.X['train'] } # check predictions, likelihood and gradients gp.predict(hyperparams, Xstar_r=self.X['test'], debugging=True) gp._LML_covar(hyperparams, debugging=True) gp._LMLgrad_covar(hyperparams, debugging=True) gp._LMLgrad_lik(hyperparams, debugging=True) gp._LMLgrad_x(hyperparams, debugging=True) # optimize hyperparams = { 'lik': SP.array([0.5]), 'X_c': X0_c, 'covar_r': SP.array([0.5]), 'covar_c': SP.array([0.5]) } opts = {'gradcheck': True} hyperparams_opt, lml_opt = optimize_base.opt_hyper(gp, hyperparams, opts=opts) Kest = covar_c.K(hyperparams_opt['covar_c']) # check predictions, likelihood and gradients gp._invalidate_cache( ) # otherwise debugging parameters are not up to date! gp.predict(hyperparams_opt, debugging=True, Xstar_r=self.X['test']) gp._LML_covar(hyperparams_opt, debugging=True) gp._LMLgrad_covar(hyperparams_opt, debugging=True) gp._LMLgrad_lik(hyperparams_opt, debugging=True) gp._LMLgrad_x(hyperparams_opt, debugging=True)
def run_optimizer(method,gp,opts,Y,X_r,Icv,cv_idx,X_o=None): if 'min_iter' in opts: min_iter = opts['min_iter'] else: min_iter = 10 if 'max_iter' in opts: max_iter = opts['max_iter'] else: max_iter = 100 # initialize LG.info('Optimize %s'%method) converged = False lmltest_global = SP.inf hyperparams_global = None Ypred_global = None r2_global = -SP.inf # hold nfolds of the data out Itrain = Icv!=cv_idx Itest = Icv==cv_idx i=1 while True: LG.info('Iteration: %d'%i) converged = False # stop, if maximum number of iterations is reached if i>max_iter: break # set data if X_o==None: gp.setData(Y=Y[Itrain],X_r=X_r[Itrain]) else: gp.setData(Y=Y[Itrain],X_r=X_r[Itrain],X_o=X_o[Itrain]) hyperparams,Ifilter,bounds = initialize.init(method,Y[Itrain].T,X_r[Itrain],opts) try: [hyperparams_opt,lmltrain] = opt.opt_hyper(gp,hyperparams,opts=opts,Ifilter=Ifilter,bounds=bounds) # gradient need not to be 0, because we have bounds on the hyperparameters... gradient = SP.array([LA.norm(x) for x in gp.LMLgrad(hyperparams_opt).values()]).mean() LG.info('LMLtrain: %.3f'%gp.LML(hyperparams_opt)) LG.info('Gradient: %.3f'%(gradient)) converged = True except AssertionError, error: print 'Assertion Error: %s'%error continue except:
def measure_runtime(env,N,D,n_reps=10,time_out=10000): opts = {'messages':False} out_dir = os.path.join(env['out_dir'],'simulations_runtime') if not os.path.exists(out_dir): os.makedirs(out_dir) t_fast = SP.zeros(n_reps) t_slow = SP.zeros(n_reps) lml_fast = SP.zeros(n_reps) lml_slow = SP.zeros(n_reps) for i in range(n_reps): # load data var_signal = 0.5 data,RV = load_simulations(env,var_signal,N,D,i) # initialize covar_c = lowrank.LowRankCF(n_dimensions=RV['n_c']) covar_r = linear.LinearCF(n_dimensions=RV['n_r']) covar_s = lowrank.LowRankCF(n_dimensions=RV['n_sigma']) covar_o = fixed.FixedCF(n_dimensions=RV['n_r']) X = data.getX(standardized=False) Y = data.getY(standardized=False).T hyperparams,Ifilter,bounds = initialize.init('GPkronsum_LIN',Y.T,X,RV) covar_r.X = X covar_o.X = X covar_o._K = SP.eye(RV['N']) covar_s.X = hyperparams['X_s'] covar_c.X = hyperparams['X_c'] kgp_fast = gp_kronsum.KronSumGP(covar_r=covar_r,covar_c=covar_c,covar_s=covar_s,covar_o=covar_o) kgp_fast.setData(Y=Y) # measure time signal.signal(signal.SIGALRM,handler) signal.alarm(time_out) try: t_start = time.clock() hyperparams_opt,lmltrain = opt.opt_hyper(kgp_fast,hyperparams,Ifilter=Ifilter,bounds=bounds,opts=opts) t_stop = time.clock() signal.alarm(0) t_fast[i] = t_stop - t_start lml_fast[i] = lmltrain except Exception, e: print e t_slow += time_out break
def test_gplvm(self): covar = linear.LinearCF(n_dimensions=self.n_latent) lik = likelihood_base.GaussIsoLik() prior = priors.GaussianPrior(key='X',theta=SP.array([1.])) gp = gplvm.GPLVM(covar=covar,likelihood=lik,prior=prior) X0 = SP.random.randn(self.n_tasks, self.n_latent) X0 = self.Xlatent covar.X = X0 gp.setData(Y=self.Ylatent) # gradient with respect to X hyperparams = {'covar':SP.array([0.5]), 'lik':SP.array([0.5]),'X':X0} LML = gp.LML(hyperparams) LMLgrad = gp.LMLgrad(hyperparams) LMLgrad_x = SP.zeros((self.n_tasks, self.n_latent)) W = gp.get_covariances(hyperparams)['W'] for d in xrange(self.n_latent): for n in xrange(self.n_tasks): Knd_grad = covar.Kgrad_x(hyperparams['covar'],d,n) LMLgrad_x[n,d] = 0.5*(W*Knd_grad).sum() LMLgrad_x += prior.LMLgrad(hyperparams)['X'] assert SP.allclose(LMLgrad['X'],LMLgrad_x), 'ouch, gradient with respect to X is wrong' # optimize opts = {'gradcheck':True} hyperparams_opt,lml_opt = optimize_base.opt_hyper(gp,hyperparams,opts=opts) Ktrue = SP.dot(self.Xlatent,self.Xlatent.T) covar.X = hyperparams_opt['X'] Kest = covar.K(hyperparams_opt['covar']) # gradient with respect to X LML = gp.LML(hyperparams_opt) LMLgrad = gp.LMLgrad(hyperparams_opt) LMLgrad_x = SP.zeros((self.n_tasks, self.n_latent)) W = gp.get_covariances(hyperparams_opt)['W'] for d in xrange(self.n_latent): for n in xrange(self.n_tasks): Knd_grad = covar.Kgrad_x(hyperparams_opt['covar'],d,n) LMLgrad_x[n,d] = 0.5*(W*Knd_grad).sum() LMLgrad_x += prior.LMLgrad(hyperparams_opt)['X'] assert SP.allclose(LMLgrad['X'],LMLgrad_x), 'ouch, gradient with respect to X is wrong'
Y = data.getY(standardized=False).T hyperparams,Ifilter,bounds = initialize.init('GPkronsum_LIN',Y.T,X,RV) covar_r.X = X covar_o.X = X covar_o._K = SP.eye(RV['N']) covar_s.X = hyperparams['X_s'] covar_c.X = hyperparams['X_c'] kgp_slow = gp_kronsum_naive.KronSumGP(covar_r=covar_r,covar_c=covar_c,covar_s=covar_s,covar_o=covar_o) kgp_slow.setData(Y=Y) # measure time signal.signal(signal.SIGALRM,handler) signal.alarm(time_out) try: t_start = time.clock() hyperparams_opt,lmltrain = opt.opt_hyper(kgp_slow,hyperparams,Ifilter=Ifilter,bounds=bounds,opts=opts) t_stop = time.clock() signal.alarm(0) t_slow[i] = t_stop - t_start lml_slow[i] = lmltrain except Exception, e: print e t_slow += time_out break # save fn_out = os.path.join(out_dir,'results_runtime_signal%03d_N%d_D%d.hdf5'%(var_signal*1E3,N,D)) f = h5py.File(fn_out,'w') f['t_fast'] = t_fast f['t_slow'] = t_slow f['lml_fast'] = lml_fast
def test_gpbase(self): covar = linear.LinearCF(n_dimensions=self.n_dimensions) n_train = self.X['train'].shape[0] theta = 1E-1 prior_cov = priors.GaussianPrior(key='covar',theta=SP.array([1.])) prior_lik = priors.GaussianPrior(key='lik',theta=SP.array([1.])) prior = priors.PriorList([prior_cov,prior_lik]) lik = likelihood_base.GaussIsoLik() gp = gp_base.GP(covar_r=covar,likelihood=lik,prior=prior) gp.setData(Y=self.Yuni['train'],X=self.X['train']) # log likelihood and gradient derivation hyperparams = {'covar':SP.array([0.5]), 'lik':SP.array([0.5])} LML = gp.LML(hyperparams) LMLgrad = gp.LMLgrad(hyperparams) K = covar.K(hyperparams['covar']) + lik.K(hyperparams['lik'],n_train) Kgrad_covar = covar.Kgrad_theta(hyperparams['covar'],0) Kgrad_lik = lik.Kgrad_theta(hyperparams['lik'],n_train,0) KinvY = LA.solve(K,self.Yuni['train']) _LML = self.n_train/2*SP.log(2*SP.pi) + 0.5*SP.log(LA.det(K)) + 0.5*(self.Yuni['train']*KinvY).sum() + prior.LML(hyperparams) LMLgrad_covar = 0.5 * SP.trace(LA.solve(K, Kgrad_covar)) - 0.5*SP.dot(KinvY.T,SP.dot(Kgrad_covar,KinvY)) LMLgrad_covar+= prior_cov.LMLgrad(hyperparams)['covar'] LMLgrad_lik = 0.5 * SP.trace(LA.solve(K, Kgrad_lik)) - 0.5*SP.dot(KinvY.T,SP.dot(Kgrad_lik,KinvY)) LMLgrad_lik+= prior_lik.LMLgrad(hyperparams)['lik'] assert SP.allclose(LML,_LML), 'ouch, marginal log likelihood does not match' assert SP.allclose(LMLgrad['covar'], LMLgrad_covar), 'ouch, gradient with respect to theta does not match' assert SP.allclose(LMLgrad['lik'], LMLgrad_lik), 'ouch, gradient with respect to theta does not match' # predict Ystar = gp.predict(hyperparams,self.X['test']) Kstar = covar.Kcross(hyperparams['covar']) _Ystar = SP.dot(Kstar.T,LA.solve(K,self.Yuni['train'])).flatten() assert SP.allclose(Ystar,_Ystar), 'ouch, predictions, do not match' # optimize opts = {'gradcheck':True,'messages':False} hyperparams_opt,lml_opt = optimize_base.opt_hyper(gp,hyperparams,opts=opts) # log likelihood and gradient derivation LML = gp.LML(hyperparams_opt) LMLgrad = gp.LMLgrad(hyperparams_opt) K = covar.K(hyperparams_opt['covar']) + lik.K(hyperparams_opt['lik'],n_train) Kgrad_covar = covar.Kgrad_theta(hyperparams_opt['covar'],0) Kgrad_lik = lik.Kgrad_theta(hyperparams_opt['lik'],n_train,0) KinvY = LA.solve(K,self.Yuni['train']) _LML = self.n_train/2*SP.log(2*SP.pi) + 0.5*SP.log(LA.det(K)) + 0.5*(self.Yuni['train']*KinvY).sum() + prior.LML(hyperparams_opt) LMLgrad_covar = 0.5 * SP.trace(LA.solve(K, Kgrad_covar)) - 0.5*SP.dot(KinvY.T,SP.dot(Kgrad_covar,KinvY)) LMLgrad_covar+= prior_cov.LMLgrad(hyperparams_opt)['covar'] LMLgrad_lik = 0.5 * SP.trace(LA.solve(K, Kgrad_lik)) - 0.5*SP.dot(KinvY.T,SP.dot(Kgrad_lik,KinvY)) LMLgrad_lik+= prior_lik.LMLgrad(hyperparams_opt)['lik'] assert SP.allclose(LML,_LML), 'ouch, marginal log likelihood does not match' assert SP.allclose(LMLgrad['covar'], LMLgrad_covar), 'ouch, gradient with respect to theta does not match' assert SP.allclose(LMLgrad['lik'], LMLgrad_lik), 'ouch, gradient with respect to theta does not match' # predict Ystar = gp.predict(hyperparams_opt,self.X['test']) Kstar = covar.Kcross(hyperparams_opt['covar']) _Ystar = SP.dot(Kstar.T,LA.solve(K,self.Yuni['train'])).flatten() assert SP.allclose(Ystar,_Ystar), 'ouch, predictions, do not match'
def test_gpkronsum(self): covar_c = lowrank.LowRankCF(n_dimensions=self.n_latent) covar_r = lowrank.LowRankCF(n_dimensions=self.n_dimensions) covar_s = lowrank.LowRankCF(n_dimensions=self.n_latent) covar_o = lowrank.LowRankCF(n_dimensions = self.n_dimensions) X0_c = SP.random.randn(self.n_tasks,self.n_latent) X0_s = SP.random.randn(self.n_tasks,self.n_latent) X0_r = SP.random.randn(self.n_train,self.n_dimensions) X0_o = SP.random.randn(self.n_train,self.n_dimensions) gp = gp_kronsum.KronSumGP(covar_c=covar_c, covar_r=covar_r, covar_s=covar_s, covar_o=covar_o) gp.setData(Y=self.Ykronsum['train']) gp2 = gp_kronsum_naive.KronSumGP(covar_c=covar_c,covar_r=covar_r,covar_s=covar_s,covar_o=covar_o) gp2.setData(Y=self.Ykronsum['train']) hyperparams = {'covar_c':SP.array([0.5,0.5]), 'X_c':X0_c, 'covar_r':SP.array([0.5,0.5]), 'X_r':X0_r, 'covar_s':SP.array([0.5,0.5]), 'X_s':X0_s, 'covar_o':SP.array([0.5,0.5]), 'X_o':X0_o} yhat = gp.predict(hyperparams,Xstar_r=self.X['test'],debugging=True) lml = gp._LML_covar(hyperparams,debugging=True) grad = {} grad.update(gp._LMLgrad_c(hyperparams, debugging=True)) grad.update(gp._LMLgrad_r(hyperparams, debugging=True)) grad.update(gp._LMLgrad_o(hyperparams, debugging=True)) grad.update(gp._LMLgrad_s(hyperparams, debugging=True)) yhat2 = gp2.predict(hyperparams,Xstar_r=self.X['test']) lml2 = gp2._LML_covar(hyperparams) grad2 = {} grad2.update(gp2._LMLgrad_covar(hyperparams)) grad2.update(gp2._LMLgrad_x(hyperparams)) assert SP.allclose(yhat,yhat2), 'predictions does not match' assert SP.allclose(lml,lml2), 'log likelihood does not match' for key in grad.keys(): assert SP.allclose(grad[key],grad2[key]), 'gradient with respect to x does not match' covar_o = diag.DiagIsoCF(n_dimensions = self.n_dimensions) gp = gp_kronsum.KronSumGP(covar_c=covar_c, covar_r=covar_r, covar_s=covar_s, covar_o=covar_o) gp.setData(Y=self.Ykronsum['train'],X_r=self.X['train'],X_o=self.X['train']) gp2 = gp_kronsum_naive.KronSumGP(covar_c=covar_c, covar_r=covar_r, covar_s=covar_s, covar_o=covar_o) gp2.setData(Y=self.Ykronsum['train'],X_r=self.X['train'],X_o=self.X['train']) hyperparams = {'covar_c':SP.array([0.5,0.5]), 'X_c':X0_c, 'covar_r':SP.array([0.5,0.5]), 'covar_s':SP.array([0.5,0.5]), 'X_s':X0_s, 'covar_o':SP.array([0.5])} bounds = {'covar_c':SP.array([[-5,+5]]*2), 'covar_r':SP.array([[-5,+5]]*2), 'covar_s':SP.array([[-5,+5]]*2), 'covar_o':SP.array([[-5,+5]])} opts = {'gradcheck':True} import time t_start = time.time() hyperparams_opt,lml_opt = optimize_base.opt_hyper(gp,hyperparams,opts=opts, bounds=bounds) t_stop = time.time() print 'time(training): %.4f'%(t_stop-t_start) t_start = time.time() hyperparams_opt2, lml_opt2 = optimize_base.opt_hyper(gp2,hyperparams,opts=opts, bounds=bounds) t_stop = time.time() print 'time(training): %.4f'%(t_stop-t_start) assert SP.allclose(lml_opt,lml_opt2), 'ouch, optimization did fail' gp._invalidate_cache() # otherwise debugging parameters are not up to date! yhat = gp.predict(hyperparams_opt,Xstar_r=self.X['test'],debugging=True) lml = gp._LML_covar(hyperparams_opt,debugging=True) grad = {} grad.update(gp._LMLgrad_c(hyperparams_opt, debugging=True)) grad.update(gp._LMLgrad_r(hyperparams_opt, debugging=True)) grad.update(gp._LMLgrad_o(hyperparams_opt, debugging=True)) grad.update(gp._LMLgrad_s(hyperparams_opt, debugging=True)) yhat2 = gp2.predict(hyperparams_opt,Xstar_r=self.X['test']) lml2 = gp2._LML_covar(hyperparams_opt) grad2 = {} grad2.update(gp2._LMLgrad_covar(hyperparams_opt)) grad2.update(gp2._LMLgrad_x(hyperparams_opt)) assert SP.allclose(yhat,yhat2), 'predictions does not match' assert SP.allclose(lml,lml2), 'log likelihood does not match' for key in grad.keys(): assert SP.allclose(grad[key],grad2[key]), 'gradient with respect to x does not match'
covar_c.X = Y_train_hat.T covar_r = composite.SumCF(n_dimensions = X_train.shape[1]) covar_r.append_covar(linear.LinearCF(n_dimensions = X_train.shape[1])) covar_r.append_covar(se.SqExpCF(n_dimensions = X_train.shape[1])) covar_r.append_covar(DiagIsoCF(n_dimensions = X_train.shape[1])) covar_r.X = X_train likelihood = lik.GaussIsoLik() gp = sMTGPR.sMTGPR(covar_r = covar_r, covar_c = covar_c, likelihood = likelihood, basis = B) gp.setData(Y = Y_train, Y_hat = Y_train_hat, X = X_train) # Training: optimize hyperparameters t_start = time.clock() hyperparams_opt,lml_opt = optimize_base.opt_hyper(gp, hyperparams, bounds = bounds, Ifilter = Ifilter) t_stop = time.clock() elapsed_time_opt[r,b] = t_stop - t_start # Testing t_start = time.clock() Y_pred_MT, Y_pred_cov_MT = gp.predict(hyperparams_opt, Xstar_r = X_test, compute_cov = True) t_stop = time.clock() elapsed_time_est[r,b] = t_stop - t_start Y_pred_MT = Y_scaler.inverse_transform(Y_pred_MT) Y_pred_cov_MT = Y_pred_cov_MT * Y_scaler.var_ s_n2_MT = likelihood.Kdiag(hyperparams_opt['lik'], n_task) * Y_scaler.var_ r2_MT[r,b] = np.mean(compute_r2(Y_test, Y_pred_MT)) NPMs_MT = normative_prob_map(Y_test, Y_pred_MT, Y_pred_cov_MT, s_n2_MT)
def test_gpbase(self): covar = linear.LinearCF(n_dimensions=self.n_dimensions) n_train = self.X['train'].shape[0] theta = 1E-1 prior_cov = priors.GaussianPrior(key='covar', theta=SP.array([1.])) prior_lik = priors.GaussianPrior(key='lik', theta=SP.array([1.])) prior = priors.PriorList([prior_cov, prior_lik]) lik = likelihood_base.GaussIsoLik() gp = gp_base.GP(covar_r=covar, likelihood=lik, prior=prior) gp.setData(Y=self.Yuni['train'], X=self.X['train']) # log likelihood and gradient derivation hyperparams = {'covar': SP.array([0.5]), 'lik': SP.array([0.5])} LML = gp.LML(hyperparams) LMLgrad = gp.LMLgrad(hyperparams) K = covar.K(hyperparams['covar']) + lik.K(hyperparams['lik'], n_train) Kgrad_covar = covar.Kgrad_theta(hyperparams['covar'], 0) Kgrad_lik = lik.Kgrad_theta(hyperparams['lik'], n_train, 0) KinvY = LA.solve(K, self.Yuni['train']) _LML = self.n_train / 2 * SP.log(2 * SP.pi) + 0.5 * SP.log( LA.det(K)) + 0.5 * (self.Yuni['train'] * KinvY).sum() + prior.LML(hyperparams) LMLgrad_covar = 0.5 * SP.trace(LA.solve( K, Kgrad_covar)) - 0.5 * SP.dot(KinvY.T, SP.dot( Kgrad_covar, KinvY)) LMLgrad_covar += prior_cov.LMLgrad(hyperparams)['covar'] LMLgrad_lik = 0.5 * SP.trace(LA.solve(K, Kgrad_lik)) - 0.5 * SP.dot( KinvY.T, SP.dot(Kgrad_lik, KinvY)) LMLgrad_lik += prior_lik.LMLgrad(hyperparams)['lik'] assert SP.allclose( LML, _LML), 'ouch, marginal log likelihood does not match' assert SP.allclose( LMLgrad['covar'], LMLgrad_covar ), 'ouch, gradient with respect to theta does not match' assert SP.allclose( LMLgrad['lik'], LMLgrad_lik), 'ouch, gradient with respect to theta does not match' # predict Ystar = gp.predict(hyperparams, self.X['test']) Kstar = covar.Kcross(hyperparams['covar']) _Ystar = SP.dot(Kstar.T, LA.solve(K, self.Yuni['train'])).flatten() assert SP.allclose(Ystar, _Ystar), 'ouch, predictions, do not match' # optimize opts = {'gradcheck': True, 'messages': False} hyperparams_opt, lml_opt = optimize_base.opt_hyper(gp, hyperparams, opts=opts) # log likelihood and gradient derivation LML = gp.LML(hyperparams_opt) LMLgrad = gp.LMLgrad(hyperparams_opt) K = covar.K(hyperparams_opt['covar']) + lik.K(hyperparams_opt['lik'], n_train) Kgrad_covar = covar.Kgrad_theta(hyperparams_opt['covar'], 0) Kgrad_lik = lik.Kgrad_theta(hyperparams_opt['lik'], n_train, 0) KinvY = LA.solve(K, self.Yuni['train']) _LML = self.n_train / 2 * SP.log(2 * SP.pi) + 0.5 * SP.log( LA.det(K)) + 0.5 * (self.Yuni['train'] * KinvY).sum() + prior.LML(hyperparams_opt) LMLgrad_covar = 0.5 * SP.trace(LA.solve( K, Kgrad_covar)) - 0.5 * SP.dot(KinvY.T, SP.dot( Kgrad_covar, KinvY)) LMLgrad_covar += prior_cov.LMLgrad(hyperparams_opt)['covar'] LMLgrad_lik = 0.5 * SP.trace(LA.solve(K, Kgrad_lik)) - 0.5 * SP.dot( KinvY.T, SP.dot(Kgrad_lik, KinvY)) LMLgrad_lik += prior_lik.LMLgrad(hyperparams_opt)['lik'] assert SP.allclose( LML, _LML), 'ouch, marginal log likelihood does not match' assert SP.allclose( LMLgrad['covar'], LMLgrad_covar ), 'ouch, gradient with respect to theta does not match' assert SP.allclose( LMLgrad['lik'], LMLgrad_lik), 'ouch, gradient with respect to theta does not match' # predict Ystar = gp.predict(hyperparams_opt, self.X['test']) Kstar = covar.Kcross(hyperparams_opt['covar']) _Ystar = SP.dot(Kstar.T, LA.solve(K, self.Yuni['train'])).flatten() assert SP.allclose(Ystar, _Ystar), 'ouch, predictions, do not match'
def test_gpkronsum(self): covar_c = lowrank.LowRankCF(n_dimensions=self.n_latent) covar_r = lowrank.LowRankCF(n_dimensions=self.n_dimensions) covar_s = lowrank.LowRankCF(n_dimensions=self.n_latent) covar_o = lowrank.LowRankCF(n_dimensions=self.n_dimensions) X0_c = SP.random.randn(self.n_tasks, self.n_latent) X0_s = SP.random.randn(self.n_tasks, self.n_latent) X0_r = SP.random.randn(self.n_train, self.n_dimensions) X0_o = SP.random.randn(self.n_train, self.n_dimensions) gp = gp_kronsum.KronSumGP(covar_c=covar_c, covar_r=covar_r, covar_s=covar_s, covar_o=covar_o) gp.setData(Y=self.Ykronsum['train']) gp2 = gp_kronsum_naive.KronSumGP(covar_c=covar_c, covar_r=covar_r, covar_s=covar_s, covar_o=covar_o) gp2.setData(Y=self.Ykronsum['train']) hyperparams = { 'covar_c': SP.array([0.5, 0.5]), 'X_c': X0_c, 'covar_r': SP.array([0.5, 0.5]), 'X_r': X0_r, 'covar_s': SP.array([0.5, 0.5]), 'X_s': X0_s, 'covar_o': SP.array([0.5, 0.5]), 'X_o': X0_o } yhat = gp.predict(hyperparams, Xstar_r=self.X['test'], debugging=True) lml = gp._LML_covar(hyperparams, debugging=True) grad = {} grad.update(gp._LMLgrad_c(hyperparams, debugging=True)) grad.update(gp._LMLgrad_r(hyperparams, debugging=True)) grad.update(gp._LMLgrad_o(hyperparams, debugging=True)) grad.update(gp._LMLgrad_s(hyperparams, debugging=True)) yhat2 = gp2.predict(hyperparams, Xstar_r=self.X['test']) lml2 = gp2._LML_covar(hyperparams) grad2 = {} grad2.update(gp2._LMLgrad_covar(hyperparams)) grad2.update(gp2._LMLgrad_x(hyperparams)) assert SP.allclose(yhat, yhat2), 'predictions does not match' assert SP.allclose(lml, lml2), 'log likelihood does not match' for key in grad.keys(): assert SP.allclose( grad[key], grad2[key]), 'gradient with respect to x does not match' covar_o = diag.DiagIsoCF(n_dimensions=self.n_dimensions) gp = gp_kronsum.KronSumGP(covar_c=covar_c, covar_r=covar_r, covar_s=covar_s, covar_o=covar_o) gp.setData(Y=self.Ykronsum['train'], X_r=self.X['train'], X_o=self.X['train']) gp2 = gp_kronsum_naive.KronSumGP(covar_c=covar_c, covar_r=covar_r, covar_s=covar_s, covar_o=covar_o) gp2.setData(Y=self.Ykronsum['train'], X_r=self.X['train'], X_o=self.X['train']) hyperparams = { 'covar_c': SP.array([0.5, 0.5]), 'X_c': X0_c, 'covar_r': SP.array([0.5, 0.5]), 'covar_s': SP.array([0.5, 0.5]), 'X_s': X0_s, 'covar_o': SP.array([0.5]) } bounds = { 'covar_c': SP.array([[-5, +5]] * 2), 'covar_r': SP.array([[-5, +5]] * 2), 'covar_s': SP.array([[-5, +5]] * 2), 'covar_o': SP.array([[-5, +5]]) } opts = {'gradcheck': True} import time t_start = time.time() hyperparams_opt, lml_opt = optimize_base.opt_hyper(gp, hyperparams, opts=opts, bounds=bounds) t_stop = time.time() print 'time(training): %.4f' % (t_stop - t_start) t_start = time.time() hyperparams_opt2, lml_opt2 = optimize_base.opt_hyper(gp2, hyperparams, opts=opts, bounds=bounds) t_stop = time.time() print 'time(training): %.4f' % (t_stop - t_start) assert SP.allclose(lml_opt, lml_opt2), 'ouch, optimization did fail' gp._invalidate_cache( ) # otherwise debugging parameters are not up to date! yhat = gp.predict(hyperparams_opt, Xstar_r=self.X['test'], debugging=True) lml = gp._LML_covar(hyperparams_opt, debugging=True) grad = {} grad.update(gp._LMLgrad_c(hyperparams_opt, debugging=True)) grad.update(gp._LMLgrad_r(hyperparams_opt, debugging=True)) grad.update(gp._LMLgrad_o(hyperparams_opt, debugging=True)) grad.update(gp._LMLgrad_s(hyperparams_opt, debugging=True)) yhat2 = gp2.predict(hyperparams_opt, Xstar_r=self.X['test']) lml2 = gp2._LML_covar(hyperparams_opt) grad2 = {} grad2.update(gp2._LMLgrad_covar(hyperparams_opt)) grad2.update(gp2._LMLgrad_x(hyperparams_opt)) assert SP.allclose(yhat, yhat2), 'predictions does not match' assert SP.allclose(lml, lml2), 'log likelihood does not match' for key in grad.keys(): assert SP.allclose( grad[key], grad2[key]), 'gradient with respect to x does not match'
# initialization parameters hyperparams, Ifilter, bounds = initialize.init('GPkronsum_LIN',Y.T,X_r,{'n_c':n_latent, 'n_sigma':n_latent}) # initialize gp and its covariance functions covar_r.X = X_r covar_o.X = X_r covar_o._K = SP.eye(n_train) covar_s.X = hyperparams['X_s'] covar_c.X = hyperparams['X_c'] gp = gp_kronsum.KronSumGP(covar_c=covar_c, covar_r=covar_r, covar_s=covar_s, covar_o=covar_o) gp.setData(Y=Y) # optimize hyperparameters t_start = time.time() hyperparams_opt,lml_opt = optimize_base.opt_hyper(gp,hyperparams, bounds=bounds,Ifilter=Ifilter) t_stop = time.time() print 'time(training): %.4f'%(t_stop-t_start) # compare SigmaOpt = covar_s.K(hyperparams_opt['covar_s']) COpt = covar_c.K(hyperparams_opt['covar_c']) fig = PLT.figure(1) fig.add_subplot(221) fig.subplots_adjust(hspace=0.5) PLT.imshow(C,interpolation='nearest') PLT.title('True Signal Covariance') PLT.xlabel('Tasks'); PLT.ylabel('Tasks')