def train_cross(self): ## not sure how importatn this crap is.. ## SET (hyper)parameters n_iters = len(self.training_set) * 5 hyp = hyperParameters() sn = 0.001; hyp.lik = array([log(sn)]) conf = self.conf dimensions = len(self.training_set[0]) hyp.mean = [0.5 for d in xrange(dimensions)] hyp.mean.append(1.0) hyp.mean = array(hyp.mean) hyp.mean = array([]) # Scale inputs and particles? self.input_scaler = preprocessing.StandardScaler().fit(self.training_set) self.scaled_training_set = self.input_scaler.transform(self.training_set) # Scale training data if self.transLog: self.output_scaler = preprocessing.StandardScaler(with_std=False).fit(log(self.training_fitness - self.shift_by())) self.adjusted_training_fitness = self.output_scaler.transform(log(self.training_fitness - self.shift_by())) else: self.output_scaler = preprocessing.StandardScaler(with_std=False).fit(self.training_fitness) self.adjusted_training_fitness = self.output_scaler.transform(self.training_fitness) ## retrain a number of times and pick best likelihood press_best = None best_hyp = None i = 0 index_array = ShuffleSplit(len(self.scaled_training_set), n_iter=n_iters, train_size=0.8, test_size=0.2) ##we use 10% of example to evaluate our while i < conf.random_start: if conf.corr == "isotropic": self.covfunc = [['kernels.covSum'], [['kernels.covSEiso'],['kernels.covNoise']]] hyp.cov = [log(uniform(low=conf.thetaL, high=conf.thetaU)) for d in xrange(2)] elif conf.corr == "anisotropic": self.covfunc = [['kernels.covSum'], [['kernels.covSEard'],['kernels.covNoise']]] hyp.cov = [log(uniform(low=conf.thetaL, high=conf.thetaU)) for d in xrange(dimensions+1)] elif conf.corr == "anirat": ## todo self.covfunc = [['kernels.covSum'], [['kernels.covRQard'],['kernels.covNoise']]] hyp.cov = [log(uniform(low=conf.thetaL, high=conf.thetaU)) for d in xrange(dimensions)] hyp.cov.append(log(uniform(low=conf.thetaL, high=conf.thetaU))) hyp.cov.append(log(uniform(low=conf.thetaL, high=conf.thetaU))) elif conf.corr == "matern3": self.covfunc = [['kernels.covSum'], [['kernels.covMatern'],['kernels.covNoise']]] hyp.cov = [log(uniform(low=conf.thetaL, high=conf.thetaU)) for d in xrange(2)] hyp.cov.append(log(3)) elif conf.corr == "matern5": self.covfunc = [['kernels.covSum'], [['kernels.covMatern'],['kernels.covNoise']]] hyp.cov = [log(uniform(low=conf.thetaL, high=conf.thetaU)) for d in xrange(2)] hyp.cov.append(log(5)) elif conf.corr == "rqard": self.covfunc = [['kernels.covSum'], [['kernels.covRQard'],['kernels.covNoise']]] hyp.cov = [log(uniform(low=conf.thetaL, high=conf.thetaU)) for d in xrange(dimensions+2)] elif conf.corr == "special": self.covfunc = [['kernels.covSum'], [['kernels.covSEiso'],['kernels.covMatern'],['kernels.covNoise']]] hyp.cov = [log(uniform(low=conf.thetaL, high=conf.thetaU)) for d in xrange(2)] hyp.cov = hyp.cov + [log(uniform(low=conf.thetaL, high=conf.thetaU)) for d in xrange(2)] hyp.cov.append(log(3)) else: logging.error("The specified kernel function is not supported") return False hyp.cov.append(log(uniform(low=0.01, high=0.5))) ## 50% propability to usen the previous best hyper-parameters: if self.hyp: random_number = uniform(0.,1.) if random_number < 0.5: hyp = self.hyp try: vargout = min_wrapper(hyp,gp,'BFGS',self.inffunc,self.meanfunc,self.covfunc,self.likfunc,self.scaled_training_set ,self.adjusted_training_fitness,None,None,True) hyp = vargout[0] ### we add some sensible checking.. ## matern we dont want to overfit ## we know that the function is not just noise hence < -1 ## we know for anisotorpic that at least one parameter has to have some meaning ## press = 0.0 for train_indexes, test_indexes in index_array: test_set = self.scaled_training_set[test_indexes] training_set = self.scaled_training_set[train_indexes] test_fitness = self.adjusted_training_fitness[test_indexes] training_fitness = self.adjusted_training_fitness[train_indexes] vargout = gp(hyp, self.inffunc,self.meanfunc,self.covfunc, self.likfunc, training_set, training_fitness, test_set) predicted_fitness = vargout[2] press = press + self.calc_press(predicted_fitness, test_fitness) #if (hyp.cov[-1] < -1.) and not ((conf.corr == "matern3") and hyp.cov[0] < 0.0) and not ((conf.corr == "anisotropic") and all(hyp.cov[:-2] < 0.0)): logging.info("Press " + str(press) + " " + str(hyp.cov)) if (((not press_best) or (press < press_best))): best_hyp = hyp press_best = press except Exception, e: logging.debug("Regressor training Failed: " + str(e)) i = i + 1
xs = np.arange(2004+1./24.,2024-1./24.,1./12.) # TEST POINTS xs = xs.reshape(len(xs),1) vargout = gp(hyp,inffunc,meanfunc,covfunc,likfunc,x,y,xs) ym = vargout[0]; ys2 = vargout[1] m = vargout[2]; s2 = vargout[3] plotter(xs,ym,ys2,x,y) ##----------------------------------------------------------## ## STANDARD GP (training) ## ## OPTIMIZE HYPERPARAMETERS ## ##----------------------------------------------------------## ## -> parameter training using (off the shelf) conjugent gradient (CG) optimization (NOTE: SCG is faster) from time import clock t0 = clock() vargout = min_wrapper(hyp,gp,'SCG',inffunc,meanfunc,covfunc,likfunc,x,y,None,None,True) t1 = clock() hyp = vargout[0] vargout = gp(hyp,inffunc,meanfunc,covfunc,likfunc,x,y,xs) ym = vargout[0]; ys2 = vargout[1] m = vargout[2]; s2 = vargout[3] print 'Time to optimize = ',t1-t0 print 'Optimized mean = ',hyp.mean print 'Optimized covariance = ',hyp.cov print 'Optimized liklihood = ',hyp.lik plotter(xs,ym,ys2,x,y)
def train_nlml(self): ## not sure how importatn this crap is.. ## SET (hyper)parameters hyp = hyperParameters() sn = 0.001; hyp.lik = array([log(sn)]) conf = self.conf dimensions = len(self.training_set[0]) hyp.mean = [0.5 for d in xrange(dimensions)] hyp.mean.append(1.0) hyp.mean = array(hyp.mean) hyp.mean = array([]) # Scale inputs and particles? self.input_scaler = preprocessing.StandardScaler().fit(self.training_set) self.scaled_training_set = self.input_scaler.transform(self.training_set) # Scale training data self.output_scaler = preprocessing.StandardScaler(with_std=False).fit(log(self.training_fitness - self.shift_by())) self.adjusted_training_fitness = self.output_scaler.transform(log(self.training_fitness - self.shift_by())) ## retrain a number of times and pick best likelihood nlml_best = None i = 0 while i < conf.random_start: if conf.corr == "isotropic": self.covfunc = [['kernels.covSum'], [['kernels.covSEiso'],['kernels.covNoise']]] hyp.cov = [log(uniform(low=conf.thetaL, high=conf.thetaU)) for d in xrange(2)] elif conf.corr == "anisotropic": self.covfunc = [['kernels.covSum'], [['kernels.covSEard'],['kernels.covNoise']]] hyp.cov = [log(uniform(low=conf.thetaL, high=conf.thetaU)) for d in xrange(dimensions+1)] elif conf.corr == "anirat": ## todo self.covfunc = [['kernels.covSum'], [['kernels.covRQard'],['kernels.covNoise']]] hyp.cov = [log(uniform(low=conf.thetaL, high=conf.thetaU)) for d in xrange(dimensions)] hyp.cov.append(log(uniform(low=conf.thetaL, high=conf.thetaU))) hyp.cov.append(log(uniform(low=conf.thetaL, high=conf.thetaU))) elif conf.corr == "matern3": self.covfunc = [['kernels.covSum'], [['kernels.covMatern'],['kernels.covNoise']]] hyp.cov = [log(uniform(low=conf.thetaL, high=conf.thetaU)) for d in xrange(2)] hyp.cov.append(log(3)) elif conf.corr == "matern5": self.covfunc = [['kernels.covSum'], [['kernels.covMatern'],['kernels.covNoise']]] hyp.cov = [log(uniform(low=conf.thetaL, high=conf.thetaU)) for d in xrange(2)] hyp.cov.append(log(5)) elif conf.corr == "rqard": self.covfunc = [['kernels.covSum'], [['kernels.covRQard'],['kernels.covNoise']]] hyp.cov = [log(uniform(low=conf.thetaL, high=conf.thetaU)) for d in xrange(dimensions+2)] elif conf.corr == "special": self.covfunc = [['kernels.covSum'], [['kernels.covSEiso'],['kernels.covMatern'],['kernels.covNoise']]] hyp.cov = [log(uniform(low=conf.thetaL, high=conf.thetaU)) for d in xrange(2)] hyp.cov = hyp.cov + [log(uniform(low=conf.thetaL, high=conf.thetaU)) for d in xrange(2)] hyp.cov.append(log(3)) else: logging.error("The specified kernel function is not supported") return False hyp.cov.append(log(uniform(low=0.1, high=1.0))) try: vargout = min_wrapper(hyp,gp,'BFGS',self.inffunc,self.meanfunc,self.covfunc,self.likfunc,self.scaled_training_set ,self.adjusted_training_fitness,None,None,True) hyp = vargout[0] vargout = gp(hyp, self.inffunc,self.meanfunc,self.covfunc,self.likfunc,self.scaled_training_set ,self.adjusted_training_fitness, None,None,False) nlml = vargout[0] ### we add some sensible checking.. ## matern we dont want to overfit ## we know that the function is not just noise hence < -1 ## we know for anisotorpic that at least one parameter has to have some meaning ## if (hyp.cov[-1] < -1.) and not ((conf.corr == "matern3") and hyp.cov[0] < 0.0) and not ((conf.corr == "anisotropic") and all(hyp.cov[:-2] < 0.0)): logging.info(str(nlml) + " " + str(hyp.cov)) if (((not nlml_best) or (nlml < nlml_best))): self.hyp = hyp nlml_best = nlml else: logging.info("hyper parameter out of spec: " + str(nlml) + " " + str(hyp.cov) + " " + str(hyp.cov[-1])) i = i - 1 except Exception, e: logging.debug("Regressor training Failed: " + str(e)) i = i - 1 i = i + 1
print time.time() - t0 ### PLOT results plotter(xstar,ym,ys2,x,y,[-2, 2, -0.9, 3.9]) ### GET negative log marginal likelihood [nlml, post] = gp(hyp2,inffunc,meanfunc,covfunc,likfunc,x,y,None,None,False) print "nlml2 = ", nlml ###----------------------------------------------------------### ### STANDARD GP (example 3) ### ###----------------------------------------------------------### ### TRAINING: OPTIMIZE hyperparameters [hyp2_opt, fopt, gopt, funcCalls] = min_wrapper(hyp2,gp,'CG',inffunc,meanfunc,covfunc,likfunc,x,y,None,None,True) print "nlml_opt = ", fopt #[hyp2_opt, fopt, gopt, funcCalls] = min_wrapper(hyp2,gp,'BFGS',inffunc,meanfunc,covfunc,likfunc,x,y,None,None,True) #print fopt ### PREDICTION vargout = gp(hyp2_opt,inffunc,meanfunc,covfunc,likfunc,x,y,xstar) ym = vargout[0]; ys2 = vargout[1]; m = vargout[2]; s2 = vargout[3] ### Plot results plotter(xstar,ym,ys2,x,y,[-1.9, 1.9, -0.9, 3.9]) ###----------------------------------------------------------### ### SPARSE GP (example 4) ###
np.reshape(p2 / (p1 + p2), (t1.shape[0], t1.shape[1]))) fig.colorbar(pc) plt.grid() plt.axis([-4, 4, -4, 4]) plt.show() meanfunc = [['means.meanConst']] covfunc = [['kernels.covSEard']] likfunc = [['lik.likErf']] inffunc = [['inf.infEP']] hyp = hyperParameters() hyp.mean = np.array([0.]) hyp.cov = np.array([0.0, 0.0, 0.0]) vargout = min_wrapper(hyp, gp, 'CG', inffunc, meanfunc, covfunc, likfunc, x, y, None, None, True) hyp = vargout[0] #hyp.mean = np.array([-2.842117459073954]) #hyp.cov = np.array([0.051885508906388,0.170633324977413,1.218386482861781]) vargout = gp(hyp, inffunc, meanfunc, covfunc, likfunc, x, y, t, np.ones((n, 1))) a = vargout[0] b = vargout[1] c = vargout[2] d = vargout[3] lp = vargout[4] fig = plt.figure() plt.plot(x1[:, 0], x1[:, 1], 'b+', markersize=12)
if PLOT: plotter(xstar,ym,ys2,x,y,[-2, 2, -0.9, 3.9]) ## GET negative log marginal likelihood [nlml, post] = gp(hyp2,inffunc,meanfunc,covfunc,likfunc,x,y,None,None,False) print "nlml =", nlml ##----------------------------------------------------------## ## STANDARD GP (example 3) ## ##----------------------------------------------------------## print '...example 3: training and prediction...' ## TRAINING: OPTIMIZE HYPERPARAMETERS ## -> parameter training via off-the-shelf optimization t0 = clock() [hyp2_opt, fopt, gopt, funcCalls] = min_wrapper(hyp2,gp,'Minimize',inffunc,meanfunc,covfunc,likfunc,x,y,None,None,True) # minimize #[hyp2_opt, fopt, gopt, funcCalls] = min_wrapper(hyp2,gp,'CG',inffunc,meanfunc,covfunc,likfunc,x,y,None,None,True) # conjugent gradient #[hyp2_opt, fopt, gopt, funcCalls] = min_wrapper(hyp2,gp,'SCG',inffunc,meanfunc,covfunc,likfunc,x,y,None,None,True) # scaled conjugent gradient (faster than CG) #[hyp2_opt, fopt, gopt, funcCalls] = min_wrapper(hyp2,gp,'BFGS',inffunc,meanfunc,covfunc,likfunc,x,y,None,None,True) # quasi-Newton method of Broyden, Fletcher, Goldfarb, and Shanno (BFGS) t1 = clock() print 'Time for optimization =',t1-t0 print "Optimal nlml =", fopt ## PREDICTION vargout = gp(hyp2_opt,inffunc,meanfunc,covfunc,likfunc,x,y,xstar) ym = vargout[0]; ys2 = vargout[1]; m = vargout[2]; s2 = vargout[3] ## Plot results if PLOT: plotter(xstar,ym,ys2,x,y,[-1.9, 1.9, -0.9, 3.9])
## PREDICTION vargout = gp(hyp,inffunc,meanfunc,covfunc,likfunc,x,y,None,None,False) print "nlml = ",vargout[0] vargout = gp(hyp,inffunc,meanfunc,covfunc,likfunc,x,y,z) ym = vargout[0]; ys2 = vargout[1] m = vargout[2]; s2 = vargout[3] ## Plot results plotter(z,ym,ys2,x,y,[-1.9, 1.9, -0.9, 3.9]) ########################################################### covfunc = [ ['kernels.covSEiso'] ] ## SET (hyper)parameters hyp2 = hyperParameters() hyp2.cov = np.array([-1.0,0.0]) hyp2.lik = np.array([np.log(0.1)]) vargout = min_wrapper(hyp2,gp,'SCG',inffunc,[],covfunc,likfunc,x,y,None,None,True) hyp2 = vargout[0] #hyp2.cov = np.array([-0.993396880620537,0.685943441677086]) #hyp2.lik = np.array([-1.902546786026883]) vargout = gp(hyp2,inffunc,[],covfunc,likfunc,x,y,None,None,False) print "nlml2 = ",vargout[0] vargout = gp(hyp2,inffunc,[],covfunc,likfunc,x,y,z) ym = vargout[0]; ys2 = vargout[1] m = vargout[2]; s2 = vargout[3] ## Plot results plotter(z,ym,ys2,x,y,[-1.9, 1.9, -0.9, 3.9]) ########################################################### covfunc = [ ['kernels.covSEiso'] ] hyp = hyperParameters()