def train(self,lambda_,reguess=True): #{ self.lambd = lambda_ if (self.lambd<0.0): print "regularization parameter, lambda, in LASSO cannot be negative" sys.exit(1) if reguess: self.obtain_guess() optimizer = l_bfgs(self,40,0,0.05,0) max_iter = 5000 converged = False cur_iter = 0 print_iter = True print 'beginning minimization of LASSO objective function with lambda = '+str(self.lambd) for i in range(max_iter): cur_iter = cur_iter + 1 converged = optimizer.next_step() if (converged): break if (print_iter): print " "+str(cur_iter)+" "+"{:.12f}".format(optimizer.value)+" "+str(optimizer.error)+" "+optimizer.comment if (converged): print " "+str(cur_iter)+" "+"{:.12f}".format(optimizer.value)+" "+str(optimizer.error)+" Optimization Converged" else: print " "+str(cur_iter)+" "+"{:.12f}".format(optimizer.value)+" "+str(optimizer.error)+" Optimization Failed" return converged
def solve(self,max_iter): #{ self.obtain_guess() optimizer = l_bfgs(self) converged = False cur_iter = 0 for i in range(max_iter): cur_iter = cur_iter + 1 converged = optimizer.next_step() if (converged): break print " "+str(cur_iter)+" "+str(optimizer.value)+" "+str(optimizer.error)+" "+optimizer.comment if (converged): print " "+str(cur_iter)+" "+str(optimizer.value)+" "+str(optimizer.error)+" Optimization Converged" else: print 'optimization failed to converge'
def train(self,z_,guess_phi,guess_theta,guess_sigma_a_sq): #{ # z - training data # w - differenced training data with zero mean # mu - mean of differenced training data self.z = z_ self.w = np.array(self.z) for i in range(self.d): differenced = np.zeros(len(self.w)-1) for j in range(len(differenced)): differenced[j] = self.w[j+1] - self.w[j] self.w = np.array(differenced) self.mu = float(np.sum(self.w))/float(len(self.w)) self.w = self.w - self.mu self.obtain_guess(guess_phi,guess_theta,guess_sigma_a_sq) #self.coord_scan(1,np.arange(0.1,0.4,0.005)) optimizer = l_bfgs(self,20,0,0.05,0) self.value() max_iter = 50 converged = False cur_iter = 0 print 'beginning minimization of negative of log-likelihood function' for i in range(max_iter): cur_iter = cur_iter + 1 converged = optimizer.next_step() if (converged): break print " "+str(cur_iter)+" "+"{:.12f}".format(optimizer.value)+" "+str(optimizer.error)+" "+optimizer.comment if (converged): print " "+str(cur_iter)+" "+"{:.12f}".format(optimizer.value)+" "+str(optimizer.error)+" Optimization Converged" else: print 'optimization failed to converge' #S = self.compute_S() #print 'this is the final S ',S #print 'this is the optimal sigma_a_sq ',S/float(len(self.w)) return converged
def train(self,tol=1.0E-7,print_iter=False): #{ self.tolerance = tol optimizer = l_bfgs(self,40,0,0.05,0) max_iter = 5000 converged = False cur_iter = 0 print 'beginning minimization of negative log likelihood for logistic regression' for i in range(max_iter): cur_iter = cur_iter + 1 converged = optimizer.next_step() if (converged): break if (print_iter): print " "+str(cur_iter)+" "+"{:.12f}".format(optimizer.value)+" "+str(optimizer.error)+" "+optimizer.comment if (converged): print " "+str(cur_iter)+" "+"{:.12f}".format(optimizer.value)+" "+str(optimizer.error)+" Optimization Converged" else: print " "+str(cur_iter)+" "+"{:.12f}".format(optimizer.value)+" "+str(optimizer.error)+" Optimization Failed" return converged
def train(self, X_train, Y_train, tol=1.0E-7, algo=1, print_iter=False): # TODO reexpression of class labels as vectors self.X_train = X_train if self.is_classification: # we assume we have been passed a vector of integer labels self.Y_train = np.zeros((Y_train.shape[0], np.amax(Y_train)+1), dtype=np.int) for i in range(Y_train.shape[0]): self.Y_train[i,Y_train[i,0]] = 1 else: self.Y_train = Y_train self.tolerance = tol if algo == 0: optimizer = steepest_descent(self) elif algo == 1: optimizer = l_bfgs(self,20,0,0.5,0) else: print 'optimizer not recognized' max_iter = 5000 converged = False cur_iter = 0 print 'beginning optimization of neural network' for i in range(max_iter): cur_iter = cur_iter + 1 converged = optimizer.next_step() if (converged): break if (print_iter): print " "+str(cur_iter)+" "+"{:.12f}".format(optimizer.value)+" "+\ str(optimizer.error)+" "+optimizer.comment if (converged): print " "+str(cur_iter)+" "+"{:.12f}".format(optimizer.value)+" "+\ str(optimizer.error)+" Optimization Converged" else: print " "+str(cur_iter)+" "+"{:.12f}".format(optimizer.value)+" "+\ str(optimizer.error)+" Optimization Failed" return converged
#! /usr/bin/env python import sys import os sys.path.append(os.getcwd()+'/nonlinear_solvers/') from quadratic_test import quadratic_test from steepest_descent import steepest_descent from l_bfgs import l_bfgs max_iter = 4000 problem_size = 10 problem = quadratic_test(problem_size) problem.obtain_guess() #optimizer = steepest_descent(problem) optimizer = l_bfgs(problem) converged = False cur_iter = 0 for i in range(max_iter): cur_iter = cur_iter + 1 converged = optimizer.next_step() if (converged): break print " "+str(cur_iter)+" "+str(optimizer.value)+" "+str(optimizer.error)+" "+optimizer.comment if (converged): print " "+str(cur_iter)+" "+str(optimizer.value)+" "+str(optimizer.error)+" Optimization Converged" else: print 'optimization failed to converge' print 'doing the same with quadratic_test internal solve function'