def evaluate_lenet5(self, learning_rate=0.1, n_epochs=1, dataset='mnist.pkl.gz', nkerns=[20, 50], batch_size=500, testing=0): rng = numpy.random.RandomState(32324) datasets = self.data train_set_x, train_set_y = datasets[0] valid_set_x, valid_set_y = datasets[1] test_set_x, test_set_y = datasets[2] n_train_batches = train_set_x.get_value( borrow=True).shape[0] // batch_size n_valid_batches = valid_set_x.get_value( borrow=True).shape[0] // batch_size n_test_batches = test_set_x.get_value( borrow=True).shape[0] // batch_size index = T.lscalar() # index for each mini batch x = T.matrix('x') y = T.ivector('y') # ------------------------------- Building Model ---------------------------------- if testing == 0: print "...Building the model" # output image size = (28-5+1)/2 = 12 layer_0_input = x.reshape((batch_size, 1, 28, 28)) layer_0 = LeNetConvPoolLayer(rng, input=layer_0_input, image_shape=(batch_size, 1, 28, 28), filter_shape=(nkerns[0], 1, 5, 5), poolsize=(2, 2)) #output image size = (12-5+1)/2 = 4 layer_1 = LeNetConvPoolLayer(rng, input=layer_0.output, image_shape=(batch_size, nkerns[0], 12, 12), filter_shape=(nkerns[1], nkerns[0], 5, 5), poolsize=(2, 2)) # make the input to hidden layer 2 dimensional layer_2_input = layer_1.output.flatten(2) layer_2 = HiddenLayer(rng, input=layer_2_input, n_in=nkerns[1] * 4 * 4, n_out=500, activation=T.tanh) layer_3 = LogReg(input=layer_2.output, n_in=500, n_out=10) self.cost = layer_3.neg_log_likelihood(y) self.s = layer_3.s self.test_model = theano.function( [index], layer_3.errors(y), givens={ x: test_set_x[index * batch_size:(index + 1) * batch_size], y: test_set_y[index * batch_size:(index + 1) * batch_size] }) self.validate_model = theano.function( [index], layer_3.errors(y), givens={ x: valid_set_x[index * batch_size:(index + 1) * batch_size], y: valid_set_y[index * batch_size:(index + 1) * batch_size] }) self.train_predic = theano.function( [index], layer_3.prob_y_given_x(), givens={ x: train_set_x[index * batch_size:(index + 1) * batch_size] }) # list of parameters self.params = layer_3.params + layer_2.params + layer_1.params + layer_0.params grads = T.grad(self.cost, self.params) self.coefficient = 1 self.shapes = [i.get_value().shape for i in self.params] symbolic_types = T.scalar, T.vector, T.matrix, T.tensor3, T.tensor4 v = [symbolic_types[len(i)]() for i in self.shapes] #import pdb #pdb.set_trace() gauss_vector = Gv(self.cost, self.s, self.params, v, self.coefficient) self.get_cost = theano.function( [ index, ], self.cost, givens={ x: train_set_x[index * batch_size:(index + 1) * batch_size], y: train_set_y[index * batch_size:(index + 1) * batch_size] }, on_unused_input='ignore') self.get_grad = theano.function( [ index, ], grads, givens={ x: train_set_x[index * batch_size:(index + 1) * batch_size], y: train_set_y[index * batch_size:(index + 1) * batch_size] }, on_unused_input='ignore') self.get_s = theano.function( [ index, ], self.s, givens={ x: train_set_x[index * batch_size:(index + 1) * batch_size], }, on_unused_input='ignore') self.function_Gv = theano.function( [index], gauss_vector, givens={ x: train_set_x[index * batch_size:(index + 1) * batch_size], y: valid_set_y[index * batch_size:(index + 1) * batch_size] }, on_unused_input='ignore') # # Using stochastic gradient updates # updates = [ (param_i, param_i-learning_rate*grad_i) for param_i, grad_i in zip(params,grads) ] # train_model = theano.function([index],cost, updates=updates, # givens={ # x: train_set_x[index*batch_size:(index+1)*batch_size], # y: train_set_y[index*batch_size:(index+1)*batch_size] # }) # Using conjugate gradient updates # 'cg_ = cg(cost,output,params,coefficient,v) # updated_params = [(param_i, param_j) for param_i,param_j in zip(params,cg_)]' #self.update_parameters= theano.function([updated_params],updates=[params,updated_params]) # -----------------------------------------Starting Training ------------------------------ if testing == 0: print('..... Training ') # for early stopping patience = 10000 patience_increase = 2 improvement_threshold = 0.95 validation_frequency = min(n_train_batches, patience // 2) self.best_validation_loss = numpy.inf # initialising loss to be inifinite best_itr = 0 test_score = 0 start_time = timeit.default_timer() epoch = 0 done_looping = False while (epoch < n_epochs): epoch = epoch + 1 for minibatch_index in range(n_train_batches): iter = (epoch - 1) * n_train_batches + minibatch_index if iter % 1 == 0: print('training @ iter = ', iter) self.cg(minibatch_index) if testing == 0: print('Optimization complete') print( 'Best validation score of %f %% obtained at iteration %i,' 'with test performance %f %%' % (best_validation_loss * 100., best_itr, test_score * 100)) print('The code ran for %.2fm' % ((end_time - start_time) / 60.))
def evaluate_lenet5(learning_rate = 0.1, n_epochs=200, dataset='mnist.pkl.gz',nkerns = [20,50], batch_size = 500 , testing =0): rng = numpy.random.RandomState(32324) datasets = load_data(dataset) train_set_x,train_set_y = datasets[0] valid_set_x,valid_set_y = datasets[1] test_set_x,test_set_y = datasets[2] n_train_batches = train_set_x.get_value(borrow=True).shape[0]//batch_size n_valid_batches = valid_set_x.get_value(borrow=True).shape[0]//batch_size n_test_batches = test_set_x.get_value(borrow=True).shape[0]//batch_size index = T.lscalar() # index for each mini batch x = T.matrix('x') y = T.ivector('y') # ------------------------------- Building Model ---------------------------------- if testing ==0: print "...Building the model" # output image size = (28-5+1)/2 = 12 layer_0_input = x.reshape((batch_size,1,28,28)) layer_0 = LeNetConvPoolLayer(rng,input = layer_0_input, image_shape=(batch_size,1,28,28),filter_shape=(nkerns[0],1,5,5),poolsize=(2,2)) #output image size = (12-5+1)/2 = 4 layer_1 = LeNetConvPoolLayer(rng, input = layer_0.output, image_shape = (batch_size, nkerns[0],12,12), filter_shape = (nkerns[1],nkerns[0],5,5), poolsize=(2,2) ) # make the input to hidden layer 2 dimensional layer_2_input = layer_1.output.flatten(2) layer_2 = HiddenLayer(rng,input = layer_2_input, n_in = nkerns[1]*4*4, n_out = 500, activation = T.tanh) layer_3 = LogReg(input = layer_2.output, n_in=500, n_out = 10) cost = layer_3.neg_log_likelihood(y) test_model = theano.function([index],layer_3.errors(y), givens={ x: test_set_x[index*batch_size:(index+1)*batch_size], y: test_set_y[index*batch_size:(index+1)*batch_size] }) validate_model = theano.function([index],layer_3.errors(y), givens={ x: valid_set_x[index*batch_size:(index+1)*batch_size], y: valid_set_y[index*batch_size:(index+1)*batch_size] }) train_predic = theano.function([index], layer_3.prob_y_given_x(), givens={ x: train_set_x[index*batch_size:(index+1)*batch_size] }) # list of parameters layer_guided = theano.function([index], layer_1.output, givens={ x: train_set_x[index*batch_size:(index+1)*batch_size] }) params = layer_3.params + layer_2.params + layer_1.params + layer_0.params grads = T.grad(cost,params) updates = [ (param_i, param_i-learning_rate*grad_i) for param_i, grad_i in zip(params,grads) ] train_model = theano.function([index],cost, updates=updates, givens={ x: train_set_x[index*batch_size:(index+1)*batch_size], y: train_set_y[index*batch_size:(index+1)*batch_size] }) # -----------------------------------------Starting Training ------------------------------ if testing ==0: print ('..... Training ' ) # for early stopping patience = 10000 patience_increase = 2 improvement_threshold = 0.95 validation_frequency = min(n_train_batches, patience//2) best_validation_loss = numpy.inf # initialising loss to be inifinite best_itr = 0 test_score = 0 start_time = timeit.default_timer() epoch = 0 done_looping = False while (epoch < n_epochs) and (not done_looping) and testing ==0: epoch = epoch+1 for minibatch_index in range(n_train_batches): iter = (epoch - 1)*n_train_batches + minibatch_index if iter%100 ==0: print ('training @ iter = ', iter) cost_ij = train_model(minibatch_index) if(iter +1)%validation_frequency ==0: # compute loss on validation set validation_losses = [validate_model(i) for i in range(n_valid_batches)] this_validation_loss = numpy.mean(validation_losses) # import pdb # pdb.set_trace() print ('epoch %i, minibatch %i/%i, validation error %f %%' %(epoch,minibatch_index+1,n_train_batches,this_validation_loss*100. )) # check with best validation score till now if this_validation_loss<best_validation_loss: # improve # if this_validation_loss < best_validation_loss * improvement_threshold: # patience = max(patience, iter*patience_increase) best_validation_loss = this_validation_loss best_itr = iter test_losses = [test_model(i) for i in range(n_test_batches)] test_score = numpy.mean(test_losses) print ('epoch %i, minibatch %i/%i, testing error %f %%' %(epoch, minibatch_index+1,n_train_batches,test_score*100.)) with open('best_model.pkl', 'wb') as f: pickle.dump(params, f) with open('Results_teacher.txt','wb') as f2: f2.write(str(test_score*100) + '\n') p_y_given_x = [train_predic(i) for i in range(n_train_batches)] with open ('prob_best_model.pkl','wb') as f1: pickle.dump(p_y_given_x,f1) # if patience <= iter: # done_looping = True # break layer_2_op_dump = [layer_guided(i) for i in range(n_train_batches)] with open ('layer_guided.pkl','wb') as lg: pickle.dump(layer_2_op_dump,lg) end_time = timeit.default_timer() # p_y_given_x = [train_model(i) for i in range(n_train_batches)] # with open ('prob_best_model.pkl') as f: # pickle.dump(p_y_given_x) if testing ==0 : print ('Optimization complete') print ('Best validation score of %f %% obtained at iteration %i,' 'with test performance %f %%' % (best_validation_loss*100., best_itr, test_score*100 )) print('The code ran for %.2fm' %((end_time - start_time)/60.))