def optimize_neuralnet_msgd(learning_rate=0.001, lambda_1=0.00, lambda_2=0.0001, maximum_epochs=1000, dataset='tomtec2chamber.pkl', minibatch=200, hidden_units=1000): train_errors_list = [] validate_errors_list = [] test_errors_list = [] data = load_data(dataset) print "data unpickled" training_data_inputs, training_data_labels = data[0] validation_data_inputs, validation_data_labels = data[1] testing_data_inputs, testing_data_labels = data[2] inp_dimensions = data[3] class_labels = data[4] minibatches_training = training_data_inputs.get_value().shape[0] / minibatch minibatches_validation = validation_data_inputs.get_value().shape[0] / minibatch minibatches_testing = testing_data_inputs.get_value().shape[0] / minibatch index = Tensor.lscalar() inputs = Tensor.matrix('inputs') labels = Tensor.ivector('labels') random_val = numpy.random.RandomState() classifier = NeuralNet(random_val=random_val, input=inputs, input_dimensions=(inp_dimensions * inp_dimensions), hidden_dimensions=hidden_units, output_dimensions=2) loss_nll = classifier.loss_nll(labels) + lambda_1 * classifier.norm_L1 + lambda_2 * classifier.norm_L2 function_for_testing = theano.function(inputs=[index], outputs=classifier.prediction_accuracy(labels), givens={ inputs: testing_data_inputs[index * minibatch:(index + 1) * minibatch], labels: testing_data_labels[index * minibatch:(index + 1) * minibatch]}) function_for_validation = theano.function(inputs=[index], outputs=classifier.prediction_accuracy(labels), givens={ inputs: validation_data_inputs[index * minibatch:(index + 1) * minibatch], labels: validation_data_labels[index * minibatch:(index + 1) * minibatch]}) gradient_parameters = [] for param in classifier.parameters: gparam = Tensor.grad(loss_nll, param) gradient_parameters.append(gparam) learning = [] for parameters, gradient_parameters in zip(classifier.parameters, gradient_parameters): learning.append((parameters, parameters - learning_rate * gradient_parameters)) function_for_training = theano.function(inputs=[index], outputs=loss_nll, updates=learning, givens={ inputs: training_data_inputs[index * minibatch:(index + 1) * minibatch], labels: training_data_labels[index * minibatch:(index + 1) * minibatch]}) function_for_train_errors = theano.function(inputs=[index], outputs=classifier.prediction_accuracy(labels), givens={ inputs: training_data_inputs[index * minibatch:(index + 1) * minibatch], labels: training_data_labels[index * minibatch:(index + 1) * minibatch]}) training_loops = 240000 best_parameters = None best_validate_error = numpy.inf test_error = 0. iteration = 0 print "training with minibatch stochastic gradient Descent and Neural Network" for current_epoch in xrange(maximum_epochs): for index in xrange(minibatches_training): nll = function_for_training(index) iteration = (current_epoch) * minibatches_training + index if (iteration + 1) % minibatches_training == 0: validation_errors = [function_for_validation(i) for i in xrange(minibatches_validation)] current_validation_error = numpy.mean(validation_errors) train_errors = [function_for_train_errors(i)for i in xrange(minibatches_training)] train_error = numpy.mean(train_errors) print('training current_epoch number = %i : training error = %f :validation error = %f' %(current_epoch, train_error * 100., current_validation_error * 100.)) if current_validation_error < best_validate_error: best_validate_error = current_validation_error test_errors = [function_for_testing(i) for i in xrange(minibatches_testing)] test_error = numpy.mean(test_errors) train_errors_list.append(train_error) validate_errors_list.append(current_validation_error) test_errors_list.append(test_error) best_parameters = classifier.parameters print('In training epoch %i with validation error = %i and test error = %f' %(current_epoch, best_validate_error * 100, test_error * 100.)) if iteration >= training_loops: break print('minibatch gradient descent optimization done with validation error = %i and test error = %f' %(best_validate_error * 100, test_error * 100.)) return best_parameters, train_errors_list, validate_errors_list, test_errors_list
def evaluate_lenet5( learning_rate=0.1, n_epochs=2, momentum=0.9, dataset="../DataGeneration/tomtec2chamber.pkl", nkerns=[20, 50], batch_size=500, confusion_error=False, default_error=True, use_rmsprop=True, verbose=False, params_file=None, ): """ :type learning_rate: float :param learning_rate: learning rate used (factor for the stochastic gradient) :type n_epochs: int :param n_epochs: maximal number of epochs to run the optimizer :type dataset: string :param dataset: path to the dataset used for training / testing :type nkerns: list of ints :param nkerns: number of kernels on each layer """ train_errors_list = [] validate_errors_list = [] test_errors_list = [] confusion_train_errors_list = [] confusion_validate_errors_list = [] confusion_test_errors_list = [] rng = numpy.random.RandomState(23455) datasets = load_data(dataset) train_set_x, train_set_y = datasets[0] valid_set_x, valid_set_y = datasets[1] test_set_x, test_set_y = datasets[2] input_shape = datasets[3] class_labels = datasets[4] class_count = len(class_labels) print "Classes: \t", class_labels print "Patch size: \t", input_shape if use_rmsprop: print "Learning rate: \t%f (before adjustment for rmsprop)" % (learning_rate) else: print "Learning rate: \t%f" % (learning_rate) print "Momentum: \t", momentum if use_rmsprop: learning_rate = learning_rate * 0.01 # rmsprop needs much smaller learning rates # compute number of minibatches for training, validation and testing n_train_batches = train_set_x.get_value(borrow=True).shape[0] n_valid_batches = valid_set_x.get_value(borrow=True).shape[0] n_test_batches = test_set_x.get_value(borrow=True).shape[0] n_train_batches /= batch_size n_valid_batches /= batch_size n_test_batches /= batch_size # allocate symbolic variables for the data index = T.lscalar() # index to a [mini]batch x = T.matrix("x") # the data is presented as rasterized images y = T.ivector("y") # the labels are presented as 1D vector of # [int] labels ###################### # BUILD ACTUAL MODEL # ###################### print "... building the model" if params_file: params = None with open(params_file, "rb") as f: params = cPickle.load(f) print "Using existing weights" if not params: print "Warning: Existing weights were non-existent" layer0_W = params[0][0] layer0_b = params[0][1] layer1_W = params[1][0] layer1_b = params[1][1] layer2_W = params[2][0] layer2_b = params[2][1] layer3_p = params[3] else: layer0_W = None layer0_b = None layer1_W = None layer1_b = None layer2_W = None layer2_b = None layer3_p = None # Reshape matrix of rasterized images of shape (batch_size,28*28) # to a 4D tensor, compatible with our LeNetConvPoolLayer layer0_input = x.reshape((batch_size, 1, input_shape, input_shape)) # Construct the first convolutional pooling layer: # filtering reduces the image size to (28-5+1,28-5+1)=(24,24) # maxpooling reduces this further to (24/2,24/2) = (12,12) # 4D output tensor is thus of shape (batch_size,nkerns[0],12,12) layer0_out_size = (input_shape - 5 + 1) / 2 layer0 = LeNetConvPoolLayer( rng, input=layer0_input, image_shape=(batch_size, 1, input_shape, input_shape), filter_shape=(nkerns[0], 1, 5, 5), poolsize=(2, 2), W=layer0_W, b=layer0_b, ) # Construct the second convolutional pooling layer # filtering reduces the image size to (12-5+1,12-5+1)=(8,8) # maxpooling reduces this further to (8/2,8/2) = (4,4) # 4D output tensor is thus of shape (nkerns[0],nkerns[1],4,4) layer1_out_size = (layer0_out_size - 5 + 1) / 2 layer1 = LeNetConvPoolLayer( rng, input=layer0.output, image_shape=(batch_size, nkerns[0], layer0_out_size, layer0_out_size), filter_shape=(nkerns[1], nkerns[0], 5, 5), poolsize=(2, 2), W=layer1_W, b=layer1_b, ) # the TanhLayer being fully-connected, it operates on 2D matrices of # shape (batch_size,num_pixels) (i.e matrix of rasterized images). # This will generate a matrix of shape (20,32*4*4) = (20,512) layer2_input = layer1.output.flatten(2) # construct a fully-connected sigmoidal layer layer2 = HiddenLayer( rng, input=layer2_input, input_dimensions=nkerns[1] * layer1_out_size * layer1_out_size, output_dimensions=500, activation_function=T.tanh, Weight=layer2_W, bias=layer2_b, ) # classify the values of the fully-connected sigmoidal layer layer3 = LogisticRegression( input=layer2.output, input_dimensions=500, output_dimensions=class_count, params=layer3_p ) # the cost we minimize during training is the NLL of the model cost = layer3.loss_nll(y) # create a function to compute the mistakes that are made by the model train_errors = theano.function( inputs=[index], outputs=layer3.prediction_accuracy(y), givens={ x: train_set_x[index * batch_size : (index + 1) * batch_size], y: train_set_y[index * batch_size : (index + 1) * batch_size], }, ) test_model = theano.function( [index], layer3.prediction_accuracy(y), givens={ x: test_set_x[index * batch_size : (index + 1) * batch_size], y: test_set_y[index * batch_size : (index + 1) * batch_size], }, ) validate_model = theano.function( [index], layer3.prediction_accuracy(y), givens={ x: valid_set_x[index * batch_size : (index + 1) * batch_size], y: valid_set_y[index * batch_size : (index + 1) * batch_size], }, ) #######################Confusion matrix code###################################### confusion_model_train = theano.function( [index], layer3.getPrediction(), givens={x: train_set_x[index * batch_size : (index + 1) * batch_size]} ) confusion_model_validate = theano.function( [index], layer3.getPrediction(), givens={x: valid_set_x[index * batch_size : (index + 1) * batch_size]} ) confusion_model_test = theano.function( [index], layer3.getPrediction(), givens={x: test_set_x[index * batch_size : (index + 1) * batch_size]} ) confusion_model_train_y = theano.function( [index], y, givens={y: train_set_y[index * batch_size : (index + 1) * batch_size]} ) confusion_model_validate_y = theano.function( [index], y, givens={y: valid_set_y[index * batch_size : (index + 1) * batch_size]} ) confusion_model_test_y = theano.function( [index], y, givens={y: test_set_y[index * batch_size : (index + 1) * batch_size]} ) ################################################################################### # create a list of all model parameters to be fit by gradient descent params = layer3.params + layer2.params + layer1.params + layer0.params # create a list of gradients for all model parameters grads = T.grad(cost, params) # train_model is a function that updates the model parameters by # SGD Since this model has many parameters, it would be tedious to # manually create an update rule for each model parameter. We thus # create the updates list by automatically looping over all # (params[i],grads[i]) pairs. updates = [] if use_rmsprop: prev_step = [ theano.shared(value=numpy.zeros(params[i].get_value().shape, dtype=theano.config.floatX)) for i in xrange(len(params)) ] mean_squ = [ theano.shared(value=numpy.zeros(params[i].get_value().shape, dtype=theano.config.floatX)) for i in xrange(len(params)) ] for param_i, grad_i, step_i, mean_squ_i in zip(params, grads, prev_step, mean_squ): # note: ideally, we'd use the full gradient at the first iteration (when mean_squ_i is usually 0) new_mean_squ_i = T.add(T.mul(mean_squ_i, 0.9), T.mul(T.square(grad_i), 0.1)) new_grad_i = T.div_proxy(grad_i, T.add(T.sqrt(new_mean_squ_i), 1e-08)) new_step_i = momentum * step_i - learning_rate * new_grad_i updates.append((mean_squ_i, new_mean_squ_i)) updates.append((param_i, param_i + new_step_i)) updates.append((step_i, new_step_i)) else: for param_i, grad_i in zip(params, grads): updates.append((param_i, param_i - learning_rate * grad_i)) train_model = theano.function( [index], cost, updates=updates, givens={ x: train_set_x[index * batch_size : (index + 1) * batch_size], y: train_set_y[index * batch_size : (index + 1) * batch_size], }, ) ############### # TRAIN MODEL # ############### print "... training" # early-stopping parameters patience = 10000 # look as this many examples regardless patience_increase = 2 # wait this much longer when a new best is # found improvement_threshold = 0.995 # a relative improvement of this much is # considered significant validation_frequency = min(n_train_batches, patience / 2) # go through this many # minibatches before checking the network # on the validation set; in this case we # check every epoch best_params = None best_validation_loss = numpy.inf best_iter = 0 test_score = 0.0 ############confusion error################ confusion_best_validation_loss = numpy.inf confusion_best_iter = 0 confusion_test_score = 0.0 ########################################### start_time = time.clock() costs = [] epoch = 0 done_looping = False while (epoch < n_epochs) and (not done_looping): print epoch epoch = epoch + 1 for minibatch_index in xrange(n_train_batches): # current_learning_rate = learning_rate * np.amax([0.005, (0.8**epoch)]) # decaying learning rate iter = (epoch - 1) * n_train_batches + minibatch_index cost_ij = train_model(minibatch_index) costs.append(cost_ij) if verbose or iter % 100 == 0: print "training @ iter = %d, cost = %f" % (iter, cost_ij) if (iter + 1) % validation_frequency == 0: if confusion_error == True: confusion_validation_losses = [confusion_model_validate(i) for i in xrange(n_valid_batches)] y = [confusion_model_validate_y(i) for i in xrange(n_valid_batches)] prediction = numpy.array(confusion_validation_losses).flatten() y = numpy.array(y).flatten() conf_matrix = numpy.bincount( class_count * (y) + (prediction), minlength=class_count * class_count ).reshape(class_count, class_count) confusion_loss = 0 confusion_loss_count = 0 for i in range(conf_matrix.shape[0]): for j in range(conf_matrix.shape[0]): if i != j: confusion_loss = confusion_loss + conf_matrix[i][j] confusion_loss_count = confusion_loss_count + conf_matrix[i][j] confusion_validation_loss = float(confusion_loss) / float(confusion_loss_count) print ( "Q2 error(Using confusion matrix): epoch %i, validation error %f %%" % (epoch, confusion_validation_loss * 100.0) ) if confusion_validation_loss < confusion_best_validation_loss: confusion_train_errors = [confusion_model_train(i) for i in xrange(n_train_batches)] y = [confusion_model_train_y(i) for i in xrange(n_train_batches)] prediction = numpy.array(confusion_train_errors).flatten() y = numpy.array(y).flatten() conf_matrix = numpy.bincount( class_count * (y) + (prediction), minlength=class_count * class_count ).reshape(class_count, class_count) confusion_loss = 0 confusion_loss_count = 0 for i in range(conf_matrix.shape[0]): for j in range(conf_matrix.shape[0]): if i != j: confusion_loss = confusion_loss + conf_matrix[i][j] confusion_loss_count = confusion_loss_count + conf_matrix[i][j] confusion_train_error = float(confusion_loss) / float(confusion_loss_count) print ( "Q2 error(Using confusion matrix): epoch %i, validation error %f %%" % (epoch, confusion_validation_loss * 100.0) ) # improve patience if loss improvement is good enough if confusion_validation_loss < confusion_best_validation_loss * improvement_threshold: patience = max(patience, iter * patience_increase) # save best validation score and iteration number confusion_best_validation_loss = confusion_validation_loss confusion_best_iter = iter confusion_test_losses = [confusion_model_test(i) for i in xrange(n_test_batches)] y = [confusion_model_test_y(i) for i in xrange(n_test_batches)] prediction = numpy.array(confusion_test_losses).flatten() y = numpy.array(y).flatten() conf_matrix = numpy.bincount( class_count * (y) + (prediction), minlength=class_count * class_count ).reshape(class_count, class_count) confusion_loss = 0 confusion_loss_count = 0 for i in range(conf_matrix.shape[0]): for j in range(conf_matrix.shape[0]): if i != j: confusion_loss = confusion_loss + conf_matrix[i][j] confusion_loss_count = confusion_loss_count + conf_matrix[i][j] confusion_test_score = float(confusion_loss) / float(confusion_loss_count) print ( ("Q2 Error(Using confusion matrix)epoch %i, test error of best " "model %f %%") % (epoch, confusion_test_score * 100.0) ) print "Confusion matrix: Test Set" print_confusion_matrix(conf_matrix, class_labels) if class_count == 2: true_positive = float(conf_matrix[1][1]) / float(confusion_loss_count) true_negative = float(conf_matrix[0][0]) / float(confusion_loss_count) false_positive = float(conf_matrix[0][1]) / float(confusion_loss_count) false_negative = float(conf_matrix[1][0]) / float(confusion_loss_count) print "True Positive(Annuli) %f %%" % (true_positive * 100.0) print "True Negative(Non Annuli) %f %%" % (true_negative * 100.0) print "False Positive %f %%" % (false_positive * 100.0) print "False Negative %f %%" % (false_negative * 100.0) confusion_train_errors_list.append(confusion_train_error) confusion_validate_errors_list.append(confusion_best_validation_loss) confusion_test_errors_list.append(confusion_test_score) confusion_best_params = [layer0.params, layer1.params, layer2.params, layer3.params] with open("confusion_weights.pkl", "wb") as f: cPickle.dump(confusion_best_params, f, cPickle.HIGHEST_PROTOCOL) visualize_errors( [ (confusion_train_errors_list, "confusion_train"), (confusion_validate_errors_list, "confusion_validate"), (confusion_test_errors_list, "confusion_test"), ], costs=costs, integrated=True, show=False, ) with open("confusion_errors.pkl", "wb") as f: cPickle.dump( ( confusion_train_errors_list, confusion_validate_errors_list, confusion_test_errors_list, ), f, protocol=cPickle.HIGHEST_PROTOCOL, ) if default_error == True: # compute zero-one loss on validation set train_losses = [train_errors(i) for i in xrange(n_train_batches)] this_train_loss = numpy.mean(train_losses) validation_losses = [validate_model(i) for i in xrange(n_valid_batches)] this_validation_loss = numpy.mean(validation_losses) print ( "epoch %i, training error %f %%, validation error %f %%" % (epoch, this_train_loss * 100, this_validation_loss * 100.0) ) # if we got the best validation score until now if this_validation_loss < best_validation_loss: # improve patience if loss improvement is good enough if this_validation_loss < best_validation_loss * improvement_threshold: patience = max(patience, iter * patience_increase) # save best validation score and iteration number best_validation_loss = this_validation_loss best_iter = iter # test it on the test set test_losses = [test_model(i) for i in xrange(n_test_batches)] test_score = numpy.mean(test_losses) train_errors_list.append(this_train_loss) validate_errors_list.append(best_validation_loss) test_errors_list.append(test_score) print ( (" epoch %i, minibatch %i/%i, test error of best " "model %f %%") % (epoch, minibatch_index + 1, n_train_batches, test_score * 100.0) ) best_params = [layer0.params, layer1.params, layer2.params, layer3.params] with open("weights.pkl", "wb") as f: cPickle.dump(best_params, f, protocol=cPickle.HIGHEST_PROTOCOL) if patience <= iter: done_looping = True break end_time = time.clock() if confusion_error == True: print ("Optimization complete.") print ( "Q2 Error (using confusion matrix)Best validation score of %f %% with test performance %f %%" % (confusion_best_validation_loss * 100.0, confusion_test_score * 100.0) ) visualize_errors( [ (confusion_train_errors_list, "confusion_train"), (confusion_validate_errors_list, "confusion_validate"), (confusion_test_errors_list, "confusion_test"), ] ) with open("confusion_errors.pkl", "wb") as f: cPickle.dump( (confusion_train_errors_list, confusion_validate_errors_list, confusion_test_errors_list), f, protocol=cPickle.HIGHEST_PROTOCOL, ) if default_error == True: print ("Optimization complete.") print ( "Best validation score of %f %% with test performance %f %%" % (best_validation_loss * 100.0, test_score * 100.0) ) visualize_errors([(train_errors_list, "train"), (validate_errors_list, "validate"), (test_errors_list, "test")]) print >> sys.stderr, ( "The code for file " + os.path.split(__file__)[1] + " ran for %.2fm" % ((end_time - start_time) / 60.0) )
st.title(""" Your cryptocurrencies """) fetch_range = st.slider('Fetching numbers', 1, 4400, (1, 200)) key = st.text_input('Please provide your key here') st.write(""" List of your currencies currently watched: """) b_data = st.checkbox('Show/Hide big data') if st.button('Refresh'): df = fetch_data(fetch_range, key) #df.reset_index(drop=True,inplace=True) #st.dataframe(df.head()) b_fetch = True if st.button('Load'): df, timestamp = load_data() #df.reset_index(drop=True,inplace=True) #st.dataframe(df.head()) b_load = True if b_fetch or b_load: disp_df = df.drop([ 'id', 'slug', 'date_added', 'tags', 'platform', 'cmc_rank', 'last_updated', 'quote' ], axis=1) sliced_indcs = ['HBAR', 'HYVE', 'BTC', 'ETH'] subset_df = disp_df[disp_df['symbol'].isin(sliced_indcs)] st.write('Your subset') st.dataframe(subset_df)
def test_dropout(): #parameters initial_learning_rate = 0.1 #learning rate will be decayed by this value after each epoch learning_rate_decay = 0.98 n_epochs = 50 batch_size = 100 dataset='dataset-2ch-2class-60px-unbal_test.pkl' use_bias = True validation_errors_array = [] train_errors_array = [] datasets = load_data(dataset) #datasets = load_data(dataset) train_set_x, train_set_y = datasets[0] valid_set_x, valid_set_y = datasets[1] test_set_x, test_set_y = datasets[2] input_shape = datasets[3] class_labels = datasets[4] class_count = len(class_labels) #mention layer sizes here [input,hidden1,hidden2,..,output] layer_sizes = [ input_shape*input_shape,1200,class_count] print "Standard NN with dropout..." print "Hidden layer dropout with 0.5 probability and input layer with 0.2 Probability" print "Layers sizes : \t" , layer_sizes print "Classes: \t", class_labels print "Patch size: \t", input_shape # compute number of minibatches for training, validation and testing n_train_batches = train_set_x.get_value(borrow=True).shape[0] / batch_size n_valid_batches = valid_set_x.get_value(borrow=True).shape[0] / batch_size n_test_batches = test_set_x.get_value(borrow=True).shape[0] / batch_size logging.basicConfig(filename='logs/2ch_2class_60x60_dropout_1hidden_1200_drop_batch100_lr0.1_full_2class_final.log',level=logging.DEBUG) ###################### # BUILD ACTUAL MODEL # ###################### print '... building the model' # allocate symbolic variables for the data index = T.lscalar() # index to a [mini]batch epoch = T.scalar() x = T.matrix('x') # the data is presented as rasterized images y = T.ivector('y') # the labels are presented as 1D vector of # [int] labels learning_rate = theano.shared(np.asarray(initial_learning_rate, dtype=theano.config.floatX)) rng = np.random.RandomState(1234) # construct the MLP class classifier = MLP(rng=rng, input=x, layer_sizes=layer_sizes, use_bias=use_bias) # cost function. dropout_cost = classifier.dropout_negative_log_likelihood(y) # Compile theano function for testing. test_model = theano.function(inputs=[index], outputs=classifier.dropout_errors(y), givens={ x: test_set_x[index * batch_size:(index + 1) * batch_size], y: test_set_y[index * batch_size:(index + 1) * batch_size]}) # Compile theano function for validation. validate_model = theano.function(inputs=[index], outputs=classifier.dropout_errors(y), givens={ x: valid_set_x[index * batch_size:(index + 1) * batch_size], y: valid_set_y[index * batch_size:(index + 1) * batch_size]}) # Compute gradients of the model wrt parameters gparams = [] for param in classifier.params: # Use the cost function here to train with dropout. gparam = T.grad(dropout_cost, param) gparams.append(gparam) # allocatition of memory for momentum'd versions of the gradient gparams_mom = [] for param in classifier.params: gparam_mom = theano.shared(np.zeros(param.get_value(borrow=True).shape, dtype=theano.config.floatX)) gparams_mom.append(gparam_mom) # Compute momentum for the current epoch according to hintons paper p_t = ifelse(epoch < 500,0.5*(1. - epoch/500.) + 0.99*(epoch/500.),0.99) # Updation of the step direction using momentum updates = {} for gparam_mom, gparam in zip(gparams_mom, gparams): updates[gparam_mom] = p_t * gparam_mom + (1. - p_t) * gparam # updating of the parameter for param, gparam_mom in zip(classifier.params, gparams_mom): stepped_param = param - (1.-p_t) * learning_rate * gparam_mom updates[param] = stepped_param # Compile theano function for training. This returns the training cost and # updates the model parameters. train_output = [dropout_cost,classifier.dropout_errors(y)] output = dropout_cost train_model = theano.function(inputs=[epoch, index], outputs=train_output, updates=updates, givens={ x: train_set_x[index * batch_size:(index + 1) * batch_size], y: train_set_y[index * batch_size:(index + 1) * batch_size]}) # Theano function to decay the learning rate,after each epoch and not after one mini batch decay_learning_rate = theano.function(inputs=[], outputs=learning_rate, updates={learning_rate: learning_rate * learning_rate_decay}) ############### # TRAIN MODEL # ############### print '... training' logging.debug('training') best_params = None best_validation_errors = np.inf best_iter = 0 test_score = 0. epoch_counter = 0 start_time = time.clock() while epoch_counter < n_epochs: # Train this epoch epoch_counter = epoch_counter + 1 train_losses = [] for minibatch_index in xrange(n_train_batches): train_output = train_model(epoch_counter, minibatch_index) iteration = (epoch_counter - 1) * n_train_batches + minibatch_index train_losses.append(train_output[1]) logging.debug(('training error @ iter = %i : %f')%(iteration,train_output[1])) #compute train loss for 1 epoch train_errors_array.append(numpy.mean(train_losses)) # Compute loss on validation set validation_losses = [validate_model(i) for i in xrange(n_valid_batches)] this_validation_errors = np.mean(validation_losses) validation_errors_array.append(this_validation_errors) # Report and save progress. print (('epoch %i, test error %f %%')%( epoch_counter, this_validation_errors*100.)) logging.debug(('epoch %i, test error %f %%')%( epoch_counter, this_validation_errors*100.)) best_validation_errors = min(best_validation_errors, this_validation_errors) learning_rate = decay_learning_rate() errors_arrays = [(train_errors_array, "train"), (validation_errors_array, "validate")] end_time = time.clock() print 'Visualizing the weights and plotting the error curves' plt.figure(1) plt.subplot(211) colors = ['r','g','b','c','m','y'] i = 0 for errors_array,split in errors_arrays: fmt_str = "%s.-" % colors[i] i = i+1 plt.plot(errors_array, fmt_str, label=split) plt.legend() plt.savefig("plots/2class_errors.png") print(('Optimization complete. Best validation score of %f %% ' 'obtained at iteration %i') % (best_validation_errors * 100., best_iter)) logging.debug(('Optimization complete. Best validation score of %f %% ' 'obtained at iteration %i') % (best_validation_errors * 100., best_iter)) print >> sys.stderr, ('The code for file ' + os.path.split(__file__)[1] + ' ran for %.2fm' % ((end_time - start_time) / 60.))