def test(self): # Training error print "Training error:" evaluator = Evaluator(self.X_train, self.Y_train, self.W) evaluator.MSE() evaluator.accuracy() # Testing error print "Testing error:" evaluator = Evaluator(self.X_test, self.Y_test, self.W) evaluator.MSE() evaluator.accuracy()
def test(self, label=None): # Training error print "Training error:" evaluator = Evaluator(self.X_train, self.Y_train, self.W) #evaluator.MSE() evaluator.accuracy() #evaluator.confusion() # Testing error print "Testing error:" evaluator = Evaluator(self.X_test, self.Y_test, self.W) #evaluator.MSE() evaluator.accuracy() evaluator.confusion() FPR, TPR = evaluator.roc() plt.plot(FPR, TPR, label=label) plt.axis([0.0,0.5,0.5,1.0])
decay_threshold=decthr, stop_threshold=stopthr ) architecture = [784, 400, 400, 10] net = network.Network(architecture, lr) validation_errors, training_costs = trainer.sgd( net, tr_d, mb, momentum=mom, evaluator=evaluator, scheduler=scheduler ) best_net = scheduler.highest_accuracy_network val_err = Utils.error_fraction( evaluator.accuracy(va_d, best_net), len(va_d[0]) )*100 eva_err = Utils.error_fraction( evaluator.accuracy(te_d, best_net), len(te_d[0]) )*100 curr_time = time.strftime("%Y%m%d-%H%M%S") Io.save( best_net, "networks/" + curr_time + "_" + str(architecture).replace(' ', '') + "_valerr" + str(val_err) + "_evaerr" + str(eva_err) + "_lr" + str(lr) + "_mom" + str(mom) + "_dec" + str(dec) +
# Create a dataset of the non-test datasets. ds = SemEvalData(blank=True) for j in range(num_folds): if j != i: ds = ds + datasets[j] input, tags, order = NeuralNet.format_dataset(ds, embedder) # Train the network. network = NeuralNet(hidden=128, layers=2, input=len(input[0]), output=2) # Train on non-test datasets. loss, accuracy = network.train(input, tags, iterations=10) accuracies += [accuracy] # Test on test dataset. test, _, order = NeuralNet.format_dataset(datasets[i], embedder) predictions.update(network.predict(test, order)) from evaluator import Evaluator neural_cm = Evaluator.compare(dataset.tags, predictions) print("Neural Network Accuracy: " + str(round(Evaluator.accuracy(neural_cm) * 100, 2)) + "%") # Save the networks results. with open('output.csv', 'w') as writefile: writefile.write(NeuralNet.pred_to_csv(predictions))
def run(self, options): # sess = tf.Session() evaluator = Evaluator() data = Data() model = Model() statPlot = StatPlot() best_accuracy = 0 print('Getting/Transforming Data.') # Initialize the data pipeline images, targets = data.batches(options, train_logical=True) # Get batch test images and targets from pipline test_images, test_targets = data.batches(options, train_logical=False) # Initialize step global_step = tf.Variable(0, name='global_step', trainable=False) # Declare Model print('Creating the CIFAR10 Model.') with tf.Session() as sess: with tf.variable_scope('model_definition') as scope: # Declare the training network model model_output = model.run(images, options) #model_output = cifar_cnn_model(images, batch_size) # This is very important!!! We must set the scope to REUSE the variables, # otherwise, when we set the test network model, it will create new random # variables. Otherwise we get random evaluations on the test batches. scope.reuse_variables() test_output = model.run(test_images, options) # test_output = cifar_cnn_model(test_images, batch_size) # Declare loss function print('Declare Loss Function.') loss = model.loss(model_output, targets) # Create accuracy function accuracy = evaluator.accuracy(test_output, test_targets) # Create training operations print('Creating the Training Operation.') #generation_num = tf.Variable(0, trainable=False) train_op = self.train(loss, options, global_step) # Initialize Variables print('Initializing the Variables.') init = tf.global_variables_initializer() sess.run(init) saver = tf.train.Saver() if options.getRunOptions('restore_boolean'): path_to_restore_checkpoint_file = os.path.join( options.getRunOptions('log_dir'), options.getRunOptions('restor_file')) assert tf.train.checkpoint_exists(path_to_restore_checkpoint_file), \ '%s not found' % path_to_restore_checkpoint_file saver.restore(sess, path_to_restore_checkpoint_file) print('Model restored from file: %s' % path_to_restore_checkpoint_file) # Initialize queue (This queue will feed into the model, so no placeholders necessary) tf.train.start_queue_runners(sess=sess) print('Starting Training') train_loss = [] test_accuracy = [] start = sess.run(global_step) for i in range(start, options.getRunOptions('generations')): _, loss_value, global_step_val = sess.run( [train_op, loss, global_step]) if (i + 1) % options.getRunOptions('output_every') == 0: train_loss.append(loss_value) output = 'Generation {}: Loss = {:.5f}'.format((i + 1), loss_value) print(output) # if (i+1) % options.getRunOptions('eval_every') == 0: [temp_accuracy] = sess.run([accuracy]) if temp_accuracy > best_accuracy: test_accuracy.append(temp_accuracy) acc_output = ' --- Test Accuracy = {:.2f}%.'.format( 100. * temp_accuracy) print(acc_output) path_to_checkpoint_file = saver.save( sess, os.path.join(options.getRunOptions('log_dir'), 'model.ckpt')) print('=> Model saved to file: %s' % path_to_checkpoint_file) best_accuracy = temp_accuracy if (i + 1) % options.getRunOptions('eval_every') == 0: path_to_checkpoint_file = saver.save( sess, os.path.join(options.getRunOptions('log_dir'), 'model_eval_every.ckpt')) print('=> Model saved to file: %s' % path_to_checkpoint_file) # statPlot.plot(train_loss, test_accuracy, options) tf.summary.FileWriter(options.getRunOptions('log_dir'), sess.graph) return (train_loss, test_accuracy)
from polluted import PollutedSpambase from evaluator import Evaluator from descent import GradientDescent if __name__=="__main__": # Get data dataset = PollutedSpambase() train_data, train_labels = dataset.training() test_data, test_labels = dataset.testing() # Do Logistic Regression gd = GradientDescent(train_data, train_labels) # 200,000 iterations gives ~85% acc W = gd.logreg_stoch(it=200001) # Evaluate solution evaluator = Evaluator([test_data], [test_labels], [W]) evaluator.accuracy()