def train(): """ Performs training and evaluation of MLP model. """ ### DO NOT CHANGE SEEDS! # Set the random seeds for reproducibility np.random.seed(42) ## Prepare all functions # Get number of units in each hidden layer specified in the string such as 100,100 if FLAGS.dnn_hidden_units: dnn_hidden_units = FLAGS.dnn_hidden_units.split(",") dnn_hidden_units = [ int(dnn_hidden_unit_) for dnn_hidden_unit_ in dnn_hidden_units ] else: dnn_hidden_units = [] ######################## # PUT YOUR CODE HERE # ####################### # set up the data cifar10 = cifar10_utils.get_cifar10(FLAGS.data_dir) test_images, test_labels = cifar10['test'].images, cifar10['test'].labels test_vectors = reshape_images(test_images) # set up the model mlp_model = MLP(3072, dnn_hidden_units, 10) loss_module = CrossEntropyModule() accuracies = [] losses = [] for i in range(FLAGS.max_steps): images, labels = cifar10['train'].next_batch(FLAGS.batch_size) image_vectors = reshape_images(images) # forward pass model_pred = mlp_model.forward(image_vectors) # backward pass loss = loss_module.forward(model_pred, labels) loss_grad = loss_module.backward(model_pred, labels) mlp_model.backward(loss_grad) # update all weights and biases mlp_model.update(FLAGS.learning_rate) # evaluate the model on the data set every eval_freq steps if i % FLAGS.eval_freq == 0: test_pred = mlp_model.forward(test_vectors) test_accuracy = accuracy(test_pred, test_labels) accuracies.append(test_accuracy) losses.append(loss) plot_curve(accuracies, 'Accuracy') plot_curve(losses, 'Loss')
def train(): """ Performs training and evaluation of MLP model. TODO: Implement training and evaluation of MLP model. Evaluate your model on the whole test set each eval_freq iterations. """ ### DO NOT CHANGE SEEDS! # Set the random seeds for reproducibility np.random.seed(42) ## Prepare all functions # Get number of units in each hidden layer specified in the string such as 100,100 if FLAGS.dnn_hidden_units: dnn_hidden_units = FLAGS.dnn_hidden_units.split(",") dnn_hidden_units = [ int(dnn_hidden_unit_) for dnn_hidden_unit_ in dnn_hidden_units ] else: dnn_hidden_units = [] dataset = cifar10_utils.get_cifar10(DATA_DIR_DEFAULT) a, b, c = dataset['train'].images.shape[1:] n_classes = dataset['train'].labels.shape[1] n_inputs = a * b * c mlp = MLP(n_inputs, dnn_hidden_units, n_classes) crossentropy = CrossEntropyModule() test_input, test_labels = dataset['test'].images, dataset['test'].labels test_input = np.reshape(test_input, (test_input.shape[0], n_inputs)) for step in range(FLAGS.max_steps): input, labels = dataset['train'].next_batch(FLAGS.batch_size) input = np.reshape(input, (FLAGS.batch_size, n_inputs)) predictions = mlp.forward(input) dL = crossentropy.backward(predictions, labels) mlp.backward(dL) for layer in mlp.layers: if (layer.__class__ == LinearModule): layer.params[ 'weight'] -= FLAGS.learning_rate * layer.grads['weight'] layer.params[ 'bias'] -= FLAGS.learning_rate * layer.grads['bias'] loss = crossentropy.forward(predictions, labels) if (step % FLAGS.eval_freq == 0): test_prediction = mlp.forward(test_input) test_loss = crossentropy.forward(test_prediction, test_labels) test_accuracy = accuracy(test_prediction, test_labels) sys.stdout = open( str(FLAGS.dnn_hidden_units) + '_' + str(FLAGS.learning_rate) + '_' + str(FLAGS.max_steps) + '_' + str(FLAGS.batch_size) + '_' + str(FLAGS.batch_size) + '_mlp_numpy.csv', 'a') print("{},{:f},{:f}".format(step, test_loss, test_accuracy))
def train(): """ Performs training and evaluation of MLP model. TODO: Implement training and evaluation of MLP model. Evaluate your model on the whole test set each eval_freq iterations. """ ### DO NOT CHANGE SEEDS! # Set the random seeds for reproducibility np.random.seed(42) ## Prepare all functions # Get number of units in each hidden layer specified in the string such as 100,100 if FLAGS.dnn_hidden_units: dnn_hidden_units = FLAGS.dnn_hidden_units.split(",") dnn_hidden_units = [ int(dnn_hidden_unit_) for dnn_hidden_unit_ in dnn_hidden_units ] else: dnn_hidden_units = [] # Get Images cifar10 = cifar10_utils.read_data_sets(FLAGS.data_dir) # Create MLP Instance trainDataSet = cifar10['train'] testDataSet = cifar10['test'] #size of squeezed images size_of_images = cifar10['train'].images[0].shape[0] * cifar10[ 'train'].images[0].shape[1] * cifar10['train'].images[0].shape[2] #MLP Object & loss mlp = MLP(size_of_images, dnn_hidden_units, np.shape(cifar10['test'].labels)[1]) loss = CrossEntropyModule() for i in range(FLAGS.max_steps): # np.random.shuffle(cifar10['train']) accuracies_train = [] loss_train = [] batch = trainDataSet.next_batch(BATCH_SIZE_DEFAULT) x = batch[0] x = x.reshape(x.shape[0], (x.shape[1] * x.shape[2] * x.shape[3])) y = batch[1] prob = mlp.forward(x) predictions = (prob == prob.max(axis=1)[:, None]).astype(int) current_accuracy = accuracy(predictions, y) accuracies_train.append(current_accuracy) current_loss = loss.forward(prob, y) loss_train.append(current_loss) out_loss_back = loss.backward(prob, y) mlp.backward(out_loss_back) if i % FLAGS.eval_freq == 0: test_dataset(mlp, testDataSet, loss, i) writer.add_scalar('Train/LossIteration', current_accuracy, i) writer.add_scalar('Train/AccuracyIteration', current_loss, i) print(i) test_dataset(mlp, testDataSet, loss, FLAGS.max_steps + 1)
def train(): """ Performs training and evaluation of MLP model. TODO: Implement training and evaluation of MLP model. Evaluate your model on the whole test set each eval_freq iterations. """ ### DO NOT CHANGE SEEDS! # Set the random seeds for reproducibility np.random.seed(42) ## Prepare all functions # Get number of units in each hidden layer specified in the string such as 100,100 if FLAGS.dnn_hidden_units: dnn_hidden_units = FLAGS.dnn_hidden_units.split(",") dnn_hidden_units = [ int(dnn_hidden_unit_) for dnn_hidden_unit_ in dnn_hidden_units ] else: dnn_hidden_units = [] ######################## # PUT YOUR CODE HERE # ####################### data = cifar10_utils.get_cifar10(FLAGS.data_dir) n_inputs = 3 * 32 * 32 n_classes = 10 model = MLP(n_inputs, dnn_hidden_units, n_classes) loss_fn = CrossEntropyModule() max_accuracy = 0.0 start_time = time.perf_counter() for step in range(1, FLAGS.max_steps + 1): x, targets = data['train'].next_batch(FLAGS.batch_size) input = x.reshape((FLAGS.batch_size, -1)) predictions = model.forward(input) gradient = loss_fn.backward(predictions, targets) model.backward(gradient) model.step(FLAGS.learning_rate) if step == 1 or step % FLAGS.eval_freq == 0: training_loss = loss_fn.forward(predictions, targets) test_predictions = model.forward(data['test'].images.reshape( data['test'].num_examples, -1)) test_loss = loss_fn.forward(test_predictions, data['test'].labels) test_acc = accuracy(test_predictions, data['test'].labels) if test_acc > max_accuracy: max_accuracy = test_acc print( "step %d/%d: training loss: %.3f test loss: %.3f accuracy: %.1f%%" % (step, FLAGS.max_steps, training_loss, test_loss, test_acc * 100)) time_taken = time.perf_counter() - start_time print("Done. Scored %.1f%% in %.1f seconds." % (max_accuracy * 100, time_taken))
def train(): """ Performs training and evaluation of MLP model. """ ### DO NOT CHANGE SEEDS! # Set the random seeds for reproducibility np.random.seed(42) ## Prepare all functions # Get number of units in each hidden layer specified in the string such as 100,100 if FLAGS.dnn_hidden_units: dnn_hidden_units = FLAGS.dnn_hidden_units.split(",") dnn_hidden_units = [ int(dnn_hidden_unit_) for dnn_hidden_unit_ in dnn_hidden_units ] else: dnn_hidden_units = [] data = cifar10_utils.get_cifar10(data_dir=FLAGS.data_dir) train = data['train'] test = data['test'] n_inputs = train.images[0].flatten().shape[0] n_classes = train.labels[0].shape[0] mlp = MLP(n_inputs, dnn_hidden_units, n_classes) loss_mod = CrossEntropyModule() loss_history = [] acc_history = [] for step in range(FLAGS.max_steps): #FLAGS.max_steps x, y = train.next_batch(FLAGS.batch_size) x = x.reshape(x.shape[0], n_inputs) out = mlp.forward(x) loss = loss_mod.forward(out, y) loss_history.append(loss) dout = loss_mod.backward(out, y) mlp.backward(dout) mlp.update(FLAGS.learning_rate) if step == 0 or (step + 1) % FLAGS.eval_freq == 0: x, y = test.images, test.labels x = x.reshape(x.shape[0], n_inputs) test_out = mlp.forward(x) acc = accuracy(test_out, y) print('Accuracy:', acc) acc_history.append(acc) print('Final loss:', loss_history[-1]) print('Final acc:', acc_history[-1]) print(len(acc_history)) plt.plot(loss_history) plt.step(range(0, FLAGS.max_steps + 1, FLAGS.eval_freq), acc_history) plt.legend(['loss', 'accuracy']) plt.show()
def train(): """ Performs training and evaluation of MLP model. TODO: Implement training and evaluation of MLP model. Evaluate your model on the whole test set each eval_freq iterations. """ ### DO NOT CHANGE SEEDS! # Set the random seeds for reproducibility np.random.seed(42) ## Prepare all functions # Get number of units in each hidden layer specified in the string such as 100,100 if FLAGS.dnn_hidden_units: dnn_hidden_units = FLAGS.dnn_hidden_units.split(",") dnn_hidden_units = [ int(dnn_hidden_unit_) for dnn_hidden_unit_ in dnn_hidden_units ] else: dnn_hidden_units = [] ######################## # PUT YOUR CODE HERE # ####################### # return {'train': train, 'validation': validation, 'test': test} dataset_dict = cifar10_utils.get_cifar10(DATA_DIR_DEFAULT) train_loader = dataset_dict['train'] test_loader = dataset_dict['test'] model = MLP(n_inputs=32 * 32 * 3, n_hidden=dnn_hidden_units, n_classes=10) test_accs = [] train_accs = [] losses = [] for epoch in range(FLAGS.max_steps): batch_x, batch_y = train_loader.next_batch(FLAGS.batch_size) out = model.forward(batch_x.reshape(FLAGS.batch_size, -1)) cross_ent = CrossEntropyModule() loss = cross_ent.forward(out, batch_y) losses.append(round(loss, 3)) dout = cross_ent.backward(out, batch_y) model.backward(dout) for layer in model.layers: if type(layer) == modules.LinearModule: layer.params['weight'] = layer.params[ 'weight'] - FLAGS.learning_rate * layer.grads['weight'] layer.params['bias'] = layer.params[ 'bias'] - FLAGS.learning_rate * layer.grads['bias'] if epoch % FLAGS.eval_freq == 0: #print accuracy on test and train set train_acc = accuracy(out, batch_y) out = model.forward( test_loader.images.reshape(test_loader.images.shape[0], -1)) test_acc = accuracy(out, test_loader.labels) print( 'Train Epoch: {}/{}\tLoss: {:.6f}\tTrain accuracy: {:.6f}\tTest accuracy: {:.6f}' .format(epoch, FLAGS.max_steps, loss, train_acc, test_acc)) test_accs.append(test_acc) train_accs.append(train_acc) out = model.forward( test_loader.images.reshape(test_loader.images.shape[0], -1)) test_acc = accuracy(out, test_loader.labels) print('FINAL Test accuracy: {:.6f}'.format(test_acc)) import matplotlib.pyplot as plt plt.figure() plt.plot([i for i in range(0, MAX_STEPS_DEFAULT, EVAL_FREQ_DEFAULT)], train_accs) plt.plot([i for i in range(0, MAX_STEPS_DEFAULT, EVAL_FREQ_DEFAULT)], test_accs) plt.legend(["train", "test"]) plt.ylabel("accuracy") plt.xlabel("epoch") plt.savefig("accuracy") plt.figure() plt.plot([i for i in range(0, MAX_STEPS_DEFAULT, 1)], losses) plt.legend(["loss"]) plt.ylabel("loss") plt.xlabel("epoch") plt.savefig("loss")
def train(): """ Performs training and evaluation of MLP model. TODO: Implement training and evaluation of MLP model. Evaluate your model on the whole test set each eval_freq iterations. """ ### DO NOT CHANGE SEEDS! # Set the random seeds for reproducibility np.random.seed(42) ## Prepare all functions # Get number of units in each hidden layer specified in the string such as 100,100 if FLAGS.dnn_hidden_units: dnn_hidden_units = FLAGS.dnn_hidden_units.split(",") dnn_hidden_units = [ int(dnn_hidden_unit_) for dnn_hidden_unit_ in dnn_hidden_units ] else: dnn_hidden_units = [] # Get negative slope parameter for LeakyReLU neg_slope = FLAGS.neg_slope ######################## # PUT YOUR CODE HERE # ####################### lr = FLAGS.learning_rate eval_freq = FLAGS.eval_freq max_steps = FLAGS.max_steps batch_size = FLAGS.batch_size input_size = 32 * 32 * 3 output_size = 10 # load dataset raw_data = cifar10_utils.get_cifar10(DATA_DIR_DEFAULT) train_data = raw_data['train'] validation_data = raw_data["validation"] test_data = raw_data['test'] model = MLP(n_inputs=input_size, n_hidden=dnn_hidden_units, n_classes=output_size, neg_slope=neg_slope) loss_target = CrossEntropyModule() csv_data = [[ 'step', 'train_loss', 'test_loss', 'train_accuracy', 'test_accuracy' ]] for step in range(max_steps): x, y = train_data.next_batch(batch_size) x = x.reshape(batch_size, input_size) # train output = model.forward(x) loss_avg = loss_target.forward(output, y) dout = loss_target.backward(output, y) model.backward(dout) # only need to update weights for linear module for each step for layer in model.layers: if isinstance(layer, LinearModule): layer.params['weight'] -= lr * layer.grads['weight'] layer.params['bias'] -= lr * layer.grads['bias'] train_acc = accuracy(output, y) # with the \r and end = '' trick, we can print on the same line print('\r[{}/{}] train_loss: {} train_accuracy: {}'.format( step + 1, max_steps, round(loss_avg, 3), round(train_acc, 3)), end='') # evaluate if step % eval_freq == 0 or step >= (max_steps - 1): x, y = test_data.next_batch(test_data.num_examples) x = x.reshape(test_data.num_examples, input_size) output = model.forward(x) test_loss = loss_target.forward(output, y) test_acc = accuracy(output, y) csv_data.append([step, loss_avg, test_loss, train_acc, test_acc]) print(' test_loss: {}, test_accuracy: {}'.format( round(test_loss, 3), round(test_acc, 3))) with open('results/train_summary_np_{}.csv'.format(int(time.time())), 'w') as csv_file: writer = csv.writer(csv_file) writer.writerows(csv_data)
def train(): """ Performs training and evaluation of MLP model. """ # Set the random seeds for reproducibility np.random.seed(42) # Prepare all functions # Get number of units in each hidden layer specified in the string such as 100,100 if FLAGS.dnn_hidden_units: dnn_hidden_units = FLAGS.dnn_hidden_units.split(",") dnn_hidden_units = [ int(dnn_hidden_unit_) for dnn_hidden_unit_ in dnn_hidden_units ] else: dnn_hidden_units = [DNN_HIDDEN_UNITS_DEFAULT] data = cifar10_utils.get_cifar10(DATA_DIR_DEFAULT) n_inputs = np.prod(data['train'].images.shape[1:]) n_classes = data['train'].labels.shape[1] n_test = data['test'].images.shape[0] x_test, y_test = data['test'].next_batch(n_test) x_test = x_test.reshape((n_test, n_inputs)) net = MLP(n_inputs, dnn_hidden_units, n_classes) loss_func = CrossEntropyModule() losses = {'train': [], 'test': []} accuracies = {'train': [], 'test': []} eval_steps = [] for s in range(FLAGS.max_steps): x, y = data['train'].next_batch(FLAGS.batch_size) x = x.reshape((FLAGS.batch_size, n_inputs)) # FORWARD out = net.forward(x) # BACKWARD dloss = loss_func.backward(out, y) net.backward(dloss) # UPDATE for grad_layer in filter(lambda l: hasattr(l, 'grads'), net.layers): grad_layer.params[ 'weight'] -= FLAGS.learning_rate * grad_layer.grads['weight'] grad_layer.params[ 'bias'] -= FLAGS.learning_rate * grad_layer.grads['bias'] # Evaluation if s % FLAGS.eval_freq == 0 or s == FLAGS.max_steps - 1: eval_steps.append(s) losses['train'].append(loss_func.forward(out, y)) accuracies['train'].append(accuracy(out, y)) out = net.forward(x_test) losses['test'].append(loss_func.forward(out, y_test)) accuracies['test'].append(accuracy(out, y_test)) print('Iter {:04d}: Test: {:.2f} ({:f}), Train: {:.2f} ({:f})'. format(s, 100 * accuracies['test'][-1], losses['test'][-1], 100 * accuracies['train'][-1], losses['train'][-1])) # Plotting for d, n in [(accuracies, 'Accuracy'), (losses, 'Loss')]: plt.figure() plt.plot(eval_steps, d['train'], label='train') plt.plot(eval_steps, d['test'], label='test') plt.xlabel('Step') plt.ylabel(n) plt.legend() plt.tight_layout() plt.savefig('np_' + n.lower() + '.pdf') print('Best testing loss: {:.2f} accuracy: {:.2f}'.format( np.min(losses['test']), 100 * np.max(accuracies['test'])))
def train(): """ Performs training and evaluation of MLP model. """ ### DO NOT CHANGE SEEDS! # Set the random seeds for reproducibility np.random.seed(42) ## Prepare all functions # Get number of units in each hidden layer specified in the string such as 100,100 if FLAGS.dnn_hidden_units: dnn_hidden_units = FLAGS.dnn_hidden_units.split(",") dnn_hidden_units = [ int(dnn_hidden_unit_) for dnn_hidden_unit_ in dnn_hidden_units ] else: dnn_hidden_units = [] ######################## # PUT YOUR CODE HERE # ####################### batch_size = FLAGS.batch_size lr = FLAGS.learning_rate cifar10 = cifar10_utils.get_cifar10(FLAGS.data_dir) x_test, y_test = cifar10['test'].images, cifar10['test'].labels (test_images, height, width, colors) = x_test.shape # (32, 32, 3) n_inputs = height * width * colors # 1024 # n_classes = 10 (_, n_classes) = y_test.shape x_test = x_test.reshape((test_images, n_inputs)) model = MLP(n_inputs, dnn_hidden_units, n_classes) ce = CrossEntropyModule() cols = ['train_acc', 'test_acc', 'train_loss', 'test_loss', 'secs'] # train results = [] name = f'mlp-numpy' with SummaryWriter(name) as w: for step in tqdm(range(FLAGS.max_steps)): x_train, y_train = cifar10['train'].next_batch(batch_size) x_train = x_train.reshape((batch_size, n_inputs)) # forward predictions = model.forward(x_train) train_loss = ce.forward(predictions, y_train) # backward grad = ce.backward(predictions, y_train) model.backward(grad) train_acc = accuracy(predictions, y_train) # update model for linear in model.linears: linear.params["weight"] -= lr * linear.grads["weight"] linear.params["bias"] -= lr * np.mean( linear.grads["bias"].T, axis=1, keepdims=True) # evaluate if step % FLAGS.eval_freq == 0: time = int(step / FLAGS.eval_freq) start = timer() predictions = model.forward(x_test) end = timer() secs = end - start test_loss = ce.forward(predictions, y_test) test_acc = accuracy(predictions, y_test) vals = [train_acc, test_acc, train_loss, test_loss, secs] stats = dict( zip(cols, [ np.asscalar(i) if isinstance(i, (np.ndarray, np.generic)) else i for i in vals ])) print( yaml.dump({ k: round(i, 3) if isinstance(i, float) else i for k, i in stats.items() })) w.add_scalars('metrics', stats, time) results.append(stats) # stop if loss has converged! check = 10 if len(results) >= 2 * check: threshold = 1e-6 losses = [item['train_loss'] for item in results] current = np.mean(losses[-check:]) prev = np.mean(losses[-2 * check:-check]) if (prev - current) < threshold: break df = pd.DataFrame(results, columns=cols) meta = { 'framework': 'numpy', 'algo': 'mlp', 'optimizer': 'sgd', 'batch_size': FLAGS.batch_size, 'learning_rate': FLAGS.learning_rate, 'dnn_hidden_units': FLAGS.dnn_hidden_units, 'weight_decay': 0, 'max_steps': FLAGS.max_steps, } for k, v in meta.items(): df[k] = v output_file = 'results/results.csv' # f'{name}.csv' if os.path.isfile(output_file): df.to_csv(f'{name}.csv', header=False, mode='a') else: df.to_csv(f'{name}.csv', header=True, mode='w') print('done!') return test_loss
def train(): """ Performs training and evaluation of MLP model. TODONE: Implement training and evaluation of MLP model. Evaluate your model on the whole test set each eval_freq iterations. """ ### DO NOT CHANGE SEEDS! # Set the random seeds for reproducibility np.random.seed(42) ## Prepare all functions # Get number of units in each hidden layer specified in the string such as 100,100 if FLAGS.dnn_hidden_units: dnn_hidden_units = FLAGS.dnn_hidden_units.split(",") dnn_hidden_units = [ int(dnn_hidden_unit_) for dnn_hidden_unit_ in dnn_hidden_units ] else: dnn_hidden_units = [] ######################## # PUT YOUR CODE HERE # ####################### learning_rate = FLAGS.learning_rate max_steps = FLAGS.max_steps batch_size = FLAGS.batch_size eval_freq = FLAGS.eval_freq data_dir = FLAGS.data_dir # load the cifar10 data cifar10 = cifar10_utils.get_cifar10(data_dir) train = cifar10['train'] test = cifar10['test'] test_images, test_labels = test.images, test.labels # obtain the dimensions of the data n_test_images, depth, height, width = test_images.shape n_inputs = height * width * depth n_classes = test_labels.shape[1] # reshape the test images test_images = test_images.reshape((n_test_images, n_inputs)) # initialize the MLP mlp = MLP(n_inputs, dnn_hidden_units, n_classes) # initialize the loss function loss_function = CrossEntropyModule() # initialize empty results list results = [] output_dir = Path.cwd().parent / 'output' # train the MLP for step in range(max_steps): # obtain a new mini-batch and reshape the images train_images, train_labels = train.next_batch(batch_size) train_images = train_images.reshape((batch_size, n_inputs)) # forward pass the mini-batch predictions = mlp.forward(train_images) loss = loss_function.forward(predictions, train_labels) # backwards propogate the loss loss_grad = loss_function.backward(predictions, train_labels) mlp.backward(loss_grad) # update the weights and biases of the linear modules of the MLP for module in mlp.modules: if isinstance(module, LinearModule): # if it is a linear module module.params[ 'weight'] -= learning_rate * module.grads['weight'] module.params['bias'] -= learning_rate * module.grads['bias'] # evaluate the MLP if (step % eval_freq == 0) or (step == max_steps - 1): # compute train data metrics train_acc = accuracy(predictions, train_labels) train_loss = loss # evaluate the MLP on the test data test_predictions = mlp.forward(test_images) # compute the test data metrics test_acc = accuracy(test_predictions, test_labels) test_loss = loss_function.forward(test_predictions, test_labels) # append the results results.append( [step + 1, train_acc, train_loss, test_acc, test_loss]) print(f'Step {step + 1:0{len(str(max_steps))}}/{max_steps}:') print(f' Performance on the training data (mini-batch):') print(f' Accuracy: {train_acc}, Loss: {train_loss}') print(f' Performance on the testing data (mini-batch):') print(f' Accuracy: {test_acc}, Loss: {test_loss}') # break if train loss has converged # threshold = 1e-6 # if len(train_loss) > 20: # previous_losses = train_loss[-20:-10] # current_losses = train_loss[-10:] # if (previous_losses - current_losses) < threshold: # print(f'Loss has converged early in {step + 1} steps') # break # save the relevant metrics to disk print('Saving the results to disk...') output_path = Path.cwd().parent / 'output' / 'mlp_numpy.csv' output_path.parent.mkdir(parents=True, exist_ok=True) column_names = ['step', 'train_acc', 'train_loss', 'test_acc', 'test_loss'] with open(output_path, 'w') as csv_file: csv_file.write(';'.join(column_names) + '\n') for i in range(len(results)): csv_file.write( f'{results[i][0]};{results[i][1]};{results[i][2]};{results[i][3]};{results[i][4]}' + '\n')
def train(): """ Performs training and evaluation of MLP model. TODO: Implement training and evaluation of MLP model. Evaluate your model on the whole test set each eval_freq iterations. """ ### DO NOT CHANGE SEEDS! # Set the random seeds for reproducibility np.random.seed(42) ## Prepare all functions # Get number of units in each hidden layer specified in the string such as 100,100 if FLAGS.dnn_hidden_units: dnn_hidden_units = FLAGS.dnn_hidden_units.split(",") dnn_hidden_units = [ int(dnn_hidden_unit_) for dnn_hidden_unit_ in dnn_hidden_units ] else: dnn_hidden_units = [] #dnn_hidden_units # [100] #FLAGS.learning_rate #0.002 #FLAGS.max_steps #1500 #FLAGS.batch_size #FLAGS.eval_freq #100 #FLAGS.data_dir # ./cifar10/cifar-10-batches-py ######################## # PUT YOUR CODE HERE # ####################### ce_loss = CrossEntropyModule() n_inputs = 3 * 32 * 32 n_classes = 10 mlp = MLP(n_inputs, dnn_hidden_units, n_classes) c10 = cifar10_utils.get_cifar10(FLAGS.data_dir) test_data = c10['test'].images test_data = test_data.reshape(test_data.shape[0], -1) acc_values = [] loss_values = [] for i in range(FLAGS.max_steps): x, y = c10['train'].next_batch(FLAGS.batch_size) x = x.reshape(FLAGS.batch_size, -1) # forward and backward pass out = mlp.forward(x.T) loss = ce_loss.forward(out, y.T) dloss = ce_loss.backward(out, y.T) mlp.backward(dloss) loss_values.append(loss) # update for m in mlp.modules: if type(m).__name__ == 'LinearModule': m.params['weight'] = m.params[ 'weight'] - FLAGS.learning_rate * m.grads['weight'] m.params['bias'] = m.params[ 'bias'] - FLAGS.learning_rate * m.grads['bias'] # evaluate if i % FLAGS.eval_freq == 0: predictions = mlp.forward(test_data.T).T targets = c10['test'].labels acc = accuracy(predictions, targets) print('acc', acc, 'loss', loss) acc_values.append(acc) # save loss and accuracy to file with open('accuracy.txt', 'a') as f_acc: print(acc_values, file=f_acc) with open('loss.txt', 'a') as f_loss: print(loss_values, file=f_loss)
def train(): """ Performs training and evaluation of MLP model. TODO: Implement training and evaluation of MLP model. Evaluate your model on the whole test set each eval_freq iterations. """ ### DO NOT CHANGE SEEDS! # Set the random seeds for reproducibility np.random.seed(42) ## Prepare all functions # Get number of units in each hidden layer specified in the string such as 100,100 if FLAGS.dnn_hidden_units: dnn_hidden_units = FLAGS.dnn_hidden_units.split(",") dnn_hidden_units = [ int(dnn_hidden_unit_) for dnn_hidden_unit_ in dnn_hidden_units ] else: dnn_hidden_units = [] # Get negative slope parameter for LeakyReLU neg_slope = FLAGS.neg_slope ######################## # PUT YOUR CODE HERE # ####################### cifar10 = cifar10_utils.get_cifar10(FLAGS.data_dir) n_classes = 10 n_inputs = 32 * 32 * 3 n_batch = FLAGS.batch_size mlp = MLP(n_inputs, dnn_hidden_units, n_classes, neg_slope) loss = CrossEntropyModule() nr_iterations = FLAGS.max_steps + 1 accuracies_test = [] accuracies_train = [] losses = [] for i in range(nr_iterations): x, y = cifar10['train'].next_batch(n_batch) x = x.reshape(-1, n_inputs) prediction = mlp.forward(x) cross_entropy_loss = loss.forward(prediction, y) losses.append(cross_entropy_loss) cross_entropy_grad = loss.backward(prediction, y) mlp.backward(cross_entropy_grad) for layer in mlp.layers: if hasattr(layer, 'params'): layer.params[ 'weight'] -= FLAGS.learning_rate * layer.grads['weight'] layer.params[ 'bias'] -= FLAGS.learning_rate * layer.grads['bias'] if i % FLAGS.eval_freq == 0: x_test, y_test = cifar10['test'].images, cifar10['test'].labels x_test = x_test.reshape(x_test.shape[0], -1) pred_test = mlp.forward(x_test) acc_test = accuracy(pred_test, y_test) acc_train = accuracy(prediction, y) print('accuracy test set at step', i, ': ', acc_test) print('accuracy train set at step', i, ': ', acc_train) accuracies_test.append(acc_test * 100) accuracies_train.append((acc_train * 100))
def train(): """ Performs training and evaluation of MLP model. TODO: Implement training and evaluation of MLP model. Evaluate your model on the whole test set each eval_freq iterations. """ ### DO NOT CHANGE SEEDS! # Set the random seeds for reproducibility np.random.seed(42) ## Prepare all functions # Get number of units in each hidden layer specified in the string such as 100,100 if FLAGS.dnn_hidden_units: dnn_hidden_units = FLAGS.dnn_hidden_units.split(",") dnn_hidden_units = [ int(dnn_hidden_unit_) for dnn_hidden_unit_ in dnn_hidden_units ] else: dnn_hidden_units = [] FLAGS.eval_freq = max(FLAGS.eval_freq, 1) # Make sure eval frequency is at least 1 print("START TRAINING") cifar10 = cifar10_utils.get_cifar10(data_dir=DATA_DIR_DEFAULT, one_hot=True, validation_size=0) train = cifar10["train"] valid = cifar10["validation"] test = cifar10["test"] losses = [] train_accuracies = [] test_accuracies = [] model = MLP(3 * 32 * 32, dnn_hidden_units, 10) loss_module = CrossEntropyModule() step = 0 while step < FLAGS.max_steps: if step % FLAGS.eval_freq == 0: # Evaluate the model on the test dataset test_accuracy = eval(model, test, FLAGS.batch_size) test_accuracies.append(test_accuracy) print(f"STEP {step} - {test_accuracy}") images, labels = train.next_batch(FLAGS.batch_size) images = images.reshape(images.shape[0], -1) preds = model.forward(images) train_accuracies.append(accuracy(preds, labels)) loss = loss_module.forward(preds, labels) dloss = loss_module.backward(preds, labels) model.backward(dloss) # Update parameters for i, layer in enumerate(model.layers): if i % 2 == 0: layer.params[ "weight"] -= FLAGS.learning_rate * layer.grads["weight"] layer.params[ "bias"] -= FLAGS.learning_rate * layer.grads["bias"] step += 1 losses.append(loss) print("END TRAINING") def moving_average(a, n=3): # Taken from https://stackoverflow.com/questions/14313510/how-to-calculate-moving-average-using-numpy ret = np.cumsum(a, dtype=float) ret[n:] = ret[n:] - ret[:-n] return ret[n - 1:] / n train_accuracies = moving_average(train_accuracies, n=100) fig, ax1 = plt.subplots() ax1.set_xlabel('Training iteration') ax1.set_ylabel('Loss') l1 = ax1.plot(range(len(losses)), losses, label="training loss", color="b", alpha=0.5, linewidth=1) ax2 = ax1.twinx() ax2.set_ylabel('Accuracy') l2 = ax2.plot(np.linspace(0, len(losses), len(test_accuracies)), test_accuracies, label="test accuracy", color="r") l3 = ax2.plot(np.linspace(0, len(losses), len(train_accuracies)), train_accuracies, label="train accuracy", color="b") plots = l1 + l2 + l3 labels = [plot.get_label() for plot in plots] ax2.legend(plots, labels) plt.savefig(os.path.join("images", "numpy_loss_accuracy.png")) plt.show()
def train(): """ Performs training and evaluation of MLP model. TODO: Implement training and evaluation of MLP model. Evaluate your model on the whole test set each eval_freq iterations. """ ### DO NOT CHANGE SEEDS! # Set the random seeds for reproducibility np.random.seed(42) ## Prepare all functions # Get number of units in each hidden layer specified in the string such as 100,100 if FLAGS.dnn_hidden_units: dnn_hidden_units = FLAGS.dnn_hidden_units.split(",") dnn_hidden_units = [int(dnn_hidden_unit_) for dnn_hidden_unit_ in dnn_hidden_units] else: dnn_hidden_units = [] ######################## # PUT YOUR CODE HERE # ####################### learning_rate = FLAGS.learning_rate max_steps = FLAGS.max_steps batch_size = FLAGS.batch_size eval_freq = FLAGS.eval_freq cifar10 = cifar10_utils.get_cifar10('cifar10/cifar-10-batches-py') mlp = MLP(32*32*3, dnn_hidden_units, 10) # soft = SoftMaxModule() cross = CrossEntropyModule() train_losses = [] accuracies = [] steps = [] # for step in range(max_steps): for step in range(max_steps): total_loss = 0 x, y = cifar10['train'].next_batch(batch_size) x = np.reshape(x, [batch_size, 32 * 32 * 3]) out = mlp.forward(x) loss = cross.forward(out, y) dout = cross.backward(out, y) mlp.backward(dout) for layer in mlp.hidden: layer.params['weight'] = layer.params['weight'] - learning_rate * layer.grads['weight'] layer.params['bias'] = layer.params['bias'] - learning_rate * layer.grads['bias'] mlp.out.params['weight'] = mlp.out.params['weight'] - learning_rate * mlp.out.grads['weight'] mlp.out.params['bias'] = mlp.out.params['bias'] - learning_rate * mlp.out.grads['bias'] total_loss += loss train_losses.append(total_loss) print('Step: {} Loss: {:.4f}'.format(step + 1, total_loss)) if (step + 1) % eval_freq == 0: test_x = cifar10['test'].images test_y = cifar10['test'].labels test_x = np.reshape(test_x, [test_x.shape[0], 32 * 32 * 3]) test_out = mlp.forward(test_x) test_accuracy = accuracy(test_out, test_y) accuracies.append(test_accuracy) steps.append(step + 1) print('Step: {} Accuracy {:.2f}'.format(step + 1, test_accuracy)) plt.plot(range(max_steps), train_losses) plt.xlabel("Step") plt.ylabel("Training loss") plt.show() plt.plot(steps, accuracies) plt.xlabel("Step") plt.ylabel("Test Accuracy") plt.show()
def train(): """ Performs training and evaluation of MLP model. TODO: Implement training and evaluation of MLP model. Evaluate your model on the whole test set each eval_freq iterations. """ ### DO NOT CHANGE SEEDS! # Set the random seeds for reproducibility np.random.seed(42) ## Prepare all functions # Get number of units in each hidden layer specified in the string such as 100,100 if FLAGS.dnn_hidden_units: dnn_hidden_units = FLAGS.dnn_hidden_units.split(",") dnn_hidden_units = [ int(dnn_hidden_unit_) for dnn_hidden_unit_ in dnn_hidden_units ] else: dnn_hidden_units = [] # set training parameters lr = FLAGS.learning_rate max_steps = FLAGS.max_steps batch_size = FLAGS.batch_size eval_freq = FLAGS.eval_freq #Frequency of evaluation on the test set data_dir = FLAGS.data_dir plot_results = FLAGS.plot train_treshold = 1e-6 #if train loss below that threshold, training stops # evaluation metrics acc_train = [] acc_test = [] loss_train = [] loss_test = [] #load input data cifar10 = cifar10_utils.get_cifar10(data_dir, one_hot=True) #get test data x_test = cifar10["test"].images y_test = cifar10["test"].labels #determine dimension of data x_dim = x_test.shape n_test_samples = x_dim[0] #number of test samples #images of size 32 x 32 x 3 n_inputs = x_dim[1] * x_dim[2] * x_dim[3] #channels * height * width #reshape test images to fit MLP input x_test = x_test.reshape((n_test_samples, n_inputs)) n_classes = y_test.shape[1] #initialize MLP my_MLP = MLP(n_inputs=n_inputs, n_hidden=dnn_hidden_units, n_classes=n_classes) #define loss function loss_fn = CrossEntropyModule() print("Start training") for step in range(max_steps): #get new minibatch x_train, y_train = cifar10["train"].next_batch(batch_size) # reshape x that each sample is represented by a vector x_train = x_train.reshape((batch_size, n_inputs)) #forward pass of minibatch out = my_MLP.forward(x_train) #here output are softmax probabilities #compute loss loss_mb = loss_fn.forward(out, y_train) #compute gradient of loss grad_loss = loss_fn.backward(out, y_train) #perform backpropagation my_MLP.backward( grad_loss ) #backprop loss from layer to layer where gradients are save in each module #update parameters with SGD for l in my_MLP.lin_layer: l.params["weight"] = l.params["weight"] - lr * l.grads["weight"] l.params["bias"] = l.params["bias"] - lr * l.grads["bias"] #perhaps modify learning rate? if (step % eval_freq == 0) or (step == max_steps - 1): print(f"Step: {step}") #compute and store training metrics loss_train.append(loss_mb) acc_train.append(accuracy(out, y_train)) print("TRAIN acc: {0:.4f} & loss: {1:.4f}".format( acc_train[-1], loss_train[-1])) #compute and store test metrics out_test = my_MLP.forward(x_test) loss_test.append(loss_fn.forward(out_test, y_test)) acc_test.append(accuracy(out_test, y_test)) print("TEST acc: {0:.4f} & loss: {1:.4f}".format( acc_test[-1], loss_test[-1])) #Early stop when training loss below threshold? if len(loss_train) > 20: prev_losses = loss_test[-2] cur_losses = loss_test[-1] if (prev_losses - cur_losses) < train_treshold: print("Training stopped early at step {0}".format(step + 1)) break print("Finished training") res_path = Path.cwd().parent / 'mlp_np_results' if not res_path.exists(): res_path.mkdir(parents=True) print("Saving results to {0}".format(res_path)) #Save an array to a binary file in NumPy .npy format. #np.save(res_path / 'loss_train', loss_train) #np.save(res_path / 'acc_train', acc_train) #np.save(res_path / 'loss_test', loss_test) #np.save(res_path / 'acc_test', acc_test) #Save array to csv file np.savetxt(res_path / 'loss_train.csv', loss_train, delimiter=',') np.savetxt(res_path / 'acc_train.csv', acc_train, delimiter=',') np.savetxt(res_path / 'loss_test.csv', loss_test, delimiter=',') np.savetxt(res_path / 'acc_test.csv', acc_test, delimiter=',') if plot_results: #plot results x = range(0, max_steps + eval_freq, eval_freq) plt.plot(x, loss_train, label="Training") plt.plot(x, loss_test, label="Test") plt.xlabel('Time steps') plt.ylabel('Loss') plt.title("Loss over time steps") plt.legend() plt.show() plt.plot(x, acc_train, label="Training") plt.plot(x, acc_test, label="Test") plt.xlabel('Time steps') plt.ylabel('Accuracy') plt.title("Accuracy over time steps") plt.legend() plt.show() print(f"Last train loss: {np.round(loss_train[-1], 4)}") print(f"Best train loss: {np.round(np.min(loss_train), 4)}") print(f"Last train acc: {np.round(acc_train[-1], 4)}") print(f"Best train acc: {np.round(np.max(acc_train), 4)}") print() print(f"Last test loss: {np.round(loss_test[-1], 4)}") print(f"Best test loss: {np.round(np.min(loss_test), 4)}") print(f"Last test acc: {np.round(acc_test[-1], 4)}") print(f"Best test acc: {np.round(np.max(acc_test), 4)}")
def train(): """ Performs training and evaluation of MLP model. TODO: Implement training and evaluation of MLP model. Evaluate your model on the whole test set each eval_freq iterations. """ ### DO NOT CHANGE SEEDS! # Set the random seeds for reproducibility np.random.seed(42) ## Prepare all functions # Get number of units in each hidden layer specified in the string such as 100,100 if FLAGS.dnn_hidden_units: dnn_hidden_units = FLAGS.dnn_hidden_units.split(",") dnn_hidden_units = [int(dnn_hidden_unit_) for dnn_hidden_unit_ in dnn_hidden_units] else: dnn_hidden_units = [] output_dir = FLAGS.output_dir if not os.path.isdir(output_dir): os.makedirs(output_dir) learning_rate = FLAGS.learning_rate max_steps = FLAGS.max_steps batch_size = FLAGS.batch_size eval_freq = FLAGS.eval_freq data_dir = FLAGS.data_dir # Obtain dataset dataset = cifar10_utils.get_cifar10(data_dir) n_inputs = dataset['train'].images[0].reshape(-1).shape[0] n_classes = dataset['train'].labels[0].shape[0] n_test = dataset['test'].images.shape[0] # Initialise MLP net = MLP(n_inputs, dnn_hidden_units, n_classes) loss_module = CrossEntropyModule() print("Network architecture:\n\t{}\nLoss module:\n\t{}".format(str(net), str(loss_module))) # Evaluation vars train_loss = [] gradient_norms = [] train_acc = [] test_acc = [] iteration = 0 # Training while iteration < max_steps: iteration += 1 # Sample a mini-batch x, y = dataset['train'].next_batch(batch_size) x = x.reshape((batch_size, -1)) # Forward propagation prediction = net.forward(x) loss = loss_module.forward(prediction, y) acc = accuracy(prediction, y) train_acc.append( (iteration, acc) ) train_loss.append( (iteration, loss) ) # Backprop dpred = loss_module.backward(prediction, y) net.backward(dpred) # Weight update in linear modules norm = 0 for layer in net.linearModules: layer.params['weight'] -= learning_rate * layer.grads['weight'] layer.params['bias'] -= learning_rate * layer.grads['bias'] norm += np.einsum('ij,ij',layer.grads['weight'],layer.grads['weight']) + \ layer.grads['bias'].T @ layer.grads['bias'] gradient_norms.append( (iteration, norm.reshape(-1)) ) # Evaluation if iteration % eval_freq == 0: x = dataset['test'].images.reshape((n_test,-1)) y = dataset['test'].labels prediction = net.forward(x) acc = accuracy(prediction, y) test_acc.append( (iteration, acc) ) print("Iteration: {}\t\tTest accuracy: {}".format(iteration, acc)) # Save raw output now = datetime.datetime.now() time_stamp = "{}{}{}{}{}".format(now.year, now.month, now.day, now.hour, now.minute) net_name = "mlpNumpy" out_dir = os.path.join(output_dir, net_name, time_stamp) if not os.path.isdir(out_dir): os.makedirs(os.path.join(output_dir, net_name, time_stamp)) metrics = {"train_loss": train_loss, "gradient_norms": gradient_norms, "train_acc": train_acc, "test_acc": test_acc} raw_data = {"net": net, "metrics": metrics} pickle.dump(raw_data, open(os.path.join(out_dir, "raw_data.p"), "wb")) # Save plots # Loss fig, ax = plt.subplots() iter = [i for (i,q) in train_loss] loss = [q for (i,q) in train_loss] ax.plot(iter, loss) ax.set(xlabel='Iteration', ylabel='Loss (log)', title='Batch training loss') ax.set_yscale('log') ax.grid() fig.savefig(os.path.join(out_dir, "loss.png")) # gradient norm fig, ax = plt.subplots() iter = [i for (i,q) in gradient_norms] norm = [q for (i,q) in gradient_norms] ax.plot(iter, norm) ax.set(xlabel='Iteration', ylabel='Norm', title='Gradient norm') ax.grid() fig.savefig(os.path.join(out_dir, "gradient_norm.png")) # accuracies fig, ax = plt.subplots() iter = [i for (i,q) in train_acc] accu = [q for (i,q) in train_acc] ax.plot(iter, accu, label='Train') iter = [i for (i,q) in test_acc] accu = [q for (i,q) in test_acc] ax.plot(iter, accu, label='Test') ax.set(xlabel='Iteration', ylabel='Accuracy', title='Train and test accuracy') ax.legend() ax.grid() fig.savefig(os.path.join(out_dir, "accuracy.png"))
def train(): """ Performs training and evaluation of MLP model. TODO: Implement training and evaluation of MLP model. Evaluate your model on the whole test set each eval_freq iterations. """ ### DO NOT CHANGE SEEDS! # Set the random seeds for reproducibility np.random.seed(42) ## Prepare all functions # Get number of units in each hidden layer specified in the string such as 100,100 if FLAGS.dnn_hidden_units: dnn_hidden_units = FLAGS.dnn_hidden_units.split(",") dnn_hidden_units = [int(dnn_hidden_unit_) for dnn_hidden_unit_ in dnn_hidden_units] else: dnn_hidden_units = [] # Get negative slope parameter for LeakyReLU neg_slope = FLAGS.neg_slope ######################## # PUT YOUR CODE HERE # ####################### # initialize required arrays for saving the results train_accuracies = [] train_losses = [] test_accuracies = [] test_losses = [] steps = [] # load data from directory specified in the input cifar10 = cifar10_utils.get_cifar10(FLAGS.data_dir) # load test images test_images = cifar10['test'].images test_targets = cifar10['test'].labels # data dimensions # test_images.shape -> (10000, 3, 32, 32): n_images, channels, height, width # test_targets.shape <- (10000, 10): n_images, n_classes n_test = test_images.shape[0] # n_inputs is one vector for all channels of width and height # n_input = n_channel * width * height n_inputs = test_images.shape[1] * test_images.shape[2] * test_images.shape[3] # reshape to (n_samples, n_inputs) test_images = test_images.reshape((n_test, n_inputs)) # initialize MLP model MLP_model = MLP(n_inputs=n_inputs, n_hidden=dnn_hidden_units, n_classes=test_targets.shape[1], neg_slope=neg_slope) # loss function os loaded loss_module = CrossEntropyModule() for iteration in range(FLAGS.max_steps + 1): train_images, train_targets = cifar10['train'].next_batch(FLAGS.batch_size) # input to MLP.forward is (batch_size, n_inputs) train_input = train_images.reshape((FLAGS.batch_size, n_inputs)) # predictions by forward pass train_predictions = MLP_model.forward(train_input) # loss acc to loss module, predictions and targets loss = loss_module.forward(train_predictions, train_targets) # Apply backward pass: MLP backward takes gradients of losses = dout # dout = backward of loss module dout = loss_module.backward(train_predictions, train_targets) # backward pass from loss (dout) MLP_model.backward(dout) ## Save training statistics # save loss, acc, iteration for train evaluation afterwards train_accuracies.append(accuracy(train_predictions, train_targets)) train_losses.append(loss) steps.append(iteration) # Parameter updates with Stochastic Gradient Descent for layer in MLP_model.layers: # last module does not have params -> no gradient descent possible if hasattr(layer, 'params'): layer.params['weight'] -= FLAGS.learning_rate * layer.grads['weight'] layer.params['bias'] -= FLAGS.learning_rate * layer.grads['bias'] # Consider FLAGS.EVAL_FREQ_DEFAULT for the evaluation of the current MLP # on the test data and training data if iteration % FLAGS.eval_freq == 0: ## Test Statistics test_predictions = MLP_model.forward(test_images) test_loss = loss_module.forward(test_predictions, test_targets) print("iteration:" + str(iteration) + "train_acc:" + str(np.mean(train_accuracies))) print("iteration:" + str(iteration) + "test_acc:" + str(test_accuracies)) test_accuracies.append(accuracy(test_predictions, test_targets)) test_losses.append(test_loss) print('Training is done') print('Plot Results') plt.subplot(2, 1, 1) plt.title("Results") plt.plot(np.arange(len(train_accuracies)), train_accuracies, label="train acc") plt.plot(np.arange(len(test_accuracies) * FLAGS.eval_freq, step=FLAGS.eval_freq), test_accuracies, label="test acc") plt.ylabel('Accuracy (%)') plt.legend() # loss plt.subplot(2, 1, 2) plt.plot(np.arange(len(train_losses)), train_losses, label=" train loss") plt.plot(np.arange(len(test_losses) * FLAGS.eval_freq, step=FLAGS.eval_freq), test_losses, label=" test loss") plt.xlabel('Epoch') plt.ylabel('Loss') plt.legend() plt.savefig('numpy_results.png') plt.show()
def train(): """ Performs training and evaluation of MLP model. TODO: Implement training and evaluation of MLP model. Evaluate your model on the whole test set each eval_freq iterations. """ ### DO NOT CHANGE SEEDS! # Set the random seeds for reproducibility np.random.seed(42) ## Prepare all functions # Get number of units in each hidden layer specified in the string such as 100,100 if FLAGS.dnn_hidden_units: dnn_hidden_units = FLAGS.dnn_hidden_units.split(",") dnn_hidden_units = [ int(dnn_hidden_unit_) for dnn_hidden_unit_ in dnn_hidden_units ] else: dnn_hidden_units = [] ######################## # PUT YOUR CODE HERE # ####################### #raise NotImplementedError #Load in data set cifar10_set = cifar10_utils.get_cifar10(FLAGS.data_dir) #Init tracking arrays test_acc = [] train_loss = [] train_acc = [] # pull in the first set and get the shapes x, t = cifar10_set['train'].next_batch(FLAGS.batch_size) x = x.reshape(FLAGS.batch_size, -1) out_dim = t.shape[1] in_dim = x.shape[1] mlp = MLP(in_dim, dnn_hidden_units, out_dim, FLAGS.neg_slope) loss_funct = CrossEntropyModule() # First pass of the data y = mlp.forward(x) loss = loss_funct.forward(y, t) mlp.backward(loss_funct.backward(y, t)) for mod in mlp.modules: if type(mod) == LinearModule: mod.params['weight'] -= FLAGS.learning_rate * mod.grads['weight'] mod.params['bias'] -= FLAGS.learning_rate * mod.grads['bias'] train_loss.append(loss) train_acc.append(accuracy(y, t)) x, t = cifar10_set['test'].images, cifar10_set['test'].labels x = x.reshape(x.shape[0], -1) y = mlp.forward(x) test_acc.append(accuracy(y, t)) print("The accuracy at step, " + str(0) + " is : " + str(test_acc[-1])) # loop through till steps are all done for i in range(1, FLAGS.max_steps + 1): x, t = cifar10_set['train'].next_batch(FLAGS.batch_size) x = x.reshape(FLAGS.batch_size, -1) y = mlp.forward(x) loss = loss_funct.forward(y, t) mlp.backward(loss_funct.backward(y, t)) #Update the weights for mod in mlp.modules: if type(mod) == LinearModule: mod.params[ 'weight'] -= FLAGS.learning_rate * mod.grads['weight'] mod.params['bias'] -= FLAGS.learning_rate * mod.grads['bias'] # Evaluation and acc/loss saving if i % FLAGS.eval_freq == 0: train_loss.append(loss) train_acc.append(accuracy(y, t)) x, t = cifar10_set['test'].images, cifar10_set['test'].labels x = x.reshape(x.shape[0], -1) y = mlp.forward(x) test_acc.append(accuracy(y, t)) print("The accuracy at step, " + str(i) + " is : " + str(test_acc[-1])) #Plotting the accuracy of test and train: plt.figure(0) plt.plot(np.arange(0, len(train_acc) * FLAGS.eval_freq * FLAGS.batch_size, FLAGS.eval_freq * FLAGS.batch_size) / cifar10_set['train'].num_examples, train_acc, label='Train') plt.plot(np.arange(0, len(train_acc) * FLAGS.eval_freq * FLAGS.batch_size, FLAGS.eval_freq * FLAGS.batch_size) / cifar10_set['train'].num_examples, test_acc, label='Test') plt.xlabel('Epoch') plt.ylabel('Accuracy') plt.title('Accuracy of Train and Test Set Through Training') plt.legend() plt.savefig('Accuracy_basic1.png') # plt.show() plt.figure(1) plt.plot(np.arange(0, len(train_loss) * FLAGS.eval_freq * FLAGS.batch_size, FLAGS.eval_freq * FLAGS.batch_size) / cifar10_set['train'].num_examples, train_loss, label='Train') plt.xlabel('Epoch') plt.ylabel('Loss') plt.title('Loss Through Training') plt.savefig('Loss_basic1.png')
def train(): """ Performs training and evaluation of MLP model. TODO: Implement training and evaluation of MLP model. Evaluate your model on the whole test set each eval_freq iterations. """ ### DO NOT CHANGE SEEDS! # Set the random seeds for reproducibility np.random.seed(42) ## Prepare all functions # Get number of units in each hidden layer specified in the string such as 100,100 if FLAGS.dnn_hidden_units: dnn_hidden_units = FLAGS.dnn_hidden_units.split(",") dnn_hidden_units = [ int(dnn_hidden_unit_) for dnn_hidden_unit_ in dnn_hidden_units ] else: dnn_hidden_units = [] # Get negative slope parameter for LeakyReLU neg_slope = FLAGS.neg_slope ######################## # PUT YOUR CODE HERE # ####################### """ Initialize data module """ cifar10 = cifar10_utils.get_cifar10(DATA_DIR_DEFAULT) x, y = cifar10['train'].next_batch(1) x_test, y_test = cifar10['test'].next_batch(10000) x = x.reshape(x.shape[0], -1) x_test = x_test.reshape(x_test.shape[0], -1) """ initialize the network """ network = MLP(x.shape[1], dnn_hidden_units, y.shape[1], neg_slope) """ compute forward """ crossEntropy = CrossEntropyModule() """ batch gradient descent """ for i in range(FLAGS.max_steps): x, y = cifar10['train'].next_batch(FLAGS.batch_size) x = x.reshape(x.shape[0], -1) prediction = network.forward(x) loss = crossEntropy.forward(prediction, y) gloss = crossEntropy.backward(prediction, y) network.backward(gloss) for linearModule in network.linearModules: linearModule.params['weight'] = linearModule.params[ 'weight'] - FLAGS.learning_rate * 1 / FLAGS.batch_size * linearModule.grads[ 'weight'] linearModule.params['bias'] = linearModule.params[ 'bias'] - FLAGS.learning_rate * 1 / FLAGS.batch_size * linearModule.grads[ 'bias'] if i % FLAGS.eval_freq == 0: prediction = network.forward(x_test) print('Accuracy after ' + str(i) + ' steps ' + str(accuracy(prediction, y_test))) prediction = network.forward(x_test) print('Final accuracy') print(accuracy(prediction, y_test))
def train(_run): """ Performs training and evaluation of MLP model. Implement training and evaluation of MLP model. Evaluate your model on the whole test set each eval_freq iterations. """ ### DO NOT CHANGE SEEDS! # Set the random seeds for reproducibility np.random.seed(42) ## Prepare all functions # Get number of units in each hidden layer specified in the string such as 100,100 if FLAGS.dnn_hidden_units: dnn_hidden_units = FLAGS.dnn_hidden_units.split(",") dnn_hidden_units = [int(dnn_hidden_unit_) for dnn_hidden_unit_ in dnn_hidden_units] else: dnn_hidden_units = [] ######################## # PUT YOUR CODE HERE # ####################### datasets = cifar10_utils.read_data_sets(DATA_DIR_DEFAULT) train_data = datasets['train'] test_data = datasets['test'] model = MLP(n_inputs=3072, n_hidden=dnn_hidden_units, n_classes=10) loss_fn = CrossEntropyModule() log_every = 10 avg_loss = 0 avg_acc = 0 for step in range(FLAGS.max_steps): x, y = train_data.next_batch(FLAGS.batch_size) x = x.reshape(FLAGS.batch_size, -1) out = model.forward(x) # Forward and backward passes loss = loss_fn.forward(out, y) dout = loss_fn.backward(out, y) model.backward(dout) # Parameter updates for layer in model.layers: params = getattr(layer, 'params', None) if params is not None: grads = layer.grads layer.params = {name: params[name] - FLAGS.learning_rate * grads[name] for name in params} avg_loss += loss/log_every avg_acc += accuracy(out, y)/log_every if step % log_every == 0: print('\r[{}/{}] train loss: {:.6f} train acc: {:.6f}'.format(step + 1, FLAGS.max_steps, avg_loss, avg_acc), end='') _run.log_scalar('train-loss', avg_loss, step) _run.log_scalar('train-acc', avg_acc, step) avg_loss = 0 avg_acc = 0 # Evaluate if step % FLAGS.eval_freq == 0 or step == (FLAGS.max_steps - 1): x, y = test_data.next_batch(test_data.num_examples) x = x.reshape(test_data.num_examples, -1) out = model.forward(x) test_loss = loss_fn.forward(out, y) test_acc = accuracy(out, y) print(' test accuracy: {:6f}'.format(test_acc)) _run.log_scalar('test-loss', test_loss, step) _run.log_scalar('test-acc', test_acc, step)
def train(): """ Performs training and evaluation of MLP model. TODO: Implement training and evaluation of MLP model. Evaluate your model on the whole test set each eval_freq iterations. """ ### DO NOT CHANGE SEEDS! # Set the random seeds for reproducibility np.random.seed(42) ## Prepare all functions # Get number of units in each hidden layer specified in the string such as 100,100 if FLAGS.dnn_hidden_units: dnn_hidden_units = FLAGS.dnn_hidden_units.split(",") dnn_hidden_units = [ int(dnn_hidden_unit_) for dnn_hidden_unit_ in dnn_hidden_units ] else: dnn_hidden_units = [] # Preparation for training print('- Init parameters') data = cifar10_utils.get_cifar10(FLAGS.data_dir) train_data = data['train'] test_data = data['test'] w, h, d = train_data.images[0].shape n_classes = train_data.labels[0].shape[0] criterion = CrossEntropyModule() model = MLP(w * h * d, dnn_hidden_units, n_classes) train_losses = [] test_losses = [] accuracies = [] print('- Start training') for step in range(FLAGS.max_steps): x_batch, x_labels = train_data.next_batch(FLAGS.batch_size) x = x_batch.reshape((FLAGS.batch_size, -1)) predictions = model.forward(x) gradient = criterion.backward(predictions, x_labels) model.backward(gradient) model.step(FLAGS.learning_rate) if step % FLAGS.eval_freq == 0 or step == FLAGS.max_steps - 1: print(' - Step: {}'.format(step)) loss = criterion.forward(predictions, x_labels) out_test = model.forward( test_data.images.reshape(test_data.num_examples, -1)) test_loss = criterion.forward(out_test, test_data.labels) acc = accuracy(out_test, test_data.labels) train_losses.append(loss) test_losses.append(test_loss) accuracies.append(acc) # Save stuff print(accuracies[-1])
def train(): """ Performs training and evaluation of MLP model. TODO: Implement training and evaluation of MLP model. Evaluate your model on the whole test set each eval_freq iterations. """ ### DO NOT CHANGE SEEDS! # Set the random seeds for reproducibility np.random.seed(42) ## Prepare all functions # Get number of units in each hidden layer specified in the string such as 100,100 if FLAGS.dnn_hidden_units: dnn_hidden_units = FLAGS.dnn_hidden_units.split(",") dnn_hidden_units = [ int(dnn_hidden_unit_) for dnn_hidden_unit_ in dnn_hidden_units ] else: dnn_hidden_units = [] # Get negative slope parameter for LeakyReLU neg_slope = FLAGS.neg_slope ######################## # PUT YOUR CODE HERE # ####################### cifar10 = cifar10_utils.get_cifar10(FLAGS.data_dir) train_data = cifar10['train'] # 60000 x 3 x 32 x32 -> 60000 x 3072, input vector 3072 n_inputs = train_data.images.reshape(train_data.images.shape[0], -1).shape[1] n_hidden = dnn_hidden_units n_classes = train_data.labels.shape[1] print(f"n_inputs {n_inputs}, n_classes {n_classes}") net = MLP(n_inputs, n_hidden, n_classes, neg_slope=neg_slope) loss = CrossEntropyModule() train_acc_plot = [] test_acc_plot = [] loss_train = [] loss_test = [] rloss = 0 print('[DEBUG] start training') for i in range(0, FLAGS.max_steps): x, y = cifar10['train'].next_batch(FLAGS.batch_size) x = x.reshape(x.shape[0], -1) out = net.forward(x) loss_forward = loss.forward(out, y) loss_grad = loss.backward(out, y) net.backward(loss_grad) for n in net.net: if hasattr(n, 'params'): n.params['weight'] = n.params[ 'weight'] - FLAGS.learning_rate * n.grads['weight'] n.params['bias'] = n.params[ 'bias'] - FLAGS.learning_rate * n.grads['bias'] rloss += loss_forward if i % FLAGS.eval_freq == 0: train_accuracy = accuracy(out, y) testX, testY = cifar10['test'].images, cifar10['test'].labels testX = testX.reshape(testX.shape[0], -1) testOut = net.forward(testX) testLoss = loss.forward(testOut, testY) test_accuracy = accuracy(testOut, testY) train_acc_plot.append(train_accuracy) test_acc_plot.append(test_accuracy) loss_train.append(rloss / (i + 1)) loss_test.append(testLoss) print( f'iter {i}, avg loss train {rloss/(i + 1)}, test loss {testLoss}, train acc {train_accuracy}, test acc {test_accuracy}' ) if FLAGS.plot: print('Start plotting...') fig, (ax1, ax2) = plt.subplots(2, 1, sharex=True) ax1.plot(np.arange(len(train_acc_plot)), train_acc_plot, label='training') ax1.plot(np.arange(len(test_acc_plot)), test_acc_plot, label='testing') ax1.set_title('Training evaluation with batch size ' + str(FLAGS.batch_size) + '\n learning rate ' + str(FLAGS.learning_rate)) ax1.set_ylabel('Accuracy') ax1.legend() ax2.plot(np.arange(len(loss_train)), loss_train, label='Train Loss') ax2.plot(np.arange(len(loss_test)), loss_test, label='Test Loss') ax2.set_title('Loss evaluation') ax2.set_ylabel('Loss') ax2.legend() plt.xlabel('Iteration') plt.savefig('numpy.png')
def train(): """ Performs training and evaluation of MLP model. TODO: Implement training and evaluation of MLP model. Evaluate your model on the whole test set each eval_freq iterations. """ ### DO NOT CHANGE SEEDS! # Set the random seeds for reproducibility np.random.seed(42) ## Prepare all functions # Get number of units in each hidden layer specified in the string such as 100,100 if FLAGS.dnn_hidden_units: dnn_hidden_units = FLAGS.dnn_hidden_units.split(",") dnn_hidden_units = [ int(dnn_hidden_unit_) for dnn_hidden_unit_ in dnn_hidden_units ] else: dnn_hidden_units = [] # load dataset cifar10 = cifar10_utils.get_cifar10(FLAGS.data_dir) # get batches batches = [] # initializing loss and accuracy arrays accuracies = [] losses = [] for i in range(FLAGS.max_steps): x, y = cifar10['train'].next_batch( FLAGS.batch_size) # (batch_size, 3, 32, 32) (batch_size, 10) x = x.reshape(FLAGS.batch_size, -1) batches.append((x, y)) # get output size out_size = batches[-1][1].shape[1] # get intput size in_size = batches[-1][0].shape[1] # initialize network net = MLP(in_size, dnn_hidden_units, out_size) # intialize loss function criterion = CrossEntropyModule() # make steps for s in range(FLAGS.max_steps): x, t = batches[s] # forwardpass y = net.forward(x) # calculate loss loss = criterion.forward(y, t) losses.append(loss) # gradient for cross entropy dx = criterion.backward(y, t) # backward pass net.backward(dx) # update weights for m in net.modules: if isinstance(m, LinearModule): m.params['weight'] -= FLAGS.learning_rate * m.grads['weight'] m.params['bias'] -= FLAGS.learning_rate * m.grads['bias'] if s % FLAGS.eval_freq == 0: x, t = cifar10['test'].images, cifar10['test'].labels x = x.reshape(x.shape[0], -1) y = net.forward(x) acc = accuracy(y, t) print('accuracy at step', s, ': ', acc) accuracies.append(acc * 100)
def train(): """ Performs training and evaluation of MLP model. """ # Set the random seeds for reproducibility np.random.seed(42) ## Prepare all functions # Get number of units in each hidden layer specified in the string such as 100,100 if FLAGS.dnn_hidden_units: dnn_hidden_units = FLAGS.dnn_hidden_units.split(",") dnn_hidden_units = [ int(dnn_hidden_unit_) for dnn_hidden_unit_ in dnn_hidden_units ] else: dnn_hidden_units = [] lr = FLAGS.learning_rate cifar10 = cifar10_utils.get_cifar10('cifar10/cifar-10-batches-py') accuracies = {'train': [], 'test': []} loss_curve = {'train': [], 'test': []} x, y = cifar10['train'].next_batch(FLAGS.batch_size) num_inputs = np.prod(x.shape[1:]) num_layers = len(dnn_hidden_units) + 1 myMLP = MLP(num_inputs, dnn_hidden_units, 10) myCR = CrossEntropyModule() for j in range(FLAGS.max_steps): x = x.reshape(FLAGS.batch_size, -1) out = myMLP.forward(x) loss = myCR.forward(out, y) dout = myCR.backward(out, y) myMLP.backward(dout) for i in range(num_layers): myMLP.layers['linear' + str(i + 1)].params['weight'] = myMLP.layers[ 'linear' + str(i + 1)].params['weight'] - myMLP.layers[ 'linear' + str(i + 1)].grads['weight'] * lr myMLP.layers['linear' + str(i + 1)].params['bias'] = myMLP.layers[ 'linear' + str(i + 1)].params['bias'] - myMLP.layers[ 'linear' + str(i + 1)].grads['bias'] * lr if j % FLAGS.eval_freq == 0: ac = accuracy(out, y) accuracies['train'].append(ac) loss_curve['train'].append(loss) x, y = cifar10['test'].images, cifar10['test'].labels x = x.reshape(x.shape[0], -1) outputs = myMLP.forward(x) loss = myCR.forward(outputs, y) loss_curve['test'].append(loss) ac = accuracy(outputs, y) accuracies['test'].append(ac) print(ac) x, y = cifar10['train'].next_batch(FLAGS.batch_size)
def train(): """ Performs training and evaluation of MLP model. TODO: Implement training and evaluation of MLP model. Evaluate your model on the whole test set each eval_freq iterations. """ ### DO NOT CHANGE SEEDS! # Set the random seeds for reproducibility np.random.seed(42) ## Prepare all functions # Get number of units in each hidden layer specified in the string such as 100,100 if FLAGS.dnn_hidden_units: dnn_hidden_units = FLAGS.dnn_hidden_units.split(",") dnn_hidden_units = [ int(dnn_hidden_unit_) for dnn_hidden_unit_ in dnn_hidden_units ] else: dnn_hidden_units = [] ######################## # PUT YOUR CODE HERE # # set all flag variables, except dor dnn_hidden_units LEARNING_RATE_DEFAULT = FLAGS.learning_rate MAX_STEPS_DEFAULT = FLAGS.max_steps BATCH_SIZE_DEFAULT = FLAGS.batch_size EVAL_FREQ_DEFAULT = FLAGS.eval_freq # get test data to initialize the model with cifar10 = cifar10_utils.get_cifar10(DATA_DIR_DEFAULT) x_test, y_test = cifar10['test'].images, cifar10['test'].labels # input_size is a concatenation of the 3d image input_size = np.shape(x_test)[1] * np.shape(x_test)[2] * np.shape( x_test)[3] # this is the class size = output size class_size = np.shape(y_test)[1] # reshape the images to be 1-dimensional vectors x_test = x_test.reshape([np.shape(x_test)[0], input_size]) model = MLP(n_inputs=input_size, n_hidden=dnn_hidden_units, n_classes=class_size) calculate_loss = CrossEntropyModule() # will store are results accuracies_val = [] accuracies_train = [] loss_val = [] loss_train = [] # keep going until we reach the max steps for step in range(MAX_STEPS_DEFAULT): # get the next batch x, y = cifar10['train'].next_batch(BATCH_SIZE_DEFAULT) x = x.reshape([np.shape(x)[0], input_size]) forward_out = model.forward(x) loss = calculate_loss.forward(forward_out, y) loss_gradient = calculate_loss.backward(forward_out, y) model.backward(loss_gradient) for layer in model.layers: if hasattr(layer, 'params'): # this only holds for the linear layers # update weights and bias layer.params['weight'] = layer.params[ 'weight'] - LEARNING_RATE_DEFAULT * layer.grads['weight'] layer.params['bias'] = layer.params[ 'bias'] - LEARNING_RATE_DEFAULT * layer.grads['bias'] # evaluate every EVAL_FREQ_DEFAULT steps if step % EVAL_FREQ_DEFAULT == 0: test_forward = model.forward(x_test) accuracies_train.append(accuracy(forward_out, y)) accuracies_val.append(accuracy(test_forward, y_test)) loss_train.append(loss) loss_val.append(calculate_loss.forward(test_forward, y_test)) print("accuracies train") print(accuracies_train) print("accuracies val") print(accuracies_val) print("losses train") print(loss_train) print("losses val") print(loss_val)
def train(): """ Performs training and evaluation of MLP model. TODO: Implement training and evaluation of MLP model. Evaluate your model on the whole test set each eval_freq iterations. """ # DO NOT CHANGE SEEDS! # Set the random seeds for reproducibility np.random.seed(42) # Prepare all functions # Get number of units in each hidden layer specified in the string such as 100,100 if FLAGS.dnn_hidden_units: dnn_hidden_units = FLAGS.dnn_hidden_units.split(",") dnn_hidden_units = [ int(dnn_hidden_unit_) for dnn_hidden_unit_ in dnn_hidden_units ] else: dnn_hidden_units = [] ######################## # PUT YOUR CODE HERE # ####################### def reshape_cifar10_mlp(x): batch_size = x.shape[0] x = x.transpose([2, 3, 1, 0]) x = x.reshape([-1, batch_size]) x = x.transpose() return x cifar10 = cifar10_utils.get_cifar10(FLAGS.data_dir) x_train, y_train = cifar10['train'].next_batch(FLAGS.batch_size) x_train = reshape_cifar10_mlp(x_train) softmax = SoftMaxModule() crossent = CrossEntropyModule() mlp = MLP(x_train.shape[1], dnn_hidden_units, y_train.shape[1]) train_accs = [] train_losses = [] eval_accs = [] eval_losses = [] for i in np.arange(FLAGS.max_steps): print('\nStep: {}\n'.format(i)) print('Training: ') logits = mlp.forward(x_train) softmax_logits = softmax.forward(logits) train_loss = crossent.forward(softmax_logits, y_train) train_acc = accuracy(softmax_logits, y_train) print('loss: {:.4f}, acc: {:.4f}\n'.format(train_loss, train_acc)) dL = crossent.backward(softmax_logits, y_train) dL = softmax.backward(dL) mlp.backward(dL) for layer in mlp.layers: if isinstance(layer, LinearModule): layer.params[ 'weight'] -= FLAGS.learning_rate * layer.grads['weight'] layer.params[ 'bias'] -= FLAGS.learning_rate * layer.grads['bias'] x_train, y_train = cifar10['train'].next_batch(FLAGS.batch_size) x_train = reshape_cifar10_mlp(x_train) if i % FLAGS.eval_freq == 0: print('Evaluation: ') x_eval, y_eval = cifar10['test'].images, cifar10['test'].labels x_eval = reshape_cifar10_mlp(x_eval) logits = mlp.forward(x_eval) softmax_logits = softmax.forward(logits) eval_loss = crossent.forward(softmax_logits, y_eval) eval_acc = accuracy(softmax_logits, y_eval) train_losses.append(train_loss) train_accs.append(train_acc) eval_losses.append(eval_loss) eval_accs.append(eval_acc) print('loss: {:.4f}, acc: {:.4f}'.format(eval_loss, eval_acc)) print('Evaluation: ') x_eval, y_eval = cifar10['test'].images, cifar10['test'].labels x_eval = reshape_cifar10_mlp(x_eval) logits = mlp.forward(x_eval) softmax_logits = softmax.forward(logits) eval_loss = crossent.forward(softmax_logits, y_eval) eval_acc = accuracy(softmax_logits, y_eval) train_losses.append(train_loss) train_accs.append(train_acc) eval_losses.append(eval_loss) eval_accs.append(eval_acc) print('loss: {:.4f}, acc: {:.4f}'.format(eval_loss, eval_acc)) print('Finished training.') plt.figure(figsize=(10, 5)) plt.plot(np.arange(len(train_losses)), train_losses, label='training loss') plt.plot(np.arange(len(eval_losses)), eval_losses, label='evaluation loss') plt.legend() plt.xlabel('Iterations [x{}]'.format(FLAGS.eval_freq)) plt.savefig('results/mlp_loss.png', bbox_inches='tight') plt.figure(figsize=(10, 5)) plt.plot(np.arange(len(train_accs)), train_accs, label='training accuracy') plt.plot(np.arange(len(eval_accs)), eval_accs, label='evaluation accuracy') plt.legend() plt.xlabel('Iterations [x{}]'.format(FLAGS.eval_freq)) plt.savefig('results/mlp_acc.png', bbox_inches='tight')
def train(): """ Performs training and evaluation of MLP model. """ ### DO NOT CHANGE SEEDS! # Set the random seeds for reproducibility np.random.seed(42) # Prepare all functions # Get number of units in each hidden layer specified in the string such as 100,100 if FLAGS.dnn_hidden_units: dnn_hidden_units = FLAGS.dnn_hidden_units.split(",") dnn_hidden_units = [ int(dnn_hidden_unit_) for dnn_hidden_unit_ in dnn_hidden_units ] else: dnn_hidden_units = [] ######################## # PUT YOUR CODE HERE # ####################### # Import data cifar10_data = cifar10_utils.get_cifar10(FLAGS.data_dir) train_data = cifar10_data['train'] test_data = cifar10_data['test'] input_size = np.prod(np.array(train_data.images[0].shape)) output_size = train_data.labels.shape[1] # Create model model = MLP(input_size, dnn_hidden_units, output_size) loss_module = CrossEntropyModule() # Train & evaluate eval_loss = 0.0 full_loss = [] lossv = [] accv = [] for step in range(1, FLAGS.max_steps + 1): x_train, y_train = train_data.next_batch(FLAGS.batch_size) y_pred = model.forward(x_train) loss = loss_module.forward(y_pred, y_train) dout = loss_module.backward(y_pred, y_train) model.backward(dout) full_loss.append(loss) # Update weights for layer in model.layers: if isinstance(layer, LinearModule): layer.params[ 'weight'] -= FLAGS.learning_rate * layer.grads['weight'] layer.params[ 'bias'] -= FLAGS.learning_rate * layer.grads['bias'] # Accuracy evaluation eval_loss += loss.item() if step % FLAGS.eval_freq == 0: test_x, test_y = test_data.images, test_data.labels predicted_y = model.forward(test_x) accuracy_result = accuracy(predicted_y, test_y) lossv.append(eval_loss / FLAGS.eval_freq) accv.append(accuracy_result) print('Step %d - accuracy: %.4f - loss: %.3f' % (step, accuracy_result, eval_loss / FLAGS.eval_freq)) eval_loss = 0.0 print("Training Done")
def train(): """ Performs training and evaluation of MLP model. TODO: Implement training and evaluation of MLP model. Evaluate your model on the whole test set each eval_freq iterations. """ ### DO NOT CHANGE SEEDS! # Set the random seeds for reproducibility np.random.seed(42) ## Prepare all functions # Get number of units in each hidden layer specified in the string such as 100,100 if FLAGS.dnn_hidden_units: dnn_hidden_units = FLAGS.dnn_hidden_units.split(",") dnn_hidden_units = [ int(dnn_hidden_unit_) for dnn_hidden_unit_ in dnn_hidden_units ] else: dnn_hidden_units = [] # Get negative slope parameter for LeakyReLU neg_slope = FLAGS.neg_slope learning_rate = FLAGS.learning_rate batch_size = FLAGS.batch_size max_steps = FLAGS.max_steps results = open("results.dat", "w+") results.write( "#numpy_mlp \n#neg_slope : {}\n#learning_rate : {}\n#batch_size : {}\n#hidden_units : {}\ \n#max_steps : {}\n".format(neg_slope, learning_rate, batch_size, dnn_hidden_units, max_steps)) results.write("#epoch batch max_steps loss train_acc test_acc test_loss\n") cifar10 = cifar10_utils.get_cifar10(FLAGS.data_dir) x_test, t_test = cifar10["test"].images, cifar10["test"].labels x_test = x_test.reshape(np.size(x_test[:, 0, 0, 0]), N_INPUTS) mlp = MLP(N_INPUTS, dnn_hidden_units, N_CLASSES, neg_slope) crossEntropy = CrossEntropyModule() for batch in range(1, max_steps + 1): x, t = cifar10["train"].next_batch(batch_size) x = x.reshape(batch_size, N_INPUTS) y = mlp.forward(x) #y predictions, t targets loss = crossEntropy.forward(y, t) dout = crossEntropy.backward(y, t) mlp.backward(dout) #accuracy before updating if batch == 1: train_acc = accuracy(y, t) y_test = mlp.forward(x_test) test_loss = crossEntropy.forward(y_test, t_test) test_acc = accuracy(y_test, t_test) results.write("%d %d %d %.3f %.3f %.3f %.3f\n" % (cifar10["train"]._epochs_completed, 0, max_steps, loss, train_acc, test_acc, test_loss)) # print("Epoch: %d. Batch: %d/%d. Loss: %.3f. Train_acc: %.3f. Test_acc: %.3f" % # (cifar10["train"]._epochs_completed, 0, max_steps, loss, train_acc, test_acc)) #update weights for layer in mlp.linears: layer.params["weight"] = layer.params[ "weight"] - learning_rate * layer.grads["weight"] layer.params["bias"] = layer.params[ "bias"] - learning_rate * layer.grads["bias"] if batch % FLAGS.eval_freq == 0: train_acc = accuracy(y, t) y_test = mlp.forward(x_test) test_loss = crossEntropy.forward(y_test, t_test) test_acc = accuracy(y_test, t_test) results.write("%d %d %d %.3f %.3f %.3f %.3f\n" % (cifar10["train"]._epochs_completed, batch, max_steps, loss, train_acc, test_acc, test_loss)) # print("Epoch: %d. Batch: %d/%d. Loss: %.3f. Train_acc: %.3f. Test_acc: %.3f" % # (cifar10["train"]._epochs_completed, batch, max_steps, loss, train_acc, test_acc)) results.close()
def train(): """ Performs training and evaluation of MLP model. TODO: Implement training and evaluation of MLP model. Evaluate your model on the whole test set each eval_freq iterations. """ ### DO NOT CHANGE SEEDS! # Set the random seeds for reproducibility np.random.seed(42) ## Prepare all functions # Get number of units in each hidden layer specified in the string such as 100,100 if FLAGS.dnn_hidden_units: dnn_hidden_units = FLAGS.dnn_hidden_units.split(",") dnn_hidden_units = [ int(dnn_hidden_unit_) for dnn_hidden_unit_ in dnn_hidden_units ] else: dnn_hidden_units = [] ######################## # PUT YOUR CODE HERE # ####################### class SGD(object): def __init__(self, layers, learning_rate): self.layers = layers self.learning_rate = learning_rate def step(self): for layer in self.layers: try: layer.grads except Exception as e: continue layer.params[ 'weight'] -= self.learning_rate * layer.grads['weight'] layer.params[ 'bias'] -= self.learning_rate * layer.grads['bias'] def eval(model, accuracies, losses): x, y = test_data.next_batch(1000) preds = model.forward(np.reshape(x, (x.shape[0], -1))) loss = loss_module.forward(preds, y) print("Test Loss", loss) print("Test Accuracy: ", accuracy(preds, y)) accuracies['val_scores'].append(accuracy(preds, y)) losses['val_scores'].append(loss) x, y = train_data.next_batch(1000) preds = model.forward(np.reshape(x, (x.shape[0], -1))) loss = loss_module.forward(preds, y) accuracies['train_scores'].append(accuracy(preds, y)) losses['train_scores'].append(loss) def plot_history(results, ylabel, title="Validation performance of ", model_name=""): import matplotlib.pyplot as plt import seaborn as sns sns.set() plt.plot([i for i in range(1, len(results["train_scores"]) + 1)], results["train_scores"], label="Train") plt.plot([i for i in range(1, len(results["val_scores"]) + 1)], results["val_scores"], label="Val") plt.xlabel("Epochs") plt.ylabel(ylabel) plt.ylim(min(results["val_scores"]), max(results["train_scores"]) * 1.01) plt.title(title + model_name) plt.legend() plt.show() cifar10 = cifar10_utils.get_cifar10(FLAGS.data_dir) train_data = cifar10_utils.DataSet(cifar10['train'].images, cifar10['train'].labels) test_data = cifar10_utils.DataSet(cifar10['test'].images, cifar10['test'].labels) model = MLP(3 * 32 * 32, dnn_hidden_units, 10) loss_module = CrossEntropyModule() optimizer = SGD(model.layers, FLAGS.learning_rate) accuracies = dict(train_scores=list(), val_scores=list()) losses = dict(train_scores=list(), val_scores=list()) for i in range(FLAGS.max_steps): x, y = train_data.next_batch(FLAGS.batch_size) preds = model.forward(np.reshape(x, (FLAGS.batch_size, -1))) loss = loss_module.forward(preds, y) model.backward(loss_module.backward(preds, y)) optimizer.step() if i % FLAGS.eval_freq == FLAGS.eval_freq - 1: print("Train loss: ", loss) eval(model, accuracies=accuracies, losses=losses) plot_history(accuracies, "Accuracies", model_name="VGG19") plot_history(losses, "Losses", title="Train and Validation Losses of ", model_name="VGG19")
def train(): """ Performs training and evaluation of MLP model. TODO: Implement training and evaluation of MLP model. Evaluate your model on the whole test set each eval_freq iterations. """ ### DO NOT CHANGE SEEDS! # Set the random seeds for reproducibility np.random.seed(42) ## Prepare all functions # Get number of units in each hidden layer specified in the string such as 100,100 if FLAGS.dnn_hidden_units: dnn_hidden_units = FLAGS.dnn_hidden_units.split(",") dnn_hidden_units = [ int(dnn_hidden_unit_) for dnn_hidden_unit_ in dnn_hidden_units ] else: dnn_hidden_units = [] ######################## # PUT YOUR CODE HERE # ####################### #raise NotImplementedError #FLAGS.learning_rate #FLAGS.batch_size #FLAGS.max_steps #FLAGS.eval_freq #FLAGS.data_dir cifar10 = cifar10_utils.get_cifar10('cifar10/cifar-10-batches-py') #x_train_batch, y_train_batch = cifar10['train'].next_batch(5) train_data = cifar10['train'] x_test, y_test = cifar10['test'].images, cifar10['test'].labels nsamples_x, channels, y_size, x_size = np.shape(x_test) nsamples_y, n_classes = np.shape(y_test) input_per_image = y_size * x_size * channels MLP_classifier = MLP(input_per_image, dnn_hidden_units, n_classes) cross_entropy_loss = CrossEntropyModule() #Evaluation list_train_acc = [] list_test_acc = [] list_train_loss = [] list_test_loss = [] #Reshape here as we do multiple test while training x_test = x_test.reshape((nsamples_x, input_per_image)) for step in range(FLAGS.max_steps): #Get batch and reshape for input x_train_batch, y_train_batch = train_data.next_batch(FLAGS.batch_size) x_train_batch = x_train_batch.reshape( (FLAGS.batch_size, input_per_image)) #Feed forward, get loss and gradient of the loss function and backpropagate. output = MLP_classifier.forward(x_train_batch) train_loss = cross_entropy_loss.forward(output, y_train_batch) loss_gradient = cross_entropy_loss.backward(output, y_train_batch) MLP_classifier.backward(loss_gradient) #Gradients are defined in each layer now, update the weights with it for pre_layer, activation in MLP_classifier.layers: gradient_w = pre_layer.grads['weight'] gradient_b = pre_layer.grads['bias'] pre_layer.params['weight'] = pre_layer.params['weight'] - ( FLAGS.learning_rate * gradient_w) pre_layer.params['bias'] = pre_layer.params['bias'] - ( FLAGS.learning_rate * gradient_b) if (step % FLAGS.eval_freq) == 0 or (step == FLAGS.max_steps - 1): output_test = MLP_classifier.forward(x_test) test_acc = accuracy(output_test, y_test) test_loss = cross_entropy_loss.forward(output_test, y_test) list_test_acc.append(test_acc) list_test_loss.append(test_loss) train_acc = accuracy(output, y_train_batch) list_train_loss.append(train_loss) list_train_acc.append(train_acc) #Print and plot results print(list_test_acc) print(list_test_loss) steps_x = range(len(list_test_acc)) plt.plot(steps_x, list_test_acc, label="Test accuracy") plt.plot(steps_x, list_train_acc, label="Train accuracy") plt.xlabel("Step") plt.ylabel("Accuracy") plt.title("Train and test accuracies", fontsize=18, fontweight="bold") plt.legend() #plt.savefig('accuracies.png', bbox_inches='tight') plt.show() plt.plot(steps_x, list_test_loss, label="Test loss") plt.plot(steps_x, list_train_loss, label="Train loss") plt.xlabel("Step") plt.ylabel("Loss") plt.title("Train and test loss", fontsize=18, fontweight="bold") plt.legend() #plt.savefig('loss.png', bbox_inches='tight') plt.show()