net = TwoLayerNet(input_dim, hidden_dim, num_classes) best_net = net ################################################################################ # TODO: Train a two-layer neural network on image features. You may want to # # cross-validate various parameters as in previous sections. Store your best # # model in the best_net variable. # ################################################################################ # Train the network stats = net.train(X_train_feats, y_train, X_val_feats, y_val, num_iters=10000, batch_size=200, learning_rate=1e-3, learning_rate_decay=0.95, reg=0.9, verbose=True) ################################################################################ # END OF YOUR CODE # ################################################################################ # In[ ]: # Run your neural net classifier on the test set. You should be able to # get more than 55% accuracy. test_acc = (net.predict(X_test_feats) == y_test).mean()
# learning_rates = np.logspace(-2, 1, 5) # regularization_strengths = np.logspace(-5, -2, 5) results = {} best_val = -1 # The highest validation accuracy that we have seen so far. for lr in learning_rates: for reg in regularization_strengths: net = TwoLayerNet(input_dim, hidden_dim, num_classes) # Train the network stats = net.train(X_train_feats, y_train, X_val_feats, y_val, num_iters=2001, batch_size=200, learning_rate=lr, learning_rate_decay=0.95, reg=reg, verbose=False) y_train_pred = net.predict(X_train_feats) training_accuracy = np.mean(y_train == y_train_pred) y_val_pred = net.predict(X_val_feats) validation_accuracy = np.mean(y_val == y_val_pred) results[(lr, reg)] = (training_accuracy, validation_accuracy) if best_val < validation_accuracy: best_val = validation_accuracy best_net = net
print 'Validation data shape: ', X_val.shape print 'Validation labels shape: ', y_val.shape print 'Test data shape: ', X_test.shape print 'Test labels shape: ', y_test.shape input_size = 32 * 32 * 3 hidden_size = 50 num_classes = 10 net = TwoLayerNet(input_size, hidden_size, num_classes) # Train the network stats = net.train(X_train, y_train, X_val, y_val, num_iters=1000, batch_size=200, learning_rate=1e-4, learning_rate_decay=0.95, reg=0.5, verbose=True) # Predict on the validation set val_acc = (net.predict(X_val) == y_val).mean() print 'Validation accuracy: ', val_acc # Plot the loss function and train / validation accuracies plt.subplot(2, 1, 1) plt.plot(stats['loss_history']) plt.title('Loss history') plt.xlabel('Iteration') plt.ylabel('Loss')
lrates = [0.001] regs = [0.02] hidden_sizes = [100] best_accuracy = 0 for lrate in lrates: for reg in regs: for hidden_size in hidden_sizes: # Train the network with the combination net = TwoLayerNet(input_size, hidden_size, num_classes) stats, test_net = net.train(X_train, y_train, X_val, y_val, num_iters=10000, batch_size=200, learning_rate=lrate, learning_rate_decay=.95, reg=reg, verbose=True) if stats['val_acc_history'][-1] > best_accuracy: best_net = test_net best_accuracy = stats['val_acc_history'][-1] best_loss = np.mean(stats["loss_history"][-10:-1]) best_reg = reg best_lrate = lrate best_size = hidden_size print "Best accuracy so far is:", best_accuracy print "With an average loss of:", best_loss
################################################################################ # END OF YOUR CODE # ################################################################################ ################################################################################ # TODO: # # Use the validation set to set the learning rate and regularization strength. # # This should be identical to the validation that you did for the SVM; save # # the best trained classifer in best_svm. You might also want to play # # with different numbers of bins in the color histogram. If you are careful # # you should be able to get accuracy of near 0.44 on the validation set. # ################################################################################ for lr in learning_rates: for reg in regularization_strengths: print (lr,reg) stats = net.train(X_train_feats, y_train, X_val_feats, y_val, num_iters=1000, batch_size=200, learning_rate=lr, learning_rate_decay=0.95, reg=reg, verbose=True) y_train_pred = net.predict(X_train_feats) y_val_pred = net.predict(X_val_feats) tmp_train_accuracy=np.mean(y_train == y_train_pred) tmp_val_accuracy=np.mean(y_val == y_val_pred) results[(lr,reg)]=[tmp_train_accuracy,tmp_val_accuracy] if tmp_val_accuracy>best_val: best_val=tmp_val_accuracy best_net=copy.deepcopy(net) ################################################################################ # END OF YOUR CODE # ################################################################################ # Print out results. for lr, reg in sorted(results):
# TODO: Train a two-layer neural network on image features. You may want to # # cross-validate various parameters as in previous sections. Store your best # # model in the best_net variable. # ################################################################################ print '-------------_*****--------------' print 'Tuning Parameters for Neural Nets: ' for lr in np.arange(0.5, 2, 0.1): for reg in np.arange(0.001, 0.01, 0.002): for hid in hidden_dim: print 'At hidden size: %d, LR: %f, and REG: %f' % (hid, lr, reg) net_iter = TwoLayerNet(input_dim, hid, num_classes) stats_iter = net_iter.train(X_train_feats, y_train, X_val_feats, y_val, num_iters=3000, batch_size=200, learning_rate=lr, learning_rate_decay=0.95, reg=reg, verbose=True) y_val_pred_iter = net_iter.predict(X_val_feats) y_train_pred_iter = net_iter.predict(X_train_feats) val_acc = np.mean(y_val == y_val_pred_iter) train_acc = np.mean(y_train == y_train_pred_iter) print 'Validation accuracy: %f' % (val_acc) if (val_acc > best_val): best_val = val_acc best_net = net_iter print 'Best so far: %f' % (val_acc) ################################################################################ # END OF YOUR CODE #
# # Train a network # To train our network we will use SGD with momentum. In addition, we will adjust the learning rate with an exponential learning rate schedule as optimization proceeds; after each epoch, we will reduce the learning rate by multiplying it by a decay rate. # In[ ]: input_size = 32 * 32 * 3 hidden_size = 50 num_classes = 10 net = TwoLayerNet(input_size, hidden_size, num_classes) # Train the network stats = net.train(X_train, y_train, X_val, y_val, num_iters=1000, batch_size=200, learning_rate=1e-4, learning_rate_decay=0.95, reg=0.25, verbose=True) # Predict on the validation set val_acc = (net.predict(X_val) == y_val).mean() print('Validation accuracy: ', val_acc) # # Debug the training # With the default parameters we provided above, # you should get a validation accuracy of about 0.29 on the validation set. # This isn't very good. # # One strategy for getting insight into what's wrong is
# To train our network we will use SGD with momentum. In addition, we will adjust the learning rate with an exponential learning rate schedule as optimization proceeds; after each epoch, we will reduce the learning rate by multiplying it by a decay rate. # In[ ]: input_size = 32 * 32 * 3 hidden_size = 50 num_classes = 10 net = TwoLayerNet(input_size, hidden_size, num_classes) # Train the network stats = net.train( X_train, y_train, X_val, y_val, num_iters=1000, batch_size=200, learning_rate=1e-4, learning_rate_decay=0.95, reg=0.5, verbose=True, ) # Predict on the validation set val_acc = (net.predict(X_val) == y_val).mean() print "Validation accuracy: ", val_acc # # Debug the training # With the default parameters we provided above, you should get a validation accuracy of about 0.29 on the validation set. This isn't very good. # # One strategy for getting insight into what's wrong is to plot the loss function and the accuracies on the training and validation sets during optimization.
print('Validation data shape: ', X_val.shape) print('Validation labels shape: ', y_val.shape) print('Test data shape: ', X_test.shape) print('Test labels shape: ', y_test.shape) ################################################################################### input_size = 32 * 32 * 3 hidden_size = 50 num_classes = 10 net = TwoLayerNet(input_size, hidden_size, num_classes) # Train the network stats = net.train(X_train, y_train, X_val, y_val, num_iters=1000, batch_size=200, learning_rate=4e-3, learning_rate_decay=0.95, reg=0.25, verbose=True) # Predict on the validation set val_acc = (net.predict(X_val) == y_val).mean() print('Validation accuracy: ', val_acc) ################################################################################### # Plot the loss function and train / validation accuracies plt.subplot(2, 1, 1) plt.plot(stats['loss_history']) plt.title('Loss history') plt.xlabel('Iteration') plt.ylabel('Loss')
plt.gca().axis('off') plt.show() #show_net_weights(net) best_net = None best_loss = float('inf') rates = [0.1,0.01,0.001] for rate in rates: net = TwoLayerNet(input_size, hidden_size, num_classes) # Train the network stats = net.train(X_train, y_train, X_val, y_val, num_iters=1000, batch_size=200, learning_rate=rate, learning_rate_decay=0.98, reg=0.3, verbose=False) loss_avg = np.mean(stats['loss_history'][-1]) print "Rate: ", rate, " loss: ", loss_avg val_acc = (net.predict(X_val) == y_val).mean() print 'Validation accuracy: ', val_acc if val_acc > best_loss: best_net = net best_loss = val_acc # net = TwoLayerNet(input_size, hidden_size, num_classes) # stats = net.train(X_train, y_train, X_val, y_val, # num_iters=10000, batch_size=200,
# # Train a network # To train our network we will use SGD with momentum. In addition, we will adjust the learning rate with an exponential learning rate schedule as optimization proceeds; after each epoch, we will reduce the learning rate by multiplying it by a decay rate. # In[ ]: input_size = 32 * 32 * 3 hidden_size = 50 num_classes = 10 net = TwoLayerNet(input_size, hidden_size, num_classes) # Train the network stats = net.train(X_train, y_train, X_val, y_val, num_iters=1000, batch_size=200, learning_rate=1e-4, learning_rate_decay=0.95, reg=0.25, verbose=True) # Predict on the validation set val_acc = (net.predict(X_val) == y_val).mean() print('Validation accuracy: ', val_acc) # # Debug the training # With the default parameters we provided above, you should get a validation accuracy of about 0.29 on the validation set. This isn't very good. # # One strategy for getting insight into what's wrong is to plot the loss function and the accuracies on the training and validation sets during optimization. # # Another strategy is to visualize the weights that were learned in the first layer of the network. In most neural networks trained on visual data, the first layer weights typically show some visible structure when visualized.
X_train, y_train, X_val, y_val, X_test, y_test = get_CIFAR10_data() print('Train data shape: ', X_train.shape) print('Train labels shape: ', y_train.shape) print('Validation data shape: ', X_val.shape) print('Validation labels shape: ', y_val.shape) print('Test data shape: ', X_test.shape) print('Test labels shape: ', y_test.shape) input_size = 32 * 32 * 3 hidden_size = 50 num_classes = 10 net = TwoLayerNet(input_size, hidden_size, num_classes) # Train the network stats = net.train(X_train, y_train, X_val, y_val, num_iters=1000, batch_size=200, learning_rate=1e-4, learning_rate_decay=0.95, reg=0.25, verbose=True) # Predict on the validation set val_acc = (net.predict(X_val) == y_val).mean() print('Validation accuracy: ', val_acc) # Plot the loss function and train / validation accuracies plt.subplot(2, 1, 1) plt.plot(stats['loss_history']) plt.title('Loss history') plt.xlabel('Iteration') plt.ylabel('Loss') plt.subplot(2, 1, 2)
reg_list = [0.05, 0.1, 0.5, 0.9, 1.5] hidden_size_list = [50, 70, 90] num_iteration_increment_list = [500, 500, 1000] best_val_acc = 0 logging.info("*****begin of hyperparameter tuning*****") for i in learning_rate_list: logging.info("for learning_rate=%.4f tuning " % i) for j in reg_list: for h in hidden_size_list: net = TwoLayerNet(input_size, h, num_classes) iteration_total = 0 for k in num_iteration_increment_list: net.train(X_train, y_train, X_val, y_val, num_iters=k, batch_size=200, learning_rate=i, learning_rate_decay=0.95, reg=j, verbose=False) val_acc = (net.predict(X_val) == y_val).mean() iteration_total += k if (val_acc > best_val_acc): best_val_acc = val_acc logging.info("best net: acc=%.3f,for learning_rate=%.4f,\ reg=%.3f,num_iters=%d,hidden_size=%d" % (best_val_acc, i, j, iteration_total, h)) logging.info("*****end of hyperparameter tuning*****")
print('Test labels shape: ', y_test.shape) #%% Train a network, we will use SGD, and we will adjust the learning rate with an # exponential learning rate schedule as optimization proceeds. input_size = 32 * 32 * 3 hidden_size = 50 num_classes = 10 net = TwoLayerNet(input_size, hidden_size, num_classes) # Train the network stats = net.train(X_train, y_train, X_val, y_val, num_iters=1000, batch_size=200, learning_rate=1e-4, learning_rate_decay=0.95, reg=0.25, verbose=True) # Predict on the validation set val_acc = (net.predict(X_val) == y_val).mean() print('Validation accuracy: ', val_acc) #%% Debug the training # Plot the loss function and train / validation accuracies plt.subplot(2, 1, 1) plt.plot(stats['loss_history']) plt.title('Loss history')
print('Validation data shape: ', X_val.shape) print('Validation labels shape: ', y_val.shape) print('Test data shape: ', X_test.shape) print('Test labels shape: ', y_test.shape) input_size = 32 * 32 * 3 hidden_size = 50 num_classes = 10 net = TwoLayerNet(input_size, hidden_size, num_classes) # Train the network stats = net.train(X_train, y_train, X_val, y_val, num_iters=1000, batch_size=200, learning_rate=1e-4, learning_rate_decay=0.95, reg=0.25, verbose=True) # Predict on the validation set val_acc = (net.predict(X_val) == y_val).mean() print('Validation accuracy: ', val_acc) # Plot the loss function and train / validation accuracies plt.subplot(2, 1, 1) plt.plot(stats['loss_history']) plt.title('Loss history') plt.xlabel('Iteration') plt.ylabel('Loss')
for hid_size in hidden_size: hidden_size = hid_size for reg in regularization_strengths: for lr in learning_rates: for num_iter in number_of_iters: for bs in batch_sizes: tic = time.time() net = TwoLayerNet(input_size, hidden_size, num_classes) print( "hid_size %d / lr %.2E / reg %.2e / num_iter %d / batch_s %d" % (hid_size, lr, reg, num_iter, bs)) training_results = net.train(X_train_feats, y_train, X_val_feats, y_val, num_iters=num_iter, batch_size=bs, learning_rate=lr, learning_rate_decay=0.95, reg=reg, verbose=True) val_acc = (net.predict(X_val_feats) == y_val).mean() toc = time.time() print('That took %fs' % (toc - tic)) est_time = toc - tic results[(hid_size, lr, reg, num_iter, bs)] = (val_acc, est_time) if val_acc > best_val: best_val = val_acc best_net = net best_stats = training_results best_hid_size = hid_size
return 10**(np.random.uniform(low, high, size)) hidden_units = np.random.randint(hidden_units[0], hidden_units[1], num_iters) learning_rate = loguniform(learning_rate[0], learning_rate[1], num_iters) reg = np.random.uniform(reg[0], reg[1], num_iters) input_size = 32 * 32 * 3 num_classes = 10 val_acc_best = -float('inf') for unit, lr, rg in zip(hidden_units, learning_rate, reg): net = TwoLayerNet(input_size, unit, num_classes) # Train the network stats = net.train(X_train, y_train, X_val, y_val, num_iters=1000, batch_size=200, learning_rate=lr, learning_rate_decay=0.95, reg=rg, verbose=True) # Predict on the validation set val_acc = (net.predict(X_val) == y_val).mean() print('Validation accuracy: ', val_acc) if val_acc > val_acc_best: best_net = net val_acc_best = val_acc print(f'Best validation accuracy: {val_acc*100}%') # *****END OF YOUR CODE (DO NOT DELETE/MODIFY THIS LINE)*****
print 'Test labels shape: ', y_test.shape # # Train a network # To train our network we will use SGD with momentum. In addition, we will adjust the learning rate with an exponential learning rate schedule as optimization proceeds; after each epoch, we will reduce the learning rate by multiplying it by a decay rate. # In[11]: input_size = 32 * 32 * 3 hidden_size = 50 num_classes = 10 net = TwoLayerNet(input_size, hidden_size, num_classes) # Train the network stats = net.train(X_train, y_train, X_val, y_val, num_iters=1000, batch_size=200, learning_rate=1e-4, learning_rate_decay=0.95, reg=0.5, verbose=True) # Predict on the validation set val_acc = (net.predict(X_val) == y_val).mean() print 'Validation accuracy: ', val_acc # # Debug the training # With the default parameters we provided above, you should get a validation accuracy of about 0.29 on the validation set. This isn't very good. # # One strategy for getting insight into what's wrong is to plot the loss function and the accuracies on the training and validation sets during optimization. # # Another strategy is to visualize the weights that were learned in the first layer of the network. In most neural networks trained on visual data, the first layer weights typically show some visible structure when visualized.
# cross-validate various parameters as in previous sections. Store your best # # model in the best_net variable. # ################################################################################ pass print input_dim, hidden_dim, num_classes print X_train_feats.shape print y_train.shape rates = [1e-3, 2e-3, 3e-3] for lr in rates: net = TwoLayerNet(input_dim, hidden_dim, num_classes) net.train( X_train_feats, y_train, X_val_feats, y_val, #num_iters=1000, batch_size=200, num_iters=3, batch_size=200, learning_rate=lr, learning_rate_decay=0.95, reg=0.5, verbose=True) val_acc = (net.predict(X_val_feats) == y_val).mean() print 'Validation accuracy: ', val_acc if val_acc > best_acc: best_acc = val_acc best_net = net net = best_net ################################################################################ # END OF YOUR CODE # ################################################################################