net = TwoLayerNet(input_dim, hidden_dim, num_classes)
best_net = net
################################################################################
# TODO: Train a two-layer neural network on image features. You may want to #
# cross-validate various parameters as in previous sections. Store your best #
# model in the best_net variable. #
################################################################################
# Train the network
stats = net.train(X_train_feats,
y_train,
X_val_feats,
y_val,
num_iters=10000,
batch_size=200,
learning_rate=1e-3,
learning_rate_decay=0.95,
reg=0.9,
verbose=True)
################################################################################
# END OF YOUR CODE #
################################################################################
# In[ ]:
# Run your neural net classifier on the test set. You should be able to
# get more than 55% accuracy.
test_acc = (net.predict(X_test_feats) == y_test).mean()
# learning_rates = np.logspace(-2, 1, 5)
# regularization_strengths = np.logspace(-5, -2, 5)
results = {}
best_val = -1 # The highest validation accuracy that we have seen so far.
for lr in learning_rates:
for reg in regularization_strengths:
net = TwoLayerNet(input_dim, hidden_dim, num_classes)
# Train the network
stats = net.train(X_train_feats,
y_train,
X_val_feats,
y_val,
num_iters=2001,
batch_size=200,
learning_rate=lr,
learning_rate_decay=0.95,
reg=reg,
verbose=False)
y_train_pred = net.predict(X_train_feats)
training_accuracy = np.mean(y_train == y_train_pred)
y_val_pred = net.predict(X_val_feats)
validation_accuracy = np.mean(y_val == y_val_pred)
results[(lr, reg)] = (training_accuracy, validation_accuracy)
if best_val < validation_accuracy:
best_val = validation_accuracy
best_net = net
print 'Validation data shape: ', X_val.shape
print 'Validation labels shape: ', y_val.shape
print 'Test data shape: ', X_test.shape
print 'Test labels shape: ', y_test.shape
input_size = 32 * 32 * 3
hidden_size = 50
num_classes = 10
net = TwoLayerNet(input_size, hidden_size, num_classes)
# Train the network
stats = net.train(X_train,
y_train,
X_val,
y_val,
num_iters=1000,
batch_size=200,
learning_rate=1e-4,
learning_rate_decay=0.95,
reg=0.5,
verbose=True)
# Predict on the validation set
val_acc = (net.predict(X_val) == y_val).mean()
print 'Validation accuracy: ', val_acc
# Plot the loss function and train / validation accuracies
plt.subplot(2, 1, 1)
plt.plot(stats['loss_history'])
plt.title('Loss history')
plt.xlabel('Iteration')
plt.ylabel('Loss')
lrates = [0.001]
regs = [0.02]
hidden_sizes = [100]
best_accuracy = 0
for lrate in lrates:
for reg in regs:
for hidden_size in hidden_sizes:
# Train the network with the combination
net = TwoLayerNet(input_size, hidden_size, num_classes)
stats, test_net = net.train(X_train,
y_train,
X_val,
y_val,
num_iters=10000,
batch_size=200,
learning_rate=lrate,
learning_rate_decay=.95,
reg=reg,
verbose=True)
if stats['val_acc_history'][-1] > best_accuracy:
best_net = test_net
best_accuracy = stats['val_acc_history'][-1]
best_loss = np.mean(stats["loss_history"][-10:-1])
best_reg = reg
best_lrate = lrate
best_size = hidden_size
print "Best accuracy so far is:", best_accuracy
print "With an average loss of:", best_loss
################################################################################
# END OF YOUR CODE #
################################################################################
################################################################################
# TODO: #
# Use the validation set to set the learning rate and regularization strength. #
# This should be identical to the validation that you did for the SVM; save #
# the best trained classifer in best_svm. You might also want to play #
# with different numbers of bins in the color histogram. If you are careful #
# you should be able to get accuracy of near 0.44 on the validation set. #
################################################################################
for lr in learning_rates:
for reg in regularization_strengths:
print (lr,reg)
stats = net.train(X_train_feats, y_train, X_val_feats, y_val,
num_iters=1000, batch_size=200,
learning_rate=lr, learning_rate_decay=0.95,
reg=reg, verbose=True)
y_train_pred = net.predict(X_train_feats)
y_val_pred = net.predict(X_val_feats)
tmp_train_accuracy=np.mean(y_train == y_train_pred)
tmp_val_accuracy=np.mean(y_val == y_val_pred)
results[(lr,reg)]=[tmp_train_accuracy,tmp_val_accuracy]
if tmp_val_accuracy>best_val:
best_val=tmp_val_accuracy
best_net=copy.deepcopy(net)
################################################################################
# END OF YOUR CODE #
################################################################################
# Print out results.
for lr, reg in sorted(results):
# TODO: Train a two-layer neural network on image features. You may want to #
# cross-validate various parameters as in previous sections. Store your best #
# model in the best_net variable. #
################################################################################
print '-------------_*****--------------'
print 'Tuning Parameters for Neural Nets: '
for lr in np.arange(0.5, 2, 0.1):
for reg in np.arange(0.001, 0.01, 0.002):
for hid in hidden_dim:
print 'At hidden size: %d, LR: %f, and REG: %f' % (hid, lr, reg)
net_iter = TwoLayerNet(input_dim, hid, num_classes)
stats_iter = net_iter.train(X_train_feats,
y_train,
X_val_feats,
y_val,
num_iters=3000,
batch_size=200,
learning_rate=lr,
learning_rate_decay=0.95,
reg=reg,
verbose=True)
y_val_pred_iter = net_iter.predict(X_val_feats)
y_train_pred_iter = net_iter.predict(X_train_feats)
val_acc = np.mean(y_val == y_val_pred_iter)
train_acc = np.mean(y_train == y_train_pred_iter)
print 'Validation accuracy: %f' % (val_acc)
if (val_acc > best_val):
best_val = val_acc
best_net = net_iter
print 'Best so far: %f' % (val_acc)
################################################################################
# END OF YOUR CODE #
# # Train a network
# To train our network we will use SGD with momentum. In addition, we will adjust the learning rate with an exponential learning rate schedule as optimization proceeds; after each epoch, we will reduce the learning rate by multiplying it by a decay rate.
# In[ ]:
input_size = 32 * 32 * 3
hidden_size = 50
num_classes = 10
net = TwoLayerNet(input_size, hidden_size, num_classes)
# Train the network
stats = net.train(X_train,
y_train,
X_val,
y_val,
num_iters=1000,
batch_size=200,
learning_rate=1e-4,
learning_rate_decay=0.95,
reg=0.25,
verbose=True)
# Predict on the validation set
val_acc = (net.predict(X_val) == y_val).mean()
print('Validation accuracy: ', val_acc)
# # Debug the training
# With the default parameters we provided above,
# you should get a validation accuracy of about 0.29 on the validation set.
# This isn't very good.
#
# One strategy for getting insight into what's wrong is
# To train our network we will use SGD with momentum. In addition, we will adjust the learning rate with an exponential learning rate schedule as optimization proceeds; after each epoch, we will reduce the learning rate by multiplying it by a decay rate.
# In[ ]:
input_size = 32 * 32 * 3
hidden_size = 50
num_classes = 10
net = TwoLayerNet(input_size, hidden_size, num_classes)
# Train the network
stats = net.train(
X_train,
y_train,
X_val,
y_val,
num_iters=1000,
batch_size=200,
learning_rate=1e-4,
learning_rate_decay=0.95,
reg=0.5,
verbose=True,
)
# Predict on the validation set
val_acc = (net.predict(X_val) == y_val).mean()
print "Validation accuracy: ", val_acc
# # Debug the training
# With the default parameters we provided above, you should get a validation accuracy of about 0.29 on the validation set. This isn't very good.
#
# One strategy for getting insight into what's wrong is to plot the loss function and the accuracies on the training and validation sets during optimization.
print('Validation data shape: ', X_val.shape)
print('Validation labels shape: ', y_val.shape)
print('Test data shape: ', X_test.shape)
print('Test labels shape: ', y_test.shape)
###################################################################################
input_size = 32 * 32 * 3
hidden_size = 50
num_classes = 10
net = TwoLayerNet(input_size, hidden_size, num_classes)
# Train the network
stats = net.train(X_train, y_train, X_val, y_val,
num_iters=1000, batch_size=200,
learning_rate=4e-3, learning_rate_decay=0.95,
reg=0.25, verbose=True)
# Predict on the validation set
val_acc = (net.predict(X_val) == y_val).mean()
print('Validation accuracy: ', val_acc)
###################################################################################
# Plot the loss function and train / validation accuracies
plt.subplot(2, 1, 1)
plt.plot(stats['loss_history'])
plt.title('Loss history')
plt.xlabel('Iteration')
plt.ylabel('Loss')
plt.gca().axis('off')
plt.show()
#show_net_weights(net)
best_net = None
best_loss = float('inf')
rates = [0.1,0.01,0.001]
for rate in rates:
net = TwoLayerNet(input_size, hidden_size, num_classes)
# Train the network
stats = net.train(X_train, y_train, X_val, y_val,
num_iters=1000, batch_size=200,
learning_rate=rate, learning_rate_decay=0.98,
reg=0.3, verbose=False)
loss_avg = np.mean(stats['loss_history'][-1])
print "Rate: ", rate, " loss: ", loss_avg
val_acc = (net.predict(X_val) == y_val).mean()
print 'Validation accuracy: ', val_acc
if val_acc > best_loss:
best_net = net
best_loss = val_acc
# net = TwoLayerNet(input_size, hidden_size, num_classes)
# stats = net.train(X_train, y_train, X_val, y_val,
# num_iters=10000, batch_size=200,
# # Train a network
# To train our network we will use SGD with momentum. In addition, we will adjust the learning rate with an exponential learning rate schedule as optimization proceeds; after each epoch, we will reduce the learning rate by multiplying it by a decay rate.
# In[ ]:
input_size = 32 * 32 * 3
hidden_size = 50
num_classes = 10
net = TwoLayerNet(input_size, hidden_size, num_classes)
# Train the network
stats = net.train(X_train,
y_train,
X_val,
y_val,
num_iters=1000,
batch_size=200,
learning_rate=1e-4,
learning_rate_decay=0.95,
reg=0.25,
verbose=True)
# Predict on the validation set
val_acc = (net.predict(X_val) == y_val).mean()
print('Validation accuracy: ', val_acc)
# # Debug the training
# With the default parameters we provided above, you should get a validation accuracy of about 0.29 on the validation set. This isn't very good.
#
# One strategy for getting insight into what's wrong is to plot the loss function and the accuracies on the training and validation sets during optimization.
#
# Another strategy is to visualize the weights that were learned in the first layer of the network. In most neural networks trained on visual data, the first layer weights typically show some visible structure when visualized.
X_train, y_train, X_val, y_val, X_test, y_test = get_CIFAR10_data()
print('Train data shape: ', X_train.shape)
print('Train labels shape: ', y_train.shape)
print('Validation data shape: ', X_val.shape)
print('Validation labels shape: ', y_val.shape)
print('Test data shape: ', X_test.shape)
print('Test labels shape: ', y_test.shape)
input_size = 32 * 32 * 3
hidden_size = 50
num_classes = 10
net = TwoLayerNet(input_size, hidden_size, num_classes)
# Train the network
stats = net.train(X_train, y_train, X_val, y_val,
num_iters=1000, batch_size=200,
learning_rate=1e-4, learning_rate_decay=0.95,
reg=0.25, verbose=True)
# Predict on the validation set
val_acc = (net.predict(X_val) == y_val).mean()
print('Validation accuracy: ', val_acc)
# Plot the loss function and train / validation accuracies
plt.subplot(2, 1, 1)
plt.plot(stats['loss_history'])
plt.title('Loss history')
plt.xlabel('Iteration')
plt.ylabel('Loss')
plt.subplot(2, 1, 2)
reg_list = [0.05, 0.1, 0.5, 0.9, 1.5]
hidden_size_list = [50, 70, 90]
num_iteration_increment_list = [500, 500, 1000]
best_val_acc = 0
logging.info("*****begin of hyperparameter tuning*****")
for i in learning_rate_list:
logging.info("for learning_rate=%.4f tuning " % i)
for j in reg_list:
for h in hidden_size_list:
net = TwoLayerNet(input_size, h, num_classes)
iteration_total = 0
for k in num_iteration_increment_list:
net.train(X_train,
y_train,
X_val,
y_val,
num_iters=k,
batch_size=200,
learning_rate=i,
learning_rate_decay=0.95,
reg=j,
verbose=False)
val_acc = (net.predict(X_val) == y_val).mean()
iteration_total += k
if (val_acc > best_val_acc):
best_val_acc = val_acc
logging.info("best net: acc=%.3f,for learning_rate=%.4f,\
reg=%.3f,num_iters=%d,hidden_size=%d" %
(best_val_acc, i, j, iteration_total, h))
logging.info("*****end of hyperparameter tuning*****")
print('Test labels shape: ', y_test.shape)
#%% Train a network, we will use SGD, and we will adjust the learning rate with an
# exponential learning rate schedule as optimization proceeds.
input_size = 32 * 32 * 3
hidden_size = 50
num_classes = 10
net = TwoLayerNet(input_size, hidden_size, num_classes)
# Train the network
stats = net.train(X_train,
y_train,
X_val,
y_val,
num_iters=1000,
batch_size=200,
learning_rate=1e-4,
learning_rate_decay=0.95,
reg=0.25,
verbose=True)
# Predict on the validation set
val_acc = (net.predict(X_val) == y_val).mean()
print('Validation accuracy: ', val_acc)
#%% Debug the training
# Plot the loss function and train / validation accuracies
plt.subplot(2, 1, 1)
plt.plot(stats['loss_history'])
plt.title('Loss history')
print('Validation data shape: ', X_val.shape)
print('Validation labels shape: ', y_val.shape)
print('Test data shape: ', X_test.shape)
print('Test labels shape: ', y_test.shape)
input_size = 32 * 32 * 3
hidden_size = 50
num_classes = 10
net = TwoLayerNet(input_size, hidden_size, num_classes)
# Train the network
stats = net.train(X_train,
y_train,
X_val,
y_val,
num_iters=1000,
batch_size=200,
learning_rate=1e-4,
learning_rate_decay=0.95,
reg=0.25,
verbose=True)
# Predict on the validation set
val_acc = (net.predict(X_val) == y_val).mean()
print('Validation accuracy: ', val_acc)
# Plot the loss function and train / validation accuracies
plt.subplot(2, 1, 1)
plt.plot(stats['loss_history'])
plt.title('Loss history')
plt.xlabel('Iteration')
plt.ylabel('Loss')
for hid_size in hidden_size:
hidden_size = hid_size
for reg in regularization_strengths:
for lr in learning_rates:
for num_iter in number_of_iters:
for bs in batch_sizes:
tic = time.time()
net = TwoLayerNet(input_size, hidden_size, num_classes)
print(
"hid_size %d / lr %.2E / reg %.2e / num_iter %d / batch_s %d"
% (hid_size, lr, reg, num_iter, bs))
training_results = net.train(X_train_feats,
y_train,
X_val_feats,
y_val,
num_iters=num_iter,
batch_size=bs,
learning_rate=lr,
learning_rate_decay=0.95,
reg=reg,
verbose=True)
val_acc = (net.predict(X_val_feats) == y_val).mean()
toc = time.time()
print('That took %fs' % (toc - tic))
est_time = toc - tic
results[(hid_size, lr, reg, num_iter, bs)] = (val_acc,
est_time)
if val_acc > best_val:
best_val = val_acc
best_net = net
best_stats = training_results
best_hid_size = hid_size
return 10**(np.random.uniform(low, high, size))
hidden_units = np.random.randint(hidden_units[0], hidden_units[1], num_iters)
learning_rate = loguniform(learning_rate[0], learning_rate[1], num_iters)
reg = np.random.uniform(reg[0], reg[1], num_iters)
input_size = 32 * 32 * 3
num_classes = 10
val_acc_best = -float('inf')
for unit, lr, rg in zip(hidden_units, learning_rate, reg):
net = TwoLayerNet(input_size, unit, num_classes)
# Train the network
stats = net.train(X_train, y_train, X_val, y_val,
num_iters=1000, batch_size=200,
learning_rate=lr, learning_rate_decay=0.95,
reg=rg, verbose=True)
# Predict on the validation set
val_acc = (net.predict(X_val) == y_val).mean()
print('Validation accuracy: ', val_acc)
if val_acc > val_acc_best:
best_net = net
val_acc_best = val_acc
print(f'Best validation accuracy: {val_acc*100}%')
# *****END OF YOUR CODE (DO NOT DELETE/MODIFY THIS LINE)*****
print 'Test labels shape: ', y_test.shape
# # Train a network
# To train our network we will use SGD with momentum. In addition, we will adjust the learning rate with an exponential learning rate schedule as optimization proceeds; after each epoch, we will reduce the learning rate by multiplying it by a decay rate.
# In[11]:
input_size = 32 * 32 * 3
hidden_size = 50
num_classes = 10
net = TwoLayerNet(input_size, hidden_size, num_classes)
# Train the network
stats = net.train(X_train, y_train, X_val, y_val,
num_iters=1000, batch_size=200,
learning_rate=1e-4, learning_rate_decay=0.95,
reg=0.5, verbose=True)
# Predict on the validation set
val_acc = (net.predict(X_val) == y_val).mean()
print 'Validation accuracy: ', val_acc
# # Debug the training
# With the default parameters we provided above, you should get a validation accuracy of about 0.29 on the validation set. This isn't very good.
#
# One strategy for getting insight into what's wrong is to plot the loss function and the accuracies on the training and validation sets during optimization.
#
# Another strategy is to visualize the weights that were learned in the first layer of the network. In most neural networks trained on visual data, the first layer weights typically show some visible structure when visualized.
# cross-validate various parameters as in previous sections. Store your best #
# model in the best_net variable. #
################################################################################
pass
print input_dim, hidden_dim, num_classes
print X_train_feats.shape
print y_train.shape
rates = [1e-3, 2e-3, 3e-3]
for lr in rates:
net = TwoLayerNet(input_dim, hidden_dim, num_classes)
net.train(
X_train_feats,
y_train,
X_val_feats,
y_val,
#num_iters=1000, batch_size=200,
num_iters=3,
batch_size=200,
learning_rate=lr,
learning_rate_decay=0.95,
reg=0.5,
verbose=True)
val_acc = (net.predict(X_val_feats) == y_val).mean()
print 'Validation accuracy: ', val_acc
if val_acc > best_acc:
best_acc = val_acc
best_net = net
net = best_net
################################################################################
# END OF YOUR CODE #
################################################################################
