def test_SVM_train(sample_train, sample_test):
#this test is designed to verify that input shapes are correct
Xtrain, ytrain = sample_train(count=40)
Xtest, ytest = sample_test(count=10)
with pytest.raises(ValueError):
#Xtrain has the wrong shape (that is why there is a value error)
lin_svm = LinearSVM()
lin_svm.train(Xtrain, ytrain)
def test_SVM_train_reshape_input(sample_train, sample_test):
Xtrain, ytrain = sample_train(count=40)
Xtest, ytest = sample_test(count=10)
Xtrain = np.reshape(Xtrain, (Xtrain.shape[0], -1))
Xtest = np.reshape(Xtest, (Xtest.shape[0], -1))
lin_svm = LinearSVM()
lin_svm.train(Xtrain,ytrain)
def test_SVM_train_reshape_input(sample_train, sample_test):
Xtrain, ytrain = sample_train(count=40)
Xtest, ytest = sample_test(count=10)
Xtrain = np.reshape(Xtrain, (Xtrain.shape[0], -1))
Xtest = np.reshape(Xtest, (Xtest.shape[0], -1))
lin_svm = LinearSVM()
lin_svm.train(Xtrain, ytrain)
def test_SVM_train(sample_train, sample_test):
#this test is designed to verify that input shapes are correct
Xtrain, ytrain = sample_train(count=40)
Xtest, ytest = sample_test(count=10)
with pytest.raises(ValueError):
#Xtrain has the wrong shape (that is why there is a value error)
lin_svm = LinearSVM()
lin_svm.train(Xtrain, ytrain)
def test_SVM_train_1(sample_train, sample_test):
#this test is designed to verify that input shapes are correct
Xtrain, ytrain = sample_train(count=40)
Xtest, ytest = sample_test(count=10)
Xtrain = np.reshape(Xtrain, (Xtrain.shape[0], -1))
with pytest.raises(ValueError):
lin_svm = LinearSVM()
lin_svm.train(Xtrain, Xtrain)
def test_LinearSVM_train(sample_train, num_iters=200,
learning_rate=1e-7, regularization=5e3):
Xtrain, ytrain = sample_train()
Xtrain = np.reshape(Xtrain, (Xtrain.shape[0], -1))
Xtrain = np.hstack([Xtrain, np.ones((Xtrain.shape[0], 1))])
svm = LinearSVM()
loss_hist = svm.train(Xtrain, ytrain, learning_rate=learning_rate,
reg=regularization, num_iters=num_iters, verbose=False)
assert loss_hist[-1] < loss_hist[0]
def test_LinearSVM_train_no_reshape(sample_train, num_iters=200,
learning_rate=1e-7, regularization=5e3):
Xtrain, ytrain = sample_train()
svm = LinearSVM()
#raises a value error because Xtrain is not of the correct input shape
# (num_train, num_features)
with pytest.raises(ValueError):
loss_hist = svm.train(Xtrain, ytrain, learning_rate=learning_rate,
reg=regularization, num_iters=num_iters, verbose=False)
def test_SVM_train_1(sample_train, sample_test):
#this test is designed to verify that input shapes are correct
Xtrain, ytrain = sample_train(count=40)
Xtest, ytest = sample_test(count=10)
Xtrain = np.reshape(Xtrain, (Xtrain.shape[0], -1))
with pytest.raises(ValueError):
lin_svm = LinearSVM()
lin_svm.train(Xtrain, Xtrain)
def test_SVM_train_2(sample_train, sample_test):
#this test is designed to verify that input shapes are correct
Xtrain, ytrain = sample_train(count=40)
Xtest, ytest = sample_test(count=10)
Xtrain = np.reshape(Xtrain, (Xtrain.shape[0], -1))
with pytest.raises(ValueError):
#this will catcha ValueError associated with bad unpacking of a tuple
lin_svm = LinearSVM()
lin_svm.train(ytrain, ytrain)
def test_SVM_train_2(sample_train, sample_test):
#this test is designed to verify that input shapes are correct
Xtrain, ytrain = sample_train(count=40)
Xtest, ytest = sample_test(count=10)
Xtrain = np.reshape(Xtrain, (Xtrain.shape[0], -1))
with pytest.raises(ValueError):
#this will catcha ValueError associated with bad unpacking of a tuple
lin_svm = LinearSVM()
lin_svm.train(ytrain, ytrain)
def test_LinearSVM_train_no_reshape(sample_train,
num_iters=200,
learning_rate=1e-7,
regularization=5e3):
Xtrain, ytrain = sample_train()
svm = LinearSVM()
#raises a value error because Xtrain is not of the correct input shape
# (num_train, num_features)
with pytest.raises(ValueError):
loss_hist = svm.train(Xtrain,
ytrain,
learning_rate=learning_rate,
reg=regularization,
num_iters=num_iters,
verbose=False)
def test_LinearSVM_train(sample_train,
num_iters=200,
learning_rate=1e-7,
regularization=5e3):
Xtrain, ytrain = sample_train()
Xtrain = np.reshape(Xtrain, (Xtrain.shape[0], -1))
Xtrain = np.hstack([Xtrain, np.ones((Xtrain.shape[0], 1))])
svm = LinearSVM()
loss_hist = svm.train(Xtrain,
ytrain,
learning_rate=learning_rate,
reg=regularization,
num_iters=num_iters,
verbose=False)
assert loss_hist[-1] < loss_hist[0]
from cs231n.classifiers.linear_classifier import LinearSVM
results = {}
best_val = -1
best_svm = None
# pass
learning_rates = [5e-9, 7.5e-9, 1e-8]
regularization_strengths = [(5+i)*1e6 for i in range(-3, 4)]
for lr in learning_rates:
for rs in regularization_strengths:
svm = LinearSVM()
loss_hist = svm.train(X_train_feats, y_train, lr, rs, num_iters=1000, verbose=False)
y_train_pred = svm.predict(X_train_feats)
train_accuracy = cal_accuracy(y_train, y_train_pred)
y_val_pred = svm.predict(X_val_feats)
val_accuracy = cal_accuracy(y_val, y_val_pred)
results[(lr, rs)] = (train_accuracy, val_accuracy)
if best_val < val_accuracy:
best_val = val_accuracy
best_svm = svm
for lr, rs in sorted(results):
train_accuracy, val_accuracy = results[(lr, rs)]
print('lr %e rs %e train accuracy: %f val accuracy: %f' % (
lr, rs, train_accuracy, val_accuracy))
# In[5]:
# Use the validation set to tune the learning rate and regularization strength
from cs231n.classifiers.linear_classifier import LinearSVM
learning_rates = [1e-9, 1e-8, 1e-7]
regularization_strengths = [5e4, 5e5, 5e6]
results = {}
best_val = -1
best_svm = None
for learnRate in learning_rates:
for reg in regularization_strengths:
svm = LinearSVM()
loss_hist = svm.train(X_train_feats,
y_train,
learning_rate=learnRate,
reg=reg,
num_iters=1500,
verbose=True)
y_train_pred = svm.predict(X_train_feats)
y_val_pred = svm.predict(X_val_feats)
train_accuracy = np.mean(y_train == y_train_pred)
val_accuracy = np.mean(y_val == y_val_pred)
results[(learnRate, reg)] = (train_accuracy, val_accuracy)
if val_accuracy > best_val:
best_val = val_accuracy
# Use the validation set to set the learning rate and regularization strength. #
# This should be identical to the validation that you did for the SVM; save #
# the best trained classifer in best_svm. You might also want to play #
# with different numbers of bins in the color histogram. If you are careful #
# you should be able to get accuracy of near 0.44 on the validation set. #
################################################################################
# *****START OF YOUR CODE (DO NOT DELETE/MODIFY THIS LINE)*****
#learning_range = np.arange(learning_rates[0], learning_rates[1], (learning_rates[1]-learning_rates[0])/20)
#reg_range = np.arange(regularization_strengths[0], regularization_strengths[1], (regularization_strengths[1]-regularization_strengths[0])/20)
num_iters = 600 # start with a small number
for my_lr in learning_rates:
for my_reg in regularization_strengths:
svm = LinearSVM()
loss_hist = svm.train(X_train_feats,
y_train,
learning_rate=my_lr,
reg=my_reg,
num_iters=num_iters,
verbose=True)
y_train_pred = svm.predict(X_train_feats)
y_val_pred = svm.predict(X_val_feats)
train_acc = np.mean(y_train == y_train_pred)
val_acc = np.mean(y_val == y_val_pred)
results.update({(my_lr, my_reg): (train_acc, val_acc)})
if val_acc > best_val:
X_train_feats = np.hstack([X_train_feats, np.ones((X_train_feats.shape[0], 1))])
X_val_feats = np.hstack([X_val_feats, np.ones((X_val_feats.shape[0], 1))])
X_test_feats = np.hstack([X_test_feats, np.ones((X_test_feats.shape[0], 1))])
# Use the validation set to tune the learning rate and regularization strength
from cs231n.classifiers.linear_classifier import LinearSVM
learning_rates = [1e-9, 1e-8, 1e-7]
regularization_strengths = [1e5, 1e6, 1e7]
results = {}
best_val = -1
best_svm = None
svm=LinearSVM()
################################################################################
# TODO: #
# Use the validation set to set the learning rate and regularization strength. #
# This should be identical to the validation that you did for the SVM; save #
# the best trained classifer in best_svm. You might also want to play #
# with different numbers of bins in the color histogram. If you are careful #
# you should be able to get accuracy of near 0.44 on the validation set. #
################################################################################
for lr in learning_rates:
for reg in regularization_strengths:
loss_hist = svm.train(X_train_feats, y_train, learning_rate=lr, reg=reg,
num_iters=2000, verbose=True)
y_val_pred = svm.predict(X_val_feats)
y_train_pred = svm.predict(X_train_feats)
tmp_train_accuracy=np.mean(y_train == y_train_pred)
tic = time.time()
_, grad_vectorized = svm_loss_vectorized(W, X_dev, y_dev, 0.00001)
toc = time.time()
print 'Vectorized loss and gradient: computed in %fs' % (toc - tic)
# The loss is a single number, so it is easy to compare the values computed
# by the two implementations. The gradient on the other hand is a matrix, so
# we use the Frobenius norm to compare them.
difference = np.linalg.norm(grad_naive - grad_vectorized, ord='fro')
print 'difference: %f' % difference
# In the file linear_classifier.py, implement SGD in the function
# LinearClassifier.train() and then run it with the code below.
from cs231n.classifiers.linear_classifier import LinearSVM
svm = LinearSVM()
tic = time.time()
loss_hist = svm.train(X_train,
y_train,
learning_rate=1e-7,
reg=5e4,
num_iters=1500,
verbose=True)
toc = time.time()
print 'That took %fs' % (toc - tic)
# A useful debugging strategy is to plot the loss as a function of
# iteration number:
plt.plot(loss_hist)
plt.xlabel('Iteration number')
plt.ylabel('Loss value')
# 预处理：多加一个bias列
X_train_feats = np.hstack([X_train_feats, np.ones((X_train_feats.shape[0], 1))])
X_val_feats = np.hstack([X_val_feats, np.ones((X_val_feats.shape[0], 1))])
X_test_feats = np.hstack([X_test_feats, np.ones((X_test_feats.shape[0], 1))])
# 用多分类SVM来对抽取的特征进行训练。
from cs231n.classifiers.linear_classifier import LinearSVM
results = {}
best_val = -1
best_svm = None
learning_rate = [1e-10, 5e-10, 5e-9, 7.5e-9, 5e-8, 1e-8, 1.25e-7]
regularization_strengths = [(5 + i) * 1e6 for i in range(-5, 5)]
for rs in regularization_strengths:
for lr in learning_rate:
svm = LinearSVM()
lost_hist = svm.train(X_train_feats, y_train, lr, rs, num_iters=1000)
y_train_pred = svm.predict(X_train_feats)
train_accuracy = np.mean(y_train == y_train_pred)
y_val_pred = svm.predict(X_val_feats)
val_accuracy = np.mean(y_val == y_val_pred)
if val_accuracy > best_val:
best_val = val_accuracy
best_svm = svm
results[(lr, rs)] = train_accuracy, val_accuracy
for lr, reg in sorted(results):
train_accuracy, val_accuracy = results[(lr, reg)]
print('lr %e reg %e train accuracy: %f val accuracy: %f' % (lr, reg, train_accuracy, val_accuracy))
print('best validation accuracy achieved during cross-validation: %f' % best_val)
if (tune_SVM != 0):
################################################################################
# TODO: #
# Use the validation set to set the learning rate and regularization strength. #
# This should be identical to the validation that you did for the SVM; save #
# the best trained classifer in best_svm. You might also want to play #
# with different numbers of bins in the color histogram. If you are careful #
# you should be able to get accuracy of near 0.44 on the validation set. #
################################################################################
print '-------------_*****--------------'
print 'Tuning Parameters for SVM: '
for lr in np.arange(0.0001, 0.001,
0.0001): #(0.0000001, 0.000001, 0.0000001)
for reg in np.arange(0.1, 1, 0.1): # (1e4, 1e5, 20000)
print 'LR: %f and REG: %d' % (lr, reg)
svm_iter = LinearSVM()
svm_iter.train(X_train_feats,
y_train,
learning_rate=lr,
reg=reg,
num_iters=3000,
verbose=True)
y_val_pred_iter = svm_iter.predict(X_val_feats)
y_train_pred_iter = svm_iter.predict(X_train_feats)
val_acc = np.mean(y_val == y_val_pred_iter)
train_acc = np.mean(y_train == y_train_pred_iter)
results[(lr, reg)] = (train_acc, val_acc) # Turple Mapping
print 'Validation accuracy: %f' % (val_acc)
if (val_acc > best_val):
best_val = val_acc
best_svm = svm_iter
tic = time.time()
_, grad_vectorized = svm_loss_vectorized(W, X_dev, Y_dev, 0.000005)
toc = time.time()
print('Vectorized loss and gradient: computed in %fs' % (toc - tic))
# The loss is a single number, so it is easy to compare the values computed
# by the two implementations. The gradient on the other hand is a matrix, so
# we use the Frobenius norm to compare them.
difference = np.linalg.norm(grad_naive - grad_vectorized, ord='fro')
print('difference: %f' % difference)
# In the file linear_classifier.py, implement SGD in the function
# LinearClassifier.train() and then run it with the code below.
from cs231n.classifiers.linear_classifier import LinearSVM
svm = LinearSVM()
tic = time.time()
loss_hist = svm.train(X_train,
Y_train,
learning_rate=1e-7,
reg=2.5e4,
num_iters=1500,
verbose=True)
toc = time.time()
print('That took %fs' % (toc - tic))
# A useful debugging strategy is to plot the loss as a function of
# iteration number:
plt.plot(loss_hist)
plt.xlabel('Iteration number')
plt.ylabel('Loss value')
best_val = -1
best_svm = None
pass
################################################################################
# TODO: #
# Use the validation set to set the learning rate and regularization strength. #
# This should be identical to the validation that you did for the SVM; save #
# the best trained classifer in best_svm. You might also want to play #
# with different numbers of bins in the color histogram. If you are careful #
# you should be able to get accuracy of near 0.44 on the validation set. #
################################################################################
#pass
for _l in learning_rates:
for _r in regularization_strengths:
svm = LinearSVM()
loss_hist = svm.train(X_train_feats,
y_train,
learning_rate=_l,
reg=_r,
num_iters=1500,
verbose=False)
y_train_pred = svm.predict(X_train_feats)
train_accuracy = np.mean(y_train == y_train_pred)
print('training accuracy: {0}'.format(train_accuracy))
y_val_pred = svm.predict(X_val_feats)
val_accuracy = np.mean(y_val == y_val_pred)
print('validation accuracy: {0}'.format(val_accuracy))
best_val = -1
best_svm = None
pass
################################################################################
# TODO: #
# Use the validation set to set the learning rate and regularization strength. #
# This should be identical to the validation that you did for the SVM; save #
# the best trained classifer in best_svm. You might also want to play #
# with different numbers of bins in the color histogram. If you are careful #
# you should be able to get accuracy of near 0.44 on the validation set. #
################################################################################
#pass
for lr in learning_rates:
for rs in regularization_strengths:
svm = LinearSVM()
svm.train(X_train_feats, y_train, lr, rs, num_iters=6000)
y_train_pred = svm.predict(X_train_feats)
train_accuracy = np.mean(y_train == y_train_pred)
y_val_pred = svm.predict(X_val_feats)
val_accuracy = np.mean(y_val == y_val_pred)
if val_accuracy > best_val:
best_val = val_accuracy
best_svm = svm
results[(lr, rs)] = train_accuracy, val_accuracy
################################################################################
# END OF YOUR CODE #
################################################################################
# Print out results.
for lr, reg in sorted(results):
def svm_sgd(X_train, y_train, X_val, y_val, X_test, y_test):
#svm = LinearSVM()
#tic = time.time()
#loss_hist = svm.train(X_train, y_train, learning_rate=1e-7, reg=5e-4,num_iters=1500,verbose=True)
#toc = time.time()
#print 'That took %fs' % (toc-tic)
#plt.plot(loss_hist)
#plt.xlabel('Iteration number')
#plt.ylabel('Loss value')
#plt.savefig('figures/svm_loss_iteration.png')
#y_train_pred = svm.predict(X_train)
#print 'training accuracy: %f' %(np.mean(y_train == y_train_pred),)
#y_val_pred = svm.predict(X_val)
#print 'validation accuracy: %f' %(np.mean(y_val == y_val_pred),)
learning_rates = [1e-7, 1e-6, 5e-5]
regs = [5e-4, 5e-5, 1e-5]
results = {}
best_val = -1
bets_svm = None
for l in learning_rates:
for r in regs:
svm = LinearSVM()
loss_hist = svm.train(X_train,
y_train,
learning_rate=l,
reg=r,
num_iters=500,
verbose=True)
y_train_pred = svm.predict(X_train)
train_acc = np.mean(y_train == y_train_pred)
y_val_pred = svm.predict(X_val)
val_acc = np.mean(y_val == y_val_pred)
if best_val < val_acc:
best_val = val_acc
best_svm = svm
results[(l, r)] = (train_acc, val_acc)
for lr, reg in sorted(results):
train_accuracy, val_accuracy = results[(lr, reg)]
print 'lr %e reg %e train accuracy: %f val accuracy: %f' % (
lr, reg, train_accuracy, val_accuracy)
return
import math
x_scatter = [math.log10(x[0]) for x in results]
y_scatter = [math.log10(x[1]) for x in results]
# plot training accuracy
sz = [results[x][0] * 1500
for x in results] # default size of markers is 20
plt.subplot(1, 2, 1)
plt.scatter(x_scatter, y_scatter, sz)
plt.xlabel('log learning rate')
plt.ylabel('log regularization strength')
plt.title('CIFAR-10 training accuracy')
# plot validation accuracy
sz = [results[x][1] * 1500
for x in results] # default size of markers is 20
plt.subplot(1, 2, 2)
plt.scatter(x_scatter, y_scatter, sz)
plt.xlabel('log learning rate')
plt.ylabel('log regularization strength')
plt.title('CIFAR-10 validation accuracy')
print 'best validation accuracy achieved during cross-validation: %f' % best_val
plt.savefig('./figures/learn_rate_reg_SVM.png')
plt.close()
# Evaluate the best svm on test set
y_test_pred = best_svm.predict(X_test)
test_accuracy = np.mean(y_test == y_test_pred)
print 'linear SVM on raw pixels final test set accuracy: %f' % test_accuracy
w = best_svm.W[:, :-1] # strip out the bias
w = w.reshape(10, 32, 32, 3)
w_min, w_max = np.min(w), np.max(w)
classes = [
'plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship',
'truck'
]
for i in xrange(10):
plt.subplot(2, 5, i + 1)
# Rescale the weights to be between 0 and 255
wimg = 255.0 * (w[i].squeeze() - w_min) / (w_max - w_min)
plt.imshow(wimg.astype('uint8'))
plt.axis('off')
plt.title(classes[i])
plt.savefig('./figures/visulize_Wg_SVM.png')
################################################################################
# Write code that chooses the best hyper-parameters by tuning on the validation #
# set. For each combination of hyper-parameters, train a linear SVM on the #
# training set, compute its accuracy on the training and validation sets, and #
# store these numbers in the results dictionary. In addition, store the best #
# validation accuracy in best_val and the LinearSVM object that achieves this #
# accuracy in best_svm. #
# #
# Hint: You should use a small value for num_iters as you develop your #
# validation code so that the SVMs don't take much time to train; once you are #
# confident that your validation code works, you should rerun the validation #
# code with a larger value for num_iters. #
################################################################################
for rate in learning_rates:
for regularization in regularization_strengths:
svm = LinearSVM()
lost_history = svm.train(x_train, y_train, rate, regularization,
2000)
y_train_pred = svm.predict(x_train)
accuracy_train = np.mean(y_train == y_train_pred)
y_val_pred = svm.predict(x_val)
accuracy_val = np.mean(y_val == y_val_pred)
results[(rate, regularization)] = (accuracy_train, accuracy_val)
if best_val < accuracy_val:
best_val = accuracy_val
best_svm = svm
################################################################################
# END OF YOUR CODE #
################################################################################
# 打印结果
for lr, reg in sorted(results):
best_val = -1
best_svm = None
################################################################################
# TODO: #
# Use the validation set to set the learning rate and regularization strength. #
# This should be identical to the validation that you did for the SVM; save #
# the best trained classifer in best_svm. You might also want to play #
# with different numbers of bins in the color histogram. If you are careful #
# you should be able to get accuracy of near 0.44 on the validation set. #
################################################################################
from cs231n.classifiers import LinearSVM
for lr in learning_rates:
for reg in regularization_strengths:
svm = LinearSVM()
svm.train(X_train_feats,
y_train,
learning_rate=lr,
reg=reg,
num_iters=1000,
verbose=False)
y_train_pred = svm.predict(X_train_feats)
train_accuracy = np.mean(y_train == y_train_pred)
y_val_pred = svm.predict(X_val_feats)
val_accuracy = np.mean(y_val == y_val_pred)
results[(lr, reg)] = (train_accuracy, val_accuracy)
if best_val < val_accuracy:
best_val = val_accuracy
best_svm = svm
################################################################################
# print 'Naive loss and gradient: computed in %fs' % (toc - tic)
# tic = time.time()
# _, grad_vectorized = svm_loss_vectorized(W, X_train, y_train, 0.00001)
# toc = time.time()
# print 'Vectorized loss and gradient: computed in %fs' % (toc - tic)
# The loss is a single number, so it is easy to compare the values computed
# by the two implementations. The gradient on the other hand is a matrix, so
# we use the Frobenius norm to compare them.
# difference = np.linalg.norm(grad_naive - grad_vectorized, ord='fro')
# print 'difference: %f' % difference
######SGD
from cs231n.classifiers.linear_classifier import LinearSVM
svm = LinearSVM()
tic = time.time()
loss_hist = svm.train(X_train, y_train, learning_rate=1e-7, reg=5e4,
num_iters=1500, verbose=True)
toc = time.time()
print 'That took %fs' % (toc - tic)
# A useful debugging strategy is to plot the loss as a function of
# iteration number:
plt.plot(loss_hist)
plt.xlabel('Iteration number')
plt.ylabel('Loss value')
# plt.show()
# Write the LinearSVM.predict function and evaluate the performance on both the
# training and validation set
y_train_pred = svm.predict(X_train)
loss, grad = svm_loss_naive(W, X_dev, y_dev, 0)
# # Numerically compute the gradient along several randomly chosen dimensions
# # and compare with analytically computed gradient (grad)
# f = lambda w: svm_loss_naive(w, X_dev, y_dev, 0.0)[0] # Returns the loss
# grad_numerical = grad_check_sparse(f, W, grad)
# # Again with the regularization turned on
# loss, grad = svm_loss_naive(W, X_dev, y_dev, 5e1)
# f = lambda w: svm_loss_naive(w, X_dev, y_dev, 5e1)[0] # Returns the loss
# grad_numerical = grad_check_sparse(f, W, grad)
''' Evaluate vectorized implementation of loss '''
loss_v, grad_v = svm_loss_vectorized(W, X_dev, y_dev, 0)
print("Gradient difference", np.linalg.norm(grad - grad_v))
print("Loss difference", loss - loss_v)
''' Implement Stochastic Gradient Descent to minimize loss '''
svm = LinearSVM()
tic = time.time()
# Get list of loss history over training and visualize
loss_hist = svm.train(X_train,
y_train,
learning_rate=1e-7,
reg=2.5e4,
num_iters=1500,
verbose=True)
toc = time.time()
print("Time", (toc - tic))
plt.plot(loss_hist)
plt.xlabel('Iteration number')
plt.ylabel('Loss value')
plt.show()
''' Generate predictions on test and validation sets'''
W, X_train, y_train, 0.00001)
toc = time.time()
print('Vectorized loss:%e computed in %s' % (loss_vectorized, toc - tic))
# The losses should match but your vectorized implementation should be much faster.
print('loss difference: %f' % (loss_naive - loss_vectorized))
difference = np.linalg.norm(grad_naive - grad_vectorized, ord='fro')
print('Gradient difference: %f' % difference)
#######################################################################################
# Stochastic Gradient Descent #
#######################################################################################
# Now implement SGD in LinearSVM.train() function and run it with the code below
svm = LinearSVM()
tic = time.time()
loss_hist = svm.train(X_train,
y_train,
learning_rate=1e-7,
reg=5e4,
num_iters=1500,
verbose=True)
toc = time.time()
print('That took %fs' % (toc - tic))
# A useful debugging strategy is to plot the loss as a function of
# iteration number:
plt.plot(loss_hist)
# print 'Naive loss and gradient: computed in %fs' % (toc - tic)
# tic = time.time()
# _, grad_vectorized = svm_loss_vectorized(W, X_train, y_train, 0.00001)
# toc = time.time()
# print 'Vectorized loss and gradient: computed in %fs' % (toc - tic)
# The loss is a single number, so it is easy to compare the values computed
# by the two implementations. The gradient on the other hand is a matrix, so
# we use the Frobenius norm to compare them.
# difference = np.linalg.norm(grad_naive - grad_vectorized, ord='fro')
# print 'difference: %f' % difference
######SGD
from cs231n.classifiers.linear_classifier import LinearSVM
svm = LinearSVM()
tic = time.time()
loss_hist = svm.train(X_train,
y_train,
learning_rate=1e-7,
reg=5e4,
num_iters=1500,
verbose=True)
toc = time.time()
print 'That took %fs' % (toc - tic)
# A useful debugging strategy is to plot the loss as a function of
# iteration number:
plt.plot(loss_hist)
plt.xlabel('Iteration number')
plt.ylabel('Loss value')
results = {}
best_val = -1
best_svm = None
pass
################################################################################
# TODO: #
# Use the validation set to set the learning rate and regularization strength. #
# This should be identical to the validation that you did for the SVM; save #
# the best trained softmax classifer in best_svm. You might also want to play #
# with different numbers of bins in the color histogram. If you are careful #
# you should be able to get accuracy of near 0.44 on the validation set. #
################################################################################
for l_r_cv in learning_rates:
for r_s_cv in regularization_strengths:
svm = LinearSVM()
svm.train(X_train_feats, y_train, learning_rate=l_r_cv, reg=r_s_cv,
num_iters=1500, verbose=True)
y_train_pred = svm.predict(X_train_feats)
t_ac= np.mean(y_train == y_train_pred)
y_val_pred = svm.predict(X_val_feats)
v_ac = np.mean(y_val == y_val_pred)
if v_ac > best_val:
best_val = v_ac
best_svm = svm
results[(l_r_cv,r_s_cv)]=(t_ac,v_ac)
################################################################################
# END OF YOUR CODE #
################################################################################
# Print out results.
best_svm = None
################################################################################
# TODO: #
# Use the validation set to set the learning rate and regularization strength. #
# This should be identical to the validation that you did for the SVM; save #
# the best trained classifer in best_svm. You might also want to play #
# with different numbers of bins in the color histogram. If you are careful #
# you should be able to get accuracy of near 0.44 on the validation set. #
################################################################################
for i in range(len(learning_rates)):
for j in range(len(regularization_strengths)):
print(
'Initializing SVM with Learning Rate = %f and Regularization Strength = %f'
% (learning_rates[i], regularization_strengths[j]))
svm = LinearSVM()
loss_hist = svm.train(X_train_feats,
y_train,
learning_rates[i],
regularization_strengths[j],
num_iters=1500,
verbose=True)
y_train_pred = svm.predict(X_train_feats)
train_acc = np.mean(y_train == y_train_pred)
y_val_pred = svm.predict(X_val_feats)
val_acc = np.mean(y_val == y_val_pred)
results[(learning_rates[i], regularization_strengths[j])] = (train_acc,
val_acc)
if val_acc > best_val:
best_val = val_acc
best_svm = svm
###################################################################################
# Use the validation set to tune the learning rate and regularization strength
from cs231n.classifiers.linear_classifier import LinearSVM
learning_rates = [1e-7, 1.5e-7]
regularization_strengths = [3e5, 5e5, 8e5]
results = {}
best_val = -1
best_svm = None
for learning_rate in learning_rates:
for reg in regularization_strengths:
model = LinearSVM()
losses = model.train(X_train_feats, y_train, learning_rate=learning_rate,
reg=reg, verbose=False)
train_result = (model.predict(X_train_feats)==y_train).mean()
val_result = (model.predict(X_val_feats)==y_val).mean()
results[(learning_rate, reg)] = [train_result, val_result]
print('Learning rate: %f, Regularization: %f, Validation acc: %f' %(learning_rate, reg, val_result))
if val_result > best_val:
best_val=val_result
best_svm=model
################################################################################
# TODO: #
# Use the validation set to set the learning rate and regularization strength. #
# This should be identical to the validation that you did for the SVM; save #
# the best trained classifer in best_svm. You might also want to play #
# with different numbers of bins in the color histogram. If you are careful #
results = {}
best_val = -1
best_svm = None
################################################################################
# TODO: #
# Use the validation set to set the learning rate and regularization strength. #
# This should be identical to the validation that you did for the SVM; save #
# the best trained classifer in best_svm. You might also want to play #
# with different numbers of bins in the color histogram. If you are careful #
# you should be able to get accuracy of near 0.44 on the validation set. #
################################################################################
for rate in learning_rates:
for strength in regularization_strengths:
svm = LinearSVM()
svm.train(X_train_feats,
y_train,
learning_rate=rate,
reg=strength,
num_iters=1500,
verbose=False)
learning_accuracy = np.mean(svm.predict(X_train_feats) == y_train)
validation_accuracy = np.mean(svm.predict(X_val_feats) == y_val)
if validation_accuracy > best_val:
best_val = validation_accuracy
best_svm = svm
results[(rate, strength)] = (learning_accuracy, validation_accuracy)
################################################################################
# END OF YOUR CODE #
################################################################################
#%%
# Use the validation set to tune the learning rate and regularization strength
from cs231n.classifiers.linear_classifier import LinearSVM
learning_rates = [2e-6, 7e-6, 2e-7, 5e-7]
regularization_strengths = [1e4, 8e4, 2e5]
results = {}
best_val = -1
best_svm = None
for lr in learning_rates:
for reg in regularization_strengths:
svm = LinearSVM()
svm.train(X_train_feats,
y_train,
lr,
reg,
num_iters=1000,
verbose=True)
y_train_pred = svm.predict(X_train_feats)
y_train_acc = np.mean(y_train == y_train_pred)
y_val_pred = svm.predict(X_val_feats)
y_val_acc = np.mean(y_val == y_val_pred)
results[lr, reg] = (y_train_acc, y_val_acc)
cur_val = y_val_acc
if cur_val > best_val:
best_val = cur_val
X_train_feats = np.hstack([X_train_feats, np.ones((X_train_feats.shape[0], 1))])
X_val_feats = np.hstack([X_val_feats, np.ones((X_val_feats.shape[0], 1))])
X_test_feats = np.hstack([X_test_feats, np.ones((X_test_feats.shape[0], 1))])
from cs231n.classifiers.linear_classifier import LinearSVM
learning_rates = [1e-8,3e-8,5e-8,7e-8,9e-8]
regularization_strengths = [1e5,3e5,5e5,7e5,9e5]
results = {}
best_val = -1
best_svm = None
for lr in learning_rates:
for reg in regularization_strengths:
svm = LinearSVM()
svm.train(X_train_feats, y_train, learning_rate=lr, reg=reg, num_iters=6000)
train_pred = svm.predict(X_train_feats)
val_pred = svm.predict(X_val_feats)
train_accuracy = np.mean(train_pred == y_train)
val_accuracy = np.mean(val_pred == y_val)
if val_accuracy > best_val:
best_val = val_accuracy
best_svm = svm
results[(lr, reg)] = (train_accuracy, val_accuracy)
for lr, reg in sorted(results):
train_accuracy, val_accuracy = results[(lr, reg)]
results = {}
best_val = -1
best_svm = None
pass
################################################################################
# TODO: #
# Use the validation set to set the learning rate and regularization strength. #
# This should be identical to the validation that you did for the SVM; save #
# the best trained classifer in best_svm. You might also want to play #
# with different numbers of bins in the color histogram. If you are careful #
# you should be able to get accuracy of near 0.44 on the validation set. #
################################################################################
for l in learning_rates:
for r in regularization_strengths:
svm = LinearSVM()
svm.train(X_train_f,
y_train,
learning_rate=l,
reg=r,
num_iters=1500,
batch_size=200)
y_train_pred = svm.predict(X_train_f)
y_val_pred = svm.predict(X_val_f)
training_accuracy = np.mean(y_train == y_train_pred)
validation_accuracy = np.mean(y_val == y_val_pred)
results[(l, r)] = (training_accuracy, validation_accuracy)
if validation_accuracy > best_val:
best_val = validation_accuracy
best_svm = svm
################################################################################
