def pick_hyperparams(X_train, y_train, X_val, y_val, learning_rates, regularization_strengths, iterations=4000, batches=400):
results = {}
best_val = -1
best_softmax = None
################################################################################
# TODO: #
# Use the validation set to set the learning rate and regularization strength. #
# Save the best trained softmax classifer in best_softmax. #
# Hint: about 10 lines of code expected
################################################################################
for lr in learning_rates:
for reg in regularization_strengths:
print lr, reg
softmax = SoftmaxClassifier()
softmax.train(X_train, y_train, lr, reg, iterations, batches)
predy = softmax.predict(X_val)
val = np.mean(predy == y_val)
predtrain = softmax.predict(X_train)
valtrain = np.mean(predtrain == y_val)
results[(lr, reg)] = (valtrain, val)
if val > best_val:
best_val = val
best_softmax = softmax
return best_softmax, results, best_val
# TODO: Use the validation set to tune hyperparameters for softmax classifier
# choose learning rate and regularization strength (use the code from softmax_hw.py)
<<<<<<< HEAD
results = {}
best_val = -1
best_softmax = None
learning_rates = [1e-4,1e-3,1e-2,1e-1]
regularization_strengths = [0.01,0.05,0.1,0.5,1]
for lr in learning_rates:
for rs in regularization_strengths:
print("calculating: lr=%e,reg=%e"%(lr,rs))
ns=SoftmaxClassifier()
ns.train(X_train,y_train,lr,rs,batch_size=400,num_iters=2000)
ta=np.mean(y_train == ns.predict(X_train))
va=np.mean(y_val == ns.predict(X_val))
results[lr,rs]=(ta,va)
if va>best_val:
best_val=va
best_softmax=ns
# TODO: Evaluate best softmax classifier on set aside test set (use the code from softmax_hw.py)
# Print out results.
for lr, reg in sorted(results):
train_accuracy, val_accuracy = results[(lr, reg)]
print 'lr %e reg %e train accuracy: %f val accuracy: %f' % (
lr, reg, train_accuracy, val_accuracy)
import numpy as np
from data import make_spiral_data, plot_2d_data
from softmax import SoftmaxClassifier
if __name__ == '__main__':
# generate spiral data
n_classes = 3
data, y = make_spiral_data(100, n_classes, 2)
fig = plot_2d_data(data, y)
fig.show()
# train model
model = SoftmaxClassifier(n_classes)
model.fit(data, y)
print("Training accuracy is {:0.2f}".format(model.training_accuracy))
print("Training loss is {:0.3f}".format(model.training_loss))
boundaries = model.plot_boundaries(data, y)
boundaries.show()
loss_vs_epoch = model.plot_training_loss()
loss_vs_epoch.show()
# test model against 'new' data
new_data, new_y = make_spiral_data(50, n_classes, 2)
predictions = model.predict(new_data)
print("Test accuracy is {:0.2f}".format(model.accuracy(new_data, new_y)))
# choose learning rate and regularization strength (use the code from softmax_hw.py)
batch_sizes = [200, 300, 400]
iterations = [1000,2000,3000]
learning_rates = [5e-7, 1e-6, 5e-6]
regularization_strengths = [1e2, 1e3,1e4, 1e5]
best_val = -1
best_softmax = None
for batch_idx, batch_size in enumerate(batch_sizes):
for it_idx, iteration in enumerate(iterations):
for learningrate in learning_rates:
for regularization in regularization_strengths:
softmax=SoftmaxClassifier()
softmax.train(X_train,y_train,learningrate,reg=regularization, num_iters=iteration,batch_size=batch_size, verbose=True)
y_pred_val=softmax.predict(X_val)
current_val = np.mean(y_pred_val==y_val)
if(current_val>best_val):
best_val = current_val
best_softmax = softmax
best_learning_rate = learningrate
best_reg = regularization
best_iteration = iteration
best_batch_size = batch_size
print "best batch size is ", best_batch_size
print "best iteration is ", best_iteration
print "best reg is ", best_reg
print "best learning rate is ", best_learning_rate
#regularization_strengths = [ 5e4, 1e5, 5e5, 1e8]
learning_rates = [5e-6]
regularization_strengths = [1e5]
cl = SoftmaxClassifier()
loss_hist = []
for lr, rs in itertools.product(learning_rates, regularization_strengths):
_ = cl.train(X_train,
y_train,
lr,
rs,
num_iters=4000,
batch_size=400,
verbose=True)
loss, _ = cl.loss(X_val, y_val, rs)
pred_t = cl.predict(X_train)
pred_v = cl.predict(X_val)
#embed()
train_match = np.where(pred_t == y_train)
train_accuracy = float(len(train_match[0])) / len(y_train)
val_match = np.where(pred_v == y_val)
val_accuracy = float(len(val_match[0])) / len(y_val)
results[(lr, rs)] = (train_accuracy, val_accuracy)
loss_hist.append(loss)
# print("For lr,rs = ",lr,rs, "Loss value = ",loss)
#embed()
ind = np.where(loss_hist == np.min(loss_hist))[0][0]
parameters = list(itertools.product(learning_rates,
regularization_strengths))[ind]
best_softmax = SoftmaxClassifier()
# TODO: Split into train, validation and test sets
X_train, X_test, y_train, y_test = cross_validation.train_test_split(X, y, test_size=0.2)
X_train, X_val, y_train, y_val = cross_validation.train_test_split(X_train, y_train, test_size=0.1)
# TODO: Use the validation set to tune hyperparameters for softmax classifier
# choose learning rate and regularization strength (use the code from softmax_hw.py)
results = {}
best_val = -1
best_params = None
best_softmax = SoftmaxClassifier()
learning_rates = [0.005]
regularization_strengths = [1]
classifier = SoftmaxClassifier()
accuracy = lambda x, y: np.mean(classifier.predict(x) == y)
for lr in learning_rates:
for reg in regularization_strengths:
classifier.train(X_train, y_train, learning_rate=lr, reg=reg, num_iters=10000, batch_size=y_train.size / 4)
val_accuracy = accuracy(X_val, y_val)
train_accuracy = accuracy(X_train, y_train)
results[(lr, reg)] = (train_accuracy, val_accuracy)
print "lr %e reg %e train accuracy: %f val accuracy: %f" % (lr, reg, train_accuracy, val_accuracy)
if val_accuracy > best_val:
best_softmax.theta = classifier.theta
best_val = val_accuracy
best_params = (lr, reg)
#best_params, best_classifier = utils.getBestRegOVA(classify, X_train, y_train, X_val, y_val, regularization_strengths, pen='l1') """ #best LR = [15] learning_rates = [15] # [11,12,13,14,15,16,17,18,19,20] #Best RS = [1e-3] regularization_strengths = [2e-3] #[1e-3,2e-3,3e-3,4e-3,5e-3,6e-3,7e-3] classify = SoftmaxClassifier() # takes classifier, X_train, y_train, X_val, y_val, learning_rates, regularization_strengths and optionally print_train and print_val best_params, best_classifier = utils.getBestRegAndLearnSoftMax(classify, X_train, y_train, X_val, y_val, learning_rates, regularization_strengths) # end """ print "\nTraining the classifier..." best_classifier = SoftmaxClassifier() best_classifier.train(X_train, y_train, reg=1e1, learning_rate=15) print np.mean(best_classifier.predict(X_val) == y_val) print theta.shape hogimg = best_classifier.theta[1025:1537,:].reshape(8,8,8,10) utils.visualizeHOGTheta(hogimg) print "\nMaking the final prediction..." sys.stdout.flush() labels = [] ids = [] batch = 50000 for j in range(3, 6): print "\nPart ", j + 1, " of 3" sys.stdout.flush() name = "batch_" + str(j) + ".npz"
print 'Gradient difference: %f' % grad_difference
results = {}
best_val = -1
best_softmax = None
#learning_rates = [1e-7, 5e-7, 1e-6, 5e-6]
#regularization_strengths = [ 5e4, 1e5, 5e5, 1e8]
learning_rates = [ 5e-6]
regularization_strengths = [1e5]
cl = SoftmaxClassifier()
loss_hist = []
for lr,rs in itertools.product(learning_rates,regularization_strengths):
_ = cl.train(X_train, y_train, lr, rs, num_iters = 4000, batch_size = 400, verbose = True)
loss,_ = cl.loss(X_val,y_val, rs)
pred_t = cl.predict(X_train)
pred_v = cl.predict(X_val)
#embed()
train_match = np.where(pred_t == y_train)
train_accuracy = float(len(train_match[0]))/len(y_train)
val_match = np.where(pred_v == y_val)
val_accuracy = float(len(val_match[0]))/len(y_val)
results[(lr,rs)] = (train_accuracy,val_accuracy)
loss_hist.append(loss)
# print("For lr,rs = ",lr,rs, "Loss value = ",loss)
#embed()
ind = np.where(loss_hist == np.min(loss_hist))[0][0]
parameters = list(itertools.product(learning_rates,regularization_strengths))[ind]
best_softmax = SoftmaxClassifier()
_ = best_softmax.train(X_train, y_train, parameters[0], parameters[1], num_iters = 4000, batch_size = 400, verbose = True)
