def finding_best_net():
best_net = None
result = {}
best_val = -1
hidden_sizes = [40, 60, 80, 100, 120]
lrs = [1e-4, 5e-4, 1e-3, 5e-3]
regs = [1e-6, 5e-6, 1e-5, 5e-5]
lr_decays = [0.95, 0.99, 0.999]
input_size = 32 * 32 * 3
num_classes = 10
nets = {}
i = 0
grid_search = [(x, y, z) for x in lrs for y in regs for z in lr_decays]
for hidden_size in hidden_sizes:
for lr, reg, lr_decay in grid_search:
print('hidden {} -- lr {} -- reg {} -- lr_decay {}'.format(
hidden_size, lr, reg, lr_decay))
print('Done {:2} out of {}'.format(
i + 1,
len(grid_search) * len(hidden_sizes)))
net = TwoLayerNet(input_size, hidden_size, num_classes)
# Training
net.train(x_train,
y_train,
x_val,
y_val,
num_iters=2000,
lr=lr,
lr_decay=lr_decay,
reg=reg,
verbose=False)
# Predict
y_train_pred = net.predict(x_train)
y_val_pred = net.predict(x_val)
# Scoring
train_accu = np.mean(y_train_pred == y_train)
val_accu = np.mean(y_val_pred == y_val)
# Store results
result[(hidden_size, lr, reg, lr_decay)] = (train_accu, val_accu)
nets[(hidden_size, lr, reg, lr_decay)] = net
if val_accu > best_val:
best_val = val_accu
best_net = net
i += 1
class NeuralNetwork():
def __init__(self):
self.input_size = 28 * 28
self.hidden_size = 50
self.num_classes = 10
self.net = TwoLayerNet(self.input_size, self.hidden_size,
self.num_classes)
self.fit()
def fit(self):
minst = tf.keras.datasets.mnist
(X_train, y_train), (X_test, y_test) = minst.load_data()
num_train = 30000
num_val = 1000
# Train set
mask = list(range(num_train, num_train + num_val))
X_val = X_train[mask]
y_val = y_train[mask]
mask = list(range(num_train))
X_train = X_train[mask]
y_train = y_train[mask]
# Preprocessing: reshape the images data into rows
X_train = np.reshape(X_train, (X_train.shape[0], -1))
X_val = np.reshape(X_val, (X_val.shape[0], -1))
stats = self.net.train(X_train,
y_train,
X_val,
y_val,
num_iters=1500,
batch_size=200,
learning_rate=1e-4,
learning_rate_decay=0.95,
reg=0.25,
verbose=True)
# val_acc = (self.net.predict(X_val) == y_val).mean()
# print('Validation accuracy: ', val_acc)
# plt.subplot(2, 1, 1)
# plt.plot(stats['loss_history'])
# plt.title('Loss history')
# plt.xlabel('Iteration')
# plt.ylabel('Loss')
#
# plt.subplot(2, 1, 2)
# plt.plot(stats['train_acc_history'], label='train')
# plt.plot(stats['val_acc_history'], label='val')
# plt.title('Classification accuracy history')
# plt.xlabel('Epoch')
# plt.ylabel('Clasification accuracy')
# plt.show()
def predict(self, x):
return self.net.predict(x)
def main():
x_train = ucihar.load_data('train', 'X')
y_train = ucihar.load_data('train', 'y')
x_test = ucihar.load_data('test', 'X')
y_test = ucihar.load_data('test', 'y')
net = TwoLayerNet(input_size=ucihar.X_DIMMENSIONS,
hidden_size=50,
output_size=ucihar.Y_CLASSES)
stats = net.train(x_train,
y_train,
x_test,
y_test,
num_iters=100000,
batch_size=64,
learning_rate=1e-2,
learning_rate_decay=1.)
predictions = net.predict(x_test)
val_acc = (predictions == y_test).mean()
print('Validation accuracy: ', val_acc)
try:
from sklearn.metrics import classification_report, confusion_matrix
print(confusion_matrix(y_test, predictions))
print(classification_report(y_test, predictions))
except ImportError:
pass
try:
import matplotlib.pyplot as plt
# Plot the loss function and train / validation accuracies
plt.subplot(1, 2, 1)
plt.plot(stats['loss_history'])
plt.title('Loss history')
plt.xlabel('Iteration')
plt.ylabel('Loss')
plt.subplot(1, 2, 2)
train_plot, = plt.plot(stats['train_acc_history'], label='train')
val_plot, = plt.plot(stats['val_acc_history'], label='val')
plt.legend(handles=[train_plot, val_plot])
plt.title('Classification accuracy history')
plt.xlabel('Epoch')
plt.ylabel('Clasification accuracy')
plt.show()
except ImportError:
pass
# iteration = [5500]
iteration = [3500, 4500, 5500] # >2500
# batch = [40, 50, 60, 70, 80]
batch = [50]
bestValAcc = 0
bestNN = None
bestTrain = None
bestParams = []
for hidden in hidden_size:
for eta in learningRates:
for r in regularization:
for t in iteration:
for b in batch:
net = TwoLayerNet(input_size, hidden, NUM_CLASS)
train = net.train(X_train, y_train, X_train, y_train,\
num_iters=t, batch_size=b, learning_rate=eta, learning_rate_decay=0.95, reg=r, verbose=False)
y_train_pred = net.predict(X_train)
y_val_pred = net.predict(X_val)
trainAcc = np.mean(y_train == y_train_pred)
valAcc = np.mean(y_val == y_val_pred)
print 'learning rate: %e reg: %.2f iteration: %d batch: %d train accuracy: %.4f val accuracy: %.4f'\
% (eta, r, t, b, trainAcc, valAcc)
if valAcc > bestValAcc:
bestValAcc = valAcc
bestNN = net
bestTrain = train
bestParams = [hidden, eta, r, t, b]
print 'best validation accuracy achieved: %.4f' % bestValAcc
print bestParams
f1 = open('./res.txt', 'w+')
print 'Test labels shape: ', y_test.shape
# # Train a network
# To train our network we will use SGD with momentum. In addition, we will adjust the learning rate with an exponential learning rate schedule as optimization proceeds; after each epoch, we will reduce the learning rate by multiplying it by a decay rate.
# In[ ]:
input_size = 32 * 32 * 3
hidden_size = 50
num_classes = 10
net = TwoLayerNet(input_size, hidden_size, num_classes)
# Train the network
stats = net.train(X_train, y_train, X_val, y_val,
num_epochs=30, batch_size=500,
learning_rate=1e-4, learning_rate_decay=0.9,
reg=0.2, verbose=True)
# Predict on the validation set
val_acc = (net.predict(X_val) == y_val).mean()
print 'Validation accuracy: ', val_acc
# # Debug the training
# With the default parameters we provided above, you should get a validation accuracy of about 0.29 on the validation set. This isn't very good.
#
# One strategy for getting insight into what's wrong is to plot the loss function and the accuracies on the training and validation sets during optimization.
#
# Another strategy is to visualize the weights that were learned in the first layer of the network. In most neural networks trained on visual data, the first layer weights typically show some visible structure when visualized.
# In[ ]:
'''
###### uncomment the following section to print the value of parameters ######
print "params values:"
print "hiddenlayer_size="+str(hiddenlayer_size_arg)
print "batch_size="+str(batch_size_arg)
print "num_iters="+str(num_iters_arg)
print "learning_rate"+str(learning_rate_arg)
print "learning_rate_decay="+str(learning_rate_decay_arg)
print "regularisation="+str(reg_arg)
print "\n\n\n"
'''
nnet = TwoLayerNet(inputsize, hiddenlayer_size_arg, outputsize)
start_time = time.clock()
train_result = nnet.train(Xtr, Ytr, Xval, Yval, learning_rate=learning_rate_arg,
learning_rate_decay=learning_rate_decay_arg,
reg=reg_arg, num_iters=num_iters_arg,
batch_size=batch_size_arg, verbose=verbose)
end_time = time.clock()
print "first loss:"+str(train_result['loss_history'][0])+" last loss:"+str(train_result['loss_history'][-1])
#print "loss:\n\n"+str(train_result['loss_history'])
print "Training accuracy:"+str(nnet.accuracy(Xtr, Ytr))
print "Validation accuracy:"+str(nnet.accuracy(Xval, Yval))
print "Testing accuracy:"+str(nnet.accuracy(Xte, Yte))
time_taken = end_time-start_time
print "Time taken for training is "+str(time_taken)
print "\n\n"
'''
###### uncomment the following section to see the effect of different parameters ######
hidden_size = 230
num_classes = 10
num_iters = 3200
best_val = -1
best_nn = None
learning_rates = [1e-3]
regularization_strengths = [7e-6]
learning_history = {}
for lr in learning_rates:
for reg in regularization_strengths:
nn = TwoLayerNet(input_size, hidden_size, num_classes)
report = nn.train(X_train,
y_train,
X_val,
y_val,
num_iters=num_iters,
batch_size=200,
learning_rate=lr,
learning_rate_decay=0.98,
reg=reg)
train_acc = np.mean(nn.predict(X_train) == y_train)
val_acc = np.mean(nn.predict(X_val) == y_val)
learning_history[(lr, reg)] = (train_acc, val_acc)
if val_acc > best_val:
best_val = val_acc
best_nn = nn
for lr, reg in sorted(learning_history):
train_acc, val_acc = learning_history[(lr, reg)]
print('lr %e reg %e train accuracy: %f val accuracy: %f' %
(lr, reg, train_acc, val_acc))
# Invoke the above function to get our data.
X_train, y_train, X_val, y_val, X_test, y_test = get_CIFAR10_data()
print 'Train data shape: ', X_train.shape
print 'Train labels shape: ', y_train.shape
print 'Validation data shape: ', X_val.shape
print 'Validation labels shape: ', y_val.shape
print 'Test data shape: ', X_test.shape
print 'Test labels shape: ', y_test.shape
input_size = 32 * 32 * 3
hidden_size = 60
num_classes = 10
net = TwoLayerNet(input_size, hidden_size, num_classes)
t_start = time.time()
# Train the network
stats = net.train(X_train, y_train, X_val, y_val,
num_iters=10000, batch_size=200,
learning_rate=1e-3, learning_rate_decay=0.95,
reg=0.1, verbose=True)
t_end = time.time()
train_acc = (net.predict(X_train) == y_train).mean()
print 'Training accuracy: ', train_acc
val_acc = (net.predict(X_val) == y_val).mean()
print 'Validation accuracy: ', val_acc
test_acc = (net.predict(X_test) == y_test).mean()
print 'Test accuracy: ', test_acc
time_taken = t_end - t_start
print 'Training time (in mins): ', time_taken/60
results = {}
best_val_acc = 0
best_net = None
learning_rate = np.array([0.7, 0.8, 0.9, 1.0, 1.1]) * 1e-3
regularization_strengths = [0.75, 1, 1.25]
print('running')
tis = time.time()
for hs in hidden_size:
for lr in learning_rate:
for reg in regularization_strengths:
net = TwoLayerNet(input_size, hs, num_classes)
states = net.train(x_train,
y_train,
x_val,
y_val,
num_iters=1500,
batch_size=200,
learning_rate=lr,
learning_rate_decay=0.95,
reg=reg,
verbose=False)
val_acc = (net.predict(x_val) == y_val).mean()
if val_acc > best_val_acc:
best_val_acc = val_acc
best_net = net
results[(hs, lr, reg)] = val_acc
print('hs %d lr %e reg %e val accuracy: %f ' %
(hs, lr, reg, val_acc))
tie = time.time()
print('finished. total time : %f s' % (tie - tis))
for hs, lr, reg in sorted(results):
# Train a network
# To train our network we will use SGD with momentum. In addition,
# we will adjust the learning rate with an exponential learning rate schedule as optimization proceeds;
# after each epoch, we will reduce the learning rate by multiplying it by a decay rate.
input_size = 32 * 32 * 3
hidden_size = 50
num_classes = 10
net = TwoLayerNet(input_size, hidden_size, num_classes)
# Train the network
stats = net.train(X_train,
y_train,
X_val,
y_val,
num_iters=1000,
batch_size=200,
learning_rate=1e-4,
learning_rate_decay=0.95,
reg=0.25,
verbose=True)
# Predict on the validation set
val_acc = (net.predict(X_val) == y_val).mean()
print('Validation accuracy: ', val_acc)
# # Debug the training
# With the default parameters we provided above, you should get a validation accuracy of about 0.29 on the validation set. This isn't very good.
#
# One strategy for getting insight into what's wrong is to plot the loss function and the accuracies on the training and validation sets during optimization.
#
# Another strategy is to visualize the weights that were learned in the first layer of the network. In most neural networks trained on visual data, the first layer weights typically show some visible structure when visualized.
print('Train labels shape: ', y_train.shape)
print('Validation data shape: ', X_val.shape)
print('Validation labels shape: ', y_val.shape)
print('Test data shape: ', X_test.shape)
print('Test labels shape: ', y_test.shape)
input_size = 32 * 32 * 3
hidden_size = 50
num_classes = 10
net = TwoLayerNet(input_size, hidden_size, num_classes)
# Train the network
stats = net.train(X_train,
y_train,
X_val,
y_val,
num_iters=1000,
batch_size=200,
learning_rate=1e-4,
learning_rate_decay=0.95,
reg=0.5,
verbose=True)
# Predict on the validation set
val_acc = (net.predict(X_val) == y_val).mean()
print('Validation accuracy: ', val_acc)
# Plot the loss function and train / validation accuracies
plt.subplot(2, 1, 1)
plt.plot(stats['loss_history'])
plt.title('Loss history')
plt.xlabel('Iteration')
plt.ylabel('Loss')
best_val = -1
best_svm = None
# X_val = X_train[-10000::, :]
# y_val = y_train[-10000::]
# X_train = X_train[:-10000:, :]
# y_train = y_train[:-10000:]
for lr in learning_rates:
for reg in regularization_strengths:
svm = TwoLayerNet(X_train.shape[1], 500, 10)
svm.train(X_train,
y_train,
X_val,
y_val,
learning_rate=lr,
reg=reg,
num_iters=1000,
batch_size=2000,
verbose=True)
y_train_pred = svm.predict(X_train)
accuracy_train = np.mean(y_train == y_train_pred)
y_val_pred = svm.predict(X_val)
accuracy_val = np.mean(y_val == y_val_pred)
results[(lr, reg)] = (accuracy_train, accuracy_val)
if best_val < accuracy_val:
best_val = accuracy_val
best_svm = svm
plt.show()
# find best model
best_net = None
input_size = 32 * 32 * 3
hidden_size = 100
num_classes = 10
net = TwoLayerNet(input_size, hidden_size, num_classes)
# Train the network
stats = net.train(X_train,
Y_train,
X_val,
Y_val,
num_iter=5000,
batch_size=200,
learning_rate=1e-4,
learning_rate_decay=0.95,
reg=0.3,
verbose=True)
# Predict on the validation set
val_acc = (net.predict(X_val) == Y_val).mean()
# Summary
print('Validation accuracy: ', val_acc)
show_net_graph(stats)
best_net = net
# test
best_val = -1
best_stats = None
learning_rates = [1e-2, 1e-3]
regularization_strengths = [0.4, 0.5, 0.6]
results = {}
iters = 1000 #100
for lr in learning_rates:
for rs in regularization_strengths:
net = TwoLayerNet(input_size, hidden_size, num_classes)
# Train the network
stats = net.train(X_train,
y_train,
X_val,
y_val,
num_iters=iters,
batch_size=200,
learning_rate=lr,
learning_rate_decay=0.95,
reg=rs)
y_train_pred = net.predict(X_train)
acc_train = np.mean(y_train == y_train_pred)
y_val_pred = net.predict(X_val)
acc_val = np.mean(y_val == y_val_pred)
results[(lr, rs)] = (acc_train, acc_val)
if best_val < acc_val:
best_stats = stats
best_val = acc_val
learning_rates = [1e-2] #[1e-2, 1e-3]
regularization_strengths = [0.6] #[0.4, 0.5, 0.6]
results = {}
iters = 300 #100
hidden_sizes = [50] #[25,50]
for lr in learning_rates:
for rs in regularization_strengths:
for hidden_size in hidden_sizes:
net = TwoLayerNet(input_size, hidden_size, num_classes)
# Train the network
stats = net.train(X_train,
y_train,
X_val,
y_val,
num_iters=iters,
batch_size=200,
learning_rate=lr,
learning_rate_decay=0.95,
reg=rs)
y_train_pred = net.predict(X_train)
acc_train = np.mean(y_train == y_train_pred)
y_val_pred = net.predict(X_val)
acc_val = np.mean(y_val == y_val_pred)
results[(lr, rs, hidden_size)] = (acc_train, acc_val)
if best_val < acc_val:
best_stats = stats
best_val = acc_val
x_train -= mean_img
x_test -= mean_img
x_val -= mean_img
return x_train, y_train, x_val, y_val, x_test, y_test
x_train, y_train, x_val, y_val, x_test, y_test = load_CIFAR10_data()
input_size = 32 * 32 * 3
hidden_size = 50
num_classes = 10
net = TwoLayerNet(input_size, hidden_size, num_classes)
# Training the network using SGD
stats = net.train(x_train, y_train, x_val, y_val, num_iters=1200, verbose=True)
val_acc = np.mean(net.predict(x_val) == y_val)
print("Validation accuracy: ", val_acc) # 44.5%
test_acc = (net.predict(x_test) == y_test).mean()
print('Test accuracy: ', test_acc)
# Plot the loss function and train / validation accuracies
def visualize_loss_acc(stats):
plt.subplot(2, 1, 1)
plt.plot(stats['loss_his'])
plt.title('Loss history')
plt.xlabel('Iteration')
plt.ylabel('Loss')