def finding_best_net():
best_net = None
result = {}
best_val = -1
hidden_sizes = [40, 60, 80, 100, 120]
lrs = [1e-4, 5e-4, 1e-3, 5e-3]
regs = [1e-6, 5e-6, 1e-5, 5e-5]
lr_decays = [0.95, 0.99, 0.999]
input_size = 32 * 32 * 3
num_classes = 10
nets = {}
i = 0
grid_search = [(x, y, z) for x in lrs for y in regs for z in lr_decays]
for hidden_size in hidden_sizes:
for lr, reg, lr_decay in grid_search:
print('hidden {} -- lr {} -- reg {} -- lr_decay {}'.format(
hidden_size, lr, reg, lr_decay))
print('Done {:2} out of {}'.format(
i + 1,
len(grid_search) * len(hidden_sizes)))
net = TwoLayerNet(input_size, hidden_size, num_classes)
# Training
net.train(x_train,
y_train,
x_val,
y_val,
num_iters=2000,
lr=lr,
lr_decay=lr_decay,
reg=reg,
verbose=False)
# Predict
y_train_pred = net.predict(x_train)
y_val_pred = net.predict(x_val)
# Scoring
train_accu = np.mean(y_train_pred == y_train)
val_accu = np.mean(y_val_pred == y_val)
# Store results
result[(hidden_size, lr, reg, lr_decay)] = (train_accu, val_accu)
nets[(hidden_size, lr, reg, lr_decay)] = net
if val_accu > best_val:
best_val = val_accu
best_net = net
i += 1
class NeuralNetwork():
def __init__(self):
self.input_size = 28 * 28
self.hidden_size = 50
self.num_classes = 10
self.net = TwoLayerNet(self.input_size, self.hidden_size,
self.num_classes)
self.fit()
def fit(self):
minst = tf.keras.datasets.mnist
(X_train, y_train), (X_test, y_test) = minst.load_data()
num_train = 30000
num_val = 1000
# Train set
mask = list(range(num_train, num_train + num_val))
X_val = X_train[mask]
y_val = y_train[mask]
mask = list(range(num_train))
X_train = X_train[mask]
y_train = y_train[mask]
# Preprocessing: reshape the images data into rows
X_train = np.reshape(X_train, (X_train.shape[0], -1))
X_val = np.reshape(X_val, (X_val.shape[0], -1))
stats = self.net.train(X_train,
y_train,
X_val,
y_val,
num_iters=1500,
batch_size=200,
learning_rate=1e-4,
learning_rate_decay=0.95,
reg=0.25,
verbose=True)
# val_acc = (self.net.predict(X_val) == y_val).mean()
# print('Validation accuracy: ', val_acc)
# plt.subplot(2, 1, 1)
# plt.plot(stats['loss_history'])
# plt.title('Loss history')
# plt.xlabel('Iteration')
# plt.ylabel('Loss')
#
# plt.subplot(2, 1, 2)
# plt.plot(stats['train_acc_history'], label='train')
# plt.plot(stats['val_acc_history'], label='val')
# plt.title('Classification accuracy history')
# plt.xlabel('Epoch')
# plt.ylabel('Clasification accuracy')
# plt.show()
def predict(self, x):
return self.net.predict(x)
def main():
x_train = ucihar.load_data('train', 'X')
y_train = ucihar.load_data('train', 'y')
x_test = ucihar.load_data('test', 'X')
y_test = ucihar.load_data('test', 'y')
net = TwoLayerNet(input_size=ucihar.X_DIMMENSIONS,
hidden_size=50,
output_size=ucihar.Y_CLASSES)
stats = net.train(x_train,
y_train,
x_test,
y_test,
num_iters=100000,
batch_size=64,
learning_rate=1e-2,
learning_rate_decay=1.)
predictions = net.predict(x_test)
val_acc = (predictions == y_test).mean()
print('Validation accuracy: ', val_acc)
try:
from sklearn.metrics import classification_report, confusion_matrix
print(confusion_matrix(y_test, predictions))
print(classification_report(y_test, predictions))
except ImportError:
pass
try:
import matplotlib.pyplot as plt
# Plot the loss function and train / validation accuracies
plt.subplot(1, 2, 1)
plt.plot(stats['loss_history'])
plt.title('Loss history')
plt.xlabel('Iteration')
plt.ylabel('Loss')
plt.subplot(1, 2, 2)
train_plot, = plt.plot(stats['train_acc_history'], label='train')
val_plot, = plt.plot(stats['val_acc_history'], label='val')
plt.legend(handles=[train_plot, val_plot])
plt.title('Classification accuracy history')
plt.xlabel('Epoch')
plt.ylabel('Clasification accuracy')
plt.show()
except ImportError:
pass
# batch = [40, 50, 60, 70, 80]
batch = [50]
bestValAcc = 0
bestNN = None
bestTrain = None
bestParams = []
for hidden in hidden_size:
for eta in learningRates:
for r in regularization:
for t in iteration:
for b in batch:
net = TwoLayerNet(input_size, hidden, NUM_CLASS)
train = net.train(X_train, y_train, X_train, y_train,\
num_iters=t, batch_size=b, learning_rate=eta, learning_rate_decay=0.95, reg=r, verbose=False)
y_train_pred = net.predict(X_train)
y_val_pred = net.predict(X_val)
trainAcc = np.mean(y_train == y_train_pred)
valAcc = np.mean(y_val == y_val_pred)
print 'learning rate: %e reg: %.2f iteration: %d batch: %d train accuracy: %.4f val accuracy: %.4f'\
% (eta, r, t, b, trainAcc, valAcc)
if valAcc > bestValAcc:
bestValAcc = valAcc
bestNN = net
bestTrain = train
bestParams = [hidden, eta, r, t, b]
print 'best validation accuracy achieved: %.4f' % bestValAcc
print bestParams
f1 = open('./res.txt', 'w+')
f1.write('best validation accuracy achieved: %.4f' % bestValAcc)
# In[ ]:
input_size = 32 * 32 * 3
hidden_size = 50
num_classes = 10
net = TwoLayerNet(input_size, hidden_size, num_classes)
# Train the network
stats = net.train(X_train, y_train, X_val, y_val,
num_epochs=30, batch_size=500,
learning_rate=1e-4, learning_rate_decay=0.9,
reg=0.2, verbose=True)
# Predict on the validation set
val_acc = (net.predict(X_val) == y_val).mean()
print 'Validation accuracy: ', val_acc
# # Debug the training
# With the default parameters we provided above, you should get a validation accuracy of about 0.29 on the validation set. This isn't very good.
#
# One strategy for getting insight into what's wrong is to plot the loss function and the accuracies on the training and validation sets during optimization.
#
# Another strategy is to visualize the weights that were learned in the first layer of the network. In most neural networks trained on visual data, the first layer weights typically show some visible structure when visualized.
# In[ ]:
# Plot the loss function and train / validation accuracies
if False:
plt.subplot(2, 1, 1)
plt.plot(stats['loss_history'])
learning_rates = [1e-3]
regularization_strengths = [7e-6]
learning_history = {}
for lr in learning_rates:
for reg in regularization_strengths:
nn = TwoLayerNet(input_size, hidden_size, num_classes)
report = nn.train(X_train,
y_train,
X_val,
y_val,
num_iters=num_iters,
batch_size=200,
learning_rate=lr,
learning_rate_decay=0.98,
reg=reg)
train_acc = np.mean(nn.predict(X_train) == y_train)
val_acc = np.mean(nn.predict(X_val) == y_val)
learning_history[(lr, reg)] = (train_acc, val_acc)
if val_acc > best_val:
best_val = val_acc
best_nn = nn
for lr, reg in sorted(learning_history):
train_acc, val_acc = learning_history[(lr, reg)]
print('lr %e reg %e train accuracy: %f val accuracy: %f' %
(lr, reg, train_acc, val_acc))
# print loss curve
plt.subplot(2, 1, 1)
plt.plot(report['loss_history'])
plt.title('Loss history')
# Invoke the above function to get our data.
X_train, y_train, X_val, y_val, X_test, y_test = get_CIFAR10_data()
print 'Train data shape: ', X_train.shape
print 'Train labels shape: ', y_train.shape
print 'Validation data shape: ', X_val.shape
print 'Validation labels shape: ', y_val.shape
print 'Test data shape: ', X_test.shape
print 'Test labels shape: ', y_test.shape
input_size = 32 * 32 * 3
hidden_size = 60
num_classes = 10
net = TwoLayerNet(input_size, hidden_size, num_classes)
t_start = time.time()
# Train the network
stats = net.train(X_train, y_train, X_val, y_val,
num_iters=10000, batch_size=200,
learning_rate=1e-3, learning_rate_decay=0.95,
reg=0.1, verbose=True)
t_end = time.time()
train_acc = (net.predict(X_train) == y_train).mean()
print 'Training accuracy: ', train_acc
val_acc = (net.predict(X_val) == y_val).mean()
print 'Validation accuracy: ', val_acc
test_acc = (net.predict(X_test) == y_test).mean()
print 'Test accuracy: ', test_acc
time_taken = t_end - t_start
print 'Training time (in mins): ', time_taken/60
net = TwoLayerNet(input_size, hidden_size, num_classes)
# Train the network
stats = net.train(X_train,
y_train,
X_val,
y_val,
num_iters=1000,
batch_size=200,
learning_rate=1e-4,
learning_rate_decay=0.95,
reg=0.25,
verbose=True)
# Predict on the validation set
val_acc = (net.predict(X_val) == y_val).mean()
print('Validation accuracy: ', val_acc)
##
# Plot the loss function and train / validation accuracies
plt.subplot(2, 1, 1)
plt.plot(stats['loss_history'])
plt.title('Loss history')
plt.xlabel('Iteration')
plt.ylabel('Loss')
plt.subplot(2, 1, 2)
l1 = plt.plot(stats['train_acc_history'], label='train')
l2 = plt.plot(stats['val_acc_history'], label='val')
#plt.legend(handles=[l1, l2])
plt.title('Classification accuracy history')
# X_train = X_train[:-10000:, :]
# y_train = y_train[:-10000:]
for lr in learning_rates:
for reg in regularization_strengths:
svm = TwoLayerNet(X_train.shape[1], 500, 10)
svm.train(X_train,
y_train,
X_val,
y_val,
learning_rate=lr,
reg=reg,
num_iters=1000,
batch_size=2000,
verbose=True)
y_train_pred = svm.predict(X_train)
accuracy_train = np.mean(y_train == y_train_pred)
y_val_pred = svm.predict(X_val)
accuracy_val = np.mean(y_val == y_val_pred)
results[(lr, reg)] = (accuracy_train, accuracy_val)
if best_val < accuracy_val:
best_val = accuracy_val
best_svm = svm
for lr, reg in sorted(results):
accuracy_train, accuracy_val = results[(lr, reg)]
print('lr %e reg %e train accuracy: %f val accuracy: %f' %
(lr, reg, accuracy_train, accuracy_val))
x_val -= mean_img
return x_train, y_train, x_val, y_val, x_test, y_test
x_train, y_train, x_val, y_val, x_test, y_test = load_CIFAR10_data()
input_size = 32 * 32 * 3
hidden_size = 50
num_classes = 10
net = TwoLayerNet(input_size, hidden_size, num_classes)
# Training the network using SGD
stats = net.train(x_train, y_train, x_val, y_val, num_iters=1200, verbose=True)
val_acc = np.mean(net.predict(x_val) == y_val)
print("Validation accuracy: ", val_acc) # 44.5%
test_acc = (net.predict(x_test) == y_test).mean()
print('Test accuracy: ', test_acc)
# Plot the loss function and train / validation accuracies
def visualize_loss_acc(stats):
plt.subplot(2, 1, 1)
plt.plot(stats['loss_his'])
plt.title('Loss history')
plt.xlabel('Iteration')
plt.ylabel('Loss')
plt.subplot(2, 1, 2)