def __init__(self):
self.input_size = 28 * 28
self.hidden_size = 50
self.num_classes = 10
self.net = TwoLayerNet(self.input_size, self.hidden_size,
self.num_classes)
self.fit()
class NeuralNetwork():
def __init__(self):
self.input_size = 28 * 28
self.hidden_size = 50
self.num_classes = 10
self.net = TwoLayerNet(self.input_size, self.hidden_size,
self.num_classes)
self.fit()
def fit(self):
minst = tf.keras.datasets.mnist
(X_train, y_train), (X_test, y_test) = minst.load_data()
num_train = 30000
num_val = 1000
# Train set
mask = list(range(num_train, num_train + num_val))
X_val = X_train[mask]
y_val = y_train[mask]
mask = list(range(num_train))
X_train = X_train[mask]
y_train = y_train[mask]
# Preprocessing: reshape the images data into rows
X_train = np.reshape(X_train, (X_train.shape[0], -1))
X_val = np.reshape(X_val, (X_val.shape[0], -1))
stats = self.net.train(X_train,
y_train,
X_val,
y_val,
num_iters=1500,
batch_size=200,
learning_rate=1e-4,
learning_rate_decay=0.95,
reg=0.25,
verbose=True)
# val_acc = (self.net.predict(X_val) == y_val).mean()
# print('Validation accuracy: ', val_acc)
# plt.subplot(2, 1, 1)
# plt.plot(stats['loss_history'])
# plt.title('Loss history')
# plt.xlabel('Iteration')
# plt.ylabel('Loss')
#
# plt.subplot(2, 1, 2)
# plt.plot(stats['train_acc_history'], label='train')
# plt.plot(stats['val_acc_history'], label='val')
# plt.title('Classification accuracy history')
# plt.xlabel('Epoch')
# plt.ylabel('Clasification accuracy')
# plt.show()
def predict(self, x):
return self.net.predict(x)
def main():
x_train = ucihar.load_data('train', 'X')
y_train = ucihar.load_data('train', 'y')
x_test = ucihar.load_data('test', 'X')
y_test = ucihar.load_data('test', 'y')
net = TwoLayerNet(input_size=ucihar.X_DIMMENSIONS,
hidden_size=50,
output_size=ucihar.Y_CLASSES)
stats = net.train(x_train,
y_train,
x_test,
y_test,
num_iters=100000,
batch_size=64,
learning_rate=1e-2,
learning_rate_decay=1.)
predictions = net.predict(x_test)
val_acc = (predictions == y_test).mean()
print('Validation accuracy: ', val_acc)
try:
from sklearn.metrics import classification_report, confusion_matrix
print(confusion_matrix(y_test, predictions))
print(classification_report(y_test, predictions))
except ImportError:
pass
try:
import matplotlib.pyplot as plt
# Plot the loss function and train / validation accuracies
plt.subplot(1, 2, 1)
plt.plot(stats['loss_history'])
plt.title('Loss history')
plt.xlabel('Iteration')
plt.ylabel('Loss')
plt.subplot(1, 2, 2)
train_plot, = plt.plot(stats['train_acc_history'], label='train')
val_plot, = plt.plot(stats['val_acc_history'], label='val')
plt.legend(handles=[train_plot, val_plot])
plt.title('Classification accuracy history')
plt.xlabel('Epoch')
plt.ylabel('Clasification accuracy')
plt.show()
except ImportError:
pass
def finding_best_net():
best_net = None
result = {}
best_val = -1
hidden_sizes = [40, 60, 80, 100, 120]
lrs = [1e-4, 5e-4, 1e-3, 5e-3]
regs = [1e-6, 5e-6, 1e-5, 5e-5]
lr_decays = [0.95, 0.99, 0.999]
input_size = 32 * 32 * 3
num_classes = 10
nets = {}
i = 0
grid_search = [(x, y, z) for x in lrs for y in regs for z in lr_decays]
for hidden_size in hidden_sizes:
for lr, reg, lr_decay in grid_search:
print('hidden {} -- lr {} -- reg {} -- lr_decay {}'.format(
hidden_size, lr, reg, lr_decay))
print('Done {:2} out of {}'.format(
i + 1,
len(grid_search) * len(hidden_sizes)))
net = TwoLayerNet(input_size, hidden_size, num_classes)
# Training
net.train(x_train,
y_train,
x_val,
y_val,
num_iters=2000,
lr=lr,
lr_decay=lr_decay,
reg=reg,
verbose=False)
# Predict
y_train_pred = net.predict(x_train)
y_val_pred = net.predict(x_val)
# Scoring
train_accu = np.mean(y_train_pred == y_train)
val_accu = np.mean(y_val_pred == y_val)
# Store results
result[(hidden_size, lr, reg, lr_decay)] = (train_accu, val_accu)
nets[(hidden_size, lr, reg, lr_decay)] = net
if val_accu > best_val:
best_val = val_accu
best_net = net
i += 1
regularization = [0.25, 0.5]
# iteration = [5500]
iteration = [3500, 4500, 5500] # >2500
# batch = [40, 50, 60, 70, 80]
batch = [50]
bestValAcc = 0
bestNN = None
bestTrain = None
bestParams = []
for hidden in hidden_size:
for eta in learningRates:
for r in regularization:
for t in iteration:
for b in batch:
net = TwoLayerNet(input_size, hidden, NUM_CLASS)
train = net.train(X_train, y_train, X_train, y_train,\
num_iters=t, batch_size=b, learning_rate=eta, learning_rate_decay=0.95, reg=r, verbose=False)
y_train_pred = net.predict(X_train)
y_val_pred = net.predict(X_val)
trainAcc = np.mean(y_train == y_train_pred)
valAcc = np.mean(y_val == y_val_pred)
print 'learning rate: %e reg: %.2f iteration: %d batch: %d train accuracy: %.4f val accuracy: %.4f'\
% (eta, r, t, b, trainAcc, valAcc)
if valAcc > bestValAcc:
bestValAcc = valAcc
bestNN = net
bestTrain = train
bestParams = [hidden, eta, r, t, b]
print 'best validation accuracy achieved: %.4f' % bestValAcc
print 'Train labels shape: ', y_train.shape
print 'Validation data shape: ', X_val.shape
print 'Validation labels shape: ', y_val.shape
print 'Test data shape: ', X_test.shape
print 'Test labels shape: ', y_test.shape
# # Train a network
# To train our network we will use SGD with momentum. In addition, we will adjust the learning rate with an exponential learning rate schedule as optimization proceeds; after each epoch, we will reduce the learning rate by multiplying it by a decay rate.
# In[ ]:
input_size = 32 * 32 * 3
hidden_size = 50
num_classes = 10
net = TwoLayerNet(input_size, hidden_size, num_classes)
# Train the network
stats = net.train(X_train, y_train, X_val, y_val,
num_epochs=30, batch_size=500,
learning_rate=1e-4, learning_rate_decay=0.9,
reg=0.2, verbose=True)
# Predict on the validation set
val_acc = (net.predict(X_val) == y_val).mean()
print 'Validation accuracy: ', val_acc
# # Debug the training
# With the default parameters we provided above, you should get a validation accuracy of about 0.29 on the validation set. This isn't very good.
#
# One strategy for getting insight into what's wrong is to plot the loss function and the accuracies on the training and validation sets during optimization.
learning_rate_decay_arg =0.98
reg_arg=1e-5
verbose = False
'''
###### uncomment the following section to print the value of parameters ######
print "params values:"
print "hiddenlayer_size="+str(hiddenlayer_size_arg)
print "batch_size="+str(batch_size_arg)
print "num_iters="+str(num_iters_arg)
print "learning_rate"+str(learning_rate_arg)
print "learning_rate_decay="+str(learning_rate_decay_arg)
print "regularisation="+str(reg_arg)
print "\n\n\n"
'''
nnet = TwoLayerNet(inputsize, hiddenlayer_size_arg, outputsize)
start_time = time.clock()
train_result = nnet.train(Xtr, Ytr, Xval, Yval, learning_rate=learning_rate_arg,
learning_rate_decay=learning_rate_decay_arg,
reg=reg_arg, num_iters=num_iters_arg,
batch_size=batch_size_arg, verbose=verbose)
end_time = time.clock()
print "first loss:"+str(train_result['loss_history'][0])+" last loss:"+str(train_result['loss_history'][-1])
#print "loss:\n\n"+str(train_result['loss_history'])
print "Training accuracy:"+str(nnet.accuracy(Xtr, Ytr))
print "Validation accuracy:"+str(nnet.accuracy(Xval, Yval))
print "Testing accuracy:"+str(nnet.accuracy(Xte, Yte))
time_taken = end_time-start_time
print "Time taken for training is "+str(time_taken)
print "\n\n"
cifar_dir = '../cifar-10-batches-py'
X_train, y_train, X_val, y_val, X_test, y_test, X_dev, y_dev = load_data(
cifar_dir, num_test=500)
input_size = 32 * 32 * 3
hidden_size = 230
num_classes = 10
num_iters = 3200
best_val = -1
best_nn = None
learning_rates = [1e-3]
regularization_strengths = [7e-6]
learning_history = {}
for lr in learning_rates:
for reg in regularization_strengths:
nn = TwoLayerNet(input_size, hidden_size, num_classes)
report = nn.train(X_train,
y_train,
X_val,
y_val,
num_iters=num_iters,
batch_size=200,
learning_rate=lr,
learning_rate_decay=0.98,
reg=reg)
train_acc = np.mean(nn.predict(X_train) == y_train)
val_acc = np.mean(nn.predict(X_val) == y_val)
learning_history[(lr, reg)] = (train_acc, val_acc)
if val_acc > best_val:
best_val = val_acc
best_nn = nn
return X_train, y_train, X_val, y_val, X_test, y_test
# Invoke the above function to get our data.
X_train, y_train, X_val, y_val, X_test, y_test = get_CIFAR10_data()
print 'Train data shape: ', X_train.shape
print 'Train labels shape: ', y_train.shape
print 'Validation data shape: ', X_val.shape
print 'Validation labels shape: ', y_val.shape
print 'Test data shape: ', X_test.shape
print 'Test labels shape: ', y_test.shape
input_size = 32 * 32 * 3
hidden_size = 60
num_classes = 10
net = TwoLayerNet(input_size, hidden_size, num_classes)
t_start = time.time()
# Train the network
stats = net.train(X_train, y_train, X_val, y_val,
num_iters=10000, batch_size=200,
learning_rate=1e-3, learning_rate_decay=0.95,
reg=0.1, verbose=True)
t_end = time.time()
train_acc = (net.predict(X_train) == y_train).mean()
print 'Training accuracy: ', train_acc
val_acc = (net.predict(X_val) == y_val).mean()
print 'Validation accuracy: ', val_acc
test_acc = (net.predict(X_test) == y_test).mean()
print 'Test accuracy: ', test_acc
print('validation accuracy :', val_acc)
'''
input_size = 32 * 32 * 3
num_classes = 10
hidden_size = [75, 100, 125]
results = {}
best_val_acc = 0
best_net = None
learning_rate = np.array([0.7, 0.8, 0.9, 1.0, 1.1]) * 1e-3
regularization_strengths = [0.75, 1, 1.25]
print('running')
tis = time.time()
for hs in hidden_size:
for lr in learning_rate:
for reg in regularization_strengths:
net = TwoLayerNet(input_size, hs, num_classes)
states = net.train(x_train,
y_train,
x_val,
y_val,
num_iters=1500,
batch_size=200,
learning_rate=lr,
learning_rate_decay=0.95,
reg=reg,
verbose=False)
val_acc = (net.predict(x_val) == y_val).mean()
if val_acc > best_val_acc:
best_val_acc = val_acc
best_net = net
results[(hs, lr, reg)] = val_acc
from vis_utils import visualize_grid
# Invoke the above function to get our data.
X_train, y_train, X_val, y_val, X_test, y_test = get_CIFAR10_data()
print('Train data shape: ', X_train.shape)
print('Train labels shape: ', y_train.shape)
print('Validation data shape: ', X_val.shape)
print('Validation labels shape: ', y_val.shape)
print('Test data shape: ', X_test.shape)
print('Test labels shape: ', y_test.shape)
X_train.astype("float64")
input_size = 32 * 32 * 3
hidden_size = 50
num_classes = 10
net = TwoLayerNet(input_size, hidden_size, num_classes)
# Train the network
stats = net.train(X_train,
y_train,
X_val,
y_val,
num_iters=1000,
batch_size=200,
learning_rate=1e-4,
learning_rate_decay=0.95,
reg=0.25,
verbose=True)
# Predict on the validation set
val_acc = (net.predict(X_val) == y_val).mean()
# adjust the hyperparameters using validation data set
learning_rates = [2e-3, 1e-3, 1e-4]
regularization_strengths = [3e-1, 5e-1, 3, 5, 10]
results = {}
best_val = -1
best_svm = None
# X_val = X_train[-10000::, :]
# y_val = y_train[-10000::]
# X_train = X_train[:-10000:, :]
# y_train = y_train[:-10000:]
for lr in learning_rates:
for reg in regularization_strengths:
svm = TwoLayerNet(X_train.shape[1], 500, 10)
svm.train(X_train,
y_train,
X_val,
y_val,
learning_rate=lr,
reg=reg,
num_iters=1000,
batch_size=2000,
verbose=True)
y_train_pred = svm.predict(X_train)
accuracy_train = np.mean(y_train == y_train_pred)
y_val_pred = svm.predict(X_val)
accuracy_val = np.mean(y_val == y_val_pred)
results[(lr, reg)] = (accuracy_train, accuracy_val)
loss, _ = self.net.loss(self.x, self.y, reg=0.05)
np.testing.assert_allclose(loss, self.expected_loss_wt, 1e-04)
def test_neural_net_gradient(self):
loss, grads = self.net.loss(self.x, self.y, reg=0.0)
f = lambda w: self.net.loss(self.x, self.y, reg=0.0)[0]
for param_name, grad in grads.iteritems():
grad_numerical = test_utils.eval_numerical_gradient(
f, self.net.params[param_name], verbose=False)
np.testing.assert_allclose(grad_numerical, grad, 1e-04)
if __name__ == '__main__':
suite = unittest.TestSuite()
np.random.seed(0)
test_net = TwoLayerNet(4, 10, 3, std=1e-1)
np.random.seed(1)
x = 10 * np.random.randn(5, 4)
y = np.array([0, 1, 2, 2, 1])
expected_loss = np.asarray([[-0.81233741, -1.27654624, -0.70335995],
[-0.17129677, -1.18803311, -0.47310444],
[-0.51590475, -1.01354314, -0.8504215],
[-0.15419291, -0.48629638, -0.52901952],
[-0.00618733, -0.12435261, -0.15226949]])
suite.addTest(
test_utils.TestCaseWithParams.get_suite(NeuralNetTest,
kwargs={
'net': test_net,
'x': x,
'y': y,
plt.subplot(2, 1, 2)
plt.plot(stats['train_acc_history'], label='train')
plt.plot(stats['val_acc_history'], label='val')
plt.title('Classification accuracy history')
plt.xlabel('Epoch')
plt.ylabel('Clasification accuracy')
plt.show()
# find best model
best_net = None
input_size = 32 * 32 * 3
hidden_size = 100
num_classes = 10
net = TwoLayerNet(input_size, hidden_size, num_classes)
# Train the network
stats = net.train(X_train,
Y_train,
X_val,
Y_val,
num_iter=5000,
batch_size=200,
learning_rate=1e-4,
learning_rate_decay=0.95,
reg=0.3,
verbose=True)
# Predict on the validation set
val_acc = (net.predict(X_val) == Y_val).mean()
return X_train, y_train, X_val, y_val, X_test, y_test
# Invoke the above function to get our data.
X_train, y_train, X_val, y_val, X_test, y_test = get_CIFAR10_data()
##print 'Train data shape: ', X_train.shape
##print 'Train labels shape: ', y_train.shape
##print 'Validation data shape: ', X_val.shape
##print 'Validation labels shape: ', y_val.shape
##print 'Test data shape: ', X_test.shape
##print 'Test labels shape: ', y_test.shape
input_size = 32 * 32 * 3
hidden_size = 50
num_classes = 10
net = TwoLayerNet(input_size, hidden_size, num_classes)
# Train the network
##stats = net.train(X_train, y_train, X_val, y_val,
## num_iters=500, batch_size=200,
## learning_rate=1e-4, learning_rate_decay=0.95,
## reg=0.5, verbose=True)
# Predict on the validation set
val_acc = (net.predict(X_val) == y_val).mean()
##print 'Validation accuracy: ', val_acc
# Plot the loss function and train / validation accuracies
##plt.subplot(2, 1, 1)
##plt.plot(stats['loss_history'])
##plt.title('Loss history')
def init_toy_model(input_size, hidden_size, num_classes):
np.random.seed(0)
return TwoLayerNet(input_size, hidden_size, num_classes, std=1e-1)
print('Train data shape: ', X_train.shape)
print('Train labels shape: ', y_train.shape)
print('Validation data shape: ', X_val.shape)
print('Validation labels shape: ', y_val.shape)
print('Test data shape: ', X_test.shape)
print('Test labels shape: ', y_test.shape)
# Train a network
# To train our network we will use SGD with momentum. In addition,
# we will adjust the learning rate with an exponential learning rate schedule as optimization proceeds;
# after each epoch, we will reduce the learning rate by multiplying it by a decay rate.
input_size = 32 * 32 * 3
hidden_size = 50
num_classes = 10
net = TwoLayerNet(input_size, hidden_size, num_classes)
# Train the network
stats = net.train(X_train,
y_train,
X_val,
y_val,
num_iters=1000,
batch_size=200,
learning_rate=1e-4,
learning_rate_decay=0.95,
reg=0.25,
verbose=True)
# Predict on the validation set
val_acc = (net.predict(X_val) == y_val).mean()
axis=0) # (9000,3072) -> (3072) all training data
# Subtract
x_train -= mean_img
x_test -= mean_img
x_val -= mean_img
return x_train, y_train, x_val, y_val, x_test, y_test
x_train, y_train, x_val, y_val, x_test, y_test = load_CIFAR10_data()
input_size = 32 * 32 * 3
hidden_size = 50
num_classes = 10
net = TwoLayerNet(input_size, hidden_size, num_classes)
# Training the network using SGD
stats = net.train(x_train, y_train, x_val, y_val, num_iters=1200, verbose=True)
val_acc = np.mean(net.predict(x_val) == y_val)
print("Validation accuracy: ", val_acc) # 44.5%
test_acc = (net.predict(x_test) == y_test).mean()
print('Test accuracy: ', test_acc)
# Plot the loss function and train / validation accuracies
def visualize_loss_acc(stats):
plt.subplot(2, 1, 1)
plt.plot(stats['loss_his'])
