def __init__(self): self.input_size = 28 * 28 self.hidden_size = 50 self.num_classes = 10 self.net = TwoLayerNet(self.input_size, self.hidden_size, self.num_classes) self.fit()
class NeuralNetwork(): def __init__(self): self.input_size = 28 * 28 self.hidden_size = 50 self.num_classes = 10 self.net = TwoLayerNet(self.input_size, self.hidden_size, self.num_classes) self.fit() def fit(self): minst = tf.keras.datasets.mnist (X_train, y_train), (X_test, y_test) = minst.load_data() num_train = 30000 num_val = 1000 # Train set mask = list(range(num_train, num_train + num_val)) X_val = X_train[mask] y_val = y_train[mask] mask = list(range(num_train)) X_train = X_train[mask] y_train = y_train[mask] # Preprocessing: reshape the images data into rows X_train = np.reshape(X_train, (X_train.shape[0], -1)) X_val = np.reshape(X_val, (X_val.shape[0], -1)) stats = self.net.train(X_train, y_train, X_val, y_val, num_iters=1500, batch_size=200, learning_rate=1e-4, learning_rate_decay=0.95, reg=0.25, verbose=True) # val_acc = (self.net.predict(X_val) == y_val).mean() # print('Validation accuracy: ', val_acc) # plt.subplot(2, 1, 1) # plt.plot(stats['loss_history']) # plt.title('Loss history') # plt.xlabel('Iteration') # plt.ylabel('Loss') # # plt.subplot(2, 1, 2) # plt.plot(stats['train_acc_history'], label='train') # plt.plot(stats['val_acc_history'], label='val') # plt.title('Classification accuracy history') # plt.xlabel('Epoch') # plt.ylabel('Clasification accuracy') # plt.show() def predict(self, x): return self.net.predict(x)
def main(): x_train = ucihar.load_data('train', 'X') y_train = ucihar.load_data('train', 'y') x_test = ucihar.load_data('test', 'X') y_test = ucihar.load_data('test', 'y') net = TwoLayerNet(input_size=ucihar.X_DIMMENSIONS, hidden_size=50, output_size=ucihar.Y_CLASSES) stats = net.train(x_train, y_train, x_test, y_test, num_iters=100000, batch_size=64, learning_rate=1e-2, learning_rate_decay=1.) predictions = net.predict(x_test) val_acc = (predictions == y_test).mean() print('Validation accuracy: ', val_acc) try: from sklearn.metrics import classification_report, confusion_matrix print(confusion_matrix(y_test, predictions)) print(classification_report(y_test, predictions)) except ImportError: pass try: import matplotlib.pyplot as plt # Plot the loss function and train / validation accuracies plt.subplot(1, 2, 1) plt.plot(stats['loss_history']) plt.title('Loss history') plt.xlabel('Iteration') plt.ylabel('Loss') plt.subplot(1, 2, 2) train_plot, = plt.plot(stats['train_acc_history'], label='train') val_plot, = plt.plot(stats['val_acc_history'], label='val') plt.legend(handles=[train_plot, val_plot]) plt.title('Classification accuracy history') plt.xlabel('Epoch') plt.ylabel('Clasification accuracy') plt.show() except ImportError: pass
def finding_best_net(): best_net = None result = {} best_val = -1 hidden_sizes = [40, 60, 80, 100, 120] lrs = [1e-4, 5e-4, 1e-3, 5e-3] regs = [1e-6, 5e-6, 1e-5, 5e-5] lr_decays = [0.95, 0.99, 0.999] input_size = 32 * 32 * 3 num_classes = 10 nets = {} i = 0 grid_search = [(x, y, z) for x in lrs for y in regs for z in lr_decays] for hidden_size in hidden_sizes: for lr, reg, lr_decay in grid_search: print('hidden {} -- lr {} -- reg {} -- lr_decay {}'.format( hidden_size, lr, reg, lr_decay)) print('Done {:2} out of {}'.format( i + 1, len(grid_search) * len(hidden_sizes))) net = TwoLayerNet(input_size, hidden_size, num_classes) # Training net.train(x_train, y_train, x_val, y_val, num_iters=2000, lr=lr, lr_decay=lr_decay, reg=reg, verbose=False) # Predict y_train_pred = net.predict(x_train) y_val_pred = net.predict(x_val) # Scoring train_accu = np.mean(y_train_pred == y_train) val_accu = np.mean(y_val_pred == y_val) # Store results result[(hidden_size, lr, reg, lr_decay)] = (train_accu, val_accu) nets[(hidden_size, lr, reg, lr_decay)] = net if val_accu > best_val: best_val = val_accu best_net = net i += 1
regularization = [0.25, 0.5] # iteration = [5500] iteration = [3500, 4500, 5500] # >2500 # batch = [40, 50, 60, 70, 80] batch = [50] bestValAcc = 0 bestNN = None bestTrain = None bestParams = [] for hidden in hidden_size: for eta in learningRates: for r in regularization: for t in iteration: for b in batch: net = TwoLayerNet(input_size, hidden, NUM_CLASS) train = net.train(X_train, y_train, X_train, y_train,\ num_iters=t, batch_size=b, learning_rate=eta, learning_rate_decay=0.95, reg=r, verbose=False) y_train_pred = net.predict(X_train) y_val_pred = net.predict(X_val) trainAcc = np.mean(y_train == y_train_pred) valAcc = np.mean(y_val == y_val_pred) print 'learning rate: %e reg: %.2f iteration: %d batch: %d train accuracy: %.4f val accuracy: %.4f'\ % (eta, r, t, b, trainAcc, valAcc) if valAcc > bestValAcc: bestValAcc = valAcc bestNN = net bestTrain = train bestParams = [hidden, eta, r, t, b] print 'best validation accuracy achieved: %.4f' % bestValAcc
print 'Train labels shape: ', y_train.shape print 'Validation data shape: ', X_val.shape print 'Validation labels shape: ', y_val.shape print 'Test data shape: ', X_test.shape print 'Test labels shape: ', y_test.shape # # Train a network # To train our network we will use SGD with momentum. In addition, we will adjust the learning rate with an exponential learning rate schedule as optimization proceeds; after each epoch, we will reduce the learning rate by multiplying it by a decay rate. # In[ ]: input_size = 32 * 32 * 3 hidden_size = 50 num_classes = 10 net = TwoLayerNet(input_size, hidden_size, num_classes) # Train the network stats = net.train(X_train, y_train, X_val, y_val, num_epochs=30, batch_size=500, learning_rate=1e-4, learning_rate_decay=0.9, reg=0.2, verbose=True) # Predict on the validation set val_acc = (net.predict(X_val) == y_val).mean() print 'Validation accuracy: ', val_acc # # Debug the training # With the default parameters we provided above, you should get a validation accuracy of about 0.29 on the validation set. This isn't very good. # # One strategy for getting insight into what's wrong is to plot the loss function and the accuracies on the training and validation sets during optimization.
learning_rate_decay_arg =0.98 reg_arg=1e-5 verbose = False ''' ###### uncomment the following section to print the value of parameters ###### print "params values:" print "hiddenlayer_size="+str(hiddenlayer_size_arg) print "batch_size="+str(batch_size_arg) print "num_iters="+str(num_iters_arg) print "learning_rate"+str(learning_rate_arg) print "learning_rate_decay="+str(learning_rate_decay_arg) print "regularisation="+str(reg_arg) print "\n\n\n" ''' nnet = TwoLayerNet(inputsize, hiddenlayer_size_arg, outputsize) start_time = time.clock() train_result = nnet.train(Xtr, Ytr, Xval, Yval, learning_rate=learning_rate_arg, learning_rate_decay=learning_rate_decay_arg, reg=reg_arg, num_iters=num_iters_arg, batch_size=batch_size_arg, verbose=verbose) end_time = time.clock() print "first loss:"+str(train_result['loss_history'][0])+" last loss:"+str(train_result['loss_history'][-1]) #print "loss:\n\n"+str(train_result['loss_history']) print "Training accuracy:"+str(nnet.accuracy(Xtr, Ytr)) print "Validation accuracy:"+str(nnet.accuracy(Xval, Yval)) print "Testing accuracy:"+str(nnet.accuracy(Xte, Yte)) time_taken = end_time-start_time print "Time taken for training is "+str(time_taken) print "\n\n"
cifar_dir = '../cifar-10-batches-py' X_train, y_train, X_val, y_val, X_test, y_test, X_dev, y_dev = load_data( cifar_dir, num_test=500) input_size = 32 * 32 * 3 hidden_size = 230 num_classes = 10 num_iters = 3200 best_val = -1 best_nn = None learning_rates = [1e-3] regularization_strengths = [7e-6] learning_history = {} for lr in learning_rates: for reg in regularization_strengths: nn = TwoLayerNet(input_size, hidden_size, num_classes) report = nn.train(X_train, y_train, X_val, y_val, num_iters=num_iters, batch_size=200, learning_rate=lr, learning_rate_decay=0.98, reg=reg) train_acc = np.mean(nn.predict(X_train) == y_train) val_acc = np.mean(nn.predict(X_val) == y_val) learning_history[(lr, reg)] = (train_acc, val_acc) if val_acc > best_val: best_val = val_acc best_nn = nn
return X_train, y_train, X_val, y_val, X_test, y_test # Invoke the above function to get our data. X_train, y_train, X_val, y_val, X_test, y_test = get_CIFAR10_data() print 'Train data shape: ', X_train.shape print 'Train labels shape: ', y_train.shape print 'Validation data shape: ', X_val.shape print 'Validation labels shape: ', y_val.shape print 'Test data shape: ', X_test.shape print 'Test labels shape: ', y_test.shape input_size = 32 * 32 * 3 hidden_size = 60 num_classes = 10 net = TwoLayerNet(input_size, hidden_size, num_classes) t_start = time.time() # Train the network stats = net.train(X_train, y_train, X_val, y_val, num_iters=10000, batch_size=200, learning_rate=1e-3, learning_rate_decay=0.95, reg=0.1, verbose=True) t_end = time.time() train_acc = (net.predict(X_train) == y_train).mean() print 'Training accuracy: ', train_acc val_acc = (net.predict(X_val) == y_val).mean() print 'Validation accuracy: ', val_acc test_acc = (net.predict(X_test) == y_test).mean() print 'Test accuracy: ', test_acc
print('validation accuracy :', val_acc) ''' input_size = 32 * 32 * 3 num_classes = 10 hidden_size = [75, 100, 125] results = {} best_val_acc = 0 best_net = None learning_rate = np.array([0.7, 0.8, 0.9, 1.0, 1.1]) * 1e-3 regularization_strengths = [0.75, 1, 1.25] print('running') tis = time.time() for hs in hidden_size: for lr in learning_rate: for reg in regularization_strengths: net = TwoLayerNet(input_size, hs, num_classes) states = net.train(x_train, y_train, x_val, y_val, num_iters=1500, batch_size=200, learning_rate=lr, learning_rate_decay=0.95, reg=reg, verbose=False) val_acc = (net.predict(x_val) == y_val).mean() if val_acc > best_val_acc: best_val_acc = val_acc best_net = net results[(hs, lr, reg)] = val_acc
from vis_utils import visualize_grid # Invoke the above function to get our data. X_train, y_train, X_val, y_val, X_test, y_test = get_CIFAR10_data() print('Train data shape: ', X_train.shape) print('Train labels shape: ', y_train.shape) print('Validation data shape: ', X_val.shape) print('Validation labels shape: ', y_val.shape) print('Test data shape: ', X_test.shape) print('Test labels shape: ', y_test.shape) X_train.astype("float64") input_size = 32 * 32 * 3 hidden_size = 50 num_classes = 10 net = TwoLayerNet(input_size, hidden_size, num_classes) # Train the network stats = net.train(X_train, y_train, X_val, y_val, num_iters=1000, batch_size=200, learning_rate=1e-4, learning_rate_decay=0.95, reg=0.25, verbose=True) # Predict on the validation set val_acc = (net.predict(X_val) == y_val).mean()
# adjust the hyperparameters using validation data set learning_rates = [2e-3, 1e-3, 1e-4] regularization_strengths = [3e-1, 5e-1, 3, 5, 10] results = {} best_val = -1 best_svm = None # X_val = X_train[-10000::, :] # y_val = y_train[-10000::] # X_train = X_train[:-10000:, :] # y_train = y_train[:-10000:] for lr in learning_rates: for reg in regularization_strengths: svm = TwoLayerNet(X_train.shape[1], 500, 10) svm.train(X_train, y_train, X_val, y_val, learning_rate=lr, reg=reg, num_iters=1000, batch_size=2000, verbose=True) y_train_pred = svm.predict(X_train) accuracy_train = np.mean(y_train == y_train_pred) y_val_pred = svm.predict(X_val) accuracy_val = np.mean(y_val == y_val_pred) results[(lr, reg)] = (accuracy_train, accuracy_val)
loss, _ = self.net.loss(self.x, self.y, reg=0.05) np.testing.assert_allclose(loss, self.expected_loss_wt, 1e-04) def test_neural_net_gradient(self): loss, grads = self.net.loss(self.x, self.y, reg=0.0) f = lambda w: self.net.loss(self.x, self.y, reg=0.0)[0] for param_name, grad in grads.iteritems(): grad_numerical = test_utils.eval_numerical_gradient( f, self.net.params[param_name], verbose=False) np.testing.assert_allclose(grad_numerical, grad, 1e-04) if __name__ == '__main__': suite = unittest.TestSuite() np.random.seed(0) test_net = TwoLayerNet(4, 10, 3, std=1e-1) np.random.seed(1) x = 10 * np.random.randn(5, 4) y = np.array([0, 1, 2, 2, 1]) expected_loss = np.asarray([[-0.81233741, -1.27654624, -0.70335995], [-0.17129677, -1.18803311, -0.47310444], [-0.51590475, -1.01354314, -0.8504215], [-0.15419291, -0.48629638, -0.52901952], [-0.00618733, -0.12435261, -0.15226949]]) suite.addTest( test_utils.TestCaseWithParams.get_suite(NeuralNetTest, kwargs={ 'net': test_net, 'x': x, 'y': y,
plt.subplot(2, 1, 2) plt.plot(stats['train_acc_history'], label='train') plt.plot(stats['val_acc_history'], label='val') plt.title('Classification accuracy history') plt.xlabel('Epoch') plt.ylabel('Clasification accuracy') plt.show() # find best model best_net = None input_size = 32 * 32 * 3 hidden_size = 100 num_classes = 10 net = TwoLayerNet(input_size, hidden_size, num_classes) # Train the network stats = net.train(X_train, Y_train, X_val, Y_val, num_iter=5000, batch_size=200, learning_rate=1e-4, learning_rate_decay=0.95, reg=0.3, verbose=True) # Predict on the validation set val_acc = (net.predict(X_val) == Y_val).mean()
return X_train, y_train, X_val, y_val, X_test, y_test # Invoke the above function to get our data. X_train, y_train, X_val, y_val, X_test, y_test = get_CIFAR10_data() ##print 'Train data shape: ', X_train.shape ##print 'Train labels shape: ', y_train.shape ##print 'Validation data shape: ', X_val.shape ##print 'Validation labels shape: ', y_val.shape ##print 'Test data shape: ', X_test.shape ##print 'Test labels shape: ', y_test.shape input_size = 32 * 32 * 3 hidden_size = 50 num_classes = 10 net = TwoLayerNet(input_size, hidden_size, num_classes) # Train the network ##stats = net.train(X_train, y_train, X_val, y_val, ## num_iters=500, batch_size=200, ## learning_rate=1e-4, learning_rate_decay=0.95, ## reg=0.5, verbose=True) # Predict on the validation set val_acc = (net.predict(X_val) == y_val).mean() ##print 'Validation accuracy: ', val_acc # Plot the loss function and train / validation accuracies ##plt.subplot(2, 1, 1) ##plt.plot(stats['loss_history']) ##plt.title('Loss history')
def init_toy_model(input_size, hidden_size, num_classes): np.random.seed(0) return TwoLayerNet(input_size, hidden_size, num_classes, std=1e-1)
print('Train data shape: ', X_train.shape) print('Train labels shape: ', y_train.shape) print('Validation data shape: ', X_val.shape) print('Validation labels shape: ', y_val.shape) print('Test data shape: ', X_test.shape) print('Test labels shape: ', y_test.shape) # Train a network # To train our network we will use SGD with momentum. In addition, # we will adjust the learning rate with an exponential learning rate schedule as optimization proceeds; # after each epoch, we will reduce the learning rate by multiplying it by a decay rate. input_size = 32 * 32 * 3 hidden_size = 50 num_classes = 10 net = TwoLayerNet(input_size, hidden_size, num_classes) # Train the network stats = net.train(X_train, y_train, X_val, y_val, num_iters=1000, batch_size=200, learning_rate=1e-4, learning_rate_decay=0.95, reg=0.25, verbose=True) # Predict on the validation set val_acc = (net.predict(X_val) == y_val).mean()
axis=0) # (9000,3072) -> (3072) all training data # Subtract x_train -= mean_img x_test -= mean_img x_val -= mean_img return x_train, y_train, x_val, y_val, x_test, y_test x_train, y_train, x_val, y_val, x_test, y_test = load_CIFAR10_data() input_size = 32 * 32 * 3 hidden_size = 50 num_classes = 10 net = TwoLayerNet(input_size, hidden_size, num_classes) # Training the network using SGD stats = net.train(x_train, y_train, x_val, y_val, num_iters=1200, verbose=True) val_acc = np.mean(net.predict(x_val) == y_val) print("Validation accuracy: ", val_acc) # 44.5% test_acc = (net.predict(x_test) == y_test).mean() print('Test accuracy: ', test_acc) # Plot the loss function and train / validation accuracies def visualize_loss_acc(stats): plt.subplot(2, 1, 1) plt.plot(stats['loss_his'])