def test_forward_pass(self): network = NeuralNetwork(3, 2, 1, 0.5) final_out, hidden_out = network.forward_pass_train(inputs) self.assertIsNotNone(final_out) self.assertIsNotNone(hidden_out) self.assertEqual((1,2),np.shape(hidden_out)) self.assertEqual((1,1),np.shape(final_out))
def test_run(self): # Test correctness of run method network = NeuralNetwork(3, 2, 1, 0.5) network.weights_input_to_hidden = test_w_i_h.copy() network.weights_hidden_to_output = test_w_h_o.copy() self.assertTrue(np.allclose(network.run(inputs), 0.09998924))
def test_backprop(self): network = NeuralNetwork(3, 4, 1, 0.5) final_out, hidden_out = network.forward_pass_train(inputs) deltawOut = np.zeros(np.shape(network.weights_hidden_to_output)) #self.assertEqual(np.shape(deltawOut),np.shape(test_w_h_o)) deltawHidden = np.zeros(np.shape(network.weights_input_to_hidden)) #self.assertEqual(np.shape(deltawHidden), np.shape(test_w_i_h)) deltaWIH, deltaWHO = network.backpropagation(final_out, hidden_out, inputs, targets, deltawHidden,deltawOut) self.assertIsNotNone(deltaWIH) self.assertIsNotNone(deltaWHO)
def get_loss(params): iterations, learning_rate, hidden_nodes = params N_i = train_features.shape[1] network = NeuralNetwork(N_i, hidden_nodes, 1, learning_rate) for ii in range(iterations): batch = np.random.choice(train_features.index, size=128) X, y = train_features.ix[batch].values, train_targets.ix[batch]['cnt'] network.train(X, y) return MSE(network.run(val_features).T, val_targets['cnt'].values)
def train(): iterations = FLAGS.iterations or 3000 learning_rate = FLAGS.learning_rate or 1 hidden_nodes = FLAGS.hidden_nodes or 12 output_nodes = 1 N_i = train_features.shape[1] network = NeuralNetwork(N_i, hidden_nodes, output_nodes, learning_rate) losses = {'train': [], 'validation': []} print('iterations', FLAGS) for ii in range(iterations): iteration(network, losses, ii)
def test_train(self): # Test that weights are updated correctly on training network = NeuralNetwork(3, 2, 1, 0.5) network.weights_input_to_hidden = test_w_i_h.copy() network.weights_hidden_to_output = test_w_h_o.copy() network.train(inputs, targets) self.assertTrue(np.allclose(network.weights_hidden_to_output, np.array([[ 0.37275328],[-0.03172939]]))) self.assertTrue(np.allclose(network.weights_input_to_hidden, np.array([[ 0.10562014, -0.20185996], [0.39775194, 0.50074398], [-0.29887597, 0.19962801]])))
def test_train(self): # Test that weights are updated correctly on training network = NeuralNetwork(3, 2, 1, 0.5) network.weights_input_to_hidden = test_w_i_h.copy() network.weights_hidden_to_output = test_w_h_o.copy() network.train(inputs, targets) self.assertTrue( np.allclose(network.weights_hidden_to_output, np.array([[0.41434416], [0.00729889], [0.32494378]]))) self.assertTrue( np.allclose( network.weights_input_to_hidden, np.array([[0.10883301, -0.20292324, 0.30587281], [0.3964668, 0.5011693, 0.29765088], [-0.2982334, 0.19941535, 0.30117456]])))
def test_activation(self): network = NeuralNetwork(3, 2, 1, 0.5) # Test that the activation function is a sigmoid self.assertTrue(np.all(network.activation_function(0.5) == 1/(1+np.exp(-0.5))))
def runTests(): import numpy as np import pandas as pd import matplotlib.pyplot as plt data_path = 'Bike-Sharing-Dataset/hour.csv' rides = pd.read_csv(data_path) rides[:24*10].plot(x='dteday', y='cnt') dummy_fields = ['season', 'weathersit', 'mnth', 'hr', 'weekday'] for each in dummy_fields: dummies = pd.get_dummies(rides[each], prefix=each, drop_first=False) rides = pd.concat([rides, dummies], axis=1) fields_to_drop = ['instant', 'dteday', 'season', 'weathersit', 'weekday', 'atemp', 'mnth', 'workingday', 'hr'] data = rides.drop(fields_to_drop, axis=1) quant_features = ['casual', 'registered', 'cnt', 'temp', 'hum', 'windspeed'] # Store scalings in a dictionary so we can convert back later scaled_features = {} for each in quant_features: mean, std = data[each].mean(), data[each].std() scaled_features[each] = [mean, std] data.loc[:, each] = (data[each] - mean)/std # Save data for approximately the last 21 days test_data = data[-21*24:] # Now remove the test data from the data set data = data[:-21*24] # Separate the data into features and targets target_fields = ['cnt', 'casual', 'registered'] features, targets = data.drop(target_fields, axis=1), data[target_fields] test_features, test_targets = test_data.drop(target_fields, axis=1), test_data[target_fields] # Hold out the last 60 days or so of the remaining data as a validation set train_features, train_targets = features[:-60*24], targets[:-60*24] val_features, val_targets = features[-60*24:], targets[-60*24:] from my_answers import NeuralNetwork def MSE(y, Y): return np.mean((y-Y)**2) inputs = np.array([[0.5, -0.2, 0.1]]) targets = np.array([[0.4]]) test_w_i_h = np.array([[0.1, -0.2], [0.4, 0.5], [-0.3, 0.2]]) test_w_h_o = np.array([[0.3], [-0.1]]) network = NeuralNetwork(3, 2, 1, 0.5) network.weights_input_to_hidden = test_w_i_h.copy() network.weights_hidden_to_output = test_w_h_o.copy() network.train(inputs, targets) network = NeuralNetwork(3, 2, 1, 0.5) network.weights_input_to_hidden = test_w_i_h.copy() network.weights_hidden_to_output = test_w_h_o.copy() network.run(inputs)
# In a model where all the weights are optimized, the more hidden nodes you have, the more accurate the predictions of the model will be. (A fully optimized model could have weights of zero, after all.) However, the more hidden nodes you have, the harder it will be to optimize the weights of the model, and the more likely it will be that suboptimal weights will lead to overfitting. With overfitting, the model will memorize the training data instead of learning the true pattern, and won't generalize well to unseen data. # # Try a few different numbers and see how it affects the performance. You can look at the losses dictionary for a metric of the network performance. If the number of hidden units is too low, then the model won't have enough space to learn and if it is too high there are too many options for the direction that the learning can take. The trick here is to find the right balance in number of hidden units you choose. You'll generally find that the best number of hidden nodes to use ends up being between the number of input and output nodes. # In[ ]: import sys #################### ### Set the hyperparameters in you myanswers.py file ### #################### from my_answers import iterations, learning_rate, hidden_nodes, output_nodes N_i = train_features.shape[1] network = NeuralNetwork(N_i, hidden_nodes, output_nodes, learning_rate) losses = {'train': [], 'validation': []} for ii in range(iterations): # Go through a random batch of 128 records from the training data set batch = np.random.choice(train_features.index, size=128) X, y = train_features.ix[batch].values, train_targets.ix[batch]['cnt'] network.train(X, y) # Printing out the training progress train_loss = MSE( network.run(train_features).T, train_targets['cnt'].values) val_loss = MSE(network.run(val_features).T, val_targets['cnt'].values) sys.stdout.write("\rProgress: {:2.1f}".format(100 * ii / float(iterations)) +
import numpy as np from my_answers import NeuralNetwork inputs = np.array([[0.5, -0.2, 0.1]]) targets = np.array([[0.4]]) test_w_i_h = np.array([[0.1, -0.2], [0.4, 0.5], [-0.3, 0.2]]) test_w_h_o = np.array([[0.3], [-0.1]]) network = NeuralNetwork(3, 2, 1, 0.5) network.weights_input_to_hidden = test_w_i_h.copy() network.weights_hidden_to_output = test_w_h_o.copy() network.train(inputs, targets) # ----------------------------- import unittest inputs = np.array([[0.5, -0.2, 0.1]]) targets = np.array([[0.4]]) test_w_i_h = np.array([[0.1, -0.2], [0.4, 0.5], [-0.3, 0.2]]) test_w_h_o = np.array([[0.3], [-0.1]]) class TestMethods(unittest.TestCase): ########## # Unit tests for data loading ##########
y=np.array(data['Attrition']) X_train, test_features, y_train, test_targets = cross_validation.train_test_split(X,y, test_size=0.2) train_features,val_features, train_targets, val_targets = cross_validation.train_test_split(X_train,y_train,test_size=0.2) print(train_features.shape) import sys from my_answers import iterations, learning_rate, hidden_nodes, output_nodes N_i = train_features.shape[1] network = NeuralNetwork(N_i, hidden_nodes, output_nodes, learning_rate) losses = {'train':[], 'validation':[]} for ii in range(iterations): # Go through a random batch of 128 records from the training data set #batch = np.random.choice(train_features, size=128) X, y = train_features, train_targets network.train(X, y) # Printing out the training progress train_loss = MSE(network.run(train_features).T, train_targets) val_loss = MSE(network.run(val_features).T, val_targets) sys.stdout.write("\rProgress: {:2.1f}".format(100 * ii/float(iterations)) \ + "% ... Training loss: " + str(train_loss)[:5] \ + " ... Validation loss: " + str(val_loss)[:5])
import sys import torch import torch.nn as nn import numpy as np from my_answers import NeuralNetwork #################### ### Set the hyperparameters in you myanswers.py file ### #################### from my_answers import iterations, learning_rate, hidden_nodes, output_nodes N_i = train_features.shape[1] network = NeuralNetwork(N_i, hidden_nodes, output_nodes, learning_rate) print(network) from torch import optim # specify loss criterion = nn.MSELoss() # specify optimizer optimizer = optim.SGD(network.parameters(), lr = learning_rate) #optimizer = optim.Adam(network.parameters(), lr = learning_rate) losses = {'train':[], 'validation':[]} last_lr = learning_rate for ii in range(iterations): # varying learning rate for SGD if ii == 1500: