def finalTest(size_training, size_test, hidden_layers, lambd, num_iterations): print "\nBeginning of the finalTest... \n" images_training, labels_training, images_test, labels_test = read_dataset(size_training, size_test) # Setup the parameters you will use for this exercise input_layer_size = 784 # 28x28 Input Images of Digits num_labels = 10 # 10 labels, from 0 to 9 (one label for each digit) layers = [input_layer_size] + hidden_layers + [num_labels] num_of_hidden_layers = len(hidden_layers) # Fill the randInitializeWeights.py in order to initialize the neural network weights. Theta = randInitializeWeights(layers) # Unroll parameters nn_weights = unroll_params(Theta) res = fmin_l_bfgs_b(costFunction, nn_weights, fprime=backwards, args=(layers, images_training, labels_training, num_labels, lambd), maxfun = num_iterations, factr = 1., disp = True) Theta = roll_params(res[0], layers) print "\nTesting Neural Network... \n" pred_training = predict(Theta, images_training) print '\nAccuracy on training set: ' + str(mean(labels_training == pred_training) * 100) pred = predict(Theta, images_test) print '\nAccuracy on test set: ' + str(mean(labels_test == pred) * 100) # Display the images where the algorithm got wrong temp = (labels_test == pred) indexes_false = [] for i in range(size_test): if temp[i] == 0: indexes_false.append(i) displayData(images_training[indexes_false, :])
def check_nn_gradients(lmd): # np.set_printoptions(precision=20) input_layer_size = 3 hidden_layer_size = 5 num_labels = 3 m = 5 # We generatesome 'random' test data theta1 = diw.randInitializeWeights(input_layer_size, hidden_layer_size) theta2 = diw.randInitializeWeights(hidden_layer_size, num_labels) # Reusing debugInitializeWeights to genete X X = diw.randInitializeWeights(input_layer_size - 1, m) X1 = np.hstack((np.ones((X.shape[0])).reshape(X.shape[0], 1), X)) y = 1 + np.mod(np.arange(1, m + 1), num_labels) # Unroll parameters nn_params = np.concatenate([theta1.flatten(), theta2.flatten()]) def cost_func(p): return ncf.ex3_nn(X, y, p, num_labels, X1, hidden_layer_size, lmd) cost, grad = cost_func(nn_params) numgrad = cng.compute_numerial_gradient(cost_func, nn_params) print(np.c_[grad, numgrad])
# Compute the cost with lambda_nn=1 print "FeedForwad Using Neural Network...(with lambda=1)" lambda_nn=1 J,grad=nnCostFunction(nn_params,input_layer_size,hidden_layer_size,num_labels,data,labels,lambda_nn) print "Cost at parameters (loaded from ex4weight): (%s)"%(J) # Sigmoid Gradient print "Evaluating sigmoid gradient..." test=np.array([1,-0.5,0,0.5,1]) g=sigmoid_gradient(test) print "sigmoid gradient with :[1,-0.5,0,0.5,1] (%s)"%(g) # Initializing Pameters print "Initializing Neural Network Parameters..." from randInitializeWeights import randInitializeWeights initial_Theta1=randInitializeWeights(input_layer_size,hidden_layer_size) initial_Theta2=randInitializeWeights(hidden_layer_size,num_labels) initial_Theta1=np.reshape(initial_Theta1,[initial_Theta1.size,1]) initial_Theta2=np.reshape(initial_Theta2,[initial_Theta2.size,1]) initial_nn_params=np.concatenate((initial_Theta1,initial_Theta2)) def costFunction(p): J,gradient=nnCostFunction(p,input_layer_size,hidden_layer_size,num_labels,data,labels,lambda_nn) print "training" print J return J def gradientFunction(p): J,gradient=nnCostFunction(p,input_layer_size,hidden_layer_size,num_labels,data,labels,lambda_nn) gradient=np.ndarray.flatten(gradient) return gradient # Training Neural Newwork...
# code in the sigmoidGradient.m file. # g = sigmoidGradient(np.array([1, -0.5, 0, 0.5, 1])) print('Sigmoid gradient evaluated at [1 -0.5 0 0.5 1]\n ', g) #% ================ Part 6: Initializing Pameters ================ # In this part of the exercise, you will be starting to implment a two # layer neural network that classifies digits. You will start by # implementing a function to initialize the weights of the neural network # (randInitializeWeights.m) initial_Theta1 = randInitializeWeights(input_layer_size, hidden_layer_size) initial_Theta2 = randInitializeWeights(hidden_layer_size, num_labels) # Unroll parameters initial_nn_params = np.hstack((initial_Theta1.flatten(),initial_Theta2.flatten())) # =============== Part 7: Implement Backpropagation =============== # Once your cost matches up with ours, you should proceed to implement the # backpropagation algorithm for the neural network. You should add to the # code you've written in nnCostFunction.m to return the partial # derivatives of the parameters. # #fprintf('\nChecking Backpropagation... \n');
num_of_hidden_layers = int(raw_input('Please select the number of hidden layers: ')) print "\n" layers = [input_layer_size] for i in range(num_of_hidden_layers): layers = layers + [int(raw_input('Please select the number nodes for the ' + str(i+1) + ' hidden layers: '))] layers = layers + [num_labels] raw_input('\nProgram paused. Press enter to continue!!!') print "\nInitializing Neural Network Parameters ...\n" # ================================ DONE ================================ # Fill the randInitializeWeights.py in order to initialize the neural network weights. Theta = randInitializeWeights(layers) # Unroll parameters nn_weights = unroll_params(Theta) raw_input('\nProgram paused. Press enter to continue!!!') # ================================ Step 3: Sigmoid ================================================ # Before you start implementing the neural network, you will first # implement the gradient for the sigmoid function. You should complete the # code in the sigmoidGradient.m file. # print "\nEvaluating sigmoid function ...\n" g = sigmoid(array([-1, -0.5, 0, 0.5, 1]))
num_of_hidden_layers = int(input('Please select the number of hidden layers: ')) print("\n") layers = [input_layer_size] for i in range(num_of_hidden_layers): layers = layers + [int(input('Please select the number nodes for the ' + str(i+1) + ' hidden layers: '))] layers = layers + [num_labels] input('\nProgram paused. Press enter to continue!!!') print("\nInitializing Neural Network Parameters ...\n") # ================================ TODO ================================ # Fill the randInitializeWeights.py in order to initialize the neural network weights. Theta = randInitializeWeights(layers) # Unroll parameters nn_weights = unroll_params(Theta) input('\nProgram paused. Press enter to continue!!!') # ================================ Step 3: Sigmoid ================================================ # Before you start implementing the neural network, you will first # implement the gradient for the sigmoid function. You should complete the # code in the sigmoidGradient.m file. # print("\nEvaluating sigmoid function ...\n") g = sigmoid(array([1, -0.5, 0, 0.5, 1]))
hidden_layer = 150 # mini batch x = np.zeros((1000, 784)) y = np.zeros((1000, 10)) # train process alpha = 0.01 epoch = 148 mini_batch = 80 l = 50 #lamda index = 5 for index in range(6): x_train = list_x_train[index] y_train = list_y_train[index] theta1 = randInitializeWeights(784, hidden_layer) theta2 = randInitializeWeights(hidden_layer, hidden_layer) theta3 = randInitializeWeights(hidden_layer, 10) print 'training... ' + str(index + 1) for i in range(epoch): # mini batch x_train, y_train = shuffle(x_train, y_train) for k in range(len(x_train)/mini_batch): x = x_train[k*mini_batch:(k + 1)*mini_batch] y = y_train[k*mini_batch:(k + 1)*mini_batch] results = nnCostFunction(theta1, theta2, theta3, x, y, lamda=l) grad1 = results[1] grad2 = results[2] grad3 = results[3] #print 'inter ' + str(i) + ' coss = ' + str(results[0])
# J, grad = ex3_nn(X1, Y, nn_parameters, K, Theta1, Theta2, X, layer, lmda) # # print('from weights import theta1,theta2 to veritify J(0.57..):', J) # # """检查sigmoid函数""" # # z1 = 100 # z2 = 0 # g_d1 = g_d(z1) # g_d2 = g_d(z2) # print('z1(0)= %f\n, z2(0.25)=%f\n'%(g_d1,g_d2)) """random_init_theta""" Theta1 = randInitializeWeights(X1.shape[1], layer) Theta2 = randInitializeWeights(layer, K) rand_nn_parameters = np.concatenate([Theta1.flatten(),Theta2.flatten()]) # print(Theta1.shape) # # 检查BP算法 # # lmda =3 # check_nn_gradients(lmda) # 训练网络 lmda = 1 def cost_func(t): return ex3_nn(X1, Y, t, K, X, layer, lmda)[0]
def main(): warnings.filterwarnings("ignore") print 'Cargando y visualizando datos...\n' input_layer_size = 784 # input_layer_size = 400 hidden_layer_size = int(neuronas.get()) num_labels = 10 lambdaP = 0 # training = scipy.io.loadmat('data1.mat') # x_training = training['X'] # y_training = training['y'] # y_training = np.squeeze(np.asarray(y_training)) (x_training, y_training) = leerTrainingSet() (x_cv, y_cv, x_test, y_test) = leerTestSet() print '\nFeedforward usando Redes Neuronales...' theta1 = randInitializeWeights(hidden_layer_size, input_layer_size) # 25 x 785 theta2 = randInitializeWeights(num_labels, hidden_layer_size) # 10 x 26 ''' thetas = scipy.io.loadmat('pesos.mat') theta1 = thetas['Theta1'] theta2 = thetas['Theta2'] theta1 = np.array(theta1) theta2 = np.array(theta2) ''' vectorTheta1 = theta1.flatten(1) # 19625 x 1 vectorTheta2 = theta2.flatten(1) # 260 x 1 nn_params = np.concatenate((vectorTheta1, vectorTheta2), axis=0) # 19885 x 1 (J, grad) = nnFuncionCosto(nn_params, input_layer_size, hidden_layer_size, num_labels, x_training, y_training, lambdaP) print 'El valor de costo inicial sin regularizar es: ' + str(J) lambdaP = float(lambdaA.get()) (J, grad) = nnFuncionCosto(nn_params, input_layer_size, hidden_layer_size, num_labels, x_training, y_training, lambdaP) print 'El valor de costo inicial regularizado es: ' + str(J) print '\nEntrenando la Red Neuronal...' extra = (input_layer_size, hidden_layer_size, num_labels, x_training, y_training, lambdaP) res = optimize.minimize(minimizarCosto, nn_params, extra, method='L-BFGS-B', jac=True, options={ 'maxiter': 25, 'disp': True }) res = res.x theta1 = np.reshape(res[:hidden_layer_size * (input_layer_size + 1)], (hidden_layer_size, (input_layer_size + 1)), order='F') # 25 x 785 theta2 = np.reshape(res[((hidden_layer_size * (input_layer_size + 1))):], (num_labels, (hidden_layer_size + 1)), order='F') pred = predecir(theta1, theta2, x_training) res_pred1 = np.mean(pred == y_training) * 100 error_pred1 = 100 - res_pred1 print 'Precision de la Red Neuronal sobre el training set: ' + str( res_pred1) print 'Error con el training set: ' + str(error_pred1) print pred = predecir(theta1, theta2, x_test) res_pred2 = np.mean(pred == y_test) * 100 error_pred2 = 100 - res_pred2 print 'Precision de la Red Neuronal sobre el test set: ' + str(res_pred2) print 'Error con el test set: ' + str(error_pred2) insertarDatosBD(1, hidden_layer_size, lambdaP, J, res_pred1, res_pred2, error_pred1, error_pred2)
def ex4(): ## Machine Learning Online Class - Exercise 4 Neural Network Learning # Instructions # ------------ # # This file contains code that helps you get started on the # linear exercise. You will need to complete the following functions # in this exericse: # # sigmoidGradient.m # randInitializeWeights.m # nnCostFunction.m # # For this exercise, you will not need to change any code in this file, # or any other files other than those mentioned above. # ## Initialization #clear ; close all; clc ## Setup the parameters you will use for this exercise input_layer_size = 400 # 20x20 Input Images of Digits hidden_layer_size = 25 # 25 hidden units num_labels = 10 # 10 labels, from 1 to 10 # (note that we have mapped "0" to label 10) ## =========== Part 1: Loading and Visualizing Data ============= # We start the exercise by first loading and visualizing the dataset. # You will be working with a dataset that contains handwritten digits. # # Load Training Data print('Loading and Visualizing Data ...') mat = scipy.io.loadmat('ex4data1.mat') X = mat['X'] y = mat['y'].ravel() m = X.shape[0] # Randomly select 100 data points to display sel = np.random.choice(m, 100, replace=False) displayData(X[sel, :]) plt.savefig('figure1.png') print('Program paused. Press enter to continue.') #pause; ## ================ Part 2: Loading Parameters ================ # In this part of the exercise, we load some pre-initialized # neural network parameters. print('\nLoading Saved Neural Network Parameters ...') # Load the weights into variables Theta1 and Theta2 mat = scipy.io.loadmat('ex4weights.mat') Theta1 = mat['Theta1'] Theta2 = mat['Theta2'] # Unroll parameters nn_params = np.concatenate([Theta1.ravel(), Theta2.ravel()]) ## ================ Part 3: Compute Cost (Feedforward) ================ # To the neural network, you should first start by implementing the # feedforward part of the neural network that returns the cost only. You # should complete the code in nnCostFunction.m to return cost. After # implementing the feedforward to compute the cost, you can verify that # your implementation is correct by verifying that you get the same cost # as us for the fixed debugging parameters. # # We suggest implementing the feedforward cost *without* regularization # first so that it will be easier for you to debug. Later, in part 4, you # will get to implement the regularized cost. # print('\nFeedforward Using Neural Network ...') # Weight regularization parameter (we set this to 0 here). lambda_value = 0 J = nnCostFunction(nn_params, input_layer_size, hidden_layer_size, num_labels, X, y, lambda_value)[0] print( 'Cost at parameters (loaded from ex4weights): %f \n(this value should be about 0.287629)' % J) print('\nProgram paused. Press enter to continue.') #pause; ## =============== Part 4: Implement Regularization =============== # Once your cost function implementation is correct, you should now # continue to implement the regularization with the cost. # print('\nChecking Cost Function (w/ Regularization) ... ') # Weight regularization parameter (we set this to 1 here). lambda_value = 1 J = nnCostFunction(nn_params, input_layer_size, hidden_layer_size, num_labels, X, y, lambda_value)[0] print( 'Cost at parameters (loaded from ex4weights): %f \n(this value should be about 0.383770)' % J) print('Program paused. Press enter to continue.') #pause; ## ================ Part 5: Sigmoid Gradient ================ # Before you start implementing the neural network, you will first # implement the gradient for the sigmoid function. You should complete the # code in the sigmoidGradient.m file. # print('\nEvaluating sigmoid gradient...') g = sigmoidGradient(np.array([1, -0.5, 0, 0.5, 1])) print('Sigmoid gradient evaluated at [1 -0.5 0 0.5 1]:') print(formatter('%f ', g)) print('\n') print('Program paused. Press enter to continue.') #pause; ## ================ Part 6: Initializing Pameters ================ # In this part of the exercise, you will be starting to implment a two # layer neural network that classifies digits. You will start by # implementing a function to initialize the weights of the neural network # (randInitializeWeights.m) print('\nInitializing Neural Network Parameters ...') initial_Theta1 = randInitializeWeights(input_layer_size, hidden_layer_size) initial_Theta2 = randInitializeWeights(hidden_layer_size, num_labels) # Unroll parameters initial_nn_params = np.concatenate( [initial_Theta1.ravel(), initial_Theta2.ravel()]) ## =============== Part 7: Implement Backpropagation =============== # Once your cost matches up with ours, you should proceed to implement the # backpropagation algorithm for the neural network. You should add to the # code you've written in nnCostFunction.m to return the partial # derivatives of the parameters. # print('\nChecking Backpropagation... ') # Check gradients by running checkNNGradients checkNNGradients() print('\nProgram paused. Press enter to continue.') #pause; ## =============== Part 8: Implement Regularization =============== # Once your backpropagation implementation is correct, you should now # continue to implement the regularization with the cost and gradient. # print('\nChecking Backpropagation (w/ Regularization) ... ') # Check gradients by running checkNNGradients lambda_value = 3 checkNNGradients(lambda_value) # Also output the costFunction debugging values debug_J = nnCostFunction(nn_params, input_layer_size, hidden_layer_size, num_labels, X, y, lambda_value)[0] print( '\n\nCost at (fixed) debugging parameters (w/ lambda = 10): %f \n(this value should be about 0.576051)\n\n' % debug_J) print('Program paused. Press enter to continue.') #pause; ## =================== Part 8: Training NN =================== # You have now implemented all the code necessary to train a neural # network. To train your neural network, we will now use "fmincg", which # is a function which works similarly to "fminunc". Recall that these # advanced optimizers are able to train our cost functions efficiently as # long as we provide them with the gradient computations. # print('\nTraining Neural Network... ') # After you have completed the assignment, change the MaxIter to a larger # value to see how more training helps. options = {'maxiter': 50} # You should also try different values of lambda lambda_value = 1 # Create "short hand" for the cost function to be minimized costFunction = lambda p: nnCostFunction( p, input_layer_size, hidden_layer_size, num_labels, X, y, lambda_value) # Now, costFunction is a function that takes in only one argument (the # neural network parameters) res = optimize.minimize(costFunction, initial_nn_params, jac=True, method='TNC', options=options) nn_params = res.x # Obtain Theta1 and Theta2 back from nn_params Theta1 = nn_params[0:hidden_layer_size * (input_layer_size + 1)].reshape( hidden_layer_size, input_layer_size + 1) Theta2 = nn_params[hidden_layer_size * (input_layer_size + 1):].reshape( num_labels, hidden_layer_size + 1) print('Program paused. Press enter to continue.') #pause; ## ================= Part 9: Visualize Weights ================= # You can now "visualize" what the neural network is learning by # displaying the hidden units to see what features they are capturing in # the data. print('\nVisualizing Neural Network... ') displayData(Theta1[:, 1:]) plt.savefig('figure2.png') print('\nProgram paused. Press enter to continue.') #pause; ## ================= Part 10: Implement Predict ================= # After training the neural network, we would like to use it to predict # the labels. You will now implement the "predict" function to use the # neural network to predict the labels of the training set. This lets # you compute the training set accuracy. pred = predict(Theta1, Theta2, X) print('\nTraining Set Accuracy: %f' % (np.mean( (pred == y).astype(int)) * 100))
def main(): ''' Main function ''' ## %% =========== Part 1: Loading and Visualizing Data ============= #% We start the exercise by first loading and visualizing the dataset. #% You will be working with a dataset that contains handwritten digits. #% # Read the Matlab data m, n, X, y = getMatlabTrainingData() # number of features input_layer_size = n # Select some random images from X print('Selecting random examples of the data to display.\n') sel = np.random.permutation(m) sel = sel[0:100] # Re-work the data orientation of each training example image_size = 20 XMatlab = np.copy(X) # Need a deep copy, not just the reference for i in range(m): XMatlab[i, :] = XMatlab[i, :].reshape(image_size, image_size).transpose().reshape(1, image_size*image_size) # display the sample images displayData(XMatlab[sel, :]) # Print Out the labels for what is being seen. print('These are the labels for the data ...\n') print(y[sel, :].reshape(10, 10)) # Pause program print("Program paused. Press Ctrl-D to continue.\n") code.interact(local=dict(globals(), **locals())) print(" ... continuing\n ") #%% ================ Part 2: Loading Parameters ================ #% In this part of the exercise, we load some pre-initialized # % neural network parameters. print('\nLoading Saved Neural Network Parameters ...\n') # Load the weights into variables Theta1 and Theta2 import scipy .io as sio fnWeights = '/home/jennym/Kaggle/DigitRecognizer/ex4/ex4weights.mat' weights = sio.loadmat(fnWeights) Theta1 = weights['Theta1'] Theta2 = weights['Theta2'] #% Unroll parameters nn_params = np.hstack((Theta1.ravel(order='F'), Theta2.ravel(order='F'))) #%% ================ Part 3: Compute Cost (Feedforward) ================ #% To the neural network, you should first start by implementing the #% feedforward part of the neural network that returns the cost only. You #% should complete the code in nnCostFunction.m to return cost. After #% implementing the feedforward to compute the cost, you can verify that #% your implementation is correct by verifying that you get the same cost #% as us for the fixed debugging parameters. #% #% We suggest implementing the feedforward cost *without* regularization #% first so that it will be easier for you to debug. Later, in part 4, you #% will get to implement the regularized cost. #% print('\nFeedforward Using Neural Network ...\n') #% Weight regularization parameter (we set this to 0 here). MLlambda = 0.0 # Cluge, put y back to matlab version, then adjust to use python # indexing later into y_matrix y[(y == 0)] = 10 y = y - 1 J, _ = nnCostFunction(nn_params, input_layer_size, hidden_layer_size, num_labels, X, y, MLlambda) print('Cost at parameters (loaded from ex4weights): ' + str(J) + '\n (this value should be about 0.287629)\n') # Pause print("Program paused. Press Ctrl-D to continue.\n") code.interact(local=dict(globals(), **locals())) print(" ... continuing\n ") #%% =============== Part 4: Implement Regularization =============== #% Once your cost function implementation is correct, you should now #% continue to implement the regularization with the cost. #% print('\nChecking Cost Function (with Regularization) ... \n') # % Weight regularization parameter (we set this to 1 here). MLlambda = 1.0 J, _ = nnCostFunction(nn_params, input_layer_size, hidden_layer_size, num_labels, X, y, MLlambda) print('Cost at parameters (loaded from ex4weights): ' + str(J) + '\n(this value should be about 0.383770)\n'); # Pause print("Program paused. Press Ctrl-D to continue.\n") code.interact(local=dict(globals(), **locals())) print(" ... continuing\n ") #%% ================ Part 5: Sigmoid Gradient ================ #% Before you start implementing the neural network, you will first #% implement the gradient for the sigmoid function. You should complete the #% code in the sigmoidGradient.m file. #% print('\nEvaluating sigmoid gradient...\n') g = sigmoidGradient(np.array([1, -0.5, 0, 0.5, 1])) print('Sigmoid gradient evaluated at [1 -0.5 0 0.5 1]:\n ') print(g) print('\n\n') # Pause print("Program paused. Press Ctrl-D to continue.\n") code.interact(local=dict(globals(), **locals())) print(" ... continuing\n ") #%% ================ Part 6: Initializing Parameters ================ #% In this part of the exercise, you will be starting to implement a two #% layer neural network that classifies digits. You will start by #% implementing a function to initialize the weights of the neural network #% (randInitializeWeights.m) print('\nInitializing Neural Network Parameters ...\n') initial_Theta1 = randInitializeWeights(input_layer_size, hidden_layer_size) initial_Theta2 = randInitializeWeights(hidden_layer_size, num_labels) #% Unroll parameters initial_nn_params = np.hstack(( initial_Theta1.ravel(order = 'F'), initial_Theta2.ravel(order = 'F'))) # Pause print("Program paused. Press Ctrl-D to continue.\n") code.interact(local=dict(globals(), **locals())) print(" ... continuing\n ") #%% =============== Part 7: Implement Backpropagation =============== #% Once your cost matches up with ours, you should proceed to implement the #% backpropagation algorithm for the neural network. You should add to the #% code you've written in nnCostFunction.m to return the partial #% derivatives of the parameters. #% print('\nChecking Backpropagation... \n') #% Check gradients by running checkNNGradients checkNNGradients() # Pause print("Program paused. Press Ctrl-D to continue.\n") code.interact(local=dict(globals(), **locals())) print(" ... continuing\n ") #%% =============== Part 8: Implement Regularization =============== #% Once your backpropagation implementation is correct, you should now #% continue to implement the regularization with the cost and gradient. #% print('\nChecking Backpropagation (w/ Regularization) ... \n') #% Check gradients by running checkNNGradients MLlambda = 3 checkNNGradients(MLlambda) # Pause print("Program paused. Press Ctrl-D to continue.\n") code.interact(local=dict(globals(), **locals())) print(" ... continuing\n ") #% Also output the costFunction debugging values debug_J, _ = nnCostFunction(nn_params, input_layer_size, hidden_layer_size, num_labels, X, y, MLlambda) print('\n\n Cost at (fixed) debugging parameters (w/ lambda = ' + '{0}): {1}'.format(MLlambda, debug_J)) print('\n (this value should be about 0.576051)\n\n') # Pause print("Program paused. Press Ctrl-D to continue.\n") code.interact(local=dict(globals(), **locals())) print(" ... continuing\n ") #%% =================== Part 8b: Training NN =================== #% You have now implemented all the code necessary to train a neural #% network. To train your neural network, we will now use "fmincg", which #% is a function which works similarly to "fminunc". Recall that these #% advanced optimizers are able to train our cost functions efficiently as #% long as we provide them with the gradient computations. #% print ('\nTraining Neural Network... \n') #% After you have completed the assignment, change the MaxIter to a larger #% value to see how more training helps. #% jkm change maxIter from 50-> 400 options = {'maxiter': MAXITER} #% You should also try different values of lambda MLlambda = 1 #% Create "short hand" for the cost function to be minimized costFunc = lambda p: nnCostFunction(p, input_layer_size, hidden_layer_size, num_labels, X, y, MLlambda) #% Now, costFunction is a function that takes in only one argument (the #% neural network parameters) ''' NOTES: Call scipy optimize minimize function method : str or callable, optional Type of solver. CG -> Minimization of scalar function of one or more variables using the conjugate gradient algorithm. jac : bool or callable, optional Jacobian (gradient) of objective function. Only for CG, BFGS, Newton-CG, L-BFGS-B, TNC, SLSQP, dogleg, trust-ncg. If jac is a Boolean and is True, fun is assumed to return the gradient along with the objective function. If False, the gradient will be estimated numerically. jac can also be a callable returning the gradient of the objective. In this case, it must accept the same arguments as fun. callback : callable, optional. Called after each iteration, as callback(xk), where xk is the current parameter vector. ''' # Setup a callback for displaying the cost at the end of each iteration class Callback(object): def __init__(self): self.it = 0 def __call__(self, p): self.it += 1 print "Iteration %5d | Cost: %e" % (self.it, costFunc(p)[0]) result = sci.minimize(costFunc, initial_nn_params, method='CG', jac=True, options=options, callback=Callback()) nn_params = result.x cost = result.fun # matlab: [nn_params, cost] = fmincg(costFunction, initial_nn_params, options); #% Obtain Theta1 and Theta2 back from nn_params Theta1 = np.reshape(nn_params[:hidden_layer_size * (input_layer_size + 1)], (hidden_layer_size, (input_layer_size + 1)), order = 'F') Theta2 = np.reshape(nn_params[hidden_layer_size * (input_layer_size + 1):], (num_labels, (hidden_layer_size + 1)), order = 'F') # Pause print("Program paused. Press Ctrl-D to continue.\n") code.interact(local=dict(globals(), **locals())) print(" ... continuing\n ") #%% ================= Part 9: Visualize Weights ================= #% You can now "visualize" what the neural network is learning by #% displaying the hidden units to see what features they are capturing in #% the data.# print('\nVisualizing Neural Network... \n') displayData(Theta1[:, 1:]) # Pause print("Program paused. Press Ctrl-D to continue.\n") code.interact(local=dict(globals(), **locals())) print(" ... continuing\n ") #%% ================= Part 10: Implement Predict ================= #% After training the neural network, we would like to use it to predict #% the labels. You will now implement the "predict" function to use the #% neural network to predict the labels of the training set. This lets #% you compute the training set accuracy. pred = predict(Theta1, Theta2, X) # JKM - my array was column stacked - don't understand why this works pp = np.row_stack(pred) accuracy = np.mean(np.double(pp == y)) * 100 print('\nTraining Set Accuracy: {0} \n'.format(accuracy)) # Pause print("Program paused. Press Ctrl-D to continue.\n") code.interact(local=dict(globals(), **locals())) print(" ... continuing\n ") # ======================================== # All Done! return