def finalTest(size_training, size_test, hidden_layers, lambd, num_iterations):
print "\nBeginning of the finalTest... \n"
images_training, labels_training, images_test, labels_test = read_dataset(size_training, size_test)
# Setup the parameters you will use for this exercise
input_layer_size = 784 # 28x28 Input Images of Digits
num_labels = 10 # 10 labels, from 0 to 9 (one label for each digit)
layers = [input_layer_size] + hidden_layers + [num_labels]
num_of_hidden_layers = len(hidden_layers)
# Fill the randInitializeWeights.py in order to initialize the neural network weights.
Theta = randInitializeWeights(layers)
# Unroll parameters
nn_weights = unroll_params(Theta)
res = fmin_l_bfgs_b(costFunction, nn_weights, fprime=backwards, args=(layers, images_training, labels_training, num_labels, lambd), maxfun = num_iterations, factr = 1., disp = True)
Theta = roll_params(res[0], layers)
print "\nTesting Neural Network... \n"
pred_training = predict(Theta, images_training)
print '\nAccuracy on training set: ' + str(mean(labels_training == pred_training) * 100)
pred = predict(Theta, images_test)
print '\nAccuracy on test set: ' + str(mean(labels_test == pred) * 100)
# Display the images where the algorithm got wrong
temp = (labels_test == pred)
indexes_false = []
for i in range(size_test):
if temp[i] == 0:
indexes_false.append(i)
displayData(images_training[indexes_false, :])
def check_nn_gradients(lmd):
# np.set_printoptions(precision=20)
input_layer_size = 3
hidden_layer_size = 5
num_labels = 3
m = 5
# We generatesome 'random' test data
theta1 = diw.randInitializeWeights(input_layer_size, hidden_layer_size)
theta2 = diw.randInitializeWeights(hidden_layer_size, num_labels)
# Reusing debugInitializeWeights to genete X
X = diw.randInitializeWeights(input_layer_size - 1, m)
X1 = np.hstack((np.ones((X.shape[0])).reshape(X.shape[0], 1), X))
y = 1 + np.mod(np.arange(1, m + 1), num_labels)
# Unroll parameters
nn_params = np.concatenate([theta1.flatten(), theta2.flatten()])
def cost_func(p):
return ncf.ex3_nn(X, y, p, num_labels, X1, hidden_layer_size, lmd)
cost, grad = cost_func(nn_params)
numgrad = cng.compute_numerial_gradient(cost_func, nn_params)
print(np.c_[grad, numgrad])
# Compute the cost with lambda_nn=1
print "FeedForwad Using Neural Network...(with lambda=1)"
lambda_nn=1
J,grad=nnCostFunction(nn_params,input_layer_size,hidden_layer_size,num_labels,data,labels,lambda_nn)
print "Cost at parameters (loaded from ex4weight): (%s)"%(J)
# Sigmoid Gradient
print "Evaluating sigmoid gradient..."
test=np.array([1,-0.5,0,0.5,1])
g=sigmoid_gradient(test)
print "sigmoid gradient with :[1,-0.5,0,0.5,1] (%s)"%(g)
# Initializing Pameters
print "Initializing Neural Network Parameters..."
from randInitializeWeights import randInitializeWeights
initial_Theta1=randInitializeWeights(input_layer_size,hidden_layer_size)
initial_Theta2=randInitializeWeights(hidden_layer_size,num_labels)
initial_Theta1=np.reshape(initial_Theta1,[initial_Theta1.size,1])
initial_Theta2=np.reshape(initial_Theta2,[initial_Theta2.size,1])
initial_nn_params=np.concatenate((initial_Theta1,initial_Theta2))
def costFunction(p):
J,gradient=nnCostFunction(p,input_layer_size,hidden_layer_size,num_labels,data,labels,lambda_nn)
print "training"
print J
return J
def gradientFunction(p):
J,gradient=nnCostFunction(p,input_layer_size,hidden_layer_size,num_labels,data,labels,lambda_nn)
gradient=np.ndarray.flatten(gradient)
return gradient
# Training Neural Newwork...
# code in the sigmoidGradient.m file.
#
g = sigmoidGradient(np.array([1, -0.5, 0, 0.5, 1]))
print('Sigmoid gradient evaluated at [1 -0.5 0 0.5 1]\n ', g)
#% ================ Part 6: Initializing Pameters ================
# In this part of the exercise, you will be starting to implment a two
# layer neural network that classifies digits. You will start by
# implementing a function to initialize the weights of the neural network
# (randInitializeWeights.m)
initial_Theta1 = randInitializeWeights(input_layer_size, hidden_layer_size)
initial_Theta2 = randInitializeWeights(hidden_layer_size, num_labels)
# Unroll parameters
initial_nn_params = np.hstack((initial_Theta1.flatten(),initial_Theta2.flatten()))
# =============== Part 7: Implement Backpropagation ===============
# Once your cost matches up with ours, you should proceed to implement the
# backpropagation algorithm for the neural network. You should add to the
# code you've written in nnCostFunction.m to return the partial
# derivatives of the parameters.
#
#fprintf('\nChecking Backpropagation... \n');
num_of_hidden_layers = int(raw_input('Please select the number of hidden layers: '))
print "\n"
layers = [input_layer_size]
for i in range(num_of_hidden_layers):
layers = layers + [int(raw_input('Please select the number nodes for the ' + str(i+1) + ' hidden layers: '))]
layers = layers + [num_labels]
raw_input('\nProgram paused. Press enter to continue!!!')
print "\nInitializing Neural Network Parameters ...\n"
# ================================ DONE ================================
# Fill the randInitializeWeights.py in order to initialize the neural network weights.
Theta = randInitializeWeights(layers)
# Unroll parameters
nn_weights = unroll_params(Theta)
raw_input('\nProgram paused. Press enter to continue!!!')
# ================================ Step 3: Sigmoid ================================================
# Before you start implementing the neural network, you will first
# implement the gradient for the sigmoid function. You should complete the
# code in the sigmoidGradient.m file.
#
print "\nEvaluating sigmoid function ...\n"
g = sigmoid(array([-1, -0.5, 0, 0.5, 1]))
num_of_hidden_layers = int(input('Please select the number of hidden layers: '))
print("\n")
layers = [input_layer_size]
for i in range(num_of_hidden_layers):
layers = layers + [int(input('Please select the number nodes for the ' + str(i+1) + ' hidden layers: '))]
layers = layers + [num_labels]
input('\nProgram paused. Press enter to continue!!!')
print("\nInitializing Neural Network Parameters ...\n")
# ================================ TODO ================================
# Fill the randInitializeWeights.py in order to initialize the neural network weights.
Theta = randInitializeWeights(layers)
# Unroll parameters
nn_weights = unroll_params(Theta)
input('\nProgram paused. Press enter to continue!!!')
# ================================ Step 3: Sigmoid ================================================
# Before you start implementing the neural network, you will first
# implement the gradient for the sigmoid function. You should complete the
# code in the sigmoidGradient.m file.
#
print("\nEvaluating sigmoid function ...\n")
g = sigmoid(array([1, -0.5, 0, 0.5, 1]))
hidden_layer = 150
# mini batch
x = np.zeros((1000, 784))
y = np.zeros((1000, 10))
# train process
alpha = 0.01
epoch = 148
mini_batch = 80
l = 50 #lamda
index = 5
for index in range(6):
x_train = list_x_train[index]
y_train = list_y_train[index]
theta1 = randInitializeWeights(784, hidden_layer)
theta2 = randInitializeWeights(hidden_layer, hidden_layer)
theta3 = randInitializeWeights(hidden_layer, 10)
print 'training... ' + str(index + 1)
for i in range(epoch):
# mini batch
x_train, y_train = shuffle(x_train, y_train)
for k in range(len(x_train)/mini_batch):
x = x_train[k*mini_batch:(k + 1)*mini_batch]
y = y_train[k*mini_batch:(k + 1)*mini_batch]
results = nnCostFunction(theta1, theta2, theta3, x, y, lamda=l)
grad1 = results[1]
grad2 = results[2]
grad3 = results[3]
#print 'inter ' + str(i) + ' coss = ' + str(results[0])
# J, grad = ex3_nn(X1, Y, nn_parameters, K, Theta1, Theta2, X, layer, lmda)
#
# print('from weights import theta1,theta2 to veritify J(0.57..):', J)
#
# """检查sigmoid函数"""
#
# z1 = 100
# z2 = 0
# g_d1 = g_d(z1)
# g_d2 = g_d(z2)
# print('z1(0)= %f\n, z2(0.25)=%f\n'%(g_d1,g_d2))
"""random_init_theta"""
Theta1 = randInitializeWeights(X1.shape[1], layer)
Theta2 = randInitializeWeights(layer, K)
rand_nn_parameters = np.concatenate([Theta1.flatten(),Theta2.flatten()])
# print(Theta1.shape)
# # 检查BP算法
#
# lmda =3
# check_nn_gradients(lmda)
# 训练网络
lmda = 1
def cost_func(t):
return ex3_nn(X1, Y, t, K, X, layer, lmda)[0]
def main():
warnings.filterwarnings("ignore")
print 'Cargando y visualizando datos...\n'
input_layer_size = 784
# input_layer_size = 400
hidden_layer_size = int(neuronas.get())
num_labels = 10
lambdaP = 0
# training = scipy.io.loadmat('data1.mat')
# x_training = training['X']
# y_training = training['y']
# y_training = np.squeeze(np.asarray(y_training))
(x_training, y_training) = leerTrainingSet()
(x_cv, y_cv, x_test, y_test) = leerTestSet()
print '\nFeedforward usando Redes Neuronales...'
theta1 = randInitializeWeights(hidden_layer_size,
input_layer_size) # 25 x 785
theta2 = randInitializeWeights(num_labels, hidden_layer_size) # 10 x 26
'''
thetas = scipy.io.loadmat('pesos.mat')
theta1 = thetas['Theta1']
theta2 = thetas['Theta2']
theta1 = np.array(theta1)
theta2 = np.array(theta2)
'''
vectorTheta1 = theta1.flatten(1) # 19625 x 1
vectorTheta2 = theta2.flatten(1) # 260 x 1
nn_params = np.concatenate((vectorTheta1, vectorTheta2),
axis=0) # 19885 x 1
(J, grad) = nnFuncionCosto(nn_params, input_layer_size, hidden_layer_size,
num_labels, x_training, y_training, lambdaP)
print 'El valor de costo inicial sin regularizar es: ' + str(J)
lambdaP = float(lambdaA.get())
(J, grad) = nnFuncionCosto(nn_params, input_layer_size, hidden_layer_size,
num_labels, x_training, y_training, lambdaP)
print 'El valor de costo inicial regularizado es: ' + str(J)
print '\nEntrenando la Red Neuronal...'
extra = (input_layer_size, hidden_layer_size, num_labels, x_training,
y_training, lambdaP)
res = optimize.minimize(minimizarCosto,
nn_params,
extra,
method='L-BFGS-B',
jac=True,
options={
'maxiter': 25,
'disp': True
})
res = res.x
theta1 = np.reshape(res[:hidden_layer_size * (input_layer_size + 1)],
(hidden_layer_size, (input_layer_size + 1)),
order='F') # 25 x 785
theta2 = np.reshape(res[((hidden_layer_size * (input_layer_size + 1))):],
(num_labels, (hidden_layer_size + 1)),
order='F')
pred = predecir(theta1, theta2, x_training)
res_pred1 = np.mean(pred == y_training) * 100
error_pred1 = 100 - res_pred1
print 'Precision de la Red Neuronal sobre el training set: ' + str(
res_pred1)
print 'Error con el training set: ' + str(error_pred1)
print
pred = predecir(theta1, theta2, x_test)
res_pred2 = np.mean(pred == y_test) * 100
error_pred2 = 100 - res_pred2
print 'Precision de la Red Neuronal sobre el test set: ' + str(res_pred2)
print 'Error con el test set: ' + str(error_pred2)
insertarDatosBD(1, hidden_layer_size, lambdaP, J, res_pred1, res_pred2,
error_pred1, error_pred2)
def ex4():
## Machine Learning Online Class - Exercise 4 Neural Network Learning
# Instructions
# ------------
#
# This file contains code that helps you get started on the
# linear exercise. You will need to complete the following functions
# in this exericse:
#
# sigmoidGradient.m
# randInitializeWeights.m
# nnCostFunction.m
#
# For this exercise, you will not need to change any code in this file,
# or any other files other than those mentioned above.
#
## Initialization
#clear ; close all; clc
## Setup the parameters you will use for this exercise
input_layer_size = 400 # 20x20 Input Images of Digits
hidden_layer_size = 25 # 25 hidden units
num_labels = 10 # 10 labels, from 1 to 10
# (note that we have mapped "0" to label 10)
## =========== Part 1: Loading and Visualizing Data =============
# We start the exercise by first loading and visualizing the dataset.
# You will be working with a dataset that contains handwritten digits.
#
# Load Training Data
print('Loading and Visualizing Data ...')
mat = scipy.io.loadmat('ex4data1.mat')
X = mat['X']
y = mat['y'].ravel()
m = X.shape[0]
# Randomly select 100 data points to display
sel = np.random.choice(m, 100, replace=False)
displayData(X[sel, :])
plt.savefig('figure1.png')
print('Program paused. Press enter to continue.')
#pause;
## ================ Part 2: Loading Parameters ================
# In this part of the exercise, we load some pre-initialized
# neural network parameters.
print('\nLoading Saved Neural Network Parameters ...')
# Load the weights into variables Theta1 and Theta2
mat = scipy.io.loadmat('ex4weights.mat')
Theta1 = mat['Theta1']
Theta2 = mat['Theta2']
# Unroll parameters
nn_params = np.concatenate([Theta1.ravel(), Theta2.ravel()])
## ================ Part 3: Compute Cost (Feedforward) ================
# To the neural network, you should first start by implementing the
# feedforward part of the neural network that returns the cost only. You
# should complete the code in nnCostFunction.m to return cost. After
# implementing the feedforward to compute the cost, you can verify that
# your implementation is correct by verifying that you get the same cost
# as us for the fixed debugging parameters.
#
# We suggest implementing the feedforward cost *without* regularization
# first so that it will be easier for you to debug. Later, in part 4, you
# will get to implement the regularized cost.
#
print('\nFeedforward Using Neural Network ...')
# Weight regularization parameter (we set this to 0 here).
lambda_value = 0
J = nnCostFunction(nn_params, input_layer_size, hidden_layer_size,
num_labels, X, y, lambda_value)[0]
print(
'Cost at parameters (loaded from ex4weights): %f \n(this value should be about 0.287629)'
% J)
print('\nProgram paused. Press enter to continue.')
#pause;
## =============== Part 4: Implement Regularization ===============
# Once your cost function implementation is correct, you should now
# continue to implement the regularization with the cost.
#
print('\nChecking Cost Function (w/ Regularization) ... ')
# Weight regularization parameter (we set this to 1 here).
lambda_value = 1
J = nnCostFunction(nn_params, input_layer_size, hidden_layer_size,
num_labels, X, y, lambda_value)[0]
print(
'Cost at parameters (loaded from ex4weights): %f \n(this value should be about 0.383770)'
% J)
print('Program paused. Press enter to continue.')
#pause;
## ================ Part 5: Sigmoid Gradient ================
# Before you start implementing the neural network, you will first
# implement the gradient for the sigmoid function. You should complete the
# code in the sigmoidGradient.m file.
#
print('\nEvaluating sigmoid gradient...')
g = sigmoidGradient(np.array([1, -0.5, 0, 0.5, 1]))
print('Sigmoid gradient evaluated at [1 -0.5 0 0.5 1]:')
print(formatter('%f ', g))
print('\n')
print('Program paused. Press enter to continue.')
#pause;
## ================ Part 6: Initializing Pameters ================
# In this part of the exercise, you will be starting to implment a two
# layer neural network that classifies digits. You will start by
# implementing a function to initialize the weights of the neural network
# (randInitializeWeights.m)
print('\nInitializing Neural Network Parameters ...')
initial_Theta1 = randInitializeWeights(input_layer_size, hidden_layer_size)
initial_Theta2 = randInitializeWeights(hidden_layer_size, num_labels)
# Unroll parameters
initial_nn_params = np.concatenate(
[initial_Theta1.ravel(),
initial_Theta2.ravel()])
## =============== Part 7: Implement Backpropagation ===============
# Once your cost matches up with ours, you should proceed to implement the
# backpropagation algorithm for the neural network. You should add to the
# code you've written in nnCostFunction.m to return the partial
# derivatives of the parameters.
#
print('\nChecking Backpropagation... ')
# Check gradients by running checkNNGradients
checkNNGradients()
print('\nProgram paused. Press enter to continue.')
#pause;
## =============== Part 8: Implement Regularization ===============
# Once your backpropagation implementation is correct, you should now
# continue to implement the regularization with the cost and gradient.
#
print('\nChecking Backpropagation (w/ Regularization) ... ')
# Check gradients by running checkNNGradients
lambda_value = 3
checkNNGradients(lambda_value)
# Also output the costFunction debugging values
debug_J = nnCostFunction(nn_params, input_layer_size, hidden_layer_size,
num_labels, X, y, lambda_value)[0]
print(
'\n\nCost at (fixed) debugging parameters (w/ lambda = 10): %f \n(this value should be about 0.576051)\n\n'
% debug_J)
print('Program paused. Press enter to continue.')
#pause;
## =================== Part 8: Training NN ===================
# You have now implemented all the code necessary to train a neural
# network. To train your neural network, we will now use "fmincg", which
# is a function which works similarly to "fminunc". Recall that these
# advanced optimizers are able to train our cost functions efficiently as
# long as we provide them with the gradient computations.
#
print('\nTraining Neural Network... ')
# After you have completed the assignment, change the MaxIter to a larger
# value to see how more training helps.
options = {'maxiter': 50}
# You should also try different values of lambda
lambda_value = 1
# Create "short hand" for the cost function to be minimized
costFunction = lambda p: nnCostFunction(
p, input_layer_size, hidden_layer_size, num_labels, X, y, lambda_value)
# Now, costFunction is a function that takes in only one argument (the
# neural network parameters)
res = optimize.minimize(costFunction,
initial_nn_params,
jac=True,
method='TNC',
options=options)
nn_params = res.x
# Obtain Theta1 and Theta2 back from nn_params
Theta1 = nn_params[0:hidden_layer_size * (input_layer_size + 1)].reshape(
hidden_layer_size, input_layer_size + 1)
Theta2 = nn_params[hidden_layer_size * (input_layer_size + 1):].reshape(
num_labels, hidden_layer_size + 1)
print('Program paused. Press enter to continue.')
#pause;
## ================= Part 9: Visualize Weights =================
# You can now "visualize" what the neural network is learning by
# displaying the hidden units to see what features they are capturing in
# the data.
print('\nVisualizing Neural Network... ')
displayData(Theta1[:, 1:])
plt.savefig('figure2.png')
print('\nProgram paused. Press enter to continue.')
#pause;
## ================= Part 10: Implement Predict =================
# After training the neural network, we would like to use it to predict
# the labels. You will now implement the "predict" function to use the
# neural network to predict the labels of the training set. This lets
# you compute the training set accuracy.
pred = predict(Theta1, Theta2, X)
print('\nTraining Set Accuracy: %f' % (np.mean(
(pred == y).astype(int)) * 100))
def main():
''' Main function '''
## %% =========== Part 1: Loading and Visualizing Data =============
#% We start the exercise by first loading and visualizing the dataset.
#% You will be working with a dataset that contains handwritten digits.
#%
# Read the Matlab data
m, n, X, y = getMatlabTrainingData()
# number of features
input_layer_size = n
# Select some random images from X
print('Selecting random examples of the data to display.\n')
sel = np.random.permutation(m)
sel = sel[0:100]
# Re-work the data orientation of each training example
image_size = 20
XMatlab = np.copy(X) # Need a deep copy, not just the reference
for i in range(m):
XMatlab[i, :] = XMatlab[i, :].reshape(image_size, image_size).transpose().reshape(1, image_size*image_size)
# display the sample images
displayData(XMatlab[sel, :])
# Print Out the labels for what is being seen.
print('These are the labels for the data ...\n')
print(y[sel, :].reshape(10, 10))
# Pause program
print("Program paused. Press Ctrl-D to continue.\n")
code.interact(local=dict(globals(), **locals()))
print(" ... continuing\n ")
#%% ================ Part 2: Loading Parameters ================
#% In this part of the exercise, we load some pre-initialized
# % neural network parameters.
print('\nLoading Saved Neural Network Parameters ...\n')
# Load the weights into variables Theta1 and Theta2
import scipy .io as sio
fnWeights = '/home/jennym/Kaggle/DigitRecognizer/ex4/ex4weights.mat'
weights = sio.loadmat(fnWeights)
Theta1 = weights['Theta1']
Theta2 = weights['Theta2']
#% Unroll parameters
nn_params = np.hstack((Theta1.ravel(order='F'), Theta2.ravel(order='F')))
#%% ================ Part 3: Compute Cost (Feedforward) ================
#% To the neural network, you should first start by implementing the
#% feedforward part of the neural network that returns the cost only. You
#% should complete the code in nnCostFunction.m to return cost. After
#% implementing the feedforward to compute the cost, you can verify that
#% your implementation is correct by verifying that you get the same cost
#% as us for the fixed debugging parameters.
#%
#% We suggest implementing the feedforward cost *without* regularization
#% first so that it will be easier for you to debug. Later, in part 4, you
#% will get to implement the regularized cost.
#%
print('\nFeedforward Using Neural Network ...\n')
#% Weight regularization parameter (we set this to 0 here).
MLlambda = 0.0
# Cluge, put y back to matlab version, then adjust to use python
# indexing later into y_matrix
y[(y == 0)] = 10
y = y - 1
J, _ = nnCostFunction(nn_params, input_layer_size, hidden_layer_size,
num_labels, X, y, MLlambda)
print('Cost at parameters (loaded from ex4weights): ' + str(J) +
'\n (this value should be about 0.287629)\n')
# Pause
print("Program paused. Press Ctrl-D to continue.\n")
code.interact(local=dict(globals(), **locals()))
print(" ... continuing\n ")
#%% =============== Part 4: Implement Regularization ===============
#% Once your cost function implementation is correct, you should now
#% continue to implement the regularization with the cost.
#%
print('\nChecking Cost Function (with Regularization) ... \n')
# % Weight regularization parameter (we set this to 1 here).
MLlambda = 1.0
J, _ = nnCostFunction(nn_params, input_layer_size, hidden_layer_size,
num_labels, X, y, MLlambda)
print('Cost at parameters (loaded from ex4weights): ' + str(J) +
'\n(this value should be about 0.383770)\n');
# Pause
print("Program paused. Press Ctrl-D to continue.\n")
code.interact(local=dict(globals(), **locals()))
print(" ... continuing\n ")
#%% ================ Part 5: Sigmoid Gradient ================
#% Before you start implementing the neural network, you will first
#% implement the gradient for the sigmoid function. You should complete the
#% code in the sigmoidGradient.m file.
#%
print('\nEvaluating sigmoid gradient...\n')
g = sigmoidGradient(np.array([1, -0.5, 0, 0.5, 1]))
print('Sigmoid gradient evaluated at [1 -0.5 0 0.5 1]:\n ')
print(g)
print('\n\n')
# Pause
print("Program paused. Press Ctrl-D to continue.\n")
code.interact(local=dict(globals(), **locals()))
print(" ... continuing\n ")
#%% ================ Part 6: Initializing Parameters ================
#% In this part of the exercise, you will be starting to implement a two
#% layer neural network that classifies digits. You will start by
#% implementing a function to initialize the weights of the neural network
#% (randInitializeWeights.m)
print('\nInitializing Neural Network Parameters ...\n')
initial_Theta1 = randInitializeWeights(input_layer_size, hidden_layer_size)
initial_Theta2 = randInitializeWeights(hidden_layer_size, num_labels)
#% Unroll parameters
initial_nn_params = np.hstack(( initial_Theta1.ravel(order = 'F'),
initial_Theta2.ravel(order = 'F')))
# Pause
print("Program paused. Press Ctrl-D to continue.\n")
code.interact(local=dict(globals(), **locals()))
print(" ... continuing\n ")
#%% =============== Part 7: Implement Backpropagation ===============
#% Once your cost matches up with ours, you should proceed to implement the
#% backpropagation algorithm for the neural network. You should add to the
#% code you've written in nnCostFunction.m to return the partial
#% derivatives of the parameters.
#%
print('\nChecking Backpropagation... \n')
#% Check gradients by running checkNNGradients
checkNNGradients()
# Pause
print("Program paused. Press Ctrl-D to continue.\n")
code.interact(local=dict(globals(), **locals()))
print(" ... continuing\n ")
#%% =============== Part 8: Implement Regularization ===============
#% Once your backpropagation implementation is correct, you should now
#% continue to implement the regularization with the cost and gradient.
#%
print('\nChecking Backpropagation (w/ Regularization) ... \n')
#% Check gradients by running checkNNGradients
MLlambda = 3
checkNNGradients(MLlambda)
# Pause
print("Program paused. Press Ctrl-D to continue.\n")
code.interact(local=dict(globals(), **locals()))
print(" ... continuing\n ")
#% Also output the costFunction debugging values
debug_J, _ = nnCostFunction(nn_params, input_layer_size,
hidden_layer_size, num_labels, X, y, MLlambda)
print('\n\n Cost at (fixed) debugging parameters (w/ lambda = ' +
'{0}): {1}'.format(MLlambda, debug_J))
print('\n (this value should be about 0.576051)\n\n')
# Pause
print("Program paused. Press Ctrl-D to continue.\n")
code.interact(local=dict(globals(), **locals()))
print(" ... continuing\n ")
#%% =================== Part 8b: Training NN ===================
#% You have now implemented all the code necessary to train a neural
#% network. To train your neural network, we will now use "fmincg", which
#% is a function which works similarly to "fminunc". Recall that these
#% advanced optimizers are able to train our cost functions efficiently as
#% long as we provide them with the gradient computations.
#%
print ('\nTraining Neural Network... \n')
#% After you have completed the assignment, change the MaxIter to a larger
#% value to see how more training helps.
#% jkm change maxIter from 50-> 400
options = {'maxiter': MAXITER}
#% You should also try different values of lambda
MLlambda = 1
#% Create "short hand" for the cost function to be minimized
costFunc = lambda p: nnCostFunction(p, input_layer_size, hidden_layer_size,
num_labels, X, y, MLlambda)
#% Now, costFunction is a function that takes in only one argument (the
#% neural network parameters)
'''
NOTES: Call scipy optimize minimize function
method : str or callable, optional Type of solver.
CG -> Minimization of scalar function of one or more variables
using the conjugate gradient algorithm.
jac : bool or callable, optional Jacobian (gradient) of objective function.
Only for CG, BFGS, Newton-CG, L-BFGS-B, TNC, SLSQP, dogleg, trust-ncg.
If jac is a Boolean and is True, fun is assumed to return the gradient
along with the objective function. If False, the gradient will be
estimated numerically. jac can also be a callable returning the
gradient of the objective. In this case, it must accept the same
arguments as fun.
callback : callable, optional. Called after each iteration, as callback(xk),
where xk is the current parameter vector.
'''
# Setup a callback for displaying the cost at the end of each iteration
class Callback(object):
def __init__(self):
self.it = 0
def __call__(self, p):
self.it += 1
print "Iteration %5d | Cost: %e" % (self.it, costFunc(p)[0])
result = sci.minimize(costFunc, initial_nn_params, method='CG',
jac=True, options=options, callback=Callback())
nn_params = result.x
cost = result.fun
# matlab: [nn_params, cost] = fmincg(costFunction, initial_nn_params, options);
#% Obtain Theta1 and Theta2 back from nn_params
Theta1 = np.reshape(nn_params[:hidden_layer_size * (input_layer_size + 1)],
(hidden_layer_size, (input_layer_size + 1)),
order = 'F')
Theta2 = np.reshape(nn_params[hidden_layer_size * (input_layer_size + 1):],
(num_labels, (hidden_layer_size + 1)),
order = 'F')
# Pause
print("Program paused. Press Ctrl-D to continue.\n")
code.interact(local=dict(globals(), **locals()))
print(" ... continuing\n ")
#%% ================= Part 9: Visualize Weights =================
#% You can now "visualize" what the neural network is learning by
#% displaying the hidden units to see what features they are capturing in
#% the data.#
print('\nVisualizing Neural Network... \n')
displayData(Theta1[:, 1:])
# Pause
print("Program paused. Press Ctrl-D to continue.\n")
code.interact(local=dict(globals(), **locals()))
print(" ... continuing\n ")
#%% ================= Part 10: Implement Predict =================
#% After training the neural network, we would like to use it to predict
#% the labels. You will now implement the "predict" function to use the
#% neural network to predict the labels of the training set. This lets
#% you compute the training set accuracy.
pred = predict(Theta1, Theta2, X)
# JKM - my array was column stacked - don't understand why this works
pp = np.row_stack(pred)
accuracy = np.mean(np.double(pp == y)) * 100
print('\nTraining Set Accuracy: {0} \n'.format(accuracy))
# Pause
print("Program paused. Press Ctrl-D to continue.\n")
code.interact(local=dict(globals(), **locals()))
print(" ... continuing\n ")
# ========================================
# All Done!
return
