def output(partId):
# Random Test Cases
X = np.reshape(3 * np.sin(np.arange(1, 31)), (3, 10), order='F')
Xm = np.reshape(np.sin(np.arange(1, 33)), (16, 2), order='F') / 5
ym = 1 + np.arange(1, 17) % 4
t1 = np.sin(np.reshape(np.arange(1, 25, 2), (4, 3), order='F'))
t2 = np.cos(np.reshape(np.arange(1, 41, 2), (4, 5), order='F'))
t = np.concatenate([t1.ravel(), t2.ravel()], axis=0)
if partId == '1':
J, _ = nnCostFunction(t, 2, 4, 4, Xm, ym, 0)
out = formatter('%0.5f ', J)
elif partId == '2':
J, _ = nnCostFunction(t, 2, 4, 4, Xm, ym, 1.5)
out = formatter('%0.5f ', J)
elif partId == '3':
out = formatter('%0.5f ', sigmoidGradient(X))
elif partId == '4':
J, grad = nnCostFunction(t, 2, 4, 4, Xm, ym, 0)
out = formatter('%0.5f ', J)
out += formatter('%0.5f ', grad)
elif partId == '5':
J, grad = nnCostFunction(t, 2, 4, 4, Xm, ym, 1.5)
out = formatter('%0.5f ', J)
out += formatter('%0.5f ', grad)
return out
def GradientCheck(nn_params, input, hidden, output, X, y, lam):
epsilion = pow(10, -4)
_, grad = nnCostFunction(nn_params, input, hidden, output, X, y, lam)
param_plus = nn_params + epsilion
param_minus = nn_params - epsilion
J1, _ = nnCostFunction(param_plus, input, hidden, output, X, y, lam)
J2, _ = nnCostFunction(param_minus, input, hidden, output, X, y, lam)
check = (J1 - J2) / (2 * epsilion)
print("\nChecking Gradients.....")
print(np.sum(grad), " ~ ", check)
def computeNumericalGradient(initial_nn_params, input_layer_size,
hidden_layer_size, num_labels, X, y, lam, D):
eps = 1e-4
sz = len(initial_nn_params)
perturb = np.zeros((sz, 1))
numgrad = np.zeros(sz)
for i in range(
sz
): #run this loop for 10 iterations and uncomment last line to see numerical and Backprob gradient
perturb[i] = eps
numgrad[i] = (nnCostFunction(
initial_nn_params + perturb.flatten(), input_layer_size,
hidden_layer_size, num_labels, X, y, lam) - nnCostFunction(
initial_nn_params - perturb.flatten(), input_layer_size,
hidden_layer_size, num_labels, X, y, lam)) / float(2 * eps)
perturb[i] = 0
print(
"Element: {0}. Numerical Gradient = {1:.9f}. BackProp Gradient = {2:.9f}."
.format(i, numgrad[i], D[i]))
return numgrad
def checkNNGradients(Lambda=0):
"""Creates a small neural network to check the
backpropagation gradients, it will output the analytical gradients
produced by your backprop code and the numerical gradients (computed
using computeNumericalGradient). These two gradient computations should
result in very similar values.
"""
#input_layer_size = 4
#hidden_layer_size = 5
#num_labels = 3
#m = 10
# We generate some 'random' test data
Theta1 = debugInitializeWeights(hidden_layer_size, input_layer_size)
Theta2 = debugInitializeWeights(num_labels, hidden_layer_size)
# Reusing debugInitializeWeights to generate X
X = debugInitializeWeights(m, input_layer_size - 1)
y = np.mod(range(1, m + 1), num_labels)
# Unroll parameters
nn_params = np.hstack((Theta1.T.ravel(), Theta2.T.ravel()))
# Short hand for cost function
costFunc = lambda p: nnCostFunction(p, input_layer_size, hidden_layer_size,
num_labels, X, y, Lambda)
#costFunc = nnCostFunction(nn_params, input_layer_size, hidden_layer_size, num_labels, X, y, Lambda)
numgrad = computeNumericalGradient(costFunc, nn_params)
grad = costFunc(nn_params)[1]
print(numgrad)
print(grad)
# Visually examine the two gradient computations. The two columns
# you get should be very similar.
print(np.column_stack((numgrad, grad)))
print('The above two columns you get should be very similar.\n' \
'(Left-Your Numerical Gradient, Right-Analytical Gradient)\n\n')
# Evaluate the norm of the difference between two solutions.
# If you have a correct implementation, and assuming you used EPSILON = 0.0001
# in computeNumericalGradient.m, then diff below should be less than 1e-9
diff = np.linalg.norm(numgrad - grad) / np.linalg.norm(numgrad + grad)
print('If your backpropagation implementation is correct, then\n ' \
'the relative difference will be small (less than 1e-9). \n' \
'\nRelative Difference: %g\n' % diff)
return (nn_params, grad)
def output(partId):
# Random Test Cases
X = reshape(3 * sin(arange(1, 31, 1)), (3,10), order='F')
Xm = reshape(sin(arange(1, 33)), (16,2), order='F') / 5
ym = 1 + arange(1, 17) % 4
t1 = sin(reshape(arange(1,25,2), (4,3), order='F'))
t2 = cos(reshape(arange(1,41,2), (4,5), order='F'))
t = hstack([t1.ravel('F'), t2.ravel('F')])
if partId == '1':
J, _ = nnCostFunction(t, 2, 4, 4, Xm, ym, 0)
return sprintf('%0.5f ', J)
elif partId == '2':
J, _ = nnCostFunction(t, 2, 4, 4, Xm, ym, 1.5)
return sprintf('%0.5f ', J)
elif partId == '3':
return sprintf('%0.5f ', sigmoidGradient(X))
elif partId == '4':
J, grad = nnCostFunction(t, 2, 4, 4, Xm, ym, 0)
out = sprintf('%0.5f ', J)
return out + sprintf('%0.5f ', grad)
elif partId == '5':
J, grad = nnCostFunction(t, 2, 4, 4, Xm, ym, 1.5)
out = sprintf('%0.5f ', J)
return out + sprintf('%0.5f ', grad)
def checkNNGradients(reg_lambda=0):
""" Creates a small neural network to check the backpropagation gradients
CHECKNNGRADIENTS(reg_lambda) Creates a small neural network to check the
backpropagation gradients, it will output the analytical gradients
produced by your backprop code and the numerical gradients (computed
using computeNumericalGradient). These two gradient computations should
result in very similar values."""
input_layer_size = 3
hidden_layer_size = 5
num_labels = 3
m = 5
# We generate some 'random' test data
Theta1 = debugInitializeWeights(hidden_layer_size, input_layer_size)
Theta2 = debugInitializeWeights(num_labels, hidden_layer_size)
# Reusing debugInitializeWeights to generate X
X = debugInitializeWeights(m, input_layer_size - 1)
y = np.mod(np.arange(m), num_labels).T.reshape(m, 1)
# Unroll parameters
nn_params = np.r_[Theta1.ravel(), Theta2.ravel()]
# Short hand for cost function
costFunc = lambda params: nnCostFunction(
params, input_layer_size, hidden_layer_size, num_labels, X, y,
reg_lambda)
cost, grad = costFunc(nn_params)
numgrad = computeNumericalGradient(costFunc, nn_params)
# Visually examine the two gradient computations. The two columns
# you get should be very similar.
print(numgrad, grad)
print('The above two columns you get should be very similar.\n',
'(Left-Your Numerical Gradient, Right-Analytical Gradient)\n\n')
# Evaluate the norm of the difference between two solutions.
# If you have a correct implementation, and assuming you used EPSILON = 0.0001
# in computeNumericalGradient.py, then diff below should be less than 1e-9
diff = np.linalg.norm(numgrad - grad) / np.linalg.norm(numgrad + grad)
print('If your backpropagation implementation is correct, then \n',
'the relative difference will be small (less than 1e-9). \n',
'\nRelative Difference: \n', diff)
def checkNNGradients(Lambda = 0):
"""Creates a small neural network to check the
backpropagation gradients, it will output the analytical gradients
produced by your backprop code and the numerical gradients (computed
using computeNumericalGradient). These two gradient computations should
result in very similar values.
"""
input_layer_size = 3
hidden_layer_size = 5
num_labels = 3
m = 5
# We generate some 'random' test data
Theta1 = debugInitializeWeights(hidden_layer_size, input_layer_size)
Theta2 = debugInitializeWeights(num_labels, hidden_layer_size)
# Reusing debugInitializeWeights to generate X
X = debugInitializeWeights(m, input_layer_size - 1)
y = np.mod(range(1, m+1), num_labels)
# Unroll parameters
nn_params = np.hstack((Theta1.T.ravel(), Theta2.T.ravel()))
# Short hand for cost function
costFunc = lambda p: nnCostFunction(p, input_layer_size, hidden_layer_size, num_labels, X, y, Lambda)
numgrad = computeNumericalGradient(costFunc, nn_params)
grad = costFunc(nn_params)[1]
# Visually examine the two gradient computations. The two columns
# you get should be very similar.
print np.column_stack((numgrad, grad))
numgrad.shape #
grad.shape #
print 'The above two columns you get should be very similar.\n' \
'(Left-Your Numerical Gradient, Right-Analytical Gradient)\n\n'
# Evaluate the norm of the difference between two solutions.
# If you have a correct implementation, and assuming you used EPSILON = 0.0001
# in computeNumericalGradient.m, then diff below should be less than 1e-9
diff = np.linalg.norm(numgrad-grad)/np.linalg.norm(numgrad+grad)
print 'If your backpropagation implementation is correct, then\n ' \
'the relative difference will be small (less than 1e-9). \n' \
'\nRelative Difference: %g\n' % diff
def checkNNGradients(lambda_=0):
#CHECKNNGRADIENTS Creates a small neural network to check the
#backpropagation gradients
# CHECKNNGRADIENTS(lambda_) Creates a small neural network to check the
# backpropagation gradients, it will output the analytical gradients
# produced by your backprop code and the numerical gradients (computed
# using computeNumericalGradient). These two gradient computations should
# result in very similar values.
#
input_layer_size = 3
hidden_layer_size = 5
num_labels = 3
m = 5
# We generate some 'random' test data
Theta1 = debugInitializeWeights(hidden_layer_size, input_layer_size)
Theta2 = debugInitializeWeights(num_labels, hidden_layer_size)
# Reusing debugInitializeWeights to generate X
X = debugInitializeWeights(m, input_layer_size - 1)
y = 1 + (arange(m)+1) % num_labels
# Unroll parameters
nn_params = hstack((Theta1.ravel(order='F'), Theta2.ravel(order='F')))
# Short hand for cost function
costFunc = lambda p: nnCostFunction(p, input_layer_size, hidden_layer_size,
num_labels, X, y, lambda_)
cost, grad = costFunc(nn_params)
numgrad = computeNumericalGradient(costFunc, nn_params)
# Visually examine the two gradient computations. The two columns
# you get should be very similar.
print column_stack((numgrad, grad))
print 'The above two columns you get should be very similar.'
print '(Left-Your Numerical Gradient, Right-Analytical Gradient)\n'
# Evaluate the norm of the difference between two solutions.
# If you have a correct implementation, and assuming you used EPSILON = 0.0001
# in computeNumericalGradient.m, then diff below should be less than 1e-9
diff = linalg.norm(numgrad-grad) / linalg.norm(numgrad+grad)
print 'If your backpropagation implementation is correct, then'
print 'the relative difference will be small (less than 1e-9).\n'
print 'Relative Difference: %g\n' % diff
def checkNNGradients(lambda_=0):
#CHECKNNGRADIENTS Creates a small neural network to check the
#backpropagation gradients
# CHECKNNGRADIENTS(lambda_) Creates a small neural network to check the
# backpropagation gradients, it will output the analytical gradients
# produced by your backprop code and the numerical gradients (computed
# using computeNumericalGradient). These two gradient computations should
# result in very similar values.
#
input_layer_size = 3
hidden_layer_size = 5
num_labels = 3
m = 5
# We generate some 'random' test data
Theta1 = debugInitializeWeights(hidden_layer_size, input_layer_size)
Theta2 = debugInitializeWeights(num_labels, hidden_layer_size)
# Reusing debugInitializeWeights to generate X
X = debugInitializeWeights(m, input_layer_size - 1)
y = 1 + (arange(m)+1) % num_labels
# Unroll parameters
nn_params = hstack((Theta1.ravel(order='F'), Theta2.ravel(order='F')))
# Short hand for cost function
costFunc = lambda p: nnCostFunction(p, input_layer_size, hidden_layer_size,
num_labels, X, y, lambda_)
cost, grad = costFunc(nn_params)
numgrad = computeNumericalGradient(costFunc, nn_params)
# Visually examine the two gradient computations. The two columns
# you get should be very similar.
print column_stack((numgrad, grad))
print 'The above two columns you get should be very similar.'
print '(Left-Your Numerical Gradient, Right-Analytical Gradient)\n'
# Evaluate the norm of the difference between two solutions.
# If you have a correct implementation, and assuming you used EPSILON = 0.0001
# in computeNumericalGradient.m, then diff below should be less than 1e-9
diff = linalg.norm(numgrad-grad) / linalg.norm(numgrad+grad)
print 'If your backpropagation implementation is correct, then'
print 'the relative difference will be small (less than 1e-9).\n'
print 'Relative Difference: %g\n' % diff
def nnGradientDescent(X, y, nn_params, iterations, alpha, input, hidden,
output, lam):
theta1 = nn_params[0:hidden * (input + 1)].reshape(hidden,
input + 1) # (25, 401)
theta2 = nn_params[(hidden * (input + 1)):].reshape(output,
hidden + 1) # (10, 26)
J_history = np.zeros([iterations, 1])
for i in range(iterations):
J_history[i], grad = nnCostFunction(nn_params, input, hidden, output,
X, y, lam)
grad1 = grad[0:hidden * (input + 1)].reshape(hidden,
input + 1) # (25, 401)
grad2 = grad[(hidden * (input + 1)):].reshape(output,
hidden + 1) # (10, 26)
theta1 = theta1 - alpha * grad1
theta2 = theta2 - alpha * grad2
nn_params = np.hstack((theta1.reshape(-1), theta2.reshape(-1)))
return nn_params, J_history
def checkNNGradients(lmbda):
#CHECKNNGRADIENTS Creates a small neural network to check the
#backpropagation gradients
# CHECKNNGRADIENTS(lmbda) Creates a small neural network to check the
# backpropagation gradients, it will output the analytical gradients
# produced by your backprop code and the numerical gradients (computed
# using computeNumericalGradient). These two gradient computations should
# result in very similar values.
#
if not 'lmbda' in locals():
lmbda = 0
input_layer_size = 3
hidden_layer_size = 5
num_labels = 3
m = 5
# We generate some 'random' test data
Theta1 = debugInitializeWeights(hidden_layer_size, input_layer_size)
Theta2 = debugInitializeWeights(num_labels, hidden_layer_size)
# Reusing debugInitializeWeights to generate X
X = debugInitializeWeights(m, input_layer_size - 1)
y = (np.reshape(np.mod(range(0,m), num_labels), (1,m))).flatten()
# Unroll parameters
nn_params = np.concatenate((Theta1.ravel(), Theta2.ravel()), axis=0)
# Short hand for cost function
costFunc = lambda p : nnCostFunction(p, input_layer_size, hidden_layer_size, num_labels, X, y, lmbda)
cost, grad = costFunc(nn_params)
numgrad = computeNumericalGradient(costFunc, nn_params)
# Visually examine the two gradient computations. The two columns
# you get should be very similar.
print np.concatenate((np.reshape(numgrad, (1, numgrad.size)).T, np.reshape(grad, (1, grad.size)).T), axis=1)
print 'The above two columns you get should be very similar.\n(Left-Your Numerical Gradient, Right-Analytical Gradient)'
# Evaluate the norm of the difference between two solutions.
# If you have a correct implementation, and assuming you used EPSILON = 0.0001
# in computeNumericalGradient.m, then diff below should be less than 1e-9
diff = np.linalg.norm(numgrad-grad) / np.linalg.norm(numgrad+grad)
print 'If your backpropagation implementation is correct, then \nthe relative difference will be small (less than 1e-9). \nRelative Difference: {}\n'.format(diff)
def checkNNGradients(_lambda=None):
if _lambda == None:
_lambda = 0
input_layer_size = 3
hidden_layer_size = 5
num_labels = 3
m = 5
# We generate some 'random' test data
Theta1 = debugInitializeWeights(hidden_layer_size, input_layer_size)
Theta2 = debugInitializeWeights(num_labels, hidden_layer_size)
# Reusing debugInitializeWeights to generate X
X = debugInitializeWeights(m, input_layer_size - 1)
y = (1 + np.arange(m)) % num_labels
y = y.reshape(-1,1)
# Unroll parameters
nn_params = np.append(Theta1.flatten(),Theta2.flatten())
# Short hand for cost function
costFunc = lambda p: nnCostFunction(p, input_layer_size, hidden_layer_size, \
num_labels, X, y, _lambda)
cost, grad = costFunc(nn_params)
numgrad = computeNumericalGradient(costFunc, nn_params)
# Visually examine the two gradient computations. The two columns
# you get should be very similar.
print(grad)
print(numgrad)
print('The above two columns you get should be very similar.\n \
(Left-Your Numerical Gradient, Right-Analytical Gradient)\n\n')
# Evaluate the norm of the difference between two solutions.
# If you have a correct implementation, and assuming you used EPSILON = 0.0001
# in computeNumericalGradient.m, then diff below should be less than 1e-9
diff = np.linalg.norm(numgrad-grad)/np.linalg.norm(numgrad+grad)
print('If your backpropagation implementation is correct, then \n \
the relative difference will be small (less than 1e-9). \n \
\nRelative Difference: %g\n'%diff)
def checkNNGradients(lambda_value=0):
input_layer_size = 3
hidden_layer_size = 5
num_labels = 3
m = 5
# We generate some 'random' test data
Theta1 = debugInitializeWeights(hidden_layer_size, input_layer_size)
Theta2 = debugInitializeWeights(num_labels, hidden_layer_size)
# Reusing debugInitializeWeights to generate X
X = debugInitializeWeights(m, input_layer_size - 1)
y = 1 + np.transpose(np.mod(range(1, m + 1), num_labels))
# y=np.expand_dims(y,axis=1)
# Unroll parameters
Theta1_1d = np.reshape(Theta1, Theta1.size, order='F')
Theta2_1d = np.reshape(Theta2, Theta2.size, order='F')
nn_params = np.hstack((Theta1_1d, Theta2_1d))
# Short hand for cost function
costFunc = lambda p: nnCostFunction(p, input_layer_size, hidden_layer_size,
num_labels, X, y, lambda_value)
cost, grad = costFunc(nn_params)
numgrad = computeNumericalGradient(costFunc,
np.expand_dims(nn_params, axis=1))
# Visually examine the two gradient computations. The two columns
# you get should be very similar.
print(numgrad, grad)
print(
'The above two columns you get should be very similar.\n (Left-Numerical Gradient, Right-(Your) Analytical Gradient)\n\n'
)
# Evaluate the norm of the difference between two solutions.
# If you have a correct implementation, and assuming you used EPSILON = 0.0001
# in computeNumericalGradient.m, then diff below should be less than 1e-9
diff = np.linalg.norm(numgrad - grad) / np.linalg.norm(numgrad + grad)
print(
'If your backpropagation implementation is correct, then \n the relative difference will be small (less than 1e-9). \n \nRelative Difference: ',
diff)
def checkNNGradients(lambda_):
# Creates a small neural network to check the
# backpropagation gradients, it will output the analytical gradients
# produced by your backprop code and the numerical gradients (computed
# using computeNumericalGradient). These two gradient computations should
# result in very similar values.
input_layer_size = 3
hidden_layer_size = 5
num_labels = 3
m = 5
# generating some 'random' test data
Theta1 = debugInitializeWeights(hidden_layer_size, input_layer_size)
Theta2 = debugInitializeWeights(num_labels, hidden_layer_size)
X = debugInitializeWeights(m, input_layer_size - 1)
y = 1 + np.arange(1, m + 1) % num_labels
Theta1 = np.reshape(Theta1, -1)
Theta2 = np.reshape(Theta2, -1)
nn_params = np.concatenate((Theta1, Theta2))
costFunction = lambda cf: nncf.nnCostFunction(
cf, input_layer_size, hidden_layer_size, num_labels, X, y, lambda_)
cost, grad = costFunction(nn_params)
numgrad = computeNumericalGradient(costFunction, nn_params)
compareGradients = np.vstack((grad, numgrad))
print(compareGradients.T)
print(
'The above two columns you get should be very similar.\nLeft-Your Numerical Gradient, Right-Analytical Gradient'
)
# Evaluate the norm of the difference between two solutions.
# If you have a correct implementation, and assuming you used EPSILON = 0.0001
# in computeNumericalGradient.m, then diff below should be less than 1e-9
diff = norm(numgrad - grad) / norm(numgrad + grad)
print(
'If your backpropagation implementation is correct, then \n the relative difference will be small (less than 1e-9). \n\nRelative Difference: ',
diff)
def checkNNGradients(lmbda=0):
input_layer_size = 3
hidden_layer_size = 5
num_labels = 3
m = 5
# We generate some 'random' test data
Theta1 = debugInitializeWeights(hidden_layer_size, input_layer_size)
Theta2 = debugInitializeWeights(num_labels, hidden_layer_size)
# Reusing debugInitializeWeights to generate X
X = debugInitializeWeights(m, input_layer_size - 1)
y = np.mod(range(m), num_labels)
# Unroll parameters
nn_params = np.hstack((Theta1.flatten(), Theta2.flatten()))
# Short hand for cost function
costFunc = lambda nn_params: nnCostFunction(
nn_params, input_layer_size, hidden_layer_size, num_labels, X, y, lmbda
)
cost, grad = costFunc(nn_params)
numgrad = computeNumericalGradient(costFunc, nn_params)
# Visually examine the two gradient computations. The two columns
# you get should be very similar.
print(np.vstack((numgrad, grad)).T)
print('The above two columns you get should be very similar.')
print('(Left-Your Numerical Gradient, Right-Analytical Gradient)')
# Evaluate the norm of the difference between two solutions.
# If you have a correct implementation, and assuming you used EPSILON = 0.0001
# in computeNumericalGradient.m, then diff below should be less than 1e-9
diff = np.linalg.norm(numgrad - grad) / np.linalg.norm(numgrad + grad)
print('If your backpropagation implementation is correct, then')
print('the relative difference will be small (less than 1e-9).')
print('Relative Difference: %g' % (diff))
def checkNNGradients(lmbda=0):
input_layer_size = 3
hidden_layer_size = 5
num_labels = 3
m = 5
# We generate some 'random' test data
Theta1 = debugInitializeWeights(hidden_layer_size, input_layer_size)
Theta2 = debugInitializeWeights(num_labels, hidden_layer_size)
# Reusing debugInitializeWeights to generate X
X = debugInitializeWeights(m, input_layer_size - 1)
y = np.mod(range(m), num_labels)
# Unroll parameters
nn_params = np.hstack((Theta1.flatten(), Theta2.flatten()))
# Short hand for cost function
costFunc = lambda nn_params: nnCostFunction(nn_params, input_layer_size, hidden_layer_size, num_labels, X, y, lmbda)
cost, grad = costFunc(nn_params)
numgrad = computeNumericalGradient(costFunc, nn_params)
# Visually examine the two gradient computations. The two columns
# you get should be very similar.
print(np.vstack((numgrad, grad)).T)
print("The above two columns you get should be very similar.")
print("(Left-Your Numerical Gradient, Right-Analytical Gradient)")
# Evaluate the norm of the difference between two solutions.
# If you have a correct implementation, and assuming you used EPSILON = 0.0001
# in computeNumericalGradient.m, then diff below should be less than 1e-9
diff = np.linalg.norm(numgrad - grad) / np.linalg.norm(numgrad + grad)
print("If your backpropagation implementation is correct, then")
print("the relative difference will be small (less than 1e-9).")
print("Relative Difference: %g" % (diff))
def checkNNGradients(lamda=0):
input_layer_size = 3
hidden_layer_size = 3
num_labels = 3
m = 5
# We generate some 'random' test data
Theta1 = debugInitializeWeights(hidden_layer_size, input_layer_size)
Theta2 = debugInitializeWeights(num_labels, hidden_layer_size)
X = debugInitializeWeights(m, input_layer_size - 1)
y = 1 + np.mod(range(1, m + 1), num_labels).reshape((m, 1))
# Unroll parameters
nn_params = np.hstack((Theta1.T.ravel(), Theta2.T.ravel()))
# Short hand for cost function
costFunc = lambda p: nnCostFunction(
p, input_layer_size, hidden_layer_size, num_labels, X, y, lamda)
cost, grad = costFunc(nn_params)
numgrad = computeNumericalGradient(costFunc, nn_params)
# Visually examine the two gradient computations. The two columns
# you get should be very similar.
print(np.vstack((numgrad, grad)).T)
print('The above two columns you get should be very similar.\n' +
'(Left-Your Numerical Gradient, Right-Analytical Gradient)')
# Evaluate the norm of the difference between two solutions.
# If you have a correct implementation, and assuming you used EPSILON = 0.0001
# in computeNumericalGradient.py, then diff below should be less than 1e-9
diff = np.linalg.norm(numgrad - grad) / np.linalg.norm(numgrad + grad)
print('If your backpropagation implementation is correct, then \n' +
'the relative difference will be small (less than 1e-9). \n' +
'Relative Difference: %g\n' % diff)
def costFunction(p):
J,gradient=nnCostFunction(p,input_layer_size,hidden_layer_size,num_labels,data,labels,lambda_nn)
print "training"
print J
return J
def costFunc(p):
return nnCostFunction(p, input_layer_size, hidden_layer_size, \
num_labels, X, y, lmbda)
## ================ Part 3: Compute Cost (Feedforward) ================
"""To the neural network, you should first start by implementing the feedforward part of the neural network
that returns the cost only. You should complete the code in nnCostFunction.m to return cost.
After implementing the feedforward to compute the cost, you can verify that your implementation
is correct by verifying that you get the same cost as us for the fixed debugging parameters.
We suggest implementing the feedforward cost *without* regularization
first so that it will be easier for you to debug. Later, in part 4, you
will get to implement the regularized cost."""
print('Feedforward Using Neural Network ...')
# Weight regularization parameter (we set this to 0 here).
xlambda = 0
J, grad = nnCostFunction.nnCostFunction(nn_params, input_layer_size,
hidden_layer_size, num_labels, X, y,
xlambda)
print(
'Cost at parameters (loaded from ex4weights): %.6f(this value should be about 0.287629)'
% J)
input('Program paused. Press enter to continue.')
## =============== Part 4: Implement Regularization ===============
# Once your cost function implementation is correct, you should now continue to implement the regularization with the cost.
print('Checking Cost Function (w/ Regularization) ... ')
# Weight regularization parameter (we set this to 1 here).
xlambda = 1
J, grad = nnCostFunction.regularization(nn_params, input_layer_size,
% implementing the feedforward to compute the cost, you can verify that
% your implementation is correct by verifying that you get the same cost
% as us for the fixed debugging parameters.
%
% We suggest implementing the feedforward cost *without* regularization
% first so that it will be easier for you to debug. Later, in part 4, you
% will get to implement the regularized cost.
%
fprintf('\nFeedforward Using Neural Network ...\n')
'''
#% Weight regularization parameter (we set this to 0 here).
_lambda = 0
J = nnCostFunction(nn_params, input_layer_size, hidden_layer_size, num_labels, X, y, _lambda)
print('Cost at parameters (loaded from ex4weights):(this value should be about 0.287629)\n', J)
# =============== Part 4: Implement Regularization ===============
# Once your cost function implementation is correct, you should now
# continue to implement the regularization with the cost.
#
#fprintf('\nChecking Cost Function (w/ Regularization) ... \n')
# Weight regularization parameter (we set this to 1 here).
_lambda = 1
#
# Once we are done, our notebook will cal **nnCostFunction** function using the set of parameters for *Theta1* and *Theta2*. We should see that the cost is about $0.287629$
# In[7]:
## ============================== Part 3: Compute Cost (Feedforward) ========================
# To the neural network, you should first start by implementing the feedforward part of the neural network
# that returns the cost only. After implementing the feedforward to compute the cost, you can verify that
# your implementation is correct by verifying that you get the same cost as us for the fixed
# debugging parameters
print('Feedforward using neural network ...\n')
# Weight regularizatoin parameter (we set this to 0 here)
lambda_ = 0
J = nnCostFunction(nn_params, input_layer_size, hidden_layer_size, num_labels,
X, y, lambda_)[0]
print('Cost at the parameters (loaded form ex4weights): %f' % (J))
print('This value should be about 0.287629\n')
# ### 1.4 Regularized cost function
# The cost function for neural networks with regularization is given by
#
# $$
# J(\theta) = \frac{1}{m}\sum_{i=1}^{m}\sum_{k=1}^{K}[-y^{(i)}_{k}\log(h_{\theta}(x^{(i)})_{k}) - (1-y^{(i)}_{k})\log(1-h_{\theta}(x^{(i)})_{k})] +
# \frac{\lambda}{2m}
# \big[
# \sum_{j=1}^{25}\sum_{k=1}^{400}\big(\Theta_{j,k}^{(1)}\big)^{2} +
# \sum_{j=1}^{10}\sum_{k=1}^{25}\big(\Theta_{j,k}^{(2)}\big)^{2}
# \big]
# $$
% feedforward part of the neural network that returns the cost only. You
% should complete the code in nnCostFunction.m to return cost. After
% implementing the feedforward to compute the cost, you can verify that
% your implementation is correct by verifying that you get the same cost
% as us for the fixed debugging parameters.
%
% We suggest implementing the feedforward cost *without* regularization
% first so that it will be easier for you to debug. Later, in part 4, you
% will get to implement the regularized cost.
%
"""
print("\nFeedforward Using Neural Network ...\n")
# Weight regularization parameter (we set this to 0 here).
lamda = 0.0;
J,grad = nn.nnCostFunction(nn_params, input_layer_size, hidden_layer_size, num_labels, X, y, lamda);
print(" The cost of at parameters (loaded from ex4weights): %f\n" %J)
print('\nProgram paused. Press enter to continue.\n');
raw_input();
"""
%% =============== Part 4: Implement Regularization ===============
% Once your cost function implementation is correct, you should now
% continue to implement the regularization with the cost.
%
"""
print('\nChecking Cost Function (w/ Regularization) ... \n')
#Weight regularization parameter (we set this to 1 here).
lamda = 1.0;
J,grad = nn.nnCostFunction(nn_params, input_layer_size, hidden_layer_size, num_labels, X, y, lamda);
def checkNNGradients(MLlambda = 0):
'''
%CHECKNNGRADIENTS Creates a small neural network to check the
%backpropagation gradients
% CHECKNNGRADIENTS(lambda) Creates a small neural network to check the
% backpropagation gradients, it will output the analytical gradients
% produced by your backprop code and the numerical gradients (computed
% using computeNumericalGradient). These two gradient computations should
% result in very similar values.
%
'''
input_layer_size = 3
hidden_layer_size = 5
num_labels = 3
m = 5
#% We generate some 'random' test data
Theta1 = debugInitializeWeights(hidden_layer_size, input_layer_size)
Theta2 = debugInitializeWeights(num_labels, hidden_layer_size)
#% Reusing debugInitializeWeights to generate X
X = debugInitializeWeights(m, input_layer_size - 1)
#y = 1 + mod(1:m, num_labels)'
y = 1 + (np.arange(m)+1) % num_labels
# JKM - need a column vector
y = y[:, None]
y = y - 1
#% Unroll parameters
nn_params = np.hstack((Theta1.ravel(order = 'F'), Theta2.ravel(order = 'F')))
#% Short hand for cost function
costFunc = lambda p: nnCostFunction(p, input_layer_size, hidden_layer_size,
num_labels, X, y, MLlambda)
cost, grad = costFunc(nn_params)
numgrad = computeNumericalGradient(costFunc, nn_params)
#% Visually examine the two gradient computations. The two columns
#% you get should be very similar.
print np.column_stack((numgrad, grad))
# disp([numgrad grad]) -> matlab
print('The above two columns you get should be very similar.\n' +
'(Left-Your Numerical Gradient, Right-Analytical Gradient)\n\n')
#% Evaluate the norm of the difference between two solutions.
#% If you have a correct implementation, and assuming you used EPSILON = 0.0001
#% in computeNumericalGradient.m, then diff below should be less than 1e-9
# diff = norm(numgrad-grad)/norm(numgrad+grad)
diff = np.linalg.norm(numgrad-grad) / np.linalg.norm(numgrad+grad)
print(" If your backpropagation implementation is correct, then")
print(" the relative difference will be small (less than 1e-9).")
print " Relative Difference: ", diff
print("\n\n")
return
# feedforward part of the neural network that returns the cost only. You
# should complete the code in nnCostFunction.m to return cost. After
# implementing the feedforward to compute the cost, you can verify that
# your implementation is correct by verifying that you get the same cost
# as us for the fixed debugging parameters.
#
# We suggest implementing the feedforward cost *without* regularization
# first so that it will be easier for you to debug. Later, in part 4, you
# will get to implement the regularized cost.
#
print 'Feedforward Using Neural Network ...'
# Weight regularization parameter (we set this to 0 here).
Lambda = 0
J, _ = nnCostFunction(nn_params, input_layer_size, hidden_layer_size,
num_labels, X, y, Lambda)
print 'Cost at parameters (loaded from ex4weights): %f \n(this value should be about 0.287629)\n' % J
raw_input("Program paused. Press Enter to continue...")
## =============== Part 4: Implement Regularization ===============
# Once your cost function implementation is correct, you should now
# continue to implement the regularization with the cost.
#
print 'Checking Cost Function (w/ Regularization) ...'
# Weight regularization parameter (we set this to 1 here).
Lambda = 1
def gradientFunction(p):
J, gradient = nnCostFunction(p, input_layer_size, hidden_layer_size,
num_labels, data, labels, lambda_nn)
gradient = np.ndarray.flatten(gradient)
return gradient
# feedforward part of the neural network that returns the cost only. You
# should complete the code in nnCostFunction.m to return cost. After
# implementing the feedforward to compute the cost, you can verify that
# your implementation is correct by verifying that you get the same cost
# as us for the fixed debugging parameters.
#
# We suggest implementing the feedforward cost *without* regularization
# first so that it will be easier for you to debug. Later, in part 4, you
# will get to implement the regularized cost.
#
print('Feedforward Using Neural Network ...')
# # Weight regularization parameter (we set this to 0 here).
lambda_reg = 0
J, _ = nncf.nnCostFunction(nn_params, input_layer_size, hidden_layer_size, \
num_labels, X, y, lambda_reg)
print('Training Set Accuracy: {:f}\n(this value should be about 0.287629)'.
format(J))
input('Program paused. Press enter to continue.\n')
## =============== Part 4: Implement Regularization ===============
# Once your cost function implementation is correct, you should now
# continue to implement the regularization with the cost.
#
print('Checking Cost Function (w/ Regularization)...')
# Weight regularization parameter (we set this to 1 here).
lambda_reg = 1
def costFunction(p):
J, gradient = nnCostFunction(p, input_layer_size, hidden_layer_size,
num_labels, data, labels, lambda_nn)
print "training"
print J
return J
data = np.array(data)
labels = a['y']
labels = np.array(labels)
# the 10s in labels convert to 0s
#labels[np.nonzero(labels==10)]=0
# load weight
b = sio.loadmat('ex4weights.mat')
Theta1 = b['Theta1']
Theta2 = b['Theta2']
Theta1 = np.reshape(Theta1, [Theta1.size, 1])
Theta2 = np.reshape(Theta2, [Theta2.size, 1])
nn_params = np.concatenate((Theta1, Theta2))
# compute cost(FeedForward) with lambda_nn=0
print "FeedForwad Using Neural Network..."
lambda_nn = 0
J, grad = nnCostFunction(nn_params, input_layer_size, hidden_layer_size,
num_labels, data, labels, lambda_nn)
print "Cost at parameters (loaded from ex4weight): (%s)" % (J)
# Compute the cost with lambda_nn=1
print "FeedForwad Using Neural Network...(with lambda=1)"
lambda_nn = 1
J, grad = nnCostFunction(nn_params, input_layer_size, hidden_layer_size,
num_labels, data, labels, lambda_nn)
print "Cost at parameters (loaded from ex4weight): (%s)" % (J)
# Sigmoid Gradient
print "Evaluating sigmoid gradient..."
test = np.array([1, -0.5, 0, 0.5, 1])
g = sigmoid_gradient(test)
print "sigmoid gradient with :[1,-0.5,0,0.5,1] (%s)" % (g)
print('Loading Saved Neural Network Parameters ...')
# Load the weights into variables Theta1 and Theta2
weights = io.loadmat('ex4weights.mat')
Theta1, Theta2 = np.matrix(weights['Theta1']), np.matrix(weights['Theta2'])
# Unroll parameters
nn_params = np.hstack((Theta1.flatten(), Theta2.flatten()))
# ================ Part 3: Compute Cost (Feedforward) ================
print('Feedforward Using Neural Network ...')
# Weight regularization parameter (we set this to 0 here).
lmbda = 0
J, grad = nnCostFunction(nn_params, input_layer_size, hidden_layer_size,
num_labels, X, y, lmbda)
print('Cost at parameters (loaded from ex4weights): %f ' % (J))
print('(this value should be about 0.287629)')
input('Program paused. Press enter to continue.')
# =============== Part 4: Implement Regularization ===============
print('Checking Cost Function (w/ Regularization) ... ')
# Weight regularization parameter (we set this to 1 here).
lmbda = 1
J, grad = nnCostFunction(nn_params, input_layer_size, hidden_layer_size,
num_labels, X, y, lmbda)
def costFunc(p):
return nnCostFunction(p, input_layer_size, hidden_layer_size, num_labels, X, y, lmbda)
def costFunc(p):
return nncf.nnCostFunction(p, input_layer_size, hidden_layer_size, \
num_labels, X, y, lambda_reg)
def costFunc(p):
return nncf.nnCostFunction(p, input_layer_size, hidden_layer1_size,hidden_layer2_size, \
num_labels, X, y, lambda_reg)
rand_indices = np.random.permutation(m)
plt.figure()
displayData(X[rand_indices[0:100], :], padding=1)
plt.show()
# =================== 1.2 Model representation ===================
# Load the weights into variables Theta1 and Theta2
mat_param = sio.loadmat('ex4weights.mat')
theta_1 = mat_param['Theta1']
theta_2 = mat_param['Theta2']
params_trained = np.hstack((theta_1.flatten(), theta_2.flatten()))
# =================== 1.3 Feedforward and cost function =============
l = 0.0
j, _ = nnCostFunction(params_trained, input_layer_size, hidden_layer_size,
num_labels, X, y, l)
print('Cost at parameters (this value should be about 0.287629):', j)
# =================== 1.4 Regularized cost function ==================
l = 1.0
j, _ = nnCostFunction(params_trained, input_layer_size, hidden_layer_size,
num_labels, X, y, l)
print('Cost at parameters (this value should be about 0.383770):', j)
# ==================== 2. Backpropagation ============================
# ==================== 2.1 Sigmoid gradient ==========================
print('Evaluating sigmoid gradient...')
g = sigmoidGradient(0)
print(g)
def costFunc(p):
return nnCostFunction(p, input_layer_size, hidden_layer_size,
num_labels, X, y, lambda_param)
from __future__ import division from nnCostFunction import nnCostFunction import numpy as np il = 2; # input layer hl = 2; # hidden layer nl = 4; # number og labels nn = np.array([range(1,18+1)]) / 10 # nn_params X = np.cos([[1, 2], [3, 4], [5, 6]]) y = np.array([[4], [2], [3]]) l = 4 J, grad = nnCostFunction(nn, il, hl, nl, X, y, l)
# Unrolling Parameters to Vector for Implementation
nn_params = []
nn_params.extend((list(Theta1.flatten()) + list(Theta2.flatten())))
# ================ Part 3: Compute Cost (Feedforward) ================
# 1. start by implementing the feedforward part of the
# neural network that returns the cost only (nnCostFunction.py)
# 2. verify that your implementation is correct by verifying that you
# get the same cost as us for the fixed debugging parameters.
print('\nFeedforward Using Neural Network ...\n')
# Weight regularization parameter (we set this to 0 here).
lambda_param = 0
J, grad = nnCostFunction(nn_params, input_layer_size, hidden_layer_size,
num_labels, X, y, lambda_param)
print('Cost at parameters (loaded from ex4weights): {:.6f} '.format(float(J)))
print('\n(this value should be about 0.287629)\n')
input('\nProgram paused. Press enter to continue.\n')
# =============== Part 4: Implement Regularization ===============
print('\nChecking Cost Function (w/ Regularization) ... \n')
# Weight regularization parameter
lambda_param = 1
J, grad = nnCostFunction(nn_params, input_layer_size, hidden_layer_size,
index = 5
for index in range(6):
x_train = list_x_train[index]
y_train = list_y_train[index]
theta1 = randInitializeWeights(784, hidden_layer)
theta2 = randInitializeWeights(hidden_layer, hidden_layer)
theta3 = randInitializeWeights(hidden_layer, 10)
print 'training... ' + str(index + 1)
for i in range(epoch):
# mini batch
x_train, y_train = shuffle(x_train, y_train)
for k in range(len(x_train)/mini_batch):
x = x_train[k*mini_batch:(k + 1)*mini_batch]
y = y_train[k*mini_batch:(k + 1)*mini_batch]
results = nnCostFunction(theta1, theta2, theta3, x, y, lamda=l)
grad1 = results[1]
grad2 = results[2]
grad3 = results[3]
#print 'inter ' + str(i) + ' coss = ' + str(results[0])
grad = np.concatenate([unRolling(grad1), unRolling(grad2)])
grad = np.concatenate([grad, unRolling(grad3)])
theta = np.concatenate([unRolling(theta1), unRolling(theta2)])
theta = np.concatenate([theta, unRolling(theta3)])
theta = theta - alpha * grad
theta1 = np.reshape(
theta[0:785 * hidden_layer], (hidden_layer, 785))
theta2 = np.reshape(
theta[785 * hidden_layer:(786 + hidden_layer) * hidden_layer], (hidden_layer, hidden_layer + 1))
theta3 = np.reshape(
theta[(786 + hidden_layer) * hidden_layer:len(theta)], (10, hidden_layer + 1))
def main():
''' Main function '''
## %% =========== Part 1: Loading and Visualizing Data =============
#% We start the exercise by first loading and visualizing the dataset.
#% You will be working with a dataset that contains handwritten digits.
#%
# Read the Matlab data
m, n, X, y = getMatlabTrainingData()
# number of features
input_layer_size = n
# Select some random images from X
print('Selecting random examples of the data to display.\n')
sel = np.random.permutation(m)
sel = sel[0:100]
# Re-work the data orientation of each training example
image_size = 20
XMatlab = np.copy(X) # Need a deep copy, not just the reference
for i in range(m):
XMatlab[i, :] = XMatlab[i, :].reshape(image_size, image_size).transpose().reshape(1, image_size*image_size)
# display the sample images
displayData(XMatlab[sel, :])
# Print Out the labels for what is being seen.
print('These are the labels for the data ...\n')
print(y[sel, :].reshape(10, 10))
# Pause program
print("Program paused. Press Ctrl-D to continue.\n")
code.interact(local=dict(globals(), **locals()))
print(" ... continuing\n ")
#%% ================ Part 2: Loading Parameters ================
#% In this part of the exercise, we load some pre-initialized
# % neural network parameters.
print('\nLoading Saved Neural Network Parameters ...\n')
# Load the weights into variables Theta1 and Theta2
import scipy .io as sio
fnWeights = '/home/jennym/Kaggle/DigitRecognizer/ex4/ex4weights.mat'
weights = sio.loadmat(fnWeights)
Theta1 = weights['Theta1']
Theta2 = weights['Theta2']
#% Unroll parameters
nn_params = np.hstack((Theta1.ravel(order='F'), Theta2.ravel(order='F')))
#%% ================ Part 3: Compute Cost (Feedforward) ================
#% To the neural network, you should first start by implementing the
#% feedforward part of the neural network that returns the cost only. You
#% should complete the code in nnCostFunction.m to return cost. After
#% implementing the feedforward to compute the cost, you can verify that
#% your implementation is correct by verifying that you get the same cost
#% as us for the fixed debugging parameters.
#%
#% We suggest implementing the feedforward cost *without* regularization
#% first so that it will be easier for you to debug. Later, in part 4, you
#% will get to implement the regularized cost.
#%
print('\nFeedforward Using Neural Network ...\n')
#% Weight regularization parameter (we set this to 0 here).
MLlambda = 0.0
# Cluge, put y back to matlab version, then adjust to use python
# indexing later into y_matrix
y[(y == 0)] = 10
y = y - 1
J, _ = nnCostFunction(nn_params, input_layer_size, hidden_layer_size,
num_labels, X, y, MLlambda)
print('Cost at parameters (loaded from ex4weights): ' + str(J) +
'\n (this value should be about 0.287629)\n')
# Pause
print("Program paused. Press Ctrl-D to continue.\n")
code.interact(local=dict(globals(), **locals()))
print(" ... continuing\n ")
#%% =============== Part 4: Implement Regularization ===============
#% Once your cost function implementation is correct, you should now
#% continue to implement the regularization with the cost.
#%
print('\nChecking Cost Function (with Regularization) ... \n')
# % Weight regularization parameter (we set this to 1 here).
MLlambda = 1.0
J, _ = nnCostFunction(nn_params, input_layer_size, hidden_layer_size,
num_labels, X, y, MLlambda)
print('Cost at parameters (loaded from ex4weights): ' + str(J) +
'\n(this value should be about 0.383770)\n');
# Pause
print("Program paused. Press Ctrl-D to continue.\n")
code.interact(local=dict(globals(), **locals()))
print(" ... continuing\n ")
#%% ================ Part 5: Sigmoid Gradient ================
#% Before you start implementing the neural network, you will first
#% implement the gradient for the sigmoid function. You should complete the
#% code in the sigmoidGradient.m file.
#%
print('\nEvaluating sigmoid gradient...\n')
g = sigmoidGradient(np.array([1, -0.5, 0, 0.5, 1]))
print('Sigmoid gradient evaluated at [1 -0.5 0 0.5 1]:\n ')
print(g)
print('\n\n')
# Pause
print("Program paused. Press Ctrl-D to continue.\n")
code.interact(local=dict(globals(), **locals()))
print(" ... continuing\n ")
#%% ================ Part 6: Initializing Parameters ================
#% In this part of the exercise, you will be starting to implement a two
#% layer neural network that classifies digits. You will start by
#% implementing a function to initialize the weights of the neural network
#% (randInitializeWeights.m)
print('\nInitializing Neural Network Parameters ...\n')
initial_Theta1 = randInitializeWeights(input_layer_size, hidden_layer_size)
initial_Theta2 = randInitializeWeights(hidden_layer_size, num_labels)
#% Unroll parameters
initial_nn_params = np.hstack(( initial_Theta1.ravel(order = 'F'),
initial_Theta2.ravel(order = 'F')))
# Pause
print("Program paused. Press Ctrl-D to continue.\n")
code.interact(local=dict(globals(), **locals()))
print(" ... continuing\n ")
#%% =============== Part 7: Implement Backpropagation ===============
#% Once your cost matches up with ours, you should proceed to implement the
#% backpropagation algorithm for the neural network. You should add to the
#% code you've written in nnCostFunction.m to return the partial
#% derivatives of the parameters.
#%
print('\nChecking Backpropagation... \n')
#% Check gradients by running checkNNGradients
checkNNGradients()
# Pause
print("Program paused. Press Ctrl-D to continue.\n")
code.interact(local=dict(globals(), **locals()))
print(" ... continuing\n ")
#%% =============== Part 8: Implement Regularization ===============
#% Once your backpropagation implementation is correct, you should now
#% continue to implement the regularization with the cost and gradient.
#%
print('\nChecking Backpropagation (w/ Regularization) ... \n')
#% Check gradients by running checkNNGradients
MLlambda = 3
checkNNGradients(MLlambda)
# Pause
print("Program paused. Press Ctrl-D to continue.\n")
code.interact(local=dict(globals(), **locals()))
print(" ... continuing\n ")
#% Also output the costFunction debugging values
debug_J, _ = nnCostFunction(nn_params, input_layer_size,
hidden_layer_size, num_labels, X, y, MLlambda)
print('\n\n Cost at (fixed) debugging parameters (w/ lambda = ' +
'{0}): {1}'.format(MLlambda, debug_J))
print('\n (this value should be about 0.576051)\n\n')
# Pause
print("Program paused. Press Ctrl-D to continue.\n")
code.interact(local=dict(globals(), **locals()))
print(" ... continuing\n ")
#%% =================== Part 8b: Training NN ===================
#% You have now implemented all the code necessary to train a neural
#% network. To train your neural network, we will now use "fmincg", which
#% is a function which works similarly to "fminunc". Recall that these
#% advanced optimizers are able to train our cost functions efficiently as
#% long as we provide them with the gradient computations.
#%
print ('\nTraining Neural Network... \n')
#% After you have completed the assignment, change the MaxIter to a larger
#% value to see how more training helps.
#% jkm change maxIter from 50-> 400
options = {'maxiter': MAXITER}
#% You should also try different values of lambda
MLlambda = 1
#% Create "short hand" for the cost function to be minimized
costFunc = lambda p: nnCostFunction(p, input_layer_size, hidden_layer_size,
num_labels, X, y, MLlambda)
#% Now, costFunction is a function that takes in only one argument (the
#% neural network parameters)
'''
NOTES: Call scipy optimize minimize function
method : str or callable, optional Type of solver.
CG -> Minimization of scalar function of one or more variables
using the conjugate gradient algorithm.
jac : bool or callable, optional Jacobian (gradient) of objective function.
Only for CG, BFGS, Newton-CG, L-BFGS-B, TNC, SLSQP, dogleg, trust-ncg.
If jac is a Boolean and is True, fun is assumed to return the gradient
along with the objective function. If False, the gradient will be
estimated numerically. jac can also be a callable returning the
gradient of the objective. In this case, it must accept the same
arguments as fun.
callback : callable, optional. Called after each iteration, as callback(xk),
where xk is the current parameter vector.
'''
# Setup a callback for displaying the cost at the end of each iteration
class Callback(object):
def __init__(self):
self.it = 0
def __call__(self, p):
self.it += 1
print "Iteration %5d | Cost: %e" % (self.it, costFunc(p)[0])
result = sci.minimize(costFunc, initial_nn_params, method='CG',
jac=True, options=options, callback=Callback())
nn_params = result.x
cost = result.fun
# matlab: [nn_params, cost] = fmincg(costFunction, initial_nn_params, options);
#% Obtain Theta1 and Theta2 back from nn_params
Theta1 = np.reshape(nn_params[:hidden_layer_size * (input_layer_size + 1)],
(hidden_layer_size, (input_layer_size + 1)),
order = 'F')
Theta2 = np.reshape(nn_params[hidden_layer_size * (input_layer_size + 1):],
(num_labels, (hidden_layer_size + 1)),
order = 'F')
# Pause
print("Program paused. Press Ctrl-D to continue.\n")
code.interact(local=dict(globals(), **locals()))
print(" ... continuing\n ")
#%% ================= Part 9: Visualize Weights =================
#% You can now "visualize" what the neural network is learning by
#% displaying the hidden units to see what features they are capturing in
#% the data.#
print('\nVisualizing Neural Network... \n')
displayData(Theta1[:, 1:])
# Pause
print("Program paused. Press Ctrl-D to continue.\n")
code.interact(local=dict(globals(), **locals()))
print(" ... continuing\n ")
#%% ================= Part 10: Implement Predict =================
#% After training the neural network, we would like to use it to predict
#% the labels. You will now implement the "predict" function to use the
#% neural network to predict the labels of the training set. This lets
#% you compute the training set accuracy.
pred = predict(Theta1, Theta2, X)
# JKM - my array was column stacked - don't understand why this works
pp = np.row_stack(pred)
accuracy = np.mean(np.double(pp == y)) * 100
print('\nTraining Set Accuracy: {0} \n'.format(accuracy))
# Pause
print("Program paused. Press Ctrl-D to continue.\n")
code.interact(local=dict(globals(), **locals()))
print(" ... continuing\n ")
# ========================================
# All Done!
return
input('Program paused. Press enter to continue')
#%% =============== Part 7: Implement Regularization ===============
# Once your backpropagation implementation is correct, you should now
# continue to implement the regularization gradient.
#
print('Checking Backpropagation (w/ Regularization) ... ')
#
## Check gradients by running checkNNGradients
lambda_value = 3
checkNNGradients(lambda_value)
# Also output the costFunction debugging values
debug_J = nnCostFunction(nn_params, input_layer_size, hidden_layer_size,
num_labels, X, y, lambda_value)
print('Cost at (fixed) debugging parameters (w/ lambda = 10): ', debug_J[0][0],
'(this value should be about 0.576051)')
input('Program paused. Press enter to continue')
#%% =================== Part 8: Training NN ===================
# You have now implemented all the code necessary to train a neural
# network. To train your neural network, we will now use "fmincg", which
# is a function which works similarly to "fminunc". Recall that these
# advanced optimizers are able to train our cost functions efficiently as
# long as we provide them with the gradient computations.
#
print('Training Neural Network...')
# feedforward part of the neural network that returns the cost only. You
# should complete the code in nnCostFunction.m to return cost. After
# implementing the feedforward to compute the cost, you can verify that
# your implementation is correct by verifying that you get the same cost
# as us for the fixed debugging parameters.
#
# We suggest implementing the feedforward cost *without* regularization
# first so that it will be easier for you to debug. Later, in part 4, you
# will get to implement the regularized cost.
#
print('\nFeedforward Using Neural Network ...')
# Weight regularization parameter (we set this to 0 here).
lambda_reg = 0
J = ncf.nnCostFunction(nn_params, input_layer_size, hidden_layer_size, num_labels, X, y, lambda_reg)
print('Cost at parameters (loaded from ex4weights): {:f} \n(this value should be about 0.287629)'.format(J))
input('\nProgram paused. Press enter to continue.')
# ## =============== Part 4: Implement Regularization ===============
# # Once your cost function implementation is correct, you should now
# # continue to implement the regularization with the cost.
# #
#
# fprintf('\nChecking Cost Function (w/ Regularization) ... \n')
#
# # Weight regularization parameter (we set this to 1 here).
# lambda = 1
#
# To the neural network, you should first start by implementing the
# feedforward part of the neural network that returns the cost only. You
# should complete the code in nnCostFunction.py to return cost. After
# implementing the feedforward to compute the cost, you can verify that
# your implementation is correct by verifying that you get the same cost
# as us for the fixed debugging parameters.
#
# We suggest implementing the feedforward cost *without* regularization
# first so that it will be easier for you to debug. Later, in part 4, you
# will get to implement the regularized cost.
print('Feedforward Using Neural Network ...')
# Weight regularization parameter (we set this to 0 here).
lamda = 0
J, _ = nnCostFunction(nn_params, input_layer_size,
hidden_layer_size, num_labels, X, y, lamda)
print('Cost at parameters (loaded from ex4weights): %f' % J)
print('(this value should be about 0.287629)\n')
# =============== Part 4: Implement Regularization ===============
# Once your cost function implementation is correct, you should now
# continue to implement the regularization with the cost.
print('Checking Cost Function (w/ Regularization) ...')
# Weight regularization parameter (we set this to 1 here).
lamda = 1
J, _ = nnCostFunction(nn_params, input_layer_size,
hidden_layer_size, num_labels, X, y, lamda)
print('Cost at parameters (loaded from ex4weights): %f' % J)
data=np.array(data)
labels=a['y']
labels=np.array(labels)
# the 10s in labels convert to 0s
#labels[np.nonzero(labels==10)]=0
# load weight
b=sio.loadmat('ex4weights.mat')
Theta1=b['Theta1']
Theta2=b['Theta2']
Theta1=np.reshape(Theta1,[Theta1.size,1])
Theta2=np.reshape(Theta2,[Theta2.size,1])
nn_params=np.concatenate((Theta1,Theta2))
# compute cost(FeedForward) with lambda_nn=0
print "FeedForwad Using Neural Network..."
lambda_nn=0
J,grad=nnCostFunction(nn_params,input_layer_size,hidden_layer_size,num_labels,data,labels,lambda_nn)
print "Cost at parameters (loaded from ex4weight): (%s)"%(J)
# Compute the cost with lambda_nn=1
print "FeedForwad Using Neural Network...(with lambda=1)"
lambda_nn=1
J,grad=nnCostFunction(nn_params,input_layer_size,hidden_layer_size,num_labels,data,labels,lambda_nn)
print "Cost at parameters (loaded from ex4weight): (%s)"%(J)
# Sigmoid Gradient
print "Evaluating sigmoid gradient..."
test=np.array([1,-0.5,0,0.5,1])
g=sigmoid_gradient(test)
print "sigmoid gradient with :[1,-0.5,0,0.5,1] (%s)"%(g)
# Initializing Pameters
def train(X, y, struc, nn_params, niters=50):
f = lambda p: nnCostFunction(p, X, y, struc, lambd=1.0)
t, j, i = fmincg.minimize(f, nn_params, maxIter=niters, verbose=False)
nn_params = t
return nn_params
def gradientFunction(p):
J,gradient=nnCostFunction(p,input_layer_size,hidden_layer_size,num_labels,data,labels,lambda_nn)
gradient=np.ndarray.flatten(gradient)
return gradient
# feedforward part of the neural network that returns the cost only. You
# should complete the code in nnCostFunction.py to return cost. After
# implementing the feedforward to compute the cost, you can verify that
# your implementation is correct by verifying that you get the same cost
# as us for the fixed debugging parameters.
#
# We suggest implementing the feedforward cost *without* regularization
# first so that it will be easier for you to debug. Later, in part 4, you
# will get to implement the regularized cost.
#
print('\nFeedforward Using Neural Network ...')
# Weight regularization parameter (we set this to 0 here).
lambda_ = 0
J, _ = nnCostFunction(nn_params, input_layer_size, hidden_layer_size,
num_labels, X, y, lambda_)
print('Cost at parameters (loaded from ex4weights): %f ' % J)
print('(this value should be about 0.287629)')
print('\nProgram paused. Press enter to continue.')
input()
## =============== Part 4: Implement Regularization ===============
# Once your cost function implementation is correct, you should now
# continue to implement the regularization with the cost.
#
print('Checking Cost Function (w/ Regularization) ...')
# Weight regularization parameter (we set this to 1 here).
# feedforward part of the neural network that returns the cost only. You
# should complete the code in nnCostFunction.m to return cost. After
# implementing the feedforward to compute the cost, you can verify that
# your implementation is correct by verifying that you get the same cost
# as us for the fixed debugging parameters.
#
# We suggest implementing the feedforward cost *without* regularization
# first so that it will be easier for you to debug. Later, in part 4, you
# will get to implement the regularized cost.
#
print('\nFeedforward Using Neural Network ...')
# Weight regularization parameter (we set this to 0 here).
lamda = 0
J, Grad = nnCostFunction(nn_params, input_layer_size, hidden_layer_size,
num_labels, X, y, lamda)
print("Cost at parameters (loaded from ex4weights): ", J,
"(this value should be about 0.287629)")
## =============== Part 4: Implement Regularization ===============
# Once your cost function implementation is correct, you should now
# continue to implement the regularization with the cost.
#
# Weight regularization parameter (we set this to 1 here).
lamda = 1
J, Grad = nnCostFunction(nn_params, input_layer_size, hidden_layer_size,
num_labels, X, y, lamda)
feedforward part of the neural network that returns the cost only. You
should complete the code in nnCostFunction.m to return cost. After
implementing the feedforward to compute the cost, you can verify that
your implementation is correct by verifying that you get the same cost
as us for the fixed debugging parameters.
We suggest implementing the feedforward cost *without* regularization
first so that it will be easier for you to debug. Later, in part 4, you
will get to implement the regularized cost."""
print('\nFeedforward Using Neural Network ...\n')
# Weight regularization parameter (we set this to 0 here).
reg_lambda = 0;
J, grad = nnCostFunction(nn_params, input_layer_size, hidden_layer_size, num_labels, X, y, reg_lambda)
print('Cost at parameters (loaded from ex4weights) should be about 10.4414339388 \n', J)
print('\nProgram paused. Press enter to continue.\n')
pause()
"""## Part 4: Implement Regularization ===============
Once your cost function implementation is correct, you should now
continue to implement the regularization with the cost.
"""
print('\nChecking Cost Function (w/ Regularization) ... \n')
# Weight regularization parameter (we set this to 1 here).
