def check_stacked_ae_cost():
"""
Check the gradients for the stacked autoencoder.
In general, we recommend that the creation of such files for checking
gradients when you write new cost functions.
"""
# Setup random data / small model
input_size = 4
hidden_size = 5
lambda_ = 0.01
data = np.random.randn(input_size, 5)
labels = np.array([0, 1, 0, 1, 0], dtype=np.uint8)
n_classes = 2
n_stack = 2
stack = [{} for i in range(n_stack)]
stack[0]['w'] = 0.1 * np.random.randn(3, input_size)
stack[0]['b'] = np.zeros(3)
stack[1]['w'] = 0.1 * np.random.randn(hidden_size, 3)
stack[1]['b'] = np.zeros(hidden_size)
softmax_theta = 0.005 * np.random.randn(hidden_size * n_classes)
stack_params, net_config = stack2params(stack)
stacked_ae_theta = np.concatenate((softmax_theta, stack_params))
cost, grad = stacked_ae_cost(stacked_ae_theta, input_size, hidden_size,
n_classes, net_config, lambda_, data, labels)
# Check that the numerical and analytic gradients are the same
J = lambda theta: stacked_ae_cost(theta, input_size, hidden_size,
n_classes, net_config, lambda_, data,
labels)[0]
nume_grad = compute_numerical_gradient(J, stacked_ae_theta)
# Use this to visually compare the gradients side by side
for i in range(grad.size):
print("{0:20.12f} {1:20.12f}".format(nume_grad[i], grad[i]))
print(
'The above two columns you get should be very similar.\n(Left-Your Numerical Gradient, Right-Analytical Gradient)\n'
)
# Compare numerically computed gradients with the ones obtained from backpropagation
# The difference should be small. In our implementation, these values are usually less than 1e-9.
# When you got this working, Congratulations!!!
diff = np.linalg.norm(nume_grad - grad) / np.linalg.norm(nume_grad + grad)
print("Norm of difference = ", diff)
print(
'Norm of the difference between numerical and analytical gradient (should be < 1e-9)\n\n'
)
def check_stacked_ae_cost():
"""
Check the gradients for the stacked autoencoder.
In general, we recommend that the creation of such files for checking
gradients when you write new cost functions.
"""
# Setup random data / small model
input_size = 4;
hidden_size = 5;
lambda_ = 0.01;
data = np.random.randn(input_size, 5)
labels = np.array([ 0, 1, 0, 1, 0], dtype=np.uint8)
n_classes = 2
n_stack = 2
stack = [{} for i in range(n_stack)]
stack[0]['w'] = 0.1 * np.random.randn(3, input_size)
stack[0]['b'] = np.zeros(3)
stack[1]['w'] = 0.1 * np.random.randn(hidden_size, 3)
stack[1]['b'] = np.zeros(hidden_size)
softmax_theta = 0.005 * np.random.randn(hidden_size * n_classes)
stack_params, net_config = stack2params(stack)
stacked_ae_theta = np.concatenate((softmax_theta, stack_params))
cost, grad = stacked_ae_cost(stacked_ae_theta, input_size, hidden_size,
n_classes, net_config, lambda_, data, labels)
# Check that the numerical and analytic gradients are the same
J = lambda theta : stacked_ae_cost(theta, input_size, hidden_size,
n_classes, net_config, lambda_, data, labels)[0]
nume_grad = compute_numerical_gradient(J, stacked_ae_theta)
# Use this to visually compare the gradients side by side
for i in range(grad.size):
print("{0:20.12f} {1:20.12f}".format(nume_grad[i], grad[i]))
print('The above two columns you get should be very similar.\n(Left-Your Numerical Gradient, Right-Analytical Gradient)\n')
# Compare numerically computed gradients with the ones obtained from backpropagation
# The difference should be small. In our implementation, these values are usually less than 1e-9.
# When you got this working, Congratulations!!!
diff = np.linalg.norm(nume_grad - grad) / np.linalg.norm(nume_grad + grad)
print("Norm of difference = ", diff)
print('Norm of the difference between numerical and analytical gradient (should be < 1e-9)\n\n')
"""
debug = False # Please switch this to True when you need to check gradient
if debug:
# First, lets make sure your numerical gradient computation is correct for a
# simple function. After you have implemented compute_numerical_gradient.py,
# run the following:
check_numerical_gradient()
# Now we can use it to check your cost function and derivative calculations
# for the sparse autoencoder.
J = lambda theta: sparse_autoencoder_cost(
theta, visible_size, hidden_size, weight_decay_param, sparsity_param,
beta, patches)[0]
numgrad = compute_numerical_gradient(J, theta)
# Use this to visually compare the gradients side by side
n = min(grad.size, 20) # Number of gradients to display
for i in range(n):
print("{0:20.12f} {1:20.12f}".format(numgrad[i], grad[i]))
print(
'The above two columns you get should be very similar.\n(Left-Your Numerical Gradient, Right-Analytical Gradient)\n'
)
# Compare numerically computed gradients with the ones obtained from backpropagation
# This should be small. In our implementation, these values are usually less than 1e-9.
# When you got this working, Congratulations!!!
diff = np.linalg.norm(numgrad - grad) / np.linalg.norm(numgrad + grad)
print("Norm of difference = ", diff)
"""
# dummy patches
debug = False
if debug:
debug_hidden_size = 5
debug_visible_size = 8
patches = np.random.rand(8, 10)
theta = initialize_parameters(debug_hidden_size, debug_visible_size)
cost, grad = sparse_autoencoder_linear_cost(theta,
debug_visible_size, debug_hidden_size, lambda_, sparsity_param, beta, patches)
# Check that the numerical and analytic gradients are the same
J = lambda theta : sparse_autoencoder_linear_cost(theta,
debug_visible_size, debug_hidden_size, lambda_, sparsity_param, beta, patches)[0]
nume_grad = compute_numerical_gradient(J, theta)
# Use this to visually compare the gradients side by side
for i in range(grad.size):
print("{0:20.12f} {1:20.12f}".format(nume_grad[i], grad[i]))
print('The above two columns you get should be very similar.\n(Left-Your Numerical Gradient, Right-Analytical Gradient)\n')
# Compare numerically computed gradients with the ones obtained from backpropagation
# The difference should be small. In our implementation, these values are usually less than 1e-9.
# When you got this working, Congratulations!!!
diff = np.linalg.norm(nume_grad - grad) / np.linalg.norm(nume_grad + grad)
print("Norm of difference = ", diff)
print('Norm of the difference between numerical and analytical gradient (should be < 1e-9)\n')
assert diff < 1e-9, 'Difference too large. Check your gradient computation again'
