def check_stacked_ae_cost(): """ Check the gradients for the stacked autoencoder. In general, we recommend that the creation of such files for checking gradients when you write new cost functions. """ # Setup random data / small model input_size = 4 hidden_size = 5 lambda_ = 0.01 data = np.random.randn(input_size, 5) labels = np.array([0, 1, 0, 1, 0], dtype=np.uint8) n_classes = 2 n_stack = 2 stack = [{} for i in range(n_stack)] stack[0]['w'] = 0.1 * np.random.randn(3, input_size) stack[0]['b'] = np.zeros(3) stack[1]['w'] = 0.1 * np.random.randn(hidden_size, 3) stack[1]['b'] = np.zeros(hidden_size) softmax_theta = 0.005 * np.random.randn(hidden_size * n_classes) stack_params, net_config = stack2params(stack) stacked_ae_theta = np.concatenate((softmax_theta, stack_params)) cost, grad = stacked_ae_cost(stacked_ae_theta, input_size, hidden_size, n_classes, net_config, lambda_, data, labels) # Check that the numerical and analytic gradients are the same J = lambda theta: stacked_ae_cost(theta, input_size, hidden_size, n_classes, net_config, lambda_, data, labels)[0] nume_grad = compute_numerical_gradient(J, stacked_ae_theta) # Use this to visually compare the gradients side by side for i in range(grad.size): print("{0:20.12f} {1:20.12f}".format(nume_grad[i], grad[i])) print( 'The above two columns you get should be very similar.\n(Left-Your Numerical Gradient, Right-Analytical Gradient)\n' ) # Compare numerically computed gradients with the ones obtained from backpropagation # The difference should be small. In our implementation, these values are usually less than 1e-9. # When you got this working, Congratulations!!! diff = np.linalg.norm(nume_grad - grad) / np.linalg.norm(nume_grad + grad) print("Norm of difference = ", diff) print( 'Norm of the difference between numerical and analytical gradient (should be < 1e-9)\n\n' )
def check_stacked_ae_cost(): """ Check the gradients for the stacked autoencoder. In general, we recommend that the creation of such files for checking gradients when you write new cost functions. """ # Setup random data / small model input_size = 4; hidden_size = 5; lambda_ = 0.01; data = np.random.randn(input_size, 5) labels = np.array([ 0, 1, 0, 1, 0], dtype=np.uint8) n_classes = 2 n_stack = 2 stack = [{} for i in range(n_stack)] stack[0]['w'] = 0.1 * np.random.randn(3, input_size) stack[0]['b'] = np.zeros(3) stack[1]['w'] = 0.1 * np.random.randn(hidden_size, 3) stack[1]['b'] = np.zeros(hidden_size) softmax_theta = 0.005 * np.random.randn(hidden_size * n_classes) stack_params, net_config = stack2params(stack) stacked_ae_theta = np.concatenate((softmax_theta, stack_params)) cost, grad = stacked_ae_cost(stacked_ae_theta, input_size, hidden_size, n_classes, net_config, lambda_, data, labels) # Check that the numerical and analytic gradients are the same J = lambda theta : stacked_ae_cost(theta, input_size, hidden_size, n_classes, net_config, lambda_, data, labels)[0] nume_grad = compute_numerical_gradient(J, stacked_ae_theta) # Use this to visually compare the gradients side by side for i in range(grad.size): print("{0:20.12f} {1:20.12f}".format(nume_grad[i], grad[i])) print('The above two columns you get should be very similar.\n(Left-Your Numerical Gradient, Right-Analytical Gradient)\n') # Compare numerically computed gradients with the ones obtained from backpropagation # The difference should be small. In our implementation, these values are usually less than 1e-9. # When you got this working, Congratulations!!! diff = np.linalg.norm(nume_grad - grad) / np.linalg.norm(nume_grad + grad) print("Norm of difference = ", diff) print('Norm of the difference between numerical and analytical gradient (should be < 1e-9)\n\n')
""" debug = False # Please switch this to True when you need to check gradient if debug: # First, lets make sure your numerical gradient computation is correct for a # simple function. After you have implemented compute_numerical_gradient.py, # run the following: check_numerical_gradient() # Now we can use it to check your cost function and derivative calculations # for the sparse autoencoder. J = lambda theta: sparse_autoencoder_cost( theta, visible_size, hidden_size, weight_decay_param, sparsity_param, beta, patches)[0] numgrad = compute_numerical_gradient(J, theta) # Use this to visually compare the gradients side by side n = min(grad.size, 20) # Number of gradients to display for i in range(n): print("{0:20.12f} {1:20.12f}".format(numgrad[i], grad[i])) print( 'The above two columns you get should be very similar.\n(Left-Your Numerical Gradient, Right-Analytical Gradient)\n' ) # Compare numerically computed gradients with the ones obtained from backpropagation # This should be small. In our implementation, these values are usually less than 1e-9. # When you got this working, Congratulations!!! diff = np.linalg.norm(numgrad - grad) / np.linalg.norm(numgrad + grad) print("Norm of difference = ", diff) """
# dummy patches debug = False if debug: debug_hidden_size = 5 debug_visible_size = 8 patches = np.random.rand(8, 10) theta = initialize_parameters(debug_hidden_size, debug_visible_size) cost, grad = sparse_autoencoder_linear_cost(theta, debug_visible_size, debug_hidden_size, lambda_, sparsity_param, beta, patches) # Check that the numerical and analytic gradients are the same J = lambda theta : sparse_autoencoder_linear_cost(theta, debug_visible_size, debug_hidden_size, lambda_, sparsity_param, beta, patches)[0] nume_grad = compute_numerical_gradient(J, theta) # Use this to visually compare the gradients side by side for i in range(grad.size): print("{0:20.12f} {1:20.12f}".format(nume_grad[i], grad[i])) print('The above two columns you get should be very similar.\n(Left-Your Numerical Gradient, Right-Analytical Gradient)\n') # Compare numerically computed gradients with the ones obtained from backpropagation # The difference should be small. In our implementation, these values are usually less than 1e-9. # When you got this working, Congratulations!!! diff = np.linalg.norm(nume_grad - grad) / np.linalg.norm(nume_grad + grad) print("Norm of difference = ", diff) print('Norm of the difference between numerical and analytical gradient (should be < 1e-9)\n') assert diff < 1e-9, 'Difference too large. Check your gradient computation again'