def test_svm_loss():
# Test svm_loss function.
num_classes, num_inputs = 10, 50
x = 0.001 * np.random.randn(num_inputs, num_classes)
y = np.random.randint(num_classes, size=num_inputs)
dx_num = eval_numerical_gradient(lambda x: svm_loss(x, y)[0], x, verbose=False)
loss, dx = svm_loss(x, y)
assert rel_error(dx, dx_num) < 1e-8
def rel_err_gradients():
"""
Return the relative error between analytic and nemerical gradients.
"""
# Number of layer units
n_samples = 100
input_size = 4 * 4
hidden_size = 4
output_size = input_size
layer_units = (input_size, hidden_size, output_size)
X_train = np.random.randn(n_samples, input_size)
reg = 1e-4
beta = 3 # weight of sparsity penalty term
sparsity_param = 1e-1 # desired average activation of the hidden units
# Define the classifier
sae = SparseAutoencoder(layer_units)
# Initialize weights
weights = sae.init_weights()
# Analytic gradients of the cost function
cost, grad = sparse_autoencoder_loss(weights,
X_train,
reg,
beta=beta,
sparsity_param=sparsity_param)
grad = sae.flatten_struct(grad) # Flattened gradients
def J(theta):
# Structured weights
weights = sae.pack_struct(theta)
return sparse_autoencoder_loss(weights,
X_train,
reg,
beta=beta,
sparsity_param=sparsity_param)[0]
theta = sae.flatten_struct(weights)
numerical_grad = eval_numerical_gradient(J, theta)
# Compare numerically computed gradients with those computed analytically
rel_err = rel_norm_diff(numerical_grad, grad)
return rel_err
def rel_err_gradients():
"""
Return the relative error between analytic and nemerical gradients.
"""
# Number of layer units
n_samples = 100
input_size = 4 * 4
hidden_size = 4
n_classes = 10
layer_units = (input_size, hidden_size, n_classes)
X_train = np.random.randn(n_samples, input_size)
y_train = np.random.randint(n_classes, size=n_samples)
reg = 1e-4
# Define the classifier
clf = NeuralNet(layer_units)
# Initialize weights
weights = clf.init_weights()
# Analytic gradients of the cost function
cost, grad = neural_net_loss(weights, X_train, y_train, reg)
grad = clf.flatten_struct(grad) # Flattened gradients
def J(theta):
# Structured weights
weights = clf.pack_struct(theta)
return neural_net_loss(weights, X_train, y_train, reg)[0]
theta = clf.flatten_struct(weights)
numerical_grad = eval_numerical_gradient(J, theta)
# Compare numerically computed gradients with those computed analytically
rel_err = rel_norm_diff(numerical_grad, grad)
return rel_err
def rel_err_gradients():
"""
Return the relative error between analytic gradients and nemerical ones.
"""
# Number of layer units
n_samples = 100
input_size = 4 * 4
hidden_size_L1 = 4
hidden_size_L2 = 4
output_size = 10
layer_units = (input_size, hidden_size_L1, hidden_size_L2, output_size)
X_train = np.random.randn(n_samples, input_size)
y_train = np.random.randint(output_size, size=n_samples)
reg = 1e-4
# Define the classifier
clf = MLP(layer_units)
# Initialize weights
weights = clf.init_weights()
# Analytic gradients of the cost function
cost, grad = mlp_loss(weights, X_train, y_train, reg)
grad = clf.flatten_struct(grad) # Flattened gradients
def J(theta):
# Structured weights
weights = clf.pack_struct(theta)
return mlp_loss(weights, X_train, y_train, reg)[0]
theta = clf.flatten_struct(weights)
numerical_grad = eval_numerical_gradient(J, theta)
# Compare numerically computed gradients with those computed analytically
rel_err = rel_norm_diff(numerical_grad, grad)
return rel_err