def test_5(self):
# Comprehensive gradient checking #
# Medium size data-set with more than two classes
arrs = []
labels = []
classes = ("cat", "dog", "bird", "turtle", "dinosaur", "human")
for m in range(0, 100):
arr = [random.random() for x in range(0, 200)]
z = random.random()
if z < 1 / 6:
label = classes[0]
elif z >= 1 / 6 and z < 2 / 6:
label = classes[1]
elif z >= 2 / 6 and z < 3 / 6:
label = classes[2]
elif z >= 3 / 6 and z < 4 / 6:
label = classes[3]
elif z >= 4 / 6 and z < 5 / 6:
label = classes[4]
else:
label = classes[5]
arrs.append(arr)
labels.append(label)
ann = Ann(arrs, labels, n_h=2) # Create Ann with these train_examples and labels
# L-1 matrices of partial derivatives for first example
J = ann.backward_batch()
T_original = copy.deepcopy(ann.Thetas)
# Just check the neuron connections between first, second, and third layer
for l in range(0, 2):
shape_J = J[l].shape
eps = 0.0001 # epsilon for a numerical approximation of the gradient
# Randomly select 100 neuron connections to check
a = random.sample(range(0, shape_J[0]), 10)
b = random.sample(range(0, shape_J[1]), 10)
for i in a:
for j in b:
T_e = np.zeros(shape_J) # Matrix of zeros
T_e[i][j] = eps
ann.Thetas[l] = T_original[l] + T_e
cost_e = ann.cost() # Cost at Theta + eps
ann.Thetas[l] = T_original[l] - T_e
cost_minus_e = ann.cost() # Cost at Theta - eps
P = (cost_e - cost_minus_e) / (2 * eps) # Numerical approximation
J_ij = J[l].item(i, j) # Backpropagation derivation
print(P, "\t", J_ij, "\t", abs(P - J_ij), (l, i, j))
# if (P < 0 and J_ij > 0 or P > 0 and J_ij < 0):
# self.fail()
self.assertAlmostEqual(P, J_ij, delta=0.001)
ann.Thetas = copy.deepcopy(T_original)
def test_5(self):
# Comprehensive gradient checking #
# Medium size data-set with more than two classes
arrs = []
labels = []
classes = ('cat', 'dog', 'bird', 'turtle', 'dinosaur', 'human')
for m in range(0, 100):
arr = [random.random() for x in range(0, 200)]
z = random.random()
if (z < 1 / 6):
label = classes[0]
elif (z >= 1 / 6 and z < 2 / 6):
label = classes[1]
elif (z >= 2 / 6 and z < 3 / 6):
label = classes[2]
elif (z >= 3 / 6 and z < 4 / 6):
label = classes[3]
elif (z >= 4 / 6 and z < 5 / 6):
label = classes[4]
else:
label = classes[5]
arrs.append(arr)
labels.append(label)
ann = Ann(arrs, labels,
n_h=2) # Create Ann with these train_examples and labels
# L-1 matrices of partial derivatives for first example
J = ann.backward_batch()
T_original = copy.deepcopy(ann.Thetas)
# Just check the neuron connections between first, second, and third layer
for l in range(0, 2):
shape_J = J[l].shape
eps = 0.0001 # epsilon for a numerical approximation of the gradient
# Randomly select 100 neuron connections to check
a = random.sample(range(0, shape_J[0]), 10)
b = random.sample(range(0, shape_J[1]), 10)
for i in a:
for j in b:
T_e = np.zeros(shape_J) # Matrix of zeros
T_e[i][j] = eps
ann.Thetas[l] = T_original[l] + T_e
cost_e = ann.cost() # Cost at Theta + eps
ann.Thetas[l] = T_original[l] - T_e
cost_minus_e = ann.cost() # Cost at Theta - eps
P = (cost_e - cost_minus_e) / (2 * eps
) # Numerical approximation
J_ij = J[l].item(i, j) # Backpropagation derivation
self.assertAlmostEqual(P, J_ij, delta=0.001)
ann.Thetas = copy.deepcopy(T_original)
def non_test_6(self):
# Test if training works by checking that training lowers the cost for random small and medium size data-sets#
# Small size random data-set with two labels
arrs = []
labels = []
classes = ('cat', 'dog')
for i in range(0, 1):
print('\nTesting data-set ' + str(i))
for m in range(0, 10):
arr = [random.random() for x in range(0, 3)]
label = classes[random.random() > 0.5]
arrs.append(arr)
labels.append(label)
ann = Ann(
arrs,
labels) # Create Ann with these train_examples and labels
cost_before = ann.cost()
ann.train()
cost_after = ann.cost()
self.assertTrue(cost_after <= cost_before)
# Medium size random data-set with three labels
arrs = []
labels = []
classes = ('cat', 'dog', 'bird')
for i in range(0, 1):
print('\nTesting data-set ' + str(i))
for m in range(0, 10):
arr = [random.random() for x in range(0, 5)]
z = random.random()
if (z < 0.33):
label = classes[0]
elif (z >= 0.33 and z < 0.66):
label = classes[1]
else:
label = classes[2]
arrs.append(arr)
labels.append(label)
ann = Ann(
arrs,
labels) # Create Ann with these train_examples and labels
cost_before = ann.cost()
ann.train()
cost_after = ann.cost()
self.assertTrue(cost_after <= cost_before)
def non_test_6(self):
# Test if training works by checking that training lowers the cost for random small and medium size data-sets#
# Small size random data-set with two labels
arrs = []
labels = []
classes = ('cat', 'dog')
for i in range(0, 1):
print('\nTesting data-set ' + str(i))
for m in range(0, 10):
arr = [random.random() for x in range(0, 3)]
label = classes[random.random() > 0.5]
arrs.append(arr)
labels.append(label)
ann = Ann(arrs, labels) # Create Ann with these train_examples and labels
cost_before = ann.cost()
ann.train()
cost_after = ann.cost()
self.assertTrue(cost_after <= cost_before)
# Medium size random data-set with three labels
arrs = []
labels = []
classes = ('cat', 'dog', 'bird')
for i in range(0, 1):
print('\nTesting data-set ' + str(i))
for m in range(0, 10):
arr = [random.random() for x in range(0, 5)]
z = random.random()
if (z < 0.33):
label = classes[0]
elif (z >= 0.33 and z < 0.66):
label = classes[1]
else:
label = classes[2]
arrs.append(arr)
labels.append(label)
ann = Ann(arrs, labels) # Create Ann with these train_examples and labels
cost_before = ann.cost()
ann.train()
cost_after = ann.cost()
self.assertTrue(cost_after <= cost_before)
def test_8(self):
# First test#
# 1 hidden layer cost test with regularization#
x1 = [1, 2, 3, 4] # Array as first example
y1 = 'yes'
arrs = []
labels = []
arrs.append(x1)
labels.append(y1)
ann1 = Ann(arrs, labels, n_h=1) # Create this architecture
# Custom Thetas weights#
M1 = np.matrix([[1, -1, 0.5, -0.3, 2],
[1, -1, 0.5, -0.3, 2],
[1, -1, 0.5, -0.3, 2],
[1, -1, 0.5, -0.3, 2]])
M2 = np.matrix([[1, 1, -1, 0.5, -1]])
ann1.Thetas[0] = M1
ann1.Thetas[1] = M2
cost_0 = ann1.cost() # lam equals 0
cost_1 = ann1.cost(lam=1) # lam equals 1
self.assertTrue(cost_1 > cost_0) # Cost with regularization penalty is always higher than without regularization
# Gradient checking (now with regularization)#
# Medium size data-set with several train_examples
lam_test = 1 # Regularization parameter
arrs = []
labels = []
classes = ('cat', 'dog')
for m in range(0, 100):
arr = [random.random() for x in range(0, 40)]
label = classes[random.random() > 0.5]
arrs.append(arr)
labels.append(label)
ann = Ann(arrs, labels, n_h=2) # Create Ann with these train_examples and labels
# L-1 matrices of partial derivatives for first example
J = ann.backward_batch(lam=lam_test, batch_size=1) # Use full-batch for gradient descent
T_original = copy.deepcopy(ann.Thetas)
for l in range(0, ann.L - 1):
shape_J = J[l].shape
eps = 0.0001 # epsilon for a numerical approximation of the gradient
a = random.sample(range(0, shape_J[0]), 2)
b = random.sample(range(0, shape_J[1]), 2)
for i in a:
for j in b:
T_e = np.zeros(shape_J) # Matrix of zeros
T_e[i][j] = eps
ann.Thetas[l] = T_original[l] + T_e
cost_e = ann.cost(lam=lam_test) # Cost at Theta + eps
ann.Thetas[l] = T_original[l] - T_e
cost_minus_e = ann.cost(lam=lam_test) # Cost at Theta - eps
P = (cost_e - cost_minus_e) / (2 * eps) # Numerical approximation
J_ij = J[l].item(i, j) # Backpropagation derivation
# print(P, '\t', J_ij, '\t', abs(P - J_ij), (l, i, j))
# if (P < 0 and J_ij > 0 or P > 0 and J_ij < 0):
# self.fail()
self.assertAlmostEqual(P, J_ij, delta=0.001)
ann.Thetas = copy.deepcopy(T_original)
def test_4(self):
# Gradient checking (check that a numerical approximation of the gradient is (almost) equal to our backpropagation derivation)#
# First data-set with one example
arrs = []
labels = []
arrs.append([1, 2, 4, 5, 5, 5])
labels.append('cat')
ann = Ann(arrs, labels, n_h=10) # Create Ann with these train_examples and labels
J = ann.backward(ann.train_examples[0].arr, ann.train_examples[0].y)
T_original = copy.deepcopy(ann.Thetas)
for l in range(0, ann.L - 1):
shape_J = J[l].shape
eps = 0.0001 # epsilon for a numerical approximation of the gradient
for i in range(0, shape_J[0]):
for j in range(0, shape_J[1]):
T_e = np.zeros(shape_J) # Matrix of zeros
T_e[i][j] = eps
ann.Thetas[l] = T_original[l] + T_e
cost_e = ann.cost() # Cost at Theta + eps
ann.Thetas[l] = T_original[l] - T_e
cost_minus_e = ann.cost() # Cost at Theta - eps
P = (cost_e - cost_minus_e) / (2 * eps) # Numerical approximation
J_ij = J[l].item(i, j) # Backpropagation derivation
# print(P, '\t', J_ij, '\t', abs(P - J_ij), (l, i, j))
# if (P < 0 and J_ij > 0 or P > 0 and J_ij < 0):
# self.fail()
self.assertAlmostEqual(P, J_ij, delta=0.001)
ann.Thetas = copy.deepcopy(T_original)
# Second data-set with several train_examples
arrs = []
labels = []
classes = ('cat', 'dog')
for m in range(0, 100):
arr = [random.random() for x in range(0, 20)]
label = classes[random.random() > 0.5]
arrs.append(arr)
labels.append(label)
ann = Ann(arrs, labels, n_h=2) # Create Ann with these train_examples and labels
# L-1 matrices of partial derivatives for first example
J = ann.backward_batch()
T_original = copy.deepcopy(ann.Thetas)
for l in range(0, ann.L - 1):
shape_J = J[l].shape
eps = 0.0001 # epsilon for a numerical approximation of the gradient
a = random.sample(range(0, shape_J[0]), 2)
b = random.sample(range(0, shape_J[1]), 2)
for i in a:
for j in b:
T_e = np.zeros(shape_J) # Matrix of zeros
T_e[i][j] = eps
ann.Thetas[l] = T_original[l] + T_e
cost_e = ann.cost() # Cost at Theta + eps
ann.Thetas[l] = T_original[l] - T_e
cost_minus_e = ann.cost() # Cost at Theta - eps
P = (cost_e - cost_minus_e) / (2 * eps) # Numerical approximation
J_ij = J[l].item(i, j) # Backpropagation derivation
self.assertAlmostEqual(P, J_ij, delta=0.001)
ann.Thetas = copy.deepcopy(T_original)
def test_8(self):
# First test#
# 1 hidden layer cost test with regularization#
x1 = [1, 2, 3, 4] # Array as first example
y1 = 'yes'
arrs = []
labels = []
arrs.append(x1)
labels.append(y1)
ann1 = Ann(arrs, labels, n_h=1) # Create this architecture
# Custom Thetas weights#
M1 = np.matrix([[1, -1, 0.5, -0.3, 2], [1, -1, 0.5, -0.3, 2],
[1, -1, 0.5, -0.3, 2], [1, -1, 0.5, -0.3, 2]])
M2 = np.matrix([[1, 1, -1, 0.5, -1]])
ann1.Thetas[0] = M1
ann1.Thetas[1] = M2
cost_0 = ann1.cost() # lam equals 0
cost_1 = ann1.cost(lam=1) # lam equals 1
self.assertTrue(
cost_1 > cost_0
) # Cost with regularization penalty is always higher than without regularization
# Gradient checking (now with regularization)#
# Medium size data-set with several train_examples
lam_test = 1 # Regularization parameter
arrs = []
labels = []
classes = ('cat', 'dog')
for m in range(0, 100):
arr = [random.random() for x in range(0, 40)]
label = classes[random.random() > 0.5]
arrs.append(arr)
labels.append(label)
ann = Ann(arrs, labels,
n_h=2) # Create Ann with these train_examples and labels
# L-1 matrices of partial derivatives for first example
J = ann.backward_batch(
lam=lam_test, batch_size=1) # Use full-batch for gradient descent
T_original = copy.deepcopy(ann.Thetas)
for l in range(0, ann.L - 1):
shape_J = J[l].shape
eps = 0.0001 # epsilon for a numerical approximation of the gradient
a = random.sample(range(0, shape_J[0]), 2)
b = random.sample(range(0, shape_J[1]), 2)
for i in a:
for j in b:
T_e = np.zeros(shape_J) # Matrix of zeros
T_e[i][j] = eps
ann.Thetas[l] = T_original[l] + T_e
cost_e = ann.cost(lam=lam_test) # Cost at Theta + eps
ann.Thetas[l] = T_original[l] - T_e
cost_minus_e = ann.cost(
lam=lam_test) # Cost at Theta - eps
P = (cost_e - cost_minus_e) / (2 * eps
) # Numerical approximation
J_ij = J[l].item(i, j) # Backpropagation derivation
# print(P, '\t', J_ij, '\t', abs(P - J_ij), (l, i, j))
# if (P < 0 and J_ij > 0 or P > 0 and J_ij < 0):
# self.fail()
self.assertAlmostEqual(P, J_ij, delta=0.001)
ann.Thetas = copy.deepcopy(T_original)
def test_4(self):
# Gradient checking (check that a numerical approximation of the gradient is (almost) equal to our backpropagation derivation)#
# First data-set with one example
arrs = []
labels = []
arrs.append([1, 2, 4, 5, 5, 5])
labels.append('cat')
ann = Ann(arrs, labels,
n_h=10) # Create Ann with these train_examples and labels
J = ann.backward(ann.train_examples[0].arr, ann.train_examples[0].y)
T_original = copy.deepcopy(ann.Thetas)
for l in range(0, ann.L - 1):
shape_J = J[l].shape
eps = 0.0001 # epsilon for a numerical approximation of the gradient
for i in range(0, shape_J[0]):
for j in range(0, shape_J[1]):
T_e = np.zeros(shape_J) # Matrix of zeros
T_e[i][j] = eps
ann.Thetas[l] = T_original[l] + T_e
cost_e = ann.cost() # Cost at Theta + eps
ann.Thetas[l] = T_original[l] - T_e
cost_minus_e = ann.cost() # Cost at Theta - eps
P = (cost_e - cost_minus_e) / (2 * eps
) # Numerical approximation
J_ij = J[l].item(i, j) # Backpropagation derivation
# print(P, '\t', J_ij, '\t', abs(P - J_ij), (l, i, j))
# if (P < 0 and J_ij > 0 or P > 0 and J_ij < 0):
# self.fail()
self.assertAlmostEqual(P, J_ij, delta=0.001)
ann.Thetas = copy.deepcopy(T_original)
# Second data-set with several train_examples
arrs = []
labels = []
classes = ('cat', 'dog')
for m in range(0, 100):
arr = [random.random() for x in range(0, 20)]
label = classes[random.random() > 0.5]
arrs.append(arr)
labels.append(label)
ann = Ann(arrs, labels,
n_h=2) # Create Ann with these train_examples and labels
# L-1 matrices of partial derivatives for first example
J = ann.backward_batch()
T_original = copy.deepcopy(ann.Thetas)
for l in range(0, ann.L - 1):
shape_J = J[l].shape
eps = 0.0001 # epsilon for a numerical approximation of the gradient
a = random.sample(range(0, shape_J[0]), 2)
b = random.sample(range(0, shape_J[1]), 2)
for i in a:
for j in b:
T_e = np.zeros(shape_J) # Matrix of zeros
T_e[i][j] = eps
ann.Thetas[l] = T_original[l] + T_e
cost_e = ann.cost() # Cost at Theta + eps
ann.Thetas[l] = T_original[l] - T_e
cost_minus_e = ann.cost() # Cost at Theta - eps
P = (cost_e - cost_minus_e) / (2 * eps
) # Numerical approximation
J_ij = J[l].item(i, j) # Backpropagation derivation
self.assertAlmostEqual(P, J_ij, delta=0.001)
ann.Thetas = copy.deepcopy(T_original)