Exemplo n.º 1
0
    def test_5(self):
        # Comprehensive gradient checking #

        # Medium size data-set with more than two classes
        arrs = []
        labels = []
        classes = ("cat", "dog", "bird", "turtle", "dinosaur", "human")
        for m in range(0, 100):
            arr = [random.random() for x in range(0, 200)]
            z = random.random()
            if z < 1 / 6:
                label = classes[0]
            elif z >= 1 / 6 and z < 2 / 6:
                label = classes[1]
            elif z >= 2 / 6 and z < 3 / 6:
                label = classes[2]
            elif z >= 3 / 6 and z < 4 / 6:
                label = classes[3]
            elif z >= 4 / 6 and z < 5 / 6:
                label = classes[4]
            else:
                label = classes[5]
            arrs.append(arr)
            labels.append(label)
        ann = Ann(arrs, labels, n_h=2)  # Create Ann with these train_examples and labels
        # L-1 matrices of partial derivatives for first example
        J = ann.backward_batch()
        T_original = copy.deepcopy(ann.Thetas)

        # Just check the neuron connections between first, second, and third layer
        for l in range(0, 2):
            shape_J = J[l].shape
            eps = 0.0001  # epsilon for a numerical approximation of the gradient
            # Randomly select 100 neuron connections to check
            a = random.sample(range(0, shape_J[0]), 10)
            b = random.sample(range(0, shape_J[1]), 10)
            for i in a:
                for j in b:
                    T_e = np.zeros(shape_J)  # Matrix of zeros
                    T_e[i][j] = eps
                    ann.Thetas[l] = T_original[l] + T_e
                    cost_e = ann.cost()  # Cost at Theta + eps
                    ann.Thetas[l] = T_original[l] - T_e
                    cost_minus_e = ann.cost()  # Cost at Theta - eps
                    P = (cost_e - cost_minus_e) / (2 * eps)  # Numerical approximation
                    J_ij = J[l].item(i, j)  # Backpropagation derivation

                    print(P, "\t", J_ij, "\t", abs(P - J_ij), (l, i, j))

                    # if (P < 0 and J_ij > 0 or P > 0 and J_ij < 0):
                    #    self.fail()

                    self.assertAlmostEqual(P, J_ij, delta=0.001)
                    ann.Thetas = copy.deepcopy(T_original)
    def test_5(self):
        # Comprehensive gradient checking #

        # Medium size data-set with more than two classes
        arrs = []
        labels = []
        classes = ('cat', 'dog', 'bird', 'turtle', 'dinosaur', 'human')
        for m in range(0, 100):
            arr = [random.random() for x in range(0, 200)]
            z = random.random()
            if (z < 1 / 6):
                label = classes[0]
            elif (z >= 1 / 6 and z < 2 / 6):
                label = classes[1]
            elif (z >= 2 / 6 and z < 3 / 6):
                label = classes[2]
            elif (z >= 3 / 6 and z < 4 / 6):
                label = classes[3]
            elif (z >= 4 / 6 and z < 5 / 6):
                label = classes[4]
            else:
                label = classes[5]
            arrs.append(arr)
            labels.append(label)
        ann = Ann(arrs, labels,
                  n_h=2)  # Create Ann with these train_examples and labels
        # L-1 matrices of partial derivatives for first example
        J = ann.backward_batch()
        T_original = copy.deepcopy(ann.Thetas)

        # Just check the neuron connections between first, second, and third layer
        for l in range(0, 2):
            shape_J = J[l].shape
            eps = 0.0001  # epsilon for a numerical approximation of the gradient
            # Randomly select 100 neuron connections to check
            a = random.sample(range(0, shape_J[0]), 10)
            b = random.sample(range(0, shape_J[1]), 10)
            for i in a:
                for j in b:
                    T_e = np.zeros(shape_J)  # Matrix of zeros
                    T_e[i][j] = eps
                    ann.Thetas[l] = T_original[l] + T_e
                    cost_e = ann.cost()  # Cost at Theta + eps
                    ann.Thetas[l] = T_original[l] - T_e
                    cost_minus_e = ann.cost()  # Cost at Theta - eps
                    P = (cost_e - cost_minus_e) / (2 * eps
                                                   )  # Numerical approximation
                    J_ij = J[l].item(i, j)  # Backpropagation derivation

                    self.assertAlmostEqual(P, J_ij, delta=0.001)
                    ann.Thetas = copy.deepcopy(T_original)
    def non_test_6(self):
        # Test if training works by checking that training lowers the cost for random small and medium size data-sets#

        # Small size random data-set with two labels
        arrs = []
        labels = []
        classes = ('cat', 'dog')
        for i in range(0, 1):
            print('\nTesting data-set ' + str(i))
            for m in range(0, 10):
                arr = [random.random() for x in range(0, 3)]
                label = classes[random.random() > 0.5]
                arrs.append(arr)
                labels.append(label)
            ann = Ann(
                arrs,
                labels)  # Create Ann with these train_examples and labels
            cost_before = ann.cost()
            ann.train()
            cost_after = ann.cost()
            self.assertTrue(cost_after <= cost_before)

        # Medium size random data-set with three labels
        arrs = []
        labels = []
        classes = ('cat', 'dog', 'bird')
        for i in range(0, 1):
            print('\nTesting data-set ' + str(i))
            for m in range(0, 10):
                arr = [random.random() for x in range(0, 5)]
                z = random.random()
                if (z < 0.33):
                    label = classes[0]
                elif (z >= 0.33 and z < 0.66):
                    label = classes[1]
                else:
                    label = classes[2]
                arrs.append(arr)
                labels.append(label)
            ann = Ann(
                arrs,
                labels)  # Create Ann with these train_examples and labels
            cost_before = ann.cost()
            ann.train()
            cost_after = ann.cost()
            self.assertTrue(cost_after <= cost_before)
Exemplo n.º 4
0
 def non_test_6(self):
     # Test if training works by checking that training lowers the cost for random small and medium size data-sets#
      
     # Small size random data-set with two labels
     arrs = []
     labels = []
     classes = ('cat', 'dog')
     for i in range(0, 1):
         print('\nTesting data-set ' + str(i))
         for m in range(0, 10):
             arr = [random.random() for x in range(0, 3)]
             label = classes[random.random() > 0.5]
             arrs.append(arr)
             labels.append(label)
         ann = Ann(arrs, labels)  # Create Ann with these train_examples and labels
         cost_before = ann.cost()
         ann.train()
         cost_after = ann.cost()
         self.assertTrue(cost_after <= cost_before)
          
     # Medium size random data-set with three labels
     arrs = []
     labels = []
     classes = ('cat', 'dog', 'bird')
     for i in range(0, 1):
         print('\nTesting data-set ' + str(i))
         for m in range(0, 10):
             arr = [random.random() for x in range(0, 5)]
             z = random.random()
             if (z < 0.33):
                 label = classes[0]
             elif (z >= 0.33 and z < 0.66):
                 label = classes[1]
             else:
                 label = classes[2]
             arrs.append(arr)
             labels.append(label)
         ann = Ann(arrs, labels)  # Create Ann with these train_examples and labels
         cost_before = ann.cost()
         ann.train()
         cost_after = ann.cost()
         self.assertTrue(cost_after <= cost_before)
Exemplo n.º 5
0
    def test_8(self):
        # First test#
        # 1 hidden layer cost test with regularization#       
        x1 = [1, 2, 3, 4]  # Array as first example
        y1 = 'yes'
        arrs = []
        labels = []
        arrs.append(x1)
        labels.append(y1)
        ann1 = Ann(arrs, labels, n_h=1)  # Create this architecture
         
        # Custom Thetas weights#
        M1 = np.matrix([[1, -1, 0.5, -0.3, 2],
                       [1, -1, 0.5, -0.3, 2],
                       [1, -1, 0.5, -0.3, 2],
                       [1, -1, 0.5, -0.3, 2]])
        M2 = np.matrix([[1, 1, -1, 0.5, -1]])
        ann1.Thetas[0] = M1
        ann1.Thetas[1] = M2
        cost_0 = ann1.cost()  # lam equals 0
        cost_1 = ann1.cost(lam=1)  # lam equals 1
        self.assertTrue(cost_1 > cost_0)  # Cost with regularization penalty is always higher than without regularization        
 
        # Gradient checking (now with regularization)#
        # Medium size data-set with several train_examples
        lam_test = 1  # Regularization parameter
        arrs = []
        labels = []
        classes = ('cat', 'dog')
        for m in range(0, 100):
            arr = [random.random() for x in range(0, 40)]
            label = classes[random.random() > 0.5]
            arrs.append(arr)
            labels.append(label)
        ann = Ann(arrs, labels, n_h=2)  # Create Ann with these train_examples and labels
        # L-1 matrices of partial derivatives for first example
        J = ann.backward_batch(lam=lam_test, batch_size=1)  # Use full-batch for gradient descent
        T_original = copy.deepcopy(ann.Thetas)
         
        for l in range(0, ann.L - 1):
            shape_J = J[l].shape
            eps = 0.0001  # epsilon for a numerical approximation of the gradient
            a = random.sample(range(0, shape_J[0]), 2)
            b = random.sample(range(0, shape_J[1]), 2)
            for i in a:
                for j in b:
                    T_e = np.zeros(shape_J)  # Matrix of zeros
                    T_e[i][j] = eps
                    ann.Thetas[l] = T_original[l] + T_e
                    cost_e = ann.cost(lam=lam_test)  # Cost at Theta + eps
                    ann.Thetas[l] = T_original[l] - T_e
                    cost_minus_e = ann.cost(lam=lam_test)  # Cost at Theta - eps
                    P = (cost_e - cost_minus_e) / (2 * eps)  # Numerical approximation
                    J_ij = J[l].item(i, j)  # Backpropagation derivation
                     
                    # print(P, '\t', J_ij, '\t', abs(P - J_ij), (l, i, j))
                     
                    # if (P < 0 and J_ij > 0 or P > 0 and J_ij < 0):
                    #    self.fail()
                     
                    self.assertAlmostEqual(P, J_ij, delta=0.001)
                    ann.Thetas = copy.deepcopy(T_original)
Exemplo n.º 6
0
 def test_4(self):
     # Gradient checking (check that a numerical approximation of the gradient is (almost) equal to our backpropagation derivation)#
      
     # First data-set with one example
     arrs = []
     labels = []
     arrs.append([1, 2, 4, 5, 5, 5])
     labels.append('cat')
     ann = Ann(arrs, labels, n_h=10)  # Create Ann with these train_examples and labels
     J = ann.backward(ann.train_examples[0].arr, ann.train_examples[0].y)
     T_original = copy.deepcopy(ann.Thetas)
      
     for l in range(0, ann.L - 1):
         shape_J = J[l].shape
         eps = 0.0001  # epsilon for a numerical approximation of the gradient
         for i in range(0, shape_J[0]):
             for j in range(0, shape_J[1]):
                 T_e = np.zeros(shape_J)  # Matrix of zeros
                 T_e[i][j] = eps
                 ann.Thetas[l] = T_original[l] + T_e
                 cost_e = ann.cost()  # Cost at Theta + eps
                 ann.Thetas[l] = T_original[l] - T_e
                 cost_minus_e = ann.cost()  # Cost at Theta - eps
                 P = (cost_e - cost_minus_e) / (2 * eps)  # Numerical approximation
                 J_ij = J[l].item(i, j)  # Backpropagation derivation
                  
                 # print(P, '\t', J_ij, '\t', abs(P - J_ij), (l, i, j))
                  
                 # if (P < 0 and J_ij > 0 or P > 0 and J_ij < 0):
                 #    self.fail()
                  
                 self.assertAlmostEqual(P, J_ij, delta=0.001)
                 ann.Thetas = copy.deepcopy(T_original)
      
     # Second data-set with several train_examples
     arrs = []
     labels = []
     classes = ('cat', 'dog')
     for m in range(0, 100):
         arr = [random.random() for x in range(0, 20)]
         label = classes[random.random() > 0.5]
         arrs.append(arr)
         labels.append(label)
     ann = Ann(arrs, labels, n_h=2)  # Create Ann with these train_examples and labels
     # L-1 matrices of partial derivatives for first example
     J = ann.backward_batch()
     T_original = copy.deepcopy(ann.Thetas)
      
     for l in range(0, ann.L - 1):
         shape_J = J[l].shape
         eps = 0.0001  # epsilon for a numerical approximation of the gradient
         a = random.sample(range(0, shape_J[0]), 2)
         b = random.sample(range(0, shape_J[1]), 2)
         for i in a:
             for j in b:
                 T_e = np.zeros(shape_J)  # Matrix of zeros
                 T_e[i][j] = eps
                 ann.Thetas[l] = T_original[l] + T_e
                 cost_e = ann.cost()  # Cost at Theta + eps
                 ann.Thetas[l] = T_original[l] - T_e
                 cost_minus_e = ann.cost()  # Cost at Theta - eps
                 P = (cost_e - cost_minus_e) / (2 * eps)  # Numerical approximation
                 J_ij = J[l].item(i, j)  # Backpropagation derivation
                  
                 self.assertAlmostEqual(P, J_ij, delta=0.001)
                 ann.Thetas = copy.deepcopy(T_original)
    def test_8(self):
        # First test#
        # 1 hidden layer cost test with regularization#
        x1 = [1, 2, 3, 4]  # Array as first example
        y1 = 'yes'
        arrs = []
        labels = []
        arrs.append(x1)
        labels.append(y1)
        ann1 = Ann(arrs, labels, n_h=1)  # Create this architecture

        # Custom Thetas weights#
        M1 = np.matrix([[1, -1, 0.5, -0.3, 2], [1, -1, 0.5, -0.3, 2],
                        [1, -1, 0.5, -0.3, 2], [1, -1, 0.5, -0.3, 2]])
        M2 = np.matrix([[1, 1, -1, 0.5, -1]])
        ann1.Thetas[0] = M1
        ann1.Thetas[1] = M2
        cost_0 = ann1.cost()  # lam equals 0
        cost_1 = ann1.cost(lam=1)  # lam equals 1
        self.assertTrue(
            cost_1 > cost_0
        )  # Cost with regularization penalty is always higher than without regularization

        # Gradient checking (now with regularization)#
        # Medium size data-set with several train_examples
        lam_test = 1  # Regularization parameter
        arrs = []
        labels = []
        classes = ('cat', 'dog')
        for m in range(0, 100):
            arr = [random.random() for x in range(0, 40)]
            label = classes[random.random() > 0.5]
            arrs.append(arr)
            labels.append(label)
        ann = Ann(arrs, labels,
                  n_h=2)  # Create Ann with these train_examples and labels
        # L-1 matrices of partial derivatives for first example
        J = ann.backward_batch(
            lam=lam_test, batch_size=1)  # Use full-batch for gradient descent
        T_original = copy.deepcopy(ann.Thetas)

        for l in range(0, ann.L - 1):
            shape_J = J[l].shape
            eps = 0.0001  # epsilon for a numerical approximation of the gradient
            a = random.sample(range(0, shape_J[0]), 2)
            b = random.sample(range(0, shape_J[1]), 2)
            for i in a:
                for j in b:
                    T_e = np.zeros(shape_J)  # Matrix of zeros
                    T_e[i][j] = eps
                    ann.Thetas[l] = T_original[l] + T_e
                    cost_e = ann.cost(lam=lam_test)  # Cost at Theta + eps
                    ann.Thetas[l] = T_original[l] - T_e
                    cost_minus_e = ann.cost(
                        lam=lam_test)  # Cost at Theta - eps
                    P = (cost_e - cost_minus_e) / (2 * eps
                                                   )  # Numerical approximation
                    J_ij = J[l].item(i, j)  # Backpropagation derivation

                    # print(P, '\t', J_ij, '\t', abs(P - J_ij), (l, i, j))

                    # if (P < 0 and J_ij > 0 or P > 0 and J_ij < 0):
                    #    self.fail()

                    self.assertAlmostEqual(P, J_ij, delta=0.001)
                    ann.Thetas = copy.deepcopy(T_original)
    def test_4(self):
        # Gradient checking (check that a numerical approximation of the gradient is (almost) equal to our backpropagation derivation)#

        # First data-set with one example
        arrs = []
        labels = []
        arrs.append([1, 2, 4, 5, 5, 5])
        labels.append('cat')
        ann = Ann(arrs, labels,
                  n_h=10)  # Create Ann with these train_examples and labels
        J = ann.backward(ann.train_examples[0].arr, ann.train_examples[0].y)
        T_original = copy.deepcopy(ann.Thetas)

        for l in range(0, ann.L - 1):
            shape_J = J[l].shape
            eps = 0.0001  # epsilon for a numerical approximation of the gradient
            for i in range(0, shape_J[0]):
                for j in range(0, shape_J[1]):
                    T_e = np.zeros(shape_J)  # Matrix of zeros
                    T_e[i][j] = eps
                    ann.Thetas[l] = T_original[l] + T_e
                    cost_e = ann.cost()  # Cost at Theta + eps
                    ann.Thetas[l] = T_original[l] - T_e
                    cost_minus_e = ann.cost()  # Cost at Theta - eps
                    P = (cost_e - cost_minus_e) / (2 * eps
                                                   )  # Numerical approximation
                    J_ij = J[l].item(i, j)  # Backpropagation derivation

                    # print(P, '\t', J_ij, '\t', abs(P - J_ij), (l, i, j))

                    # if (P < 0 and J_ij > 0 or P > 0 and J_ij < 0):
                    #    self.fail()

                    self.assertAlmostEqual(P, J_ij, delta=0.001)
                    ann.Thetas = copy.deepcopy(T_original)

        # Second data-set with several train_examples
        arrs = []
        labels = []
        classes = ('cat', 'dog')
        for m in range(0, 100):
            arr = [random.random() for x in range(0, 20)]
            label = classes[random.random() > 0.5]
            arrs.append(arr)
            labels.append(label)
        ann = Ann(arrs, labels,
                  n_h=2)  # Create Ann with these train_examples and labels
        # L-1 matrices of partial derivatives for first example
        J = ann.backward_batch()
        T_original = copy.deepcopy(ann.Thetas)

        for l in range(0, ann.L - 1):
            shape_J = J[l].shape
            eps = 0.0001  # epsilon for a numerical approximation of the gradient
            a = random.sample(range(0, shape_J[0]), 2)
            b = random.sample(range(0, shape_J[1]), 2)
            for i in a:
                for j in b:
                    T_e = np.zeros(shape_J)  # Matrix of zeros
                    T_e[i][j] = eps
                    ann.Thetas[l] = T_original[l] + T_e
                    cost_e = ann.cost()  # Cost at Theta + eps
                    ann.Thetas[l] = T_original[l] - T_e
                    cost_minus_e = ann.cost()  # Cost at Theta - eps
                    P = (cost_e - cost_minus_e) / (2 * eps
                                                   )  # Numerical approximation
                    J_ij = J[l].item(i, j)  # Backpropagation derivation

                    self.assertAlmostEqual(P, J_ij, delta=0.001)
                    ann.Thetas = copy.deepcopy(T_original)