def test_supervised_gradient_descent(self):
        def linear_regression_cost_gradient(parameters, input, output):
            prediction = np.dot(parameters, input)
            cost = (prediction - output) ** 2
            gradient = 2.0 * (prediction - output) * input
            return cost, gradient

        inputs = np.random.normal(0.0, size=(10, 2))
        outputs = np.random.normal(0.0, size=10)
        initial_parameters = np.random.uniform(-1.0, 1.0, size=2)

        # Create cost and gradient function for supervised SGD and check its gradient
        cost_gradient = bind_cost_gradient(linear_regression_cost_gradient,
                                           inputs, outputs, sampler=batch_sampler)
        result = gradient_check(cost_gradient, initial_parameters)
        self.assertTrue(result)

        # Run gradient descent on the function and see if it minimizes cost function
        actual, cost_history = gradient_descent(cost_gradient, initial_parameters, 10)

        # Compute exact solution of linear regression by closed form
        expected = np.linalg.solve(np.dot(inputs.T, inputs), np.dot(inputs.T, outputs))

        for e, a in zip(expected, actual):
            self.assertAlmostEqual(e, a, places=0)
    def assertMultinomialLogisticRegression(self, sampler):
        data_size = 3
        input_size = 5
        output_size = 4
        inputs = np.random.uniform(-10.0, 10.0, size=(data_size, input_size))
        outputs = np.random.randint(0, output_size, size=data_size)
        initial_parameters = np.random.normal(size=(input_size, output_size))

        # Create cost and gradient function for gradient descent and check its gradient
        cost_gradient = bind_cost_gradient(
            multinomial_logistic_regression_cost_gradient,
            inputs,
            outputs,
            sampler=sampler)
        result = gradient_check(cost_gradient, initial_parameters)
        self.assertTrue(result)

        # Train multinomial logistic regression and see if it predicts correct labels
        final_parameters, cost_history = gradient_descent(
            cost_gradient, initial_parameters, 100)
        predictions = np.argmax(softmax(np.dot(final_parameters.T, inputs.T)),
                                axis=0)

        for output, prediction in zip(outputs, predictions):
            self.assertEqual(output, prediction)
    def test_supervised_gradient_descent(self):
        def linear_regression_cost_gradient(parameters, input, output):
            prediction = np.dot(parameters, input)
            cost = (prediction - output)**2
            gradient = 2.0 * (prediction - output) * input
            return cost, gradient

        inputs = np.random.normal(0.0, size=(10, 2))
        outputs = np.random.normal(0.0, size=10)
        initial_parameters = np.random.uniform(-1.0, 1.0, size=2)

        # Create cost and gradient function for supervised SGD and check its gradient
        cost_gradient = bind_cost_gradient(linear_regression_cost_gradient,
                                           inputs,
                                           outputs,
                                           sampler=batch_sampler)
        result = gradient_check(cost_gradient, initial_parameters)
        self.assertTrue(result)

        # Run gradient descent on the function and see if it minimizes cost function
        actual, cost_history = gradient_descent(cost_gradient,
                                                initial_parameters, 10)

        # Compute exact solution of linear regression by closed form
        expected = np.linalg.solve(np.dot(inputs.T, inputs),
                                   np.dot(inputs.T, outputs))

        for e, a in zip(expected, actual):
            self.assertAlmostEqual(e, a, places=0)
    def test_gradient_descent(self):
        expected = 0.0

        def quadratic_cost_gradient(x):
            cost = (x - expected)**2
            gradient = 2.0 * (x - expected)
            return cost, gradient

        initial_parameters = np.random.normal()

        # Test gradient descent with constant learning rate
        final_parameters, cost_history = \
            gradient_descent(quadratic_cost_gradient, initial_parameters, 100, get_constant(0.5))
        self.assertAlmostEqual(expected, final_parameters, places=1)

        # Test gradient descent with AdaGrad learning rate
        final_parameters, cost_history = \
            gradient_descent(quadratic_cost_gradient, initial_parameters, 100, get_adagrad(0.5))
        self.assertAlmostEqual(expected, final_parameters, places=1)
Esempio n. 5
0
    def test_gradient_descent(self):
        expected = 0.0

        def quadratic_cost_gradient(x):
            cost = (x - expected) ** 2
            gradient = 2.0 * (x - expected)
            return cost, gradient

        initial_parameters = np.random.normal()

        # Test gradient descent with constant learning rate
        final_parameters, cost_history = \
            gradient_descent(quadratic_cost_gradient, initial_parameters, 100, get_constant(0.5))
        self.assertAlmostEqual(expected, final_parameters, places=1)

        # Test gradient descent with AdaGrad learning rate
        final_parameters, cost_history = \
            gradient_descent(quadratic_cost_gradient, initial_parameters, 100, get_adagrad(0.5))
        self.assertAlmostEqual(expected, final_parameters, places=1)
Esempio n. 6
0
    def test_skip_gram(self):
        vocabulary_size = 5
        vector_size = 2
        context_size = 3
        data_size = 4
        inputs = np.random.randint(0, vocabulary_size, size=data_size)
        outputs = np.random.randint(0, vocabulary_size, size=(data_size, context_size))
        initial_parameters = np.random.normal(size=(2, vocabulary_size, vector_size))

        # Create cost and gradient function for supervised SGD and check its gradient
        cost_gradient = bind_cost_gradient(skip_gram_cost_gradient, inputs, outputs, sampler=batch_sampler)
        result = gradient_check(cost_gradient, initial_parameters)
        self.assertEqual([], result)
        final_parameters, cost_history = gradient_descent(cost_gradient, initial_parameters, 10)
Esempio n. 7
0
    def test_word2vec(self):
        vocabulary_size = 5
        vector_size = 2
        context_size = 3
        data_size = 2
        sentence_size = 2
        sentences = np.random.randint(0, vocabulary_size, size=(data_size, sentence_size))
        initial_parameters = np.random.normal(size=(2, vocabulary_size, vector_size))

        # Create cost and gradient function for supervised SGD and check its gradient
        word2vec_cost_gradient = get_word2vec_cost_gradient(context_size)
        cost_gradient = bind_cost_gradient(word2vec_cost_gradient, sentences, None, sampler=batch_sampler)
        result = gradient_check(cost_gradient, initial_parameters)
        self.assertTrue(result)
        final_parameters, cost_history = gradient_descent(cost_gradient, initial_parameters, 10)
    def test_logistic_regression(self):
        input = np.random.uniform(-10.0, 10.0, size=10)
        output = np.random.randint(0, 2)

        def logistic_regression_wrapper(parameters):
            return logistic_regression_cost_gradient(parameters, input, output)

        initial_parameters = np.random.normal(scale=1e-5, size=10)
        result = gradient_check(logistic_regression_wrapper, initial_parameters)
        self.assertTrue(result)

        # Train logistic regression and see if it predicts correct label
        final_parameters, cost_history = gradient_descent(logistic_regression_wrapper, initial_parameters, 100)
        prediction = sigmoid(np.dot(input, final_parameters)) > 0.5
        self.assertEqual(output, prediction)
    def test_logistic_regression(self):
        input = np.random.uniform(-10.0, 10.0, size=10)
        output = np.random.randint(0, 2)

        def logistic_regression_wrapper(parameters):
            return logistic_regression_cost_gradient(parameters, input, output)

        initial_parameters = np.random.normal(scale=1e-5, size=10)
        result = gradient_check(logistic_regression_wrapper,
                                initial_parameters)
        self.assertTrue(result)

        # Train logistic regression and see if it predicts correct label
        final_parameters, cost_history = gradient_descent(
            logistic_regression_wrapper, initial_parameters, 100)
        prediction = sigmoid(np.dot(input, final_parameters)) > 0.5
        self.assertEqual(output, prediction)
Esempio n. 10
0
    def assertMultinomialLogisticRegression(self, sampler):
        data_size = 3
        input_size = 5
        output_size = 4
        inputs = np.random.uniform(-10.0, 10.0, size=(data_size, input_size))
        outputs = np.random.randint(0, output_size, size=data_size)
        initial_parameters = np.random.normal(size=(input_size, output_size))

        # Create cost and gradient function for gradient descent and check its gradient
        cost_gradient = bind_cost_gradient(multinomial_logistic_regression_cost_gradient,
                                           inputs, outputs, sampler=sampler)
        result = gradient_check(cost_gradient, initial_parameters)
        self.assertTrue(result)

        # Train multinomial logistic regression and see if it predicts correct labels
        final_parameters, cost_history = gradient_descent(cost_gradient, initial_parameters, 100)
        predictions = np.argmax(softmax(np.dot(final_parameters.T, inputs.T)), axis=0)

        for output, prediction in zip(outputs, predictions):
            self.assertEqual(output, prediction)
Esempio n. 11
0
    def assertLogisticRegression(self, sampler):
        data_size = 3
        input_size = 5
        inputs = np.random.uniform(-10.0, 10.0, size=(data_size, input_size))
        outputs = np.random.randint(0, 2, size=data_size)
        initial_parameters = np.random.normal(scale=1e-5, size=input_size)

        # Create cost and gradient function for gradient descent and check its gradient
        cost_gradient = bind_cost_gradient(logistic_regression_cost_gradient,
                                           inputs, outputs, sampler=sampler)
        result = gradient_check(cost_gradient, initial_parameters)
        self.assertTrue(result)

        # Train logistic regression and see if it predicts correct labels
        final_parameters, cost_history = gradient_descent(cost_gradient, initial_parameters, 100)
        predictions = sigmoid(np.dot(inputs, final_parameters)) > 0.5

        # Binary classification of 3 data points with 5 dimension is always linearly separable
        for output, prediction in zip(outputs, predictions):
            self.assertEqual(output, prediction)
Esempio n. 12
0
    def test_skip_gram(self):
        vocabulary_size = 5
        vector_size = 2
        context_size = 3
        data_size = 4
        inputs = np.random.randint(0, vocabulary_size, size=data_size)
        outputs = np.random.randint(0,
                                    vocabulary_size,
                                    size=(data_size, context_size))
        initial_parameters = np.random.normal(size=(2, vocabulary_size,
                                                    vector_size))

        # Create cost and gradient function for supervised SGD and check its gradient
        cost_gradient = bind_cost_gradient(skip_gram_cost_gradient,
                                           inputs,
                                           outputs,
                                           sampler=batch_sampler)
        result = gradient_check(cost_gradient, initial_parameters)
        self.assertEqual([], result)
        final_parameters, cost_history = gradient_descent(
            cost_gradient, initial_parameters, 10)
Esempio n. 13
0
    def test_multinomial_logistic_regression(self):
        input_size = 10
        output_size = 5
        input = np.random.normal(size=(input_size,))
        output = np.random.randint(0, output_size)

        def multinomial_logistic_regression_wrapper(parameters):
            return multinomial_logistic_regression_cost_gradient(parameters, input, output)

        initial_parameters = np.random.normal(size=(input_size, output_size))
        result = gradient_check(multinomial_logistic_regression_wrapper, initial_parameters)
        self.assertTrue(result)

        # Train multinomial logistic regression and see if it predicts correct label
        final_parameters, cost_history = gradient_descent(
            multinomial_logistic_regression_wrapper, initial_parameters, 100)
        prediction = softmax(np.dot(final_parameters.T, input)) > 0.5
        for i in range(len(prediction)):
            if output == i:
                self.assertEqual(1, prediction[i])
            else:
                self.assertEqual(0, prediction[i])
Esempio n. 14
0
    def test_word2vec(self):
        vocabulary_size = 5
        vector_size = 2
        context_size = 3
        data_size = 2
        sentence_size = 2
        sentences = np.random.randint(0,
                                      vocabulary_size,
                                      size=(data_size, sentence_size))
        initial_parameters = np.random.normal(size=(2, vocabulary_size,
                                                    vector_size))

        # Create cost and gradient function for supervised SGD and check its gradient
        word2vec_cost_gradient = get_word2vec_cost_gradient(context_size)
        cost_gradient = bind_cost_gradient(word2vec_cost_gradient,
                                           sentences,
                                           None,
                                           sampler=batch_sampler)
        result = gradient_check(cost_gradient, initial_parameters)
        self.assertTrue(result)
        final_parameters, cost_history = gradient_descent(
            cost_gradient, initial_parameters, 10)
Esempio n. 15
0
    def assertLogisticRegression(self, sampler):
        data_size = 3
        input_size = 5
        inputs = np.random.uniform(-10.0, 10.0, size=(data_size, input_size))
        outputs = np.random.randint(0, 2, size=data_size)
        initial_parameters = np.random.normal(scale=1e-5, size=input_size)

        # Create cost and gradient function for gradient descent and check its gradient
        cost_gradient = bind_cost_gradient(logistic_regression_cost_gradient,
                                           inputs,
                                           outputs,
                                           sampler=sampler)
        result = gradient_check(cost_gradient, initial_parameters)
        self.assertTrue(result)

        # Train logistic regression and see if it predicts correct labels
        final_parameters, cost_history = gradient_descent(
            cost_gradient, initial_parameters, 100)
        predictions = sigmoid(np.dot(inputs, final_parameters)) > 0.5

        # Binary classification of 3 data points with 5 dimension is always linearly separable
        for output, prediction in zip(outputs, predictions):
            self.assertEqual(output, prediction)
Esempio n. 16
0
    def test_multinomial_logistic_regression(self):
        input_size = 10
        output_size = 5
        input = np.random.normal(size=(input_size, ))
        output = np.random.randint(0, output_size)

        def multinomial_logistic_regression_wrapper(parameters):
            return multinomial_logistic_regression_cost_gradient(
                parameters, input, output)

        initial_parameters = np.random.normal(size=(input_size, output_size))
        result = gradient_check(multinomial_logistic_regression_wrapper,
                                initial_parameters)
        self.assertTrue(result)

        # Train multinomial logistic regression and see if it predicts correct label
        final_parameters, cost_history = gradient_descent(
            multinomial_logistic_regression_wrapper, initial_parameters, 100)
        prediction = softmax(np.dot(final_parameters.T, input)) > 0.5
        for i in range(len(prediction)):
            if output == i:
                self.assertEqual(1, prediction[i])
            else:
                self.assertEqual(0, prediction[i])