class TestLassoRegression(unittest.TestCase): """Tests for TestLassoRegression. Uses housing data to test LassoRegression. Statics: _multiprocess_can_split_ (bool): Flag for nose tests to run tests in parallel. """ _multiprocess_can_split_ = True def setUp(self): """Constructor for TestLassoRegression. Loads housing data, and creates training and testing data. """ self.convert_numpy = ConvertNumpy() self.normalize_features = NormalizeFeatures() self.lasso = LassoRegression() self.predict_output = PredictOutput() self.residual_sum_squares = ResidualSumSquares() self.k_fold_cross_validation = KFoldCrossValidation() # Create a dictionary type to store relevant data types so that our pandas # will read the correct information dtype_dict = {'bathrooms': float, 'waterfront': int, 'sqft_above': int, 'sqft_living15': float, 'grade': int, 'yr_renovated': int, 'price': float, 'bedrooms': float, 'zipcode': str, 'long': float, 'sqft_lot15': float, 'sqft_living': float, 'floors': str, 'condition': int, 'lat': float, 'date': str, 'sqft_basement': int, 'yr_built': int, 'id': str, 'sqft_lot': int, 'view': int} # Create a kc_house that encompasses all test and train data self.kc_house = pd.read_csv('./unit_tests/test_data/regression/kc_house/kc_house_data.csv', dtype=dtype_dict) # Create a kc_house_test_frame that encompasses only train data self.kc_house_train = pd.read_csv('./unit_tests/test_data/regression/kc_house/kc_house_train_data.csv', dtype=dtype_dict) # Create a kc_house_frames that encompasses only test data self.kc_house_test = pd.read_csv('./unit_tests/test_data/regression/kc_house/kc_house_test_data.csv', dtype=dtype_dict) # Convert all the frames with the floors to float type self.kc_house['floors'] = self.kc_house['floors'].astype(float) self.kc_house_train['floors'] = self.kc_house['floors'].astype(float) self.kc_house_test['floors'] = self.kc_house['floors'].astype(float) # Then back to int type self.kc_house['floors'] = self.kc_house['floors'].astype(int) self.kc_house_train['floors'] = self.kc_house['floors'].astype(int) self.kc_house_test['floors'] = self.kc_house['floors'].astype(int) def test_01_normalize_features(self): """Tests normalizing features. Test normalization features, and compare it with known values. """ # Normalize the features, and also return the norms features, norms = self.normalize_features.l2_norm(np.array([[3., 6., 9.], [4., 8., 12.]])) # Assert that the np array is equal to features self.assertTrue(np.array_equal(np.array([[0.6, 0.6, 0.6], [0.8, 0.8, 0.8]]), features), True) # Assert that the np array is equal to norms self.assertTrue(np.array_equal(np.array([5., 10., 15.]), norms), True) def test_02_compute_ro(self): """Test compute ro Test compute one round of ro. """ # We will use sqft_iving, and sqft_living15 features = ['sqft_living', 'bedrooms'] # Output will use price output = ['price'] # Convert our pandas frame to numpy feature_matrix, output = self.convert_numpy.convert_to_numpy(self.kc_house, features, output, 1) # Create our initial weights normalized_feature_matrix, _ = self.normalize_features.l2_norm(feature_matrix) # Set initial weights weights = np.array([1., 4., 1.]) # Compute ro_j ro_j = self.lasso.compute_ro_j(normalized_feature_matrix, output, weights) # Assert the output of ro_j self.assertTrue(np.allclose(ro_j, np.array([79400300.03492916, 87939470.77299108, 80966698.67596565]))) def test_03_compute_coordinate_descent_step(self): """Test one coordinate descent step. Test one coordinate descent step and compare it with known values. """ # Assert that both are equal self.assertEquals(round(self.lasso.lasso_coordinate_descent_step({"i": 1, "weights": np.array([1., 4.])}, np.array([[3./math.sqrt(13), 1./math.sqrt(10)], [2./math.sqrt(13), 3./math.sqrt(10)]]), np.array([1., 1.]), {"l1_penalty": 0.1}), 8), round(0.425558846691, 8)) def test_04_coordinate_descent(self): """Test coordinate descent. Test coordinate descent and compare with known values. """ # We will use sqft_iving, and sqft_living15 features = ['sqft_living', 'bedrooms'] # Output will use price output = ['price'] # Convert our pandas frame to numpy feature_matrix, output = self.convert_numpy.convert_to_numpy(self.kc_house, features, output, 1) # Create our initial weights normalized_feature_matrix, _ = self.normalize_features.l2_norm(feature_matrix) # Set initial weights initial_weights = np.zeros(3) # Set l1 penalty l1_penalty = 1e7 # Set tolerance tolerance = 1.0 # Compute the weights using coordinate descent weights = self.lasso.lasso_cyclical_coordinate_descent(normalized_feature_matrix, output, {"initial_weights": initial_weights, "l1_penalty": l1_penalty, "tolerance": tolerance}) # Assert that these two numpy arrays are the same self.assertTrue(np.allclose(weights, np.array([21624998.3663629, 63157246.78545423, 0.]), True)) # Predict the output predicted_output = self.predict_output.regression(normalized_feature_matrix, weights) # Assert that the RSS is what we wanted self.assertEquals(round(self.residual_sum_squares.residual_sum_squares_regression(output, predicted_output), -10), round(1.63049248148e+15, -10)) def test_05_coordinate_descent_with_normalization(self): """Test coordinate descent with normalization. Test coordinate descent and then normalize the result, so that we can use the weights on a test set. """ # We will use multiple features features = ['bedrooms', 'bathrooms', 'sqft_living', 'sqft_lot', 'floors', 'waterfront', 'view', 'condition', 'grade', 'sqft_above', 'sqft_basement', 'yr_built', 'yr_renovated'] # Output will use price output = ['price'] # Convert our pandas frame to numpy feature_matrix, output = self.convert_numpy.convert_to_numpy(self.kc_house_train, features, output, 1) # Create our initial weights normalized_feature_matrix, norms = self.normalize_features.l2_norm(feature_matrix) # Compute Multiple Weights weights1e7 = self.lasso.lasso_cyclical_coordinate_descent(normalized_feature_matrix, output, {"initial_weights": np.zeros(len(features)+1), "l1_penalty": 1e7, "tolerance": 1}) weights1e8 = self.lasso.lasso_cyclical_coordinate_descent(normalized_feature_matrix, output, {"initial_weights": np.zeros(len(features)+1), "l1_penalty": 1e8, "tolerance": 1}) weights1e4 = self.lasso.lasso_cyclical_coordinate_descent(normalized_feature_matrix, output, {"initial_weights": np.zeros(len(features)+1), "l1_penalty": 1e4, "tolerance": 5e5}) # Compute multiple normalized normalized_weights1e4 = weights1e4 / norms normalized_weights1e7 = weights1e7 / norms normalized_weights1e8 = weights1e8 / norms # We will use multiple features features = ['bedrooms', 'bathrooms', 'sqft_living', 'sqft_lot', 'floors', 'waterfront', 'view', 'condition', 'grade', 'sqft_above', 'sqft_basement', 'yr_built', 'yr_renovated'] # Output will use price output = ['price'] # Convert our test pandas frame to numpy test_feature_matrix, test_output = self.convert_numpy.convert_to_numpy(self.kc_house_test, features, output, 1) # Predict the output predicted_output = self.predict_output.regression(test_feature_matrix, normalized_weights1e4) # Assert that the RSS is what we wanted self.assertEquals(round(self.residual_sum_squares.residual_sum_squares_regression(test_output, predicted_output), -12), round(2.2778100476e+14, -12)) # Predict the output predicted_output = self.predict_output.regression(test_feature_matrix, normalized_weights1e7) # Assert that the RSS is what we wanted self.assertEquals(round(self.residual_sum_squares.residual_sum_squares_regression(test_output, predicted_output), -12), round(2.75962079909e+14, -12)) # Predict the output predicted_output = self.predict_output.regression(test_feature_matrix, normalized_weights1e8) # Assert that the RSS is what we wanted self.assertEquals(round(self.residual_sum_squares.residual_sum_squares_regression(test_output, predicted_output), -12), round(5.37049248148e+14, -12))
class TestWeightedLogisticRegression(unittest.TestCase): """Tests WeightedLogisticRegression class. Uses Amazon data to test WeightedLogisticRegression class. Statics: _multiprocess_can_split_ (bool): Flag for nose tests to run tests in parallel. """ _multiprocess_can_split_ = True def setUp(self): """Constructor for TestWeightedLogisticRegression. Loads Amazon data, and creates training and testing data. """ self.convert_numpy = ConvertNumpy() self.predict = PredictOutput() self.ada = AdaBoost() self.accuracy = Accuracy() self.weighted_logistic_regression = WeightedLogisticRegression() # Load the important words self.important_words = json.load(open('./unit_tests/test_data/classification/amazon/important_words.json', 'r')) # Load the amazon baby subset self.review_frame = pd.read_csv('./unit_tests/test_data/classification/amazon/amazon_baby_subset.csv') # Review needs to be text self.review_frame['review'].astype(str) # Clean up the punctuations self.review_frame['review_clean'] = self.review_frame.apply( axis=1, func=lambda row: str(row["review"]).translate(str.maketrans({key: None for key in string.punctuation}))) # Remove any nan text self.review_frame['review_clean'] = self.review_frame.apply( axis=1, func=lambda row: '' if row["review_clean"] == "nan" else row["review_clean"]) # Count the number of words that appears in each review, and make an indepedent column for word in self.important_words: self.review_frame[word] = self.review_frame['review_clean'].apply(lambda s, w=word: s.split().count(w)) def test_01_gradient_ascent(self): """Tests gradient ascent algorithm. Tests the gradient ascent algorithm and compare it with known values. """ # We will use important words for the output features = self.important_words # Output will use sentiment output = ['sentiment'] # Convert our pandas frame to numpy feature_matrix, sentiment = self.convert_numpy.convert_to_numpy(self.review_frame, features, output, 1) # Create weight list for training data weights_list = np.array([1]*len(self.review_frame)) # Compute the coefficients coefficients = self.weighted_logistic_regression.gradient_ascent(feature_matrix, sentiment, {"initial_coefficients": np.zeros(194), "weights_list": weights_list, "step_size": 1e-7, "max_iter": 30}) # Assert the coefficients self.assertEqual([round(i, 5) for i in coefficients[0:20]], [round(i, 5) for i in [0.00020000000000000001, 0.0014300000000000001, -0.00131, 0.0068900000000000003, 0.0068500000000000002, 0.00034000000000000002, -0.0062399999999999999, -0.00059000000000000003, 0.0067099999999999998, 0.0046600000000000001, 0.00042999999999999999, 0.0020300000000000001, 0.0030300000000000001, -0.00332, 0.0015, -0.00011, 0.00115, -0.0021700000000000001, -0.00139, -0.0046600000000000001]]) # Compute predictions predictions = self.predict.logistic_regression(feature_matrix, coefficients) # Accuracy has to match 0.74356999999999995 self.assertEqual(round(self.accuracy.general(predictions, sentiment), 5), round(0.74356999999999995, 5)) def test_02_adaboost(self): """Tests adaboost algorithm. Tests the adaboost algorithm with weighted logistic regression. """ # We will use important words for the output features = self.important_words # Output will use sentiment output = ['sentiment'] # Convert our pandas frame to numpy feature_matrix, sentiment = self.convert_numpy.convert_to_numpy(self.review_frame, features, output, 1) # Create 15 weighted logistic regression weights, models = self.ada.logistic_regression(feature_matrix, sentiment, iterations=15, model_dict={"predict_method": self.predict.logistic_regression, "model": self.weighted_logistic_regression, "model_method": "gradient_ascent", "model_parameters": {"step_size": 1e-7, "max_iter": 30, "initial_coefficients": np.zeros(194)}}) # Get the predictions of each dataset in the test data predictions = self.predict.adaboost_logistic_regression(self.predict.logistic_regression, models, weights, feature_matrix) # Assert the predictions self.assertEqual(list(predictions)[0:20], [1, -1, 1, 1, 1, 1, 1, 1, 1, -1, -1, 1, 1, -1, 1, -1, 1, 1, -1, 1]) # Accuracy has to match 0.77612999999999999 self.assertEqual(round(self.accuracy.general(predictions, sentiment), 5), round(0.77612999999999999, 5))
class TestLogisticRegressionL2Norm(unittest.TestCase): """Tests for LogisticRegressionL2Norm class. Uses Amazon data to test logistic regression. Statics: _multiprocess_can_split_ (bool): Flag for nose tests to run tests in parallel. """ _multiprocess_can_split_ = True def setUp(self): """Constructor for TestLogisticRegression. Loads Amazon data, and creates training and testing data. """ # Create an instance of the Convert Numpy class self.convert_numpy = ConvertNumpy() # Create an instance of log likelihood self.log_likelhood = LogLikelihood() # Create an instance of the accuracy class self.accuracy = Accuracy() # Load the important words self.important_words = json.load(open('./unit_tests/test_data/classification/amazon/important_words.json', 'r')) # Create an instance of the Logistic Regression with L2 Norm class self.logistic_regression_l2_norm = LogisticRegressionL2Norm() # Load the amazon baby train subset self.training_data = pd.read_csv('./unit_tests/test_data/classification/amazon/amazon_baby_subset_train.csv') # Load the amazon baby train subset self.validation_data = pd.read_csv('./unit_tests/test_data/' 'classification/amazon/amazon_baby_subset_validation.csv') def test_01_gradient_ascent_no_penalty(self): """Tests gradient ascent algorithm. Tests the gradient ascent algorithm but with no l2 penalty. """ # We will use important words for the output features = self.important_words # Output will use sentiment output = ['sentiment'] # Convert our pandas frame to numpy feature_matrix_train, label_train = self.convert_numpy.convert_to_numpy(self.training_data, features, output, 1) feature_matrix_valid, label_valid = self.convert_numpy.convert_to_numpy(self.validation_data, features, output, 1) # Compute the coefficients coefficients = self.logistic_regression_l2_norm.gradient_ascent(feature_matrix_train, label_train, {"initial_coefficients": np.zeros(194), "step_size": 5e-6, "l2_penalty": 0, "max_iter": 501}) # Get the accuracy train_accuracy = self.accuracy.logistic_regression(feature_matrix_train, label_train, coefficients) validation_accuracy = self.accuracy.logistic_regression(feature_matrix_valid, label_valid, coefficients) # Make sure the accuraries are correct self.assertEqual(round(0.785156157787, 5), round(train_accuracy, 5)) self.assertEqual(round(0.78143964149, 5), round(validation_accuracy, 5)) def test_02_gradient_ascent_10_penalty(self): """Test gradient ascent algorithm. Tests the gradient ascent algorithm with penalty. """ # We will use important words for the output features = self.important_words # Output will use sentiment output = ['sentiment'] # Convert our pandas frame to numpy feature_matrix_train, label_train = self.convert_numpy.convert_to_numpy(self.training_data, features, output, 1) feature_matrix_valid, label_valid = self.convert_numpy.convert_to_numpy(self.validation_data, features, output, 1) # Compute the coefficients coefficients = self.logistic_regression_l2_norm.gradient_ascent(feature_matrix_train, label_train, {"initial_coefficients": np.zeros(194), "step_size": 5e-6, "l2_penalty": 10, "max_iter": 501}) # Get the accuracy train_accuracy = self.accuracy.logistic_regression(feature_matrix_train, label_train, coefficients) validation_accuracy = self.accuracy.logistic_regression(feature_matrix_valid, label_valid, coefficients) # Make sure the accuracies are correct self.assertEqual(round(0.784990911452, 5), round(train_accuracy, 5)) self.assertEqual(round(0.781719727383, 5), round(validation_accuracy, 5)) def test_03_log_likelihood(self): """Tests log likelihood with l2 norm. Tests the log likelihood with l2 norm and compare it with known values. """ # Generate test feature, coefficients, and label feature_matrix = np.array([[1., 2., 3.], [1., -1., -1]]) coefficients = np.array([1., 3., -1.]) label = np.array([-1, 1]) # Compute the log likelihood lg = self.log_likelhood.log_likelihood_l2_norm(feature_matrix, label, coefficients, 10) # Assert the value self.assertEqual(round(lg, 5), round(-105.33141000000001, 5))
class TestRidgeRegression(unittest.TestCase): """Test for RidgeRegression. Uses housing data to test RidgeRegression. Statics: _multiprocess_can_split_ (bool): Flag for nose tests to run tests in parallel. """ _multiprocess_can_split_ = True def setUp(self): """Constructor for TestRidgeRegression. Loads housing data, and creates training and testing data. """ self.convert_numpy = ConvertNumpy() self.ridge_regression = RidgeRegression() self.predict_output = PredictOutput() self.residual_sum_squares = ResidualSumSquares() self.k_fold_cross_validation = KFoldCrossValidation() # Create a dictionary type to store relevant data types so that our pandas # will read the correct information dtype_dict = {'bathrooms': float, 'waterfront': int, 'sqft_above': int, 'sqft_living15': float, 'grade': int, 'yr_renovated': int, 'price': float, 'bedrooms': float, 'zipcode': str, 'long': float, 'sqft_lot15': float, 'sqft_living': float, 'floors': str, 'condition': int, 'lat': float, 'date': str, 'sqft_basement': int, 'yr_built': int, 'id': str, 'sqft_lot': int, 'view': int} # Create a kc_house that encompasses all test and train data self.kc_house = pd.read_csv('./unit_tests/test_data/regression/kc_house/kc_house_data.csv', dtype=dtype_dict) # Create a kc_house_test_frame that encompasses only train data self.kc_house_train = pd.read_csv('./unit_tests/test_data/regression/kc_house/kc_house_train_data.csv', dtype=dtype_dict) # Create a kc_house_frames that encompasses only test data self.kc_house_test = pd.read_csv('./unit_tests/test_data/regression/kc_house/kc_house_test_data.csv', dtype=dtype_dict) # Create a kc_house_train_valid_shuffled that encompasses both train and valid data and shuffled self.kc_house_train_valid_shuffled = pd.read_csv('./unit_tests/test_data/regression/' 'kc_house_with_validation_k_fold/' 'wk3_kc_house_train_valid_shuffled.csv', dtype=dtype_dict) def test_01_gradient_descent_no_penalty(self): """Tests gradient descent algorithm. Tests the result on gradient descent with low penalty. """ # We will use sqft_living for our features features = ['sqft_living'] # Output will use price output = ['price'] # Convert our pandas frame to numpy feature_matrix, output = self.convert_numpy.convert_to_numpy(self.kc_house_train, features, output, 1) # Create our initial weights initial_weights = np.array([0., 0.]) # Step size step_size = 1e-12 # Max Iterations to Run max_iterations = 1000 # Tolerance tolerance = None # L2 Penalty l2_penalty = 0.0 # Compute our gradient descent value final_weights = self.ridge_regression.gradient_descent(feature_matrix, output, {"initial_weights": initial_weights, "step_size": step_size, "tolerance": tolerance, "l2_penalty": l2_penalty, "max_iteration": max_iterations}) # We will use sqft_iving, and sqft_living15 test_features = ['sqft_living'] # Output will be price test_output = ['price'] # Convert our test pandas frame to numpy test_feature_matrix, test_output = self.convert_numpy.convert_to_numpy(self.kc_house_test, test_features, test_output, 1) # Predict the output of test features predicted_output = self.predict_output.regression(test_feature_matrix, final_weights) # Compute RSS rss = self.residual_sum_squares.residual_sum_squares_regression(test_output, predicted_output) # Assert that the weights is correct self.assertEquals(round(-0.16311351478746433, 5), round(final_weights[0], 5)) self.assertEquals(round(263.02436896538489, 3), round(final_weights[1], 3)) # Assert that rss is correct self.assertEquals(round(275723632153607.72, -5), round(rss, -5)) def test_02_gradient_descent_high_penalty(self): """Tests gradient descent. Tests the result on gradient descent with high penalty. """ # We will use sqft_living for our features features = ['sqft_living'] # Output will use price output = ['price'] # Convert our pandas frame to numpy feature_matrix, output = self.convert_numpy.convert_to_numpy(self.kc_house_train, features, output, 1) # Create our initial weights initial_weights = np.array([0., 0.]) # Step size step_size = 1e-12 # Max Iterations to Run max_iterations = 1000 # Tolerance tolerance = None # L2 Penalty l2_penalty = 1e11 # Compute our gradient descent value final_weights = self.ridge_regression.gradient_descent(feature_matrix, output, {"initial_weights": initial_weights, "step_size": step_size, "tolerance": tolerance, "l2_penalty": l2_penalty, "max_iteration": max_iterations}) # We will use sqft_iving test_features = ['sqft_living'] # Output will be price test_output = ['price'] # Convert our test pandas frame to numpy test_feature_matrix, test_output = self.convert_numpy.convert_to_numpy(self.kc_house_test, test_features, test_output, 1) # Predict the output of test features predicted_output = self.predict_output.regression(test_feature_matrix, final_weights) # Compute RSS rss = self.residual_sum_squares.residual_sum_squares_regression(test_output, predicted_output) # Assert that the weights is correct self.assertEquals(round(9.7673000000000005, 5), round(final_weights[0], 5)) self.assertEquals(round(124.572, 3), round(final_weights[1], 3)) # Assert that rss is correct self.assertEquals(round(694642101500000.0, -5), round(rss, -5)) def test_03_gradient_descent_multiple_high_penalty(self): """Tests gradient descent. Tests gradient descent with multiple features, and high penalty. """ # We will use sqft_iving, and sqft_living15 features = ['sqft_living', 'sqft_living15'] # Output will use price output = ['price'] # Convert our pandas frame to numpy feature_matrix, output = self.convert_numpy.convert_to_numpy(self.kc_house_train, features, output, 1) # Create our initial weights initial_weights = np.array([0.0, 0.0, 0.0]) # Step size step_size = 1e-12 # Max Iterations to Run max_iterations = 1000 # Tolerance tolerance = None # L2 Penalty l2_penalty = 1e11 # Compute our gradient descent value final_weights = self.ridge_regression.gradient_descent(feature_matrix, output, {"initial_weights": initial_weights, "step_size": step_size, "tolerance": tolerance, "l2_penalty": l2_penalty, "max_iteration": max_iterations}) # We will use sqft_iving, and sqft_living15 test_features = ['sqft_living', 'sqft_living15'] # Output will be price test_output = ['price'] # Convert our test pandas frame to numpy test_feature_matrix, test_output = self.convert_numpy.convert_to_numpy(self.kc_house_test, test_features, test_output, 1) # Predict the output of test features predicted_output = self.predict_output.regression(test_feature_matrix, final_weights) # Compute RSS rss = self.residual_sum_squares.residual_sum_squares_regression(test_output, predicted_output) # Assert that the weights is correct self.assertEquals(round(6.7429699999999997, 5), round(final_weights[0], 5)) self.assertEquals(round(91.489000000000004, 3), round(final_weights[1], 3)) self.assertEquals(round(78.437490333967176, 3), round(final_weights[2], 3)) # Assert that rss is correct self.assertEquals(round(500404800500842.0, -5), round(rss, -5)) # Look at the first predicted output self.assertEquals(round(270453.53000000003, 3), round(predicted_output[0], 3)) # The first output should be 310000 in the test set self.assertEquals(310000.0, test_output[0]) def test_04_gradient_descent_k_fold(self): """Tests gradient descent with K fold cross validation. Tests best l2_penalty for ridge regression using gradient descent. """ # We will use sqft_living for our features features = ['sqft_living'] # Output will use price output = ['price'] # Create our initial weights initial_weights = np.array([0., 0.]) # Step size step_size = 1e-12 # Tolerance tolerance = None # Max Iterations to Run max_iterations = 1000 # Number of Folds folds = 10 # Store Cross Validation results cross_validation_results = [] # We want to test l2 penalty values in [10^1, 10^2, 10^3, 10^4, ..., 10^11] for l2_penalty in np.logspace(1, 11, num=11): # Create a dictionary of model_parameters model_parameters = {'step_size': step_size, 'max_iteration': max_iterations, 'initial_weights': initial_weights, 'tolerance': tolerance, 'l2_penalty': l2_penalty} # Compute the cross validation results cv = self.k_fold_cross_validation.k_fold_cross_validation(folds, self.ridge_regression.gradient_descent, model_parameters, {"data": self.kc_house_train, "output": output, "features": features}) # Append it into the results cross_validation_results.append((l2_penalty, cv)) # Lowest Result lowest = sorted(cross_validation_results, key=lambda x: x[1])[0] # Assert True that 10000000 is the l2_penalty that gives the lowest cross validation error self.assertEquals(10000000.0, lowest[0]) # Assert True that is the lowest l2_penalty self.assertEquals(round(120916225809145.0, 0), round(lowest[1], 0)) def test_05_gradient_ascent(self): """Tests gradient ascent. Tests gradient ascent and compare it with known values. """ # We will use sqft_living for our features features = ['sqft_living'] # Output will be price output = ['price'] # Convert our pandas frame to numpy feature_matrix, output = self.convert_numpy.convert_to_numpy(self.kc_house_train, features, output, 1) # Create our initial weights initial_weights = np.array([0., 0.]) # Step size step_size = 1e-12 # Max Iterations to Run max_iterations = 1000 # Tolerance tolerance = None # L2 Penalty l2_penalty = 0.0 # Compute our hill climbing value final_weights = self.ridge_regression.gradient_ascent(feature_matrix, output, {"initial_weights": initial_weights, "step_size": step_size, "tolerance": tolerance, "l2_penalty": l2_penalty, "max_iteration": max_iterations}) # Assert that the weights is correct self.assertEquals(round(-7.7535764461428101e+70, -68), round(final_weights[0], -68)) self.assertEquals(round(-1.9293745396177612e+74, -70), round(final_weights[1], -70)) def test_07_gradient_ascent_high_tolerance(self): """Tests gradient ascent. Tests gradient ascent and compare it with known values. """ # We will use sqft_living for our features features = ['sqft_living'] # Output will be price output = ['price'] # Convert our pandas frame to numpy feature_matrix, output = self.convert_numpy.convert_to_numpy(self.kc_house_train, features, output, 1) # Create our initial weights initial_weights = np.array([0., 0.]) # Step size step_size = 1e-12 # Max Iterations to Run max_iterations = 1000 # Tolerance tolerance = 1 # L2 Penalty l2_penalty = 0.0 # Compute our hill climbing value final_weights = self.ridge_regression.gradient_ascent(feature_matrix, output, {"initial_weights": initial_weights, "step_size": step_size, "tolerance": tolerance, "l2_penalty": l2_penalty, "max_iteration": max_iterations}) # Assert that the weights is correct self.assertEquals(0, round(final_weights[0], -68)) self.assertEquals(0, round(final_weights[1], -70)) def test_08_gradient_descent_no_penalty_high_tolerance(self): """Tests gradient descent algorithm. Tests the result on gradient descent with low penalty. """ # We will use sqft_living for our features features = ['sqft_living'] # Output will use price output = ['price'] # Convert our pandas frame to numpy feature_matrix, output = self.convert_numpy.convert_to_numpy(self.kc_house_train, features, output, 1) # Create our initial weights initial_weights = np.array([0., 0.]) # Step size step_size = 1e-12 # Max Iterations to Run max_iterations = 100000 # Tolerance tolerance = 10000000000 # L2 Penalty l2_penalty = 0.0 # Compute our gradient descent value final_weights = self.ridge_regression.gradient_descent(feature_matrix, output, {"initial_weights": initial_weights, "step_size": step_size, "tolerance": tolerance, "l2_penalty": l2_penalty, "max_iteration": max_iterations}) # We will use sqft_iving, and sqft_living15 test_features = ['sqft_living'] # Output will be price test_output = ['price'] # Convert our test pandas frame to numpy test_feature_matrix, test_output = self.convert_numpy.convert_to_numpy(self.kc_house_test, test_features, test_output, 1) # Predict the output of test features predicted_output = self.predict_output.regression(test_feature_matrix, final_weights) # Compute RSS rss = self.residual_sum_squares.residual_sum_squares_regression(test_output, predicted_output) # Assert that the weights is correct self.assertEquals(round(0.093859999999999999, 5), round(final_weights[0], 5)) self.assertEquals(round(262.98200000000003, 3), round(final_weights[1], 3)) # Assert that rss is correct self.assertEquals(round(275724298300000.0, -5), round(rss, -5))
class TestKNearestNeighborRegression(unittest.TestCase): """Tests for TestKNearestNeighborRegression. Uses housing data to test KNearestNeighborRegression. Statics: _multiprocess_can_split_ (bool): Flag for nose tests to run tests in parallel. """ _multiprocess_can_split_ = True def setUp(self): """Constructor for TestKNearestNeighborRegression. Loads housing data, and creates training and testing data. """ self.convert_numpy = ConvertNumpy() self.normalize_features = NormalizeFeatures() self.knn = KNearestNeighborRegression() self.euclidean_distance = EuclideanDistance() self.determine_k_knn = DetermineKKnn() # Create a dictionary type to store relevant data types so that our pandas # will read the correct information dtype_dict = {'bathrooms': float, 'waterfront': int, 'sqft_above': int, 'sqft_living15': float, 'grade': int, 'yr_renovated': int, 'price': float, 'bedrooms': float, 'zipcode': str, 'long': float, 'sqft_lot15': float, 'sqft_living': float, 'floors': str, 'condition': int, 'lat': float, 'date': str, 'sqft_basement': int, 'yr_built': int, 'id': str, 'sqft_lot': int, 'view': int} # Create a kc_house that encompasses all test and train data self.kc_house = pd.read_csv('./unit_tests/test_data/regression/kc_house_knn/kc_house_data_small.csv', dtype=dtype_dict) # Create a kc_house_test_frame that encompasses only train data self.kc_house_train = pd.read_csv('./unit_tests/test_data/regression/kc_house_knn/' 'kc_house_data_small_train.csv', dtype=dtype_dict) # Create a kc_house_frames that encompasses only test data self.kc_house_test = pd.read_csv('./unit_tests/test_data/regression/kc_house_knn/kc_house_data_small_test.csv', dtype=dtype_dict) # Create a kc_house_frames that encompasses only validation data self.kc_house_valid = pd.read_csv('./unit_tests/test_data/regression/kc_house_knn/kc_house_data_validation.csv', dtype=dtype_dict) # Convert all the frames with the floors to float type self.kc_house['floors'] = self.kc_house['floors'].astype(float) self.kc_house_train['floors'] = self.kc_house_train['floors'].astype(float) self.kc_house_test['floors'] = self.kc_house_test['floors'].astype(float) self.kc_house_valid['floors'] = self.kc_house_valid['floors'].astype(float) # Then back to int type self.kc_house['floors'] = self.kc_house['floors'].astype(int) self.kc_house_train['floors'] = self.kc_house_train['floors'].astype(int) self.kc_house_test['floors'] = self.kc_house_test['floors'].astype(int) self.kc_house_valid['floors'] = self.kc_house_valid['floors'].astype(int) def test_01_compute_euclidean_distance(self): """Tests Euclidean distance. Tests Euclidean distance and compare it with known values. """ # List of features to convert to numpy feature_list = ['bedrooms', 'bathrooms', 'sqft_living', 'sqft_lot', 'floors', 'waterfront', 'view', 'condition', 'grade', 'sqft_above', 'sqft_basement', 'yr_built', 'yr_renovated', 'lat', 'long', 'sqft_living15', 'sqft_lot15'] # Output to convert to numpy output = ['price'] # Extract features and output for train, test, and validation set features_train, _ = self.convert_numpy.convert_to_numpy(self.kc_house_train, feature_list, output, 1) features_test, _ = self.convert_numpy.convert_to_numpy(self.kc_house_test, feature_list, output, 1) # features_valid, output_valid = self.convert_numpy.convert_to_numpy(self.kc_house_valid, feature_list, # output, 1) # Normalize our training features, and then normalize the test set and valid set features_train, norms = self.normalize_features.l2_norm(features_train) features_test = features_test / norms # features_valid = features_valid / norms # Compute the euclidean distance distance = self.euclidean_distance.euclidean_distance(features_test[0], features_train[9]) # Assert that both are equal self.assertEqual(round(distance, 3), round(0.059723593716661257, 3)) def test_02_compute_euclidean_distance_query_point(self): """Tests Euclidean distance with a set of query points. Test to compute euclidean distance from a query point to multiple points in the training set """ # List of features to convert to numpy feature_list = ['bedrooms', 'bathrooms', 'sqft_living', 'sqft_lot', 'floors', 'waterfront', 'view', 'condition', 'grade', 'sqft_above', 'sqft_basement', 'yr_built', 'yr_renovated', 'lat', 'long', 'sqft_living15', 'sqft_lot15'] # Output to convert to numpy output = ['price'] # Extract features and output for train, test, and validation set features_train, output_train = self.convert_numpy.convert_to_numpy(self.kc_house_train, feature_list, output, 1) features_test, _ = self.convert_numpy.convert_to_numpy(self.kc_house_test, feature_list, output, 1) # features_valid, output_valid = self.convert_numpy.convert_to_numpy(self.kc_house_valid, feature_list, # output, 1) # Normalize our training features, and then normalize the test set and valid set features_train, norms = self.normalize_features.l2_norm(features_train) features_test = features_test / norms # features_valid = features_valid / norms # Determine the smallest euclidean distance set we get smallest = sys.maxsize smallest_index = 0 for index, val in enumerate(self.euclidean_distance.euclidean_distance_cmp_one_value(features_train, features_test[2])): if val < smallest: smallest = val smallest_index = index # Assert that we are getting the right prediction (for 1-NN neighbor) self.assertEqual(round(smallest, 8), round(0.00286049526751, 8)) self.assertEqual(output_train[smallest_index], 249000) self.assertEqual(smallest_index, 382) def test_03_compute_knn(self): """Tests knn regression algorithm. Tests the knn algorithm and compare it with known values. """ # List of features to convert to numpy feature_list = ['bedrooms', 'bathrooms', 'sqft_living', 'sqft_lot', 'floors', 'waterfront', 'view', 'condition', 'grade', 'sqft_above', 'sqft_basement', 'yr_built', 'yr_renovated', 'lat', 'long', 'sqft_living15', 'sqft_lot15'] # Output to convert to numpy output = ['price'] # Extract features and output for train, test, and validation set features_train, output_train = self.convert_numpy.convert_to_numpy(self.kc_house_train, feature_list, output, 1) features_test, _ = self.convert_numpy.convert_to_numpy(self.kc_house_test, feature_list, output, 1) # features_valid, output_valid = self.convert_numpy.convert_to_numpy(self.kc_house_valid, feature_list, # output, 1) # Normalize our training features, and then normalize the test set and valid set features_train, norms = self.normalize_features.l2_norm(features_train) features_test = features_test / norms # features_valid = features_valid / norms # Assert that the array is the closest with the 3rd house in features_test self.assertTrue(np.array_equal(self.knn.k_nearest_neighbor_regression(4, features_train, features_test[2]), np.array([382, 1149, 4087, 3142]))) # Assert that the 413987.5 is the correct prediction self.assertEqual(self.knn.predict_k_nearest_neighbor_regression(4, features_train, output_train, features_test[2]), 413987.5) # Compute the lowest predicted value lowest_predicted = sys.maxsize lowest_predicted_index = 0 for index, val in enumerate(self.knn.predict_k_nearest_neighbor_all_regression(10, features_train, output_train, features_test[0:10])): if val < lowest_predicted: lowest_predicted = val lowest_predicted_index = index # Assert that the few values such as lowest predicted values and index are the one we expect self.assertEqual(lowest_predicted, 350032.0) self.assertEqual(lowest_predicted_index, 6) def test_03_compute_best_k(self): """Compute best K for KNN Regression. Compute best K using K Fold Cross Validation. """ # List of features to convert to numpy feature_list = ['bedrooms', 'bathrooms', 'sqft_living', 'sqft_lot', 'floors', 'waterfront', 'view', 'condition', 'grade', 'sqft_above', 'sqft_basement', 'yr_built', 'yr_renovated', 'lat', 'long', 'sqft_living15', 'sqft_lot15'] # Output to convert to numpy output = ['price'] # Extract features and output for train, test, and validation set features_train, output_train = self.convert_numpy.convert_to_numpy(self.kc_house_train, feature_list, output, 1) # features_test, output_test = self.convert_numpy.convert_to_numpy(self.kc_house_test, feature_list, # output, 1) features_valid, output_valid = self.convert_numpy.convert_to_numpy(self.kc_house_valid, feature_list, output, 1) # Normalize our training features, and then normalize the test set and valid set features_train, norms = self.normalize_features.l2_norm(features_train) # features_test = features_test / norms features_valid = features_valid / norms # Compute the lowest K and lowest K's RSS low_rss, low_idx = self.determine_k_knn.determine_k_knn(self.knn.predict_k_nearest_neighbor_all_regression, 1, 16, {"features_train": features_train, "features_valid": features_valid, "output_train": output_train, "output_valid": output_valid}) # Assert that the lowest k and rss is correct self.assertEqual(round(low_rss, -13), round(6.73616787355e+13, -13)) self.assertEqual(low_idx, 8)
class KFoldCrossValidation: """Class for K Fold Cross Validation. Class for K Fold Cross Validation for selecting best parameters. Attributes: convert_numpy (ConvertNumpy): Pandas to Numpy conversion class. predict_output (PredictOutput): Output prediction. residual_sum_squares (ResidualSumSquares): Computes residual sum of squares. """ def __init__(self): """Constructor for KFoldCrossValidation. Constructor for KFoldCrossValidation, used to setup numpy conversion, output prediction, and residual sum of squares. """ self.convert_numpy = ConvertNumpy() self.predict_output = PredictOutput() self.residual_sum_squares = ResidualSumSquares() def k_fold_cross_validation(self, k, model, model_parameters, data_parameters): """Performs K Fold Cross Validation. Takes in our data, and splits the data to smaller subsets, and these smaller subsets are used as validation sets, and everything else not included in the validation set is used as training sets. The model will be trained using the training set, and the performance assessment such as RSS would be used on the validation set against the model. Args: k (int): Number of folds.= model (obj): Model used for k folds cross validation. model_parameters (dict): Model parameters to train the specified model. data_parameters (dict): A dictionary of data information: { data (pandas.DataFrame): Data used for k folds cross validation, output (str): Output name, features (list of str): A list of feature names. } Returns: float: Average validation error. """ # Sum of the validation error, will divide by k (fold) later validation_error_sum = 0 # Loop through each fold for i in range(k): # Computes validation, and training set validation_set, training_set = self.create_validation_training_set(data_parameters["data"], k, i) # Convert our pandas frame to numpy to create validation set validation_set_matrix, validation_output = self.convert_numpy.convert_to_numpy(validation_set, data_parameters["features"], data_parameters["output"], 1) # Create a model with Train Set 1 + Train Set 2 final_weights = self.create_weights(model, model_parameters, training_set, data_parameters) # Predict the output of test features predicted_output = self.predict_output.regression(validation_set_matrix, final_weights) # compute squared error (in other words, rss) validation_error_sum += self.residual_sum_squares.residual_sum_squares_regression(validation_output, predicted_output) # Return the validation_error_sum divided by fold return validation_error_sum/k @staticmethod def create_validation_training_set(data, k, iteration): """Slice data according to k, iteration, and size of data. Computes the validation, and training set according to the k number of folds, and the current iteration. Args: data (pandas.DataFrame): Data used for k folds cross validation. k (int): Number of folds. iteration (int): Current K fold validation iteration. Returns: A tuple that contains training set, and validation set: ( validation_set (pandas.DataFrame): Validation set. training_set (pandas.DataFrame): Training set. ) """ length_data = len(data) # Compute the start section of the current fold start = int((length_data * iteration) / k) # Compute the end section of the current fold end = int((length_data * (iteration + 1)) / k - 1) # Get our validation set from the start to the end+1 (+1 since we need to include the end) # <Start : end + 1> Validation Set validation_set = data[start:end + 1] # The Training set the left and the right parts of the validation set # < 0 : Start > Train Set 1 # < Start : End + 1 > Validation Set # < End + 1 : n > Train Set 2 # Train Set 1 + Train Set 2 = All data excluding validation set training_set = data[0:start].append(data[end + 1:length_data]) return validation_set, training_set def create_weights(self, model, model_parameters, training_set, data_parameters): """Use model to create weights. Use model, model parameters, and training set, create a set of coefficients. Args: model (obj): Model that can be run. model_parameters (dict): A dictionary of model parameters. training_set (pandas.DataFrame): Train set used for k folds cross validation. data_parameters (dict): A dictionary of data information: { data (pandas.DataFrame): Data used for k folds cross validation, output (str): Output name, features (list of str): A list of feature names. } Returns: numpy.array: numpy array of weights created by running model. """ # Convert our pandas frame to numpy to create training set training_feature_matrix, training_output = self.convert_numpy.convert_to_numpy(training_set, data_parameters["features"], data_parameters["output"], 1) # Create a model with Train Set 1 + Train Set 2 return model(model_parameters=model_parameters, feature_matrix=training_feature_matrix, output=training_output)
class TestLogisticRegression(unittest.TestCase): """Tests for LogisticRegression class. Uses Amazon data to test logistic regression. Statics: _multiprocess_can_split_ (bool): Flag for nose tests to run tests in parallel. """ _multiprocess_can_split_ = True def setUp(self): """Constructor for TestLogisticRegression. Loads Amazon data, and creates training and testing data. """ self.convert_numpy = ConvertNumpy() self.log_likelhood = LogLikelihood() self.predict_output = PredictOutput() self.logistic_regression = LogisticRegression() self.confusion_matrix = ConfusionMatrix() # Load the important words self.important_words = json.load(open('./unit_tests/test_data/classification/amazon/important_words.json', 'r')) # Load the amazon baby subset self.review_frame = pd.read_csv('./unit_tests/test_data/classification/amazon/amazon_baby_subset.csv') # Review needs to be text self.review_frame['review'].astype(str) # Clean up the punctuations self.review_frame['review_clean'] = self.review_frame.apply( axis=1, func=lambda row: str(row["review"]).translate(str.maketrans({key: None for key in string.punctuation}))) # Remove any nan text self.review_frame['review_clean'] = self.review_frame.apply( axis=1, func=lambda row: '' if row["review_clean"] == "nan" else row["review_clean"]) # Count the number of words that appears in each review, and make an indepedent column for word in self.important_words: self.review_frame[word] = self.review_frame['review_clean'].apply(lambda s, w=word: s.split().count(w)) # Load training data self.train_frame = pd.read_csv('./unit_tests/test_data/classification/amazon/amazon_baby_subset_train_mod2.csv') def test_01_gradient_ascent(self): """Test gradient ascent algorithm. Tests the gradient ascent algorithm and compare it with known values. """ # We will use important words for the output features = self.important_words # Output will use sentiment output = ['sentiment'] # Convert our pandas frame to numpy feature_matrix, sentiment = self.convert_numpy.convert_to_numpy(self.review_frame, features, output, 1) # Compute the coefficients coefficients = self.logistic_regression.gradient_ascent(feature_matrix, sentiment, {"initial_coefficients": np.zeros(194), "step_size": 1e-7, "max_iter": 301}) # Real coefficients that we need to compare with the computed coefficients real_coef = [5.16220157e-03, 1.55656966e-02, -8.50204675e-03, 6.65460842e-02, 6.58907629e-02, 5.01743882e-03, -5.38601484e-02, -3.50488413e-03, 6.47945868e-02, 4.54356263e-02, 3.98353364e-03, 2.00775410e-02, 3.01350011e-02, -2.87115530e-02, 1.52161964e-02, 2.72592062e-04, 1.19448177e-02, -1.82461935e-02, -1.21706420e-02, -4.15110334e-02, 2.76820391e-03, 1.77031999e-02, -4.39700067e-03, 4.49764014e-02, 9.90916464e-03, 8.99239081e-04, -1.36219516e-03, 1.26859357e-02, 8.26466695e-03, -2.77426972e-02, 6.10128809e-04, 1.54084501e-02, -1.32134753e-02, -3.00512492e-02, 2.97399371e-02, 1.84087080e-02, 2.86178752e-03, -1.05768015e-02, -6.57350362e-04, -1.01476555e-02, -4.79579528e-03, 7.50891810e-03, 4.27938289e-03, 3.06785501e-03, -2.20317661e-03, 9.57273354e-03, 9.91666827e-05, -1.98462567e-02, 1.75702722e-02, 1.55478612e-03, -1.77375440e-02, 9.78324102e-03, 1.17031606e-02, -7.35345937e-03, -6.08714030e-03, 6.43766808e-03, 1.07159665e-02, -3.05345476e-03, 7.17190727e-03, 5.73320003e-03, 4.60661876e-03, -5.20588421e-03, 6.71012331e-03, 9.03281814e-03, 1.74563147e-03, 6.00279979e-03, 1.20181744e-02, -1.83594607e-02, -6.91010811e-03, -1.38687273e-02, -1.50406590e-02, 5.92353611e-03, 5.67478991e-03, -5.28786220e-03, 3.08147864e-03, 5.53751236e-03, 1.49917916e-02, -3.35666000e-04, -3.30695153e-02, -4.78990943e-03, -6.41368859e-03, 7.99938935e-03, -8.61390444e-04, 1.68052959e-02, 1.32539901e-02, 1.72307051e-03, 2.98030675e-03, 8.58284300e-03, 1.17082481e-02, 2.80825907e-03, 2.18724016e-03, 1.68824711e-02, -4.65973741e-03, 1.51368285e-03, -1.09509122e-02, 9.17842898e-03, -1.88572281e-04, -3.89820373e-02, -2.44821005e-02, -1.87023714e-02, -2.13943485e-02, -1.29690465e-02, -1.71378670e-02, -1.37566767e-02, -1.49770449e-02, -5.10287978e-03, -2.89789761e-02, -1.48663194e-02, -1.28088380e-02, -1.07709355e-02, -6.95286915e-03, -5.04082164e-03, -9.25914404e-03, -2.40427481e-02, -2.65927785e-02, -1.97320937e-03, -5.04127508e-03, -7.00791912e-03, -3.48088523e-03, -6.40958916e-03, -4.07497010e-03, -6.30054296e-03, -1.09187932e-02, -1.26051900e-02, -1.66895314e-03, -7.76418781e-03, -5.15960485e-04, -1.94199551e-03, -1.24761586e-03, -5.01291731e-03, -9.12049191e-03, -7.22098801e-03, -8.31782981e-03, -5.60573348e-03, -1.47098335e-02, -9.31520819e-03, -2.22034402e-03, -7.07573098e-03, -5.10115608e-03, -8.93572862e-03, -1.27545713e-02, -7.04171991e-03, -9.76219676e-04, 4.12091713e-04, 8.29251160e-04, 2.64661064e-03, -7.73228782e-03, 1.53471164e-03, -7.37263060e-03, -3.73694386e-03, -3.81416409e-03, -1.64575145e-03, -3.31887732e-03, 1.22257832e-03, 1.36699286e-05, -3.01866601e-03, -1.02826343e-02, -1.06691327e-02, 2.23639046e-03, -9.87424798e-03, -1.02192048e-02, -3.41330929e-03, 3.34489960e-03, -3.50984516e-03, -6.26283150e-03, -7.22419943e-03, -5.47016154e-03, -1.25063947e-02, -2.47805699e-03, -1.60017985e-02, -6.40098934e-03, -4.26644386e-03, -1.55376990e-02, 2.31349237e-03, -9.06653337e-03, -6.30012672e-03, -1.21010303e-02, -3.02578875e-03, -6.76289718e-03, -5.65498722e-03, -6.87050239e-03, -1.18950595e-02, -1.86489236e-04, -1.15230476e-02, 2.81533219e-03, -8.10150295e-03, -1.00062131e-02, 4.02037651e-03, -5.44300346e-03, 2.85818985e-03, 1.19885003e-04, -6.47587687e-03, -1.14493516e-03, -7.09205934e-03] # Loop through each value, the coefficients must be the same for pred_coef, coef in zip(coefficients, real_coef): # Assert that both values are the same self.assertEqual(round(pred_coef, 5), round(coef, 5)) # Get the output of the logistic regression with threshold 0 output = self.predict_output.logistic_regression(feature_matrix, coefficients, 0) # Generate a confusion matrix confusion_matrix = self.confusion_matrix.confusion_matrix(sentiment, output) # Assert the values are to be expected self.assertEqual(confusion_matrix, {'false_negatives': 7311, 'true_negatives': 20635, 'true_positives': 19268, 'false_positives': 5858}) # Assert that the precision is correct self.assertEqual(round(self.confusion_matrix.precision(sentiment, output), 5), round(0.7249332179540239, 5)) # Assert that the recall is correct self.assertEqual(round(self.confusion_matrix.recall(sentiment, output), 5), round(0.7668550505452519, 5)) def test_02_stochastic_gradient_ascent(self): """Test stochastic gradient descent for logistic regression. Tests stochastic gradient descent and test it with some known values. """ # We will use important words for the output features = self.important_words # Output will use sentiment output = ['sentiment'] # Convert our pandas frame to numpy feature_matrix, sentiment = self.convert_numpy.convert_to_numpy(self.train_frame, features, output, 1) # Compute the coefficients coefficients = self.logistic_regression.stochastic_gradient_ascent(feature_matrix, sentiment, {"initial_coefficients": np.zeros(194), "step_size": 5e-1, "batch_size": 1, "max_iter": 10}) # Real coefficients that we need to compare with the computed coefficients real_coef = [0.26845909, 0.05510662, -0.78232359, 0.24929641, 0.1213813, -0.13194118, -0.42110769, 0.23944013, 0.52334226, 0.30746343, 1.46697311, 0.15734639, 0.24112255, -0.22849175, -0.48095714, 0., 0.05984944, -0.41942527, -0.48095714, 0.10654088, 0., 0.06153186, -0.41942527, 0.43843464, 0., 0.21719583, 0., 0.84326475, 0.28108825, 0.28108825, 0., 0., 0.24611428, -0.19986888, 0.15734639, 0., 0., -0.48095714, 0.12623269, 0., 0.28108825, 0.07542718, 0., -0.42110769, 0.15734639, -0.48095714, 0.24611428, -0.48095714, 0., 0., 0.06153186, 0.28108825, 0., 0., 0., 0.05984944, 0.5932902, 0.5621765, -0.48095714, 0., 0.05984944, 0.05984944, 0.31220195, 0.11805882, 0., 0.15085436, 0.24611428, 0., 0., 0., 0.06153186, 0.12623269, 0., 0., 0., 0., 0., 0., -0.35472444, 0.12623269, 0., 0., 0.68023532, 0.28108825, 0.06153186, 0.0311137, 0.35651543, 0., 0.28108825, 0., 0.05984944, 0., 0.35651543, 0.28108825, 0., 0., 0., -0.90206483, 0.07542718, -0.48095714, 0., 0., -0.48095714, 0., 0., 0., -0.25, 0.0311137, 0., 0.28108825, 0., 0., 0., 0., 0., 0., 0.34262011, -0.48095714, 0.28108825, 0., 0., 0., 0., 0., 0.06153186, 0.12623269, 0.05984944, 0., 0., 0., 0., 0.12623269, 0., 0., 0.12623269, 0.07542718, 0.15085436, 0.07542718, -0.68082602, 0., 0., 0., 0.05984944, 0., 0., 0.28108825, 0., -0.25, 0., 0., 0.07542718, 0., 0., 0.28108825, 0., 0., 0., 0., 0., 0., 0.06153186, 0.0311137, 0., -0.48095714, 0., 0., 0., 0., 0., 0., 0., 0.40732094, 0., 0., 0.05984944, 0., 0., 0., 0., 0., 0., 0., 0.06153186, 0., 0.06153186, 0., -0.25, 0.05984944, 0., 0., 0., 0., -0.96191427, 0.] # Loop through each value, the coefficients must be the same for pred_coef, coef in zip(coefficients, real_coef): # Assert that both values are the same self.assertEqual(round(pred_coef, 5), round(coef, 5)) # Get the output of the logistic regression with threshold 0 output = self.predict_output.logistic_regression(feature_matrix, coefficients, 0) # Generate a confusion matrix confusion_matrix = self.confusion_matrix.confusion_matrix(sentiment, output) # Assert the values are to be expected self.assertEqual(confusion_matrix, {'false_negatives': 6517, 'true_negatives': 11707, 'true_positives': 17331, 'false_positives': 12225}) # Assert that the precision is correct self.assertEqual(round(self.confusion_matrix.precision(sentiment, output), 5), round(0.72673, 5)) # Assert that the recall is correct self.assertEqual(round(self.confusion_matrix.recall(sentiment, output), 5), round(0.58638, 5)) def test_02_stochastic_gradient_ascent_high_iteration(self): """Test stochastic gradient descent for logistic regression. Tests stochastic gradient descent and test it with some known values. """ # We will use important words for the output features = self.important_words # Output will use sentiment output = ['sentiment'] # Convert our pandas frame to numpy feature_matrix, sentiment = self.convert_numpy.convert_to_numpy(self.train_frame, features, output, 1) # Compute the coefficients coefficients = self.logistic_regression.stochastic_gradient_ascent(feature_matrix, sentiment, {"initial_coefficients": np.zeros(194), "step_size": 5e-1, "batch_size": 1000, "max_iter": 1000}) # Real coefficients that we need to compare with the computed coefficients real_coef = [-0.06659918, 0.07516305, 0.02337901, 0.91476437, 1.25935729, -0.01093744, -0.29808423, 0.00724611, 1.14319635, 0.58421811, -0.10388794, 0.25341405, 0.51935047, -0.16643157, 0.1581433, -0.01678466, 0.11023426, -0.07801531, -0.11943521, -0.23901842, 0.19961916, 0.26962603, 0.00726172, 1.58116946, -0.04749877, -0.01222728, -0.12452547, 0.2408741, 0.23996495, -0.27318487, 0.16391931, 0.46141695, -0.00520781, -0.41720674, 1.3914436, 0.59286041, -0.01877455, -0.1177062, 0.04522629, -0.05050944, -0.1872891, 0.1119123, 0.05552736, 0.018883, -0.28821684, 0.35454167, 0.09146771, -0.15185966, 0.45980111, 0.13696004, -0.27719711, 0.37826182, 0.51482099, -0.12707594, -0.08043197, 0.27088589, 0.20836676, -0.22217221, 0.34308818, 0.05011724, 0.01336183, -0.00422257, 0.25914879, 0.18971367, 0.11804381, 0.06478439, 0.13413068, -0.35940054, -0.04225724, -0.23574987, -0.26178573, 0.37077618, 0.266064, 0.0552738, 0.25274691, 0.15248314, 0.9721445, 0.03951392, -0.59577998, -0.09680726, -0.13168621, 0.42806047, 0.03576358, 1.03088019, 0.52916025, -0.09516351, 0.23544152, 0.31386904, 0.50647271, 0.25383116, 0.1369185, 0.93673001, -0.06280486, 0.1670564, -0.20573152, 0.2201837, 0.12892914, -0.9711816, -0.24387714, -0.3566874, -0.65956699, -0.28473646, -0.34083222, -0.44708957, -0.29828401, -0.52797307, -1.92693359, -0.33116364, -0.43025271, -0.21284617, 0.16375567, -0.0299845, -0.30294927, -1.25019619, -1.55092776, -0.09266983, -0.08014312, -0.07565967, -0.00950432, 0.00327247, 0.03190358, -0.04247063, -0.28205865, -0.45678176, 0.06141561, -0.2690871, -0.05979329, -0.0019354, -0.01279985, 0.05323391, -0.35513613, -0.26639425, -0.41094467, -0.14117863, -0.90001241, -0.33279773, 0.01621988, -0.08709595, -0.10450457, -0.12567406, -0.61727551, -0.18663497, 0.17636203, 0.09316913, -0.06829369, 0.1880183, -0.5078543, 0.03964466, -0.26089197, -0.07480237, -0.05556211, -0.1450303, -0.04780934, 0.08911386, -0.15163772, 0.06213261, -0.34512242, -0.33522342, 0.06580618, -0.44499204, -0.68623426, -0.12564489, 0.2609755, 0.09998045, -0.25098629, -0.29549973, -0.15944276, -0.47408765, -0.03058168, -1.42253269, -0.49855378, 0.05835175, -1.17789127, -0.08226967, -0.56793665, -0.35814271, -0.98559717, -0.16918106, -0.12477773, -0.23457722, -0.13170106, -0.64351485, -0.01773532, -0.2686544, 0.047442, -0.34218929, -0.48340895, 0.37866335, -0.25162177, 0.05277577, 0.01545386, -0.26267815, -0.09903819, -0.54500151] # Loop through each value, the coefficients must be the same for pred_coef, coef in zip(coefficients, real_coef): # Assert that both values are the same self.assertEqual(round(pred_coef, 5), round(coef, 5)) # Get the output of the logistic regression with threshold 0 output = self.predict_output.logistic_regression(feature_matrix, coefficients, 0) # Generate a confusion matrix confusion_matrix = self.confusion_matrix.confusion_matrix(sentiment, output) # Assert the values are to be expected self.assertEqual(confusion_matrix, {'false_negatives': 5018, 'true_negatives': 18995, 'true_positives': 18830, 'false_positives': 4937}) # Assert that the precision is correct self.assertEqual(round(self.confusion_matrix.precision(sentiment, output), 5), round(0.78958, 5)) # Assert that the recall is correct self.assertEqual(round(self.confusion_matrix.recall(sentiment, output), 5), round(0.79228, 5)) def test_04_log_likelihood(self): """Test log likelihood. Test the log likelihood algorithm, and compare it with some known values. """ # Generate test feature, coefficients, and label feature_matrix = np.array([[1., 2., 3.], [1., -1., -1]]) coefficients = np.array([1., 3., -1.]) label = np.array([-1, 1]) # Compute the log likelihood lg = self.log_likelhood.log_likelihood(feature_matrix, label, coefficients) # Assert the value self.assertEqual(round(lg, 5), round(-5.33141161544, 5)) def test_05_average_log_likelihood(self): """Test average log likelihood. Test the average log likelihood algorithm, and compare it with some known values. """ # Generate test feature, coefficients, and label feature_matrix = np.array([[1., 2., 3.], [1., -1., -1]]) coefficients = np.array([1., 3., -1.]) label = np.array([-1, 1]) # Compute the log likelihood lg = self.log_likelhood.average_log_likelihood(feature_matrix, label, coefficients) # Assert the value self.assertEqual(round(lg, 5), round(-2.6657099999999998, 5))
class TestLinearRegression(unittest.TestCase): """Test for LinearRegression. Uses housing data to test LinearRegression. Statics: _multiprocess_can_split_ (bool): Flag for nose tests to run tests in parallel. """ _multiprocess_can_split_ = True def setUp(self): """Constructor for TestLinearRegression. Loads housing data, and creates training and testing data. """ self.convert_numpy = ConvertNumpy() self.linear_regression = LinearRegression() self.predict_output = PredictOutput() self.residual_sum_squares = ResidualSumSquares() # Create a dictionary type to store relevant data types so that our pandas # will read the correct information dtype_dict = {'bathrooms': float, 'waterfront': int, 'sqft_above': int, 'sqft_living15': float, 'grade': int, 'yr_renovated': int, 'price': float, 'bedrooms': float, 'zipcode': str, 'long': float, 'sqft_lot15': float, 'sqft_living': float, 'floors': str, 'condition': int, 'lat': float, 'date': str, 'sqft_basement': int, 'yr_built': int, 'id': str, 'sqft_lot': int, 'view': int} # Create a kc_house that encompasses all test and train data self.kc_house = pd.read_csv('./unit_tests/test_data/regression/kc_house/kc_house_data.csv', dtype=dtype_dict) # Create a kc_house_test_frame that encompasses only train data self.kc_house_train = pd.read_csv('./unit_tests/test_data/regression/kc_house/kc_house_train_data.csv', dtype=dtype_dict) # Create a kc_house_frames that encompasses only test data self.kc_house_test = pd.read_csv('./unit_tests/test_data/regression/kc_house/kc_house_test_data.csv', dtype=dtype_dict) def test_01_gradient_descent(self): """Test gradient descent. Tests gradient descent and compare it to known values. """ # We will use sqft_living for our features features = ['sqft_living'] # Output will use price output = ['price'] # Convert our pandas frame to numpy feature_matrix, output = self.convert_numpy.convert_to_numpy(self.kc_house_train, features, output, 1) # Create our initial weights initial_weights = np.array([-47000., 1.]) # Step size step_size = 7e-12 # Tolerance tolerance = 2.5e7 # Compute our gradient descent value final_weights = self.linear_regression.gradient_descent(feature_matrix, output, {"initial_weights": initial_weights, "step_size": step_size, "tolerance": tolerance}) # Assert that the weights is correct self.assertEquals(round(-46999.887165546708, 3), round(final_weights[0], 3)) self.assertEquals(round(281.91211917520917, 3), round(final_weights[1], 3)) def test_02_gradient_descent_multiple(self): """Tests gradient descent on multiple features. Computes gradient descent on multiple input, and computes predicted model and RSS. """ # We will use sqft_iving, and sqft_living15 features = ['sqft_living', 'sqft_living15'] # Output will be price output = ['price'] # Convert our pandas frame to numpy feature_matrix, output = self.convert_numpy.convert_to_numpy(self.kc_house_train, features, output, 1) # Create our initial weights initial_weights = np.array([-100000., 1., 1.]) # Step size step_size = 4e-12 # Tolerance tolerance = 1e9 # Compute our gradient descent value final_weights = self.linear_regression.gradient_descent(feature_matrix, output, {"initial_weights": initial_weights, "step_size": step_size, "tolerance": tolerance}) # We will use sqft_iving, and sqft_living15 test_features = ['sqft_living', 'sqft_living15'] # Output will be price test_output = ['price'] # Convert our test pandas frame to numpy test_feature_matrix, test_output = self.convert_numpy.convert_to_numpy(self.kc_house_test, test_features, test_output, 1) # Predict the output of test features predicted_output = self.predict_output.regression(test_feature_matrix, final_weights) # Compute RSS rss = self.residual_sum_squares.residual_sum_squares_regression(test_output, predicted_output) # Assert that rss is correct self.assertEquals(round(270263443629803.41, -3), round(rss, -3)) def test_03_gradient_ascent(self): """Test gradient ascent. Test gradient ascent and compare it to known values. """ # We will use sqft_living for our features features = ['sqft_living'] # Output will be price output = ['price'] # Convert our pandas frame to numpy feature_matrix, output = self.convert_numpy.convert_to_numpy(self.kc_house_train, features, output, 1) # Create our initial weights initial_weights = np.array([-47000., 1.]) # Step size step_size = 7e-12 # Tolerance tolerance = 2.5e7 # Compute our hill climbing value final_weights = self.linear_regression.gradient_ascent(feature_matrix, output, {"initial_weights": initial_weights, "step_size": step_size, "tolerance": tolerance}) # Assert that the weights is correct self.assertEquals(round(-47000.142201335177, 3), round(final_weights[0], 3)) self.assertEquals(round(-352.86068692252599, 3), round(final_weights[1], 3))