class Error: """Computes error for classification. Computes error for classification algorithms, such as binary tree. Attributes: predict_output (PredictOutput): Class for predicting output. """ def __init__(self): """Constructor for Error. Constructor for Error, sets up output prediction class. """ self.predict_output = PredictOutput() def binary_tree(self, tree, data, target): """Computes classification error for binary tree. Computes classification error for binary tree classification. Classification error = # Mistakes ---------------- # Total examples Args: tree (dict): The top node of a binary tree, with the following dict format: { 'is_leaf' (bool): False, 'prediction' (NoneType): None, 'splitting_feature' (str): splitting_feature, 'left' (dict): left_tree, 'right' (dict): right_tree } data (pandas.DataFrame): A pandas frame that has the same features binary tree. target (str): The target we want to predict. Returns: float: Clarification error. """ # Apply the classify(tree, x) to each row in your data prediction = data.apply(lambda x: self.predict_output.binary_tree(tree, x), axis=1) # Once you've made the predictions, calculate the classification error and return it data["prediction"] = prediction mistakes = data.apply(lambda x: x[target] != x["prediction"], axis=1).sum() # Return mistakes/total examples return float(mistakes)/float(len(data))
class TestBinaryDecisionTrees(unittest.TestCase): """Tests for the BinaryDecisionTrees class. Uses lending club data to test binary decision trees. Attributes: binary_decision_trees (BinaryDecisionTrees): Binary decision tree class. predict_output (PredictOutput): Predict output class to predict output for decision tree. accuracy (Accuracy): Measures the accuracy of algorithms. error (Error): Measure the accuracy of algorithms. loans_data (pandas.DataFrame): Lending Club Data. features (list of str): List of features to build decision tree on. target (str): The target that we are predicting. train_data (pandas.DataFrame): Lending Club training Data. test_data (pandas.DataFrame): Lending Club testing data. Statics: _multiprocess_can_split_ (bool): Flag for nose tests to run tests in parallel. """ _multiprocess_can_split_ = True def setUp(self): """Constructor for TestBinaryDecisionTrees. We will clean up the loans_data by doing one hot encoding our features list, however, in the end we will use some pre-built data for training and testing, but it uses the same method for one hot encoding. """ self.binary_decision_trees = BinaryDecisionTrees() self.predict_output = PredictOutput() self.accuracy = Accuracy() self.error = Error() # Pandas type set dtype_dict = {'grade': str, 'term': str, 'emp_length': str, 'bad_loans': int} # Load the lending club data self.loans_data = pd.read_csv('./unit_tests/test_data/classification/lending_club/lending_club_data.csv', dtype=dtype_dict) # List features and targets that we are interested self.features = ['grade', 'term', 'home_ownership', 'emp_length'] self.target = 'safe_loans' # Do a one hot encoding of features for feature in self.features: # One hot encode loans_data_one_hot_encoded = pd.get_dummies(self.loans_data[feature].apply(lambda x: x), prefix=feature, prefix_sep='.') # Drop the feature self.loans_data.drop(feature, axis=1, inplace=True) # Join the feature with the new one encoded features self.loans_data = pd.concat([self.loans_data, loans_data_one_hot_encoded], axis=1) # Update our features self.features = list(self.loans_data.columns.values) self.features.remove('safe_loans') # Load our training and testing data self.train_data = pd.read_csv('./unit_tests/test_data/classification/lending_club/lending_club_train.csv') self.test_data = pd.read_csv('./unit_tests/test_data/classification/lending_club/lending_club_test.csv') def test_01_greedy_recursive(self): """Tests greedy recursive function for BinaryDecisionTrees class We will use the training data to build a decision tree, and measure the accuracy with some known good values. """ # Create a decision tree decision_tree = self.binary_decision_trees.greedy_recursive(self.train_data, self.features, self.target, {"current_depth": 0, "max_depth": 6}) # Get the classification result of the first row classification = self.predict_output.binary_tree(decision_tree, self.test_data.iloc[0]) # Assert that the classification should be -1 self.assertEqual(classification, -1) # Compute the accuracy of the decision tree accuracy = self.error.binary_tree(decision_tree, self.test_data, self.target) # Assert that the classification should be 0.3837785437311504 self.assertEqual(round(accuracy, 5), round(0.3837785437311504, 5)) def test_02_greedy_recursive_high_depth_low_feature(self): """Tests greedy recursive function for BinaryDecisionTrees class We will use the training data to build a decision tree, and use high depth. """ # Create a decision tree decision_tree = self.binary_decision_trees.greedy_recursive(self.train_data, ['grade.A', 'grade.B'], self.target, {"current_depth": 0, "max_depth": 1000}) # Get the classification result of the first row classification = self.predict_output.binary_tree(decision_tree, self.test_data.iloc[0]) # Assert that the classification should be -1 self.assertEqual(classification, -1) # Compute the accuracy of the decision tree accuracy = self.error.binary_tree(decision_tree, self.test_data, self.target) # Assert that the classification should be 0.38432 self.assertEqual(round(accuracy, 5), round(0.38432, 5)) def test_04_greedy_recursive_high_depth(self): """Tests greedy recursive function for BinaryDecisionTrees class We will use the training data to build a decision tree, and use high depth. """ # Create a decision tree decision_tree = self.binary_decision_trees.greedy_recursive(self.train_data, self.features, self.target, {"current_depth": 0, "max_depth": 10000}) # Get the classification result of the first row classification = self.predict_output.binary_tree(decision_tree, self.test_data.iloc[0]) # Assert that the classification should be -1 self.assertEqual(classification, -1) # Compute the accuracy of the decision tree accuracy = self.error.binary_tree(decision_tree, self.test_data, self.target) # Assert that the classification should be 0.37732 self.assertEqual(round(accuracy, 5), round(0.37732, 5)) def test_03_greedy_recursive_early_stop(self): """Tests for greedy recursive with early stopping for BinaryDecisionTrees class We will use early stopping for greedy recursive, and measure performance. """ # Create a model with max_depth=6, min_node_size=100, min_error_reduction=0 model_1 = self.binary_decision_trees.greedy_recursive_early_stop(self.train_data, self.features, self.target, {"current_depth": 0, "max_depth": 6, "min_node_size": 100, "min_error_reduction": 0.0}) # Get the classification result of the first row classification = self.predict_output.binary_tree(model_1, self.test_data.iloc[0]) # Assert that the classification should be -1 self.assertEqual(classification, -1) # Compute the accuracy of the decision tree accuracy = self.error.binary_tree(model_1, self.test_data, self.target) # Assert that the classification should be 0.38367083153813014 self.assertEqual(round(accuracy, 5), round(0.38367083153813014, 5)) # Create a model with max_depth=6, min_node_size=0, min_error_reduction=-1 model_2 = self.binary_decision_trees.greedy_recursive_early_stop(self.train_data, self.features, self.target, {"current_depth": 0, "max_depth": 6, "min_node_size": 0, "min_error_reduction": -1}) # Get the classification result of the first row classification = self.predict_output.binary_tree(model_2, self.test_data.iloc[0]) # Assert that the classification should be -1 self.assertEqual(classification, -1) # Compute the accuracy of the decision tree accuracy = self.error.binary_tree(model_2, self.test_data, self.target) # Assert that the classification should be 0.3837785437311504 self.assertEqual(round(accuracy, 5), round(0.3837785437311504, 5)) def test_04_greedy_recursive_early_stop_high_depth(self): """Tests for greedy recursive with early stopping for BinaryDecisionTrees class We will use early stopping for greedy recursive, and measure performance. """ # Create a model with max_depth=5000, min_node_size=0, min_error_reduction=0 model_1 = self.binary_decision_trees.greedy_recursive_early_stop(self.train_data, ['grade.A', 'grade.B'], self.target, {"current_depth": 0, "max_depth": 5000, "min_node_size": 0, "min_error_reduction": 0.0}) # Get the classification result of the first row classification = self.predict_output.binary_tree(model_1, self.test_data.iloc[0]) # Assert that the classification should be -1 self.assertEqual(classification, -1) # Compute the accuracy of the decision tree accuracy = self.error.binary_tree(model_1, self.test_data, self.target) # Assert that the classification should be 0.38432 self.assertEqual(round(accuracy, 5), round(0.38432, 5)) def test_05_greedy_recursive_early_stop_high_depth(self): """Tests for greedy recursive with early stopping for BinaryDecisionTrees class We will use early stopping for greedy recursive, and measure performance. """ # Create a model with max_depth=5000, min_node_size=0, min_error_reduction=0 model_1 = self.binary_decision_trees.greedy_recursive_early_stop(self.train_data, ['grade.A', 'grade.B', 'grade.C', 'grade.D', 'grade.E', 'grade.F', 'grade.G', 'term. 36 months'], self.target, {"current_depth": 0, "max_depth": 5000, "min_node_size": 0, "min_error_reduction": -50000}) # Get the classification result of the first row classification = self.predict_output.binary_tree(model_1, self.test_data.iloc[0]) # Assert that the classification should be -1 self.assertEqual(classification, -1) # Compute the accuracy of the decision tree accuracy = self.error.binary_tree(model_1, self.test_data, self.target) # Assert that the classification should be 0.38162 self.assertEqual(round(accuracy, 5), round(0.38162, 5))