class Error:
"""Computes error for classification.
Computes error for classification algorithms, such as binary tree.
Attributes:
predict_output (PredictOutput): Class for predicting output.
"""
def __init__(self):
"""Constructor for Error.
Constructor for Error, sets up output prediction class.
"""
self.predict_output = PredictOutput()
def binary_tree(self, tree, data, target):
"""Computes classification error for binary tree.
Computes classification error for binary tree classification.
Classification error = # Mistakes
----------------
# Total examples
Args:
tree (dict): The top node of a binary tree, with the following dict format:
{
'is_leaf' (bool): False,
'prediction' (NoneType): None,
'splitting_feature' (str): splitting_feature,
'left' (dict): left_tree,
'right' (dict): right_tree
}
data (pandas.DataFrame): A pandas frame that has the same features binary tree.
target (str): The target we want to predict.
Returns:
float: Clarification error.
"""
# Apply the classify(tree, x) to each row in your data
prediction = data.apply(lambda x: self.predict_output.binary_tree(tree, x), axis=1)
# Once you've made the predictions, calculate the classification error and return it
data["prediction"] = prediction
mistakes = data.apply(lambda x: x[target] != x["prediction"], axis=1).sum()
# Return mistakes/total examples
return float(mistakes)/float(len(data))
class TestBinaryDecisionTrees(unittest.TestCase):
"""Tests for the BinaryDecisionTrees class.
Uses lending club data to test binary decision trees.
Attributes:
binary_decision_trees (BinaryDecisionTrees): Binary decision tree class.
predict_output (PredictOutput): Predict output class to predict output for decision tree.
accuracy (Accuracy): Measures the accuracy of algorithms.
error (Error): Measure the accuracy of algorithms.
loans_data (pandas.DataFrame): Lending Club Data.
features (list of str): List of features to build decision tree on.
target (str): The target that we are predicting.
train_data (pandas.DataFrame): Lending Club training Data.
test_data (pandas.DataFrame): Lending Club testing data.
Statics:
_multiprocess_can_split_ (bool): Flag for nose tests to run tests in parallel.
"""
_multiprocess_can_split_ = True
def setUp(self):
"""Constructor for TestBinaryDecisionTrees.
We will clean up the loans_data by doing one hot encoding our features list, however, in the end we will
use some pre-built data for training and testing, but it uses the same method for one hot encoding.
"""
self.binary_decision_trees = BinaryDecisionTrees()
self.predict_output = PredictOutput()
self.accuracy = Accuracy()
self.error = Error()
# Pandas type set
dtype_dict = {'grade': str, 'term': str, 'emp_length': str, 'bad_loans': int}
# Load the lending club data
self.loans_data = pd.read_csv('./unit_tests/test_data/classification/lending_club/lending_club_data.csv',
dtype=dtype_dict)
# List features and targets that we are interested
self.features = ['grade', 'term', 'home_ownership', 'emp_length']
self.target = 'safe_loans'
# Do a one hot encoding of features
for feature in self.features:
# One hot encode
loans_data_one_hot_encoded = pd.get_dummies(self.loans_data[feature].apply(lambda x: x),
prefix=feature, prefix_sep='.')
# Drop the feature
self.loans_data.drop(feature, axis=1, inplace=True)
# Join the feature with the new one encoded features
self.loans_data = pd.concat([self.loans_data, loans_data_one_hot_encoded], axis=1)
# Update our features
self.features = list(self.loans_data.columns.values)
self.features.remove('safe_loans')
# Load our training and testing data
self.train_data = pd.read_csv('./unit_tests/test_data/classification/lending_club/lending_club_train.csv')
self.test_data = pd.read_csv('./unit_tests/test_data/classification/lending_club/lending_club_test.csv')
def test_01_greedy_recursive(self):
"""Tests greedy recursive function for BinaryDecisionTrees class
We will use the training data to build a decision tree, and measure the accuracy with some known good values.
"""
# Create a decision tree
decision_tree = self.binary_decision_trees.greedy_recursive(self.train_data, self.features, self.target,
{"current_depth": 0, "max_depth": 6})
# Get the classification result of the first row
classification = self.predict_output.binary_tree(decision_tree, self.test_data.iloc[0])
# Assert that the classification should be -1
self.assertEqual(classification, -1)
# Compute the accuracy of the decision tree
accuracy = self.error.binary_tree(decision_tree, self.test_data, self.target)
# Assert that the classification should be 0.3837785437311504
self.assertEqual(round(accuracy, 5), round(0.3837785437311504, 5))
def test_02_greedy_recursive_high_depth_low_feature(self):
"""Tests greedy recursive function for BinaryDecisionTrees class
We will use the training data to build a decision tree, and use high depth.
"""
# Create a decision tree
decision_tree = self.binary_decision_trees.greedy_recursive(self.train_data, ['grade.A', 'grade.B'],
self.target, {"current_depth": 0,
"max_depth": 1000})
# Get the classification result of the first row
classification = self.predict_output.binary_tree(decision_tree, self.test_data.iloc[0])
# Assert that the classification should be -1
self.assertEqual(classification, -1)
# Compute the accuracy of the decision tree
accuracy = self.error.binary_tree(decision_tree, self.test_data, self.target)
# Assert that the classification should be 0.38432
self.assertEqual(round(accuracy, 5), round(0.38432, 5))
def test_04_greedy_recursive_high_depth(self):
"""Tests greedy recursive function for BinaryDecisionTrees class
We will use the training data to build a decision tree, and use high depth.
"""
# Create a decision tree
decision_tree = self.binary_decision_trees.greedy_recursive(self.train_data, self.features, self.target,
{"current_depth": 0, "max_depth": 10000})
# Get the classification result of the first row
classification = self.predict_output.binary_tree(decision_tree, self.test_data.iloc[0])
# Assert that the classification should be -1
self.assertEqual(classification, -1)
# Compute the accuracy of the decision tree
accuracy = self.error.binary_tree(decision_tree, self.test_data, self.target)
# Assert that the classification should be 0.37732
self.assertEqual(round(accuracy, 5), round(0.37732, 5))
def test_03_greedy_recursive_early_stop(self):
"""Tests for greedy recursive with early stopping for BinaryDecisionTrees class
We will use early stopping for greedy recursive, and measure performance.
"""
# Create a model with max_depth=6, min_node_size=100, min_error_reduction=0
model_1 = self.binary_decision_trees.greedy_recursive_early_stop(self.train_data, self.features, self.target,
{"current_depth": 0, "max_depth": 6,
"min_node_size": 100,
"min_error_reduction": 0.0})
# Get the classification result of the first row
classification = self.predict_output.binary_tree(model_1, self.test_data.iloc[0])
# Assert that the classification should be -1
self.assertEqual(classification, -1)
# Compute the accuracy of the decision tree
accuracy = self.error.binary_tree(model_1, self.test_data, self.target)
# Assert that the classification should be 0.38367083153813014
self.assertEqual(round(accuracy, 5), round(0.38367083153813014, 5))
# Create a model with max_depth=6, min_node_size=0, min_error_reduction=-1
model_2 = self.binary_decision_trees.greedy_recursive_early_stop(self.train_data, self.features, self.target,
{"current_depth": 0, "max_depth": 6,
"min_node_size": 0,
"min_error_reduction": -1})
# Get the classification result of the first row
classification = self.predict_output.binary_tree(model_2, self.test_data.iloc[0])
# Assert that the classification should be -1
self.assertEqual(classification, -1)
# Compute the accuracy of the decision tree
accuracy = self.error.binary_tree(model_2, self.test_data, self.target)
# Assert that the classification should be 0.3837785437311504
self.assertEqual(round(accuracy, 5), round(0.3837785437311504, 5))
def test_04_greedy_recursive_early_stop_high_depth(self):
"""Tests for greedy recursive with early stopping for BinaryDecisionTrees class
We will use early stopping for greedy recursive, and measure performance.
"""
# Create a model with max_depth=5000, min_node_size=0, min_error_reduction=0
model_1 = self.binary_decision_trees.greedy_recursive_early_stop(self.train_data, ['grade.A', 'grade.B'],
self.target,
{"current_depth": 0, "max_depth": 5000,
"min_node_size": 0,
"min_error_reduction": 0.0})
# Get the classification result of the first row
classification = self.predict_output.binary_tree(model_1, self.test_data.iloc[0])
# Assert that the classification should be -1
self.assertEqual(classification, -1)
# Compute the accuracy of the decision tree
accuracy = self.error.binary_tree(model_1, self.test_data, self.target)
# Assert that the classification should be 0.38432
self.assertEqual(round(accuracy, 5), round(0.38432, 5))
def test_05_greedy_recursive_early_stop_high_depth(self):
"""Tests for greedy recursive with early stopping for BinaryDecisionTrees class
We will use early stopping for greedy recursive, and measure performance.
"""
# Create a model with max_depth=5000, min_node_size=0, min_error_reduction=0
model_1 = self.binary_decision_trees.greedy_recursive_early_stop(self.train_data, ['grade.A', 'grade.B',
'grade.C', 'grade.D',
'grade.E', 'grade.F',
'grade.G',
'term. 36 months'],
self.target,
{"current_depth": 0, "max_depth": 5000,
"min_node_size": 0,
"min_error_reduction": -50000})
# Get the classification result of the first row
classification = self.predict_output.binary_tree(model_1, self.test_data.iloc[0])
# Assert that the classification should be -1
self.assertEqual(classification, -1)
# Compute the accuracy of the decision tree
accuracy = self.error.binary_tree(model_1, self.test_data, self.target)
# Assert that the classification should be 0.38162
self.assertEqual(round(accuracy, 5), round(0.38162, 5))