def fit(self, X_train, y_train):
# Set X_train
self.X_train = X_train
# Set y_train
self.y_train = y_train
# Set header
header = ['att' + str(i) for i in range(len(self.X_train[0]))]
# Delete header
del self.X_train[0]
# Set train
train = [X_train[i] + [y_train[i]] for i in range(len(X_train))]
# Create forest array
forest = []
# Traverse
for _ in range(self.N):
# Create tree dictionary
tree = {}
# Set attributes
tree['atts'] = myutils.compute_random_subset(header[:-1], self.F)
# Get train set and test set
train_set, test_set = myutils.compute_bootstrapped_sample(train)
# Set tree
tree['tree'] = self.get_tree(train_set,
tree['atts'] + [header[-1]])
# Set accuracy
tree['accuracy'] = self.compute_tree_accuracy(tree, test_set)
# Append tree to forest
forest.append(tree)
# Sort
sort = sorted(forest, key=itemgetter('accuracy'), reverse=True)
# Set forest
self.forest = sort[:self.M]
def test_random_forest_fit():
interview_header = ["level", "lang", "tweets", "phd", "interviewed_well"]
interview_table = [["Senior", "Java", "no", "no", "False"],
["Senior", "Java", "no", "yes", "False"],
["Mid", "Python", "no", "no", "True"],
["Junior", "Python", "no", "no", "True"],
["Junior", "R", "yes", "no", "True"],
["Junior", "R", "yes", "yes", "False"],
["Mid", "R", "yes", "yes", "True"],
["Senior", "Python", "no", "no", "False"],
["Senior", "R", "yes", "no", "True"],
["Junior", "Python", "yes", "no", "True"],
["Senior", "Python", "yes", "yes", "True"],
["Mid", "Python", "no", "yes", "True"],
["Mid", "Java", "yes", "no", "True"],
["Junior", "Python", "no", "yes", "False"]]
myutils.prepend_attribute_label(interview_table, interview_header)
interview_pytable = MyPyTable(column_names=interview_header,
data=interview_table)
y_col = interview_pytable.get_column("interviewed_well", False)
x_cols = interview_pytable.drop_col("interviewed_well")
many_trees = MyRandomForestClassifier()
X_sample, y_sample = myutils.compute_bootstrapped_sample(x_cols, y_col)
X_train, X_test, y_train, y_test = myutils.train_test_split(
X_sample, y_sample, .33)
many_trees.fit(X_train, y_train, X_test, y_test)
y_predicted = many_trees.predict(X_test)
numCorrectPredictions = 0
numWrongPredictions = 0
for i in range(len(y_test)):
values = [y_predicted[i], y_test[i]] #predicted/actual
if (values[0] == values[1]):
numCorrectPredictions = numCorrectPredictions + 1
else:
numWrongPredictions = numWrongPredictions + 1
accuracy = np.round((numCorrectPredictions) /
(numCorrectPredictions + numWrongPredictions), 3)
error_rate = np.round(
(numWrongPredictions) / (numCorrectPredictions + numWrongPredictions),
3)
print("-----------------------------------------------------------")
print("Accuracy and Error Rate")
print("-----------------------------------------------------------")
print()
print("Random Forest: accuracy = {}, error rate = {}".format(
accuracy, error_rate))
print()
print(
"Because of the random aspect of this classifier, this will not always pass the tests"
)
print()
print("Predicted table: " + str(y_predicted))
print("Testing set: " + str(y_test))
for i in range(len(y_test)):
assert y_predicted[i] == y_test[i]
def fit(self, X_train, y_train):
"""Fits a decision tree classifier to X_train and y_train using the TDIDT (top down induction of decision tree) algorithm.
Args:
X_train(list of list of obj): The list of training instances (samples).
The shape of X_train is (n_train_samples, n_features)
y_train(list of obj): The target y values (parallel to X_train)
The shape of y_train is n_train_samples
Notes:
Since TDIDT is an eager learning algorithm, this method builds a decision tree model
from the training data.
Build a decision tree using the nested list representation described in class.
Store the tree in the tree attribute.
Use attribute indexes to construct default attribute names (e.g. "att0", "att1", ...).
"""
header = ['att' + str(i) for i in range(len(X_train[0]))]
attribute_domains = {}
for i, val in enumerate(header):
attribute_domains[val] = myutils.unique_index(X_train, i)
self.X_train = X_train
self.y_train = y_train
sample_X_train, sample_x_test, sample_y_train, sample_y_test = myevaluation.train_test_split(
X_train, y_train, test_size=0.33, shuffle=True)
train = [
sample_X_train[i] + [sample_y_train[i]]
for i in range(len(sample_X_train))
]
for _ in range(self.N):
available_attributes = header.copy()
self.trees.append(
myutils.tdidt_forest(
myutils.compute_bootstrapped_sample(train),
available_attributes, attribute_domains, header, self.F))
accuracies = []
for tree in self.trees:
header = ['att' + str(i) for i in range(len(sample_x_test[0]))]
prediction = []
for row in sample_x_test:
prediction.append(myutils.tdidt_predict(header, tree, row))
accuracy = 0
for i in range(len(prediction)):
if prediction[i] == sample_y_test[i]:
accuracy += 1
accuracy /= len(sample_y_test)
accuracies.append([accuracy])
# find m most accurate
m_trees = []
for i in range(len(accuracies)):
accuracies[i].append(i)
accuracies = sorted(accuracies)
for i in range(self.M):
m_trees.append(self.trees[accuracies[-(i + 1)][1]])
self.trees = m_trees
def fit(self, X_tr, y_tr):
"""Fits a decision tree classifier to X_train and y_train using the TDIDT (top down induction of decision tree) algorithm.
Args:
X_tr(list of list of obj): The list of training instances (samples).
The shape of X_train is (n_train_samples, n_features)
y_tr(list of obj): The target y values (parallel to X_train)
The shape of y_train is n_train_samples
Notes:
Since TDIDT is an eager learning algorithm, this method builds a decision tree model
from the training data.
Build a decision tree using the nested list representation described in class.
Store the tree in the tree attribute.
Use attribute indexes to construct default attribute names (e.g. "att0", "att1", ...).
"""
X_train, X_test, y_train, y_test = myutils.random_stratified_split(X_tr, y_tr)
trees = []
for jj in range(self.N):
tree = None
self.header = ["att" + str(ii) for ii in range(len(X_train[0]))]
for jj in range(len(X_train[0])):
temp = []
for ii in range(len(X_train)):
if X_train[ii][jj] not in temp:
temp.append(X_train[ii][jj])
self.domain[self.header[jj]] = temp
# my advice is to stitch together X_train and y_train
train = [X_train[i] + [y_train[i]] for i in range(len(X_train))]
bootstrapped_train = myutils.compute_bootstrapped_sample(train)
# initial call to tdidt current instances is the whole table (train)
available_attributes = self.header.copy()
tree = self.tdidt(bootstrapped_train, available_attributes)
trees.append(tree)
performances = []
for ii, tree in enumerate(trees):
counter = 0
for jj, instance in enumerate(X_test):
if self.tdidt_predict(self.header, tree, instance) == y_test[jj]:
counter+=1
performances.append(counter/len(y_test))
# Thank you stackoverflow
# https://stackoverflow.com/questions/6618515/sorting-list-based-on-values-from-another-list
sortedtrees = [x for _,x in sorted(zip(performances, trees))]
self.best_M_trees = sortedtrees[:self.M]
pass
def fit(self, X_train, y_train, X_test, y_test):
"""Fits many decision tree classifiers to X_train and y_train using the TDIDT (top down induction of decision tree) algorithm.
Args:
X_train(list of list of obj): The list of training instances (samples).
The shape of X_train is (n_train_samples, n_features)
y_train(list of obj): The target y values (parallel to X_train)
The shape of y_train is n_train_samples
X_test(list of list of obj): The list of testing samples, used to determine accuracy of trees
y_test(list of obj): The target y values (parallel to X_test), used to determine accuracy of trees
Notes:
"""
# Call train_test_split to get "test" and "remainder" data (happens before this function is called)
# Loop N times
# Call compute_bootstrapped_sample to get a subset of the "remainder" data
# Call tdidt with the bootstrapped sample to build a decision tree
# Test performance of all trees to get the M best ones
# Use majority voting to make predictions (predict method)
self.X_train = X_train
self.y_train = y_train
N_trees = []
for _ in range(self.N):
new_X_train, new_y_train = myutils.compute_bootstrapped_sample(X_train, y_train)
tree = MyDecisionTreeClassifier()
tree.fit(new_X_train, new_y_train, is_forest=True)
N_trees.append(tree)
self.trees = []
# Test tree performance
# for the M best trees, append them to self.trees
accuracy_list = []
for tree in N_trees:
accuracy = 0
predict_list = tree.predict(X_test)
for i in range(len(predict_list)):
if predict_list[i] == y_test[i]:
accuracy += 1
accuracy_list.append(myutils.calculateAccuracy(accuracy, len(y_test) - accuracy))
prev_best = 1.1
M_indexes = np.argpartition(accuracy_list, -self.M)[-self.M:]
for index in M_indexes:
self.trees.append(N_trees[index])
def fit(self, X_train, y_train, M=7, N=20, F=2):
"""Fits a random forest classifier to X_train and y_train using the TDIDT (top down induction of decision tree) algorithm.
Args:
X_train(list of list of obj): The list of training instances (samples).
The shape of X_train is (n_train_samples, n_features)
y_train(list of obj): The target y values (parallel to X_train)
The shape of y_train is n_train_samples
"""
self.X_train = copy.deepcopy(X_train)
self.y_train = copy.deepcopy(y_train)
# create random stratified test set with 2:1 ratio
X_remainder, X_test, y_remainder, y_test = myevaluation.train_test_split(
copy.deepcopy(X_train), copy.deepcopy(y_train))
for i, x in enumerate(y_remainder):
X_remainder[i].append(x)
for i, x in enumerate(y_test):
X_test[i].append(x)
# generate N random decision trees using bagging
trees = []
for i in range(N):
# print(i)
# print("getting sample and validation sets...")
# get the sample and validation sets
sample = myutils.compute_bootstrapped_sample(X_remainder)
validation_set = []
for x in X_remainder:
if x not in sample:
validation_set.append(x)
# print("length of sample and validation sets:", len(sample), len(validation_set))
# print("getting the tree...")
# get the tree from the sample
available_attributes = myutils.get_available_attributes(sample)
tree = myutils.tdidt_random_forest(
sample, [x for x in range(0,
len(sample[0]) - 1)],
available_attributes, F)
# print("testing the tree")
# test against the validation set
validation_set_x = [x[:-1] for x in validation_set]
validation_set_y = [x[-1] for x in validation_set]
predictions = []
header = []
for i in range(0, len(validation_set_x[0])):
header.append("att" + str(i))
for x, y in zip(validation_set_x, validation_set_y):
prediction = myutils.tdidt_predict(header, tree, x)
predictions.append(int(prediction == y))
# print("accuracy:", sum(predictions)/len(predictions))
trees.append({
"accuracy": sum(predictions) / len(predictions),
"tree": tree
})
# print("getting the best M trees")
# get the best M of N trees
trees = sorted(trees, key=lambda k: k["accuracy"], reverse=True)
self.trees = trees[:M]
def fit(self, X_train, y_train):
"""Fits a decision tree classifier to X_train and y_train using the TDIDT (top down induction of decision tree) algorithm.
Args:
X_train(list of list of obj): The list of training instances (samples).
The shape of X_train is (n_train_samples, n_features)
y_train(list of obj): The target y values (parallel to X_train)
The shape of y_train is n_train_samples
Notes:
Since TDIDT is an eager learning algorithm, this method builds a decision tree model
from the training data.
Build a decision tree using the nested list representation described in class.
Store the tree in the tree attribute.
Use attribute indexes to construct default attribute names (e.g. "att0", "att1", ...).
"""
# fit() accepts X_train and y_train
# TODO: calculate the attribute domains dictionary
if (self.seed != None):
random.seed(self.seed)
n_trees = []
accuracies = []
for i in range(self.N):
header = []
attribute_domains = {}
#loops through X_train and creates header
for i in range(len(X_train[0])) :
header.append("att" + str(i))
#loops though header to form attribute domains dictionairy
count = 0
for item in header:
curr_col = myutils.get_column(X_train, count)
values, counts = myutils.get_frequencies(curr_col)
attribute_domains[item] = values
count+=1
#stitching together X_train and y_train and getting available attributes
train = [X_train[i] + [y_train[i]] for i in range(len(X_train))]
available_attributes = header.copy()
boot_train = myutils.compute_bootstrapped_sample(train)
validation_set = []
for row in train:
if row not in boot_train:
validation_set.append(row)
#forming tree
tree = myutils.tdidt_forest(boot_train, available_attributes, attribute_domains, header, self.F)
#print(tree)
tree_dict = {}
tree_dict["tree"] = tree
y_test = []
for row in validation_set:
y_test.append(row.pop())
y_predict = myutils.predict_tree(validation_set, tree)
acc = myutils.get_accuracy(y_predict, y_test)
tree_dict["acc"] = acc
n_trees.append(tree_dict)
sorted_trees = sorted(n_trees, key=lambda k: k['acc'], reverse=True)
for i in range(self.M):
self.trees.append(sorted_trees[i]["tree"])
def fit(self, X_train, y_train):
self.X_train = X_train
self.y_train = y_train
# stitch together X_train and y_train so y_train is in right most column
train = [
self.X_train[i] + [self.y_train[i]]
for i in range(0, len(self.X_train))
]
trees = []
attr_sets = []
tree_accuracies = []
for i in range(self.N):
# Call bootstrap method to get different random data from data set (each instance has every column)
bootstrapped_table = myutils.compute_bootstrapped_sample(train)
bootstrapped_y = myutils.get_column_by_index(
bootstrapped_table, -1)
bootstrapped_X = myutils.remove_column(bootstrapped_table, -1)
# call train test split to get X_train, y_train, X_test, y_test
tree_X_train, tree_X_validation, tree_y_train, tree_y_validation = myutils.train_test_split(
bootstrapped_X, bootstrapped_y, 1 / 3)
# TODO: Randomly select F indices and make X_train those F columns
# print(tree_X_train[:10])
# print(tree_X_validation[:10])
# print(tree_y_train[:10])
# print(tree_y_validation[:10])
num_attributes = len(tree_X_train[0])
attr_indices = myutils.generate_F_indices(num_attributes, self.F)
attr_indices = sorted(attr_indices)
attr_sets.append(attr_indices)
subsetted_tree_X_train = myutils.attribute_subset_table(
tree_X_train, attr_indices)
# create decision tree
decision_tree = MyDecisionTreeClassifier()
# print(subsetted_tree_X_train[:10])
decision_tree.fit(subsetted_tree_X_train, tree_y_train)
trees.append(decision_tree)
# calculate tree accuracy using validation set
predicted = decision_tree.predict(tree_X_validation)
match_count = 0
for index, prediction in enumerate(predicted):
if prediction == tree_y_validation[index]:
match_count += 1
accuracy = match_count / len(predicted)
tree_accuracies.append(accuracy)
# select M best trees based on accuracies
# sort accuracies and cooresponding trees
zipped_lists = zip(trees, tree_accuracies, attr_sets)
sorted_zipped = sorted(zipped_lists, reverse=True, key=lambda x: x[1])
tuples = zip(*sorted_zipped)
sorted_trees, sorted_accuracies, sorted_attr_sets = [
list(tuple) for tuple in tuples
]
best_m_trees = sorted_trees[:self.M]
best_sorted_attr_sets = sorted_attr_sets[:self.M]
self.best_m_trees = best_m_trees
self.M_attr_sets = best_sorted_attr_sets
def fit(self, X_train, y_train):
''' Fits the random forest model to a given training set
Args:
X_train(list of list of obj): The list of training instances (samples).
The shape of X_train is (n_train_samples, n_features)
y_train(list of obj): The target y values (parallel to X_train).
The shape of y_train is n_samples
'''
self.X_train = copy.deepcopy(X_train)
self.y_train = copy.deepcopy(y_train)
self.learners = []
self.accuracies = []
# generate N learners
for i in range(self.N):
# create the bootstrap sample
if self.seed is not None:
X_sample, y_sample = myutils.compute_bootstrapped_sample(self.X_train, self.y_train, self.seed)
self.seed += 1 # increment the seed so not all the trees are the same
else:
X_sample, y_sample = myutils.compute_bootstrapped_sample(self.X_train, self.y_train)
# create the validation set
X_val = [x for x in self.X_train if x not in X_sample]
y_idxs = [self.X_train.index(x) for x in X_val]
y_val = [self.y_train[idx] for idx in y_idxs]
# get only a random subset of attributes for each sample
values = [i for i in range(len(self.X_train[0]))] # num of items in header
if self.seed is not None:
F_attributes = myutils.compute_random_subset(values, self.F, self.seed)
self.seed += 1 # increment the seed so not all the trees are the same
else:
F_attributes = myutils.compute_random_subset(values, self.F)
# get only those attributes from the training set
for i in range(len(X_sample)):
X_sample[i] = [X_sample[i][j] for j in range(len(X_sample[i])) if j in F_attributes]
# get only those attributes from the validation set
for i in range(len(X_val)):
X_val[i] = [X_val[i][j] for j in range(len(X_val[i])) if j in F_attributes]
# build a decision tree from the sample
tree = MyDecisionTreeClassifier()
tree.fit(X_sample, y_sample)
self.learners.append(tree)
# test the trees accuracy on the validation set
y_pred = tree.predict(X_val)
self.accuracies.append(myutils.compute_accuracy(y_pred, y_val))
# get only the best M learners
# sort the dists and move the indices to match the sorted list
# by combining the two lists into a list of tuples, sorting, and unpacking
sorted_accs, sorted_idxs = (list(tup) for tup in zip(*sorted(zip(self.accuracies, range(len(self.learners))))))
# slice the lists to only include the M best learners
self.learners = [self.learners[i] for i in range(len(self.learners)) if i in sorted_idxs[:self.M]]
# self.learners = sorted_learners[:M+1]
self.accuracies = sorted_accs[:self.M]