def get_ratings_genre(table, genre, rating):
"""Get list with ratings attached with given genre column
Args:
table(MyPyTable): given object of MyPyTable
genre(string): Genre to search for in table
rating(string): Service provider to pull from in get column
Returns:
list(list): list with ratings from each correctly found genre"""
genre_col = MyPyTable.get_column(table, 'Genres', True)
col = MyPyTable.get_column(table, rating, True)
list = []
for i in range(len(genre_col)):
if genre in genre_col[i]:
if rating == 'Rotten Tomatoes' and '%' in col[i]:
col[i] = float(col[i].strip('%'))
list.append(col[i])
copy_list = copy.deepcopy(list)
for value in list:
if value == '':
copy_list.remove(value)
list = copy_list
return list
def scatter_plot(table, x_column_name, y_column_name):
"""Creates a scatter plot with given data
Args:
table(MyPyTable): given table to perform operation
column_name(string): column name to get column from for scatter plot. Column on the x axis
y_column_name(string): column name to get column from for scatter plot. Column on the y axis
Returns:
coeficient(float): coeficient value
cov(float): covariance value
"""
y_col = MyPyTable.get_column(table, y_column_name, False)
x_col = MyPyTable.get_column(table, x_column_name, False)
coeficient = utils.correlation_coeficient(x_col, y_col)
cov = utils.covariance(x_col, y_col)
m, b = utils.compute_slope_intercept(x_col, y_col)
plt.scatter(x_col, y_col)
plt.plot([min(x_col), max(x_col)],
[m * min(x_col) + b, m * max(x_col) + b],
c="r",
label="corr: " + str(coeficient) + ", cov: " + str(cov))
plt.legend()
plt.plot()
plt.show()
return coeficient, cov
def get_sea_frequencies(MyPyTable, col_name):
"""Gets the frequency and count of a column by name
Args:
MyPyTable(MyPyTable): self of MyPyTable
col_name(str): name of the column
Returns:
values, counts (string, int): name of value and its frequency"""
rain_col = MyPyTable.get_column(col_name)
row_index_to_drop = []
print("range:", len(rain_col), len(MyPyTable.data))
for i in range(len(rain_col)):
if rain_col[i] == "FALSE":
row_index_to_drop.append(i)
count = 0
row_to_drop = []
for i in range(len(MyPyTable.data)):
if i in row_index_to_drop:
row_to_drop.append(MyPyTable.data[i])
MyPyTable.drop_rows(row_to_drop)
months = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']
month_col = MyPyTable.get_column('DATE')
yes_col = []
for month in months:
yes = 0
for i in range(len(month_col)):
if month in month_col[i]:
yes = yes + 1
yes_col.append(yes)
return months, yes_col
def test_random_forest_fit():
interview_header = ["level", "lang", "tweets", "phd", "interviewed_well"]
interview_table = [["Senior", "Java", "no", "no", "False"],
["Senior", "Java", "no", "yes", "False"],
["Mid", "Python", "no", "no", "True"],
["Junior", "Python", "no", "no", "True"],
["Junior", "R", "yes", "no", "True"],
["Junior", "R", "yes", "yes", "False"],
["Mid", "R", "yes", "yes", "True"],
["Senior", "Python", "no", "no", "False"],
["Senior", "R", "yes", "no", "True"],
["Junior", "Python", "yes", "no", "True"],
["Senior", "Python", "yes", "yes", "True"],
["Mid", "Python", "no", "yes", "True"],
["Mid", "Java", "yes", "no", "True"],
["Junior", "Python", "no", "yes", "False"]]
myutils.prepend_attribute_label(interview_table, interview_header)
interview_pytable = MyPyTable(column_names=interview_header,
data=interview_table)
y_col = interview_pytable.get_column("interviewed_well", False)
x_cols = interview_pytable.drop_col("interviewed_well")
many_trees = MyRandomForestClassifier()
X_sample, y_sample = myutils.compute_bootstrapped_sample(x_cols, y_col)
X_train, X_test, y_train, y_test = myutils.train_test_split(
X_sample, y_sample, .33)
many_trees.fit(X_train, y_train, X_test, y_test)
y_predicted = many_trees.predict(X_test)
numCorrectPredictions = 0
numWrongPredictions = 0
for i in range(len(y_test)):
values = [y_predicted[i], y_test[i]] #predicted/actual
if (values[0] == values[1]):
numCorrectPredictions = numCorrectPredictions + 1
else:
numWrongPredictions = numWrongPredictions + 1
accuracy = np.round((numCorrectPredictions) /
(numCorrectPredictions + numWrongPredictions), 3)
error_rate = np.round(
(numWrongPredictions) / (numCorrectPredictions + numWrongPredictions),
3)
print("-----------------------------------------------------------")
print("Accuracy and Error Rate")
print("-----------------------------------------------------------")
print()
print("Random Forest: accuracy = {}, error rate = {}".format(
accuracy, error_rate))
print()
print(
"Because of the random aspect of this classifier, this will not always pass the tests"
)
print()
print("Predicted table: " + str(y_predicted))
print("Testing set: " + str(y_test))
for i in range(len(y_test)):
assert y_predicted[i] == y_test[i]
def get_aus_frequencies(MyPyTable, col_name):
"""Gets the frequency and count of a column by name
Args:
MyPyTable(MyPyTable): self of MyPyTable
col_name(str): name of the column
Returns:
values, counts (string, int): name of value and its frequency"""
rain_col = MyPyTable.get_column("RainToday")
row_index_to_drop = []
for i in range(len(rain_col)):
if rain_col[i] == "No":
row_to_drop.append(i)
row_to_drop = []
for i in range(len(MyPyTable.data)):
if i in row_index_to_drop:
row_to_drop.append(i)
table = MyPyTable.drop_rows(rows_to_drop)
table.pretty_print()
return table
values = []
counts = []
'''for value in col:
def unique_genres(table):
"""Get list of unique genres within a table
Args:
table(MyPyTable): given object of MyPyTable
Returns:
values(list): list with unique genres"""
genre_str = ''
genre_col = MyPyTable.get_column(table, 'Genres', False)
vals, counts = get_frequencies(table, 'Genres')
for v in vals:
genre_str = genre_str + v + ','
genre_array = genre_str.split(',')
values = []
for value in genre_array:
if value != '':
if value not in values:
# haven't seen this value before
values.append(value)
elif value in values:
pass
return values
def test_random_forest_classifier_fit():
mp_table = MyPyTable(interview_header, interview_table)
# Formulate X_train and y_train
y_train = mp_table.get_column('interviewed_well')
X_train_col_names = ["level", "lang", "tweets", "phd"]
X_train = mp_table.get_rows(X_train_col_names)
myRF = MyRandomForestClassifier(N=4, M=2, F=4)
myRF.fit(X_train, y_train)
assert len(myRF.M_attr_sets) == myRF.M
def bagging(X, Y, N, M, F):
# 1. split your dataset into a test set and a "remainder set"
x_remainder, x_test, y_r, y_test = myevaluation.train_test_split(X, Y)
# 2. using the remainder set, sample N bootsrap samples and use each one to build a classifier
# for each N sample:
# ~63% of the remainder set will be sampled into training set
# ~37% will be leftover for this tree's validation set
forest = []
# accuracies = [[0] for i in range(N)]
accuracies = {}
for i in range(N):
x_train, y_train = compute_bootstrapped_sample(
x_remainder, y_r) #get the bootstrap sample
tree = my_class.MyDecisionTreeClassifier()
tree.fit(x_train, y_train, True, F) #build classifier
# get remainder of x_train and use as validation set
x_v = []
y_v = []
for j in range(len(x_remainder)):
if x_remainder[j] not in x_train:
x_v.append(x_remainder[j])
y_v.append(y_r[j])
pred = tree.predict(x_v)
accuracy = get_accuracy(y_v, pred)
accuracies[str(i)] = accuracy # {i: accuracy, }
forest.append(tree)
# 3. measure the performance of the tree on the validation set and select the best M of N
# trees based on the performance metrics
best_trees_dict = best_M(M, accuracies)
best_trees = []
for key in best_trees_dict:
best_trees.append(forest[int(key)])
# 4. using majority voting, make predictions from the M learners for each instance in the test set
all_predictions = [] # [[predictions1],[predictions2]...]
for tree in best_trees:
pred = tree.predict(x_test)
all_predictions.append(pred) #think about this like flipping a table
#get the majority for every single row
pred_header = build_header(
all_predictions) #turn all predictions into a mypy
pred_mypy = MyPyTable(pred_header, all_predictions)
voted_predictions = []
for i in range(
len(all_predictions[0])
): #loop through every x_test, create a column of predictions, pick the pred by majority rule
pred_col = pred_mypy.get_column(i)
vals, counts = get_freq_str(pred_col)
j = counts.index(max(counts))
y_predict = vals[j]
voted_predictions.append(y_predict)
forest_accuracy = get_accuracy(y_test, voted_predictions)
return best_trees, voted_predictions, forest_accuracy
def hist_graph(table, column_name):
"""Creates a histogram graph with given data
Args:
table(MyPyTable): given table to perform operation
column_name(string): column name to get column from for hist graph
"""
col = MyPyTable.get_column(table, column_name, False)
plt.hist(col, bins=10)
plt.show()
def pie_chart_dataPrep(table, cols_to_plot):
"""
"""
totals_list = []
for col_name in cols_to_plot:
column_Category = MyPyTable.get_column(table, str(col_name))
total = 0
for val in column_Category:
total = total + float(val)
totals_list.append(total)
return totals_list
def get_year_counts(table, platform):
"""Get years of occuring platform game occurences along with their individual frequencies
Args:
table(MyPyTable): given object of MyPyTable
platform(string): platform to search for in table
Returns:
values, counts (string, int): name of value and its frequency"
"""
plat_col = MyPyTable.get_column(table, 'Platform', True)
col = MyPyTable.get_column(table, "Year", True)
list = []
for i in range(len(plat_col)):
if plat_col[i] == platform:
list.append(col[i])
copy_list = copy.deepcopy(list)
for value in list:
if value == 'N/A':
copy_list.remove(value)
list = copy_list
list.sort()
values = []
counts = []
for value in list:
if value not in values:
# haven't seen this value before
values.append(value)
counts.append(1)
elif value in values:
index = values.index(value)
counts[index] += 1
return values, counts
def convert_attributes(table):
"""Converts IMDb to double digit float and Rotten Tomatoes to string without %
Args:
table(MyPyTable): given object of MyPyTable
Returns:
imbd_col(list): IMDb list in double digits
rotten_col(list): Rotten Tomatoes list stripped of %"""
#IMDb conversion
col = MyPyTable.get_column(table, 'IMDb', False)
rotten_col = MyPyTable.get_column(table, 'Rotten Tomatoes', False)
imbd_col = []
for i in col:
i = i * 10
imbd_col.append(i)
#rotten tomatoes conversion
for a, x in enumerate(rotten_col):
rotten_col[a] = float(x[:-1])
return imbd_col, rotten_col
def get_mpg_frequencies(MyPyTable, col_name):
"""Gets the frequency and count of a column by name
Args:
MyPyTable(MyPyTable): self of MyPyTable
col_name(str): name of the column
Returns:
values, counts (string, int): name of value and its frequency"""
col = MyPyTable.get_column(col_name)
values = []
counts = []
for value in col:
if value not in values:
# haven't seen this value before
if value >= 13 and value < 14:
values, counts = mpg_val_check(1, values, counts, value)
elif value == 14:
values, counts = mpg_val_check(2, values, counts, value)
elif value > 14 and value <= 16:
values, counts = mpg_val_check(3, values, counts, value)
elif value > 16 and value <= 19:
values, counts = mpg_val_check(4, values, counts, value)
elif value > 19 and value <= 23:
values, counts = mpg_val_check(5, values, counts, value)
elif value > 23 and value <= 26:
values, counts = mpg_val_check(6, values, counts, value)
elif value > 26 and value <= 30:
values, counts = mpg_val_check(7, values, counts, value)
elif value > 30 and value <= 36:
values, counts = mpg_val_check(8, values, counts, value)
elif value > 36 and value <= 44:
values, counts = mpg_val_check(9, values, counts, value)
elif value >= 45:
values, counts = mpg_val_check(10, values, counts, value)
temp_counts = copy.deepcopy(counts)
#re-order/sort values and temp_counts
for i in range(len(values)):
index = values[i]
temp_counts[index - 1] = counts[i]
values.sort()
counts = temp_counts
return values, counts
def compute_entropy(instances, available_attributes, index):
mypy = MyPyTable(available_attributes, instances)
classes = mypy.get_column(-1)
attributes = mypy.get_column(index)
temp = set(attributes)
__, tables = group_by(attributes, classes)
totals = []
sub_entropies = []
# get the class counts here
for jj, element in enumerate(temp):
totals.append(attributes.count(element))
# parallel array of counts of each att for each class
arr = []
for table in tables:
arr.append(table.count(element))
su = 0
for kk in arr:
if kk <= 0:
pass
else:
su -= kk / totals[jj] * math.log2(kk / totals[jj])
su *= totals[jj] / len(attributes)
sub_entropies.append(su)
return sum(sub_entropies)
def percent_hist_graph(table, column_name):
"""Creates a histogram graph with given data and removes the percent sign from given column_names
Args:
table(MyPyTable): given table to perform operation
column_name(string): column name to get column from for hist graph
"""
col = MyPyTable.get_column(table, column_name, False)
for i, x in enumerate(col):
col[i] = float(x[:-1])
plt.hist(col, bins=10)
plt.show()
def percentages_columns(table, column_names):
"""Gives the percentage of each column's frequency divided by total column length
Args:
table(MyPyTable): given object of MyPyTable
column_names(list): List of string column names
Returns:
percentages(list): list of percentages in each correct index matching with the given columns list"""
counts = get_occurences_given_columns(table, column_names)
percentages = []
col = MyPyTable.get_column(table, column_names[0], False)
length = len(col)
for count in counts:
percentages.append(round((count / length) * 100, 0))
return percentages
def test_My_Random_Forest_Classifier_fit():
# Object Declarations
# Tests with N = 3, M = 2, F = 2 and seed = 0
rand_forest_test = MyRandomForestClassifier(3, 2, 2, 0)
table = MyPyTable()
# Variable Assignment and Declaration
table.data = interview_table
table.column_names = interview_header
X_test = interview_table
y_train = table.get_column("interviewed_well")
# Tests on the Interview Dataset
rand_forest_test.header = interview_header
rand_forest_test.fit(X_test, y_train)
trees = rand_forest_test.trees
def test_random_forest_classifier_predict():
X_test = [["Mid", "Python", "no", "no", "True"],
["Mid", "R", "yes", "yes", "True"],
["Mid", "Python", "no", "yes", "True"]]
y_test = ["True", "True", "True"]
mp_table = MyPyTable(interview_header, interview_table)
# Formulate X_train and y_train
y_train = mp_table.get_column('interviewed_well')
X_train_col_names = ["level", "lang", "tweets", "phd"]
X_train = mp_table.get_rows(X_train_col_names)
myRF = MyRandomForestClassifier(N=4, M=2, F=4)
myRF.fit(X_train, y_train)
predictions = myRF.predict(X_test)
for i in range(0, len(predictions)):
assert predictions[i] == y_test[i]
def random_forest_predict(X_test, trees):
# 4. using majority voting, make predictions from the M learners for each instance in the test set
all_predictions = [] # [[predictions1],[predictions2]...]
for tree in trees:
pred = tree.predict(X_test)
all_predictions.append(pred) #think about this like flipping a table
#get the majority for every single row
pred_header = build_header(all_predictions) #turn all predictions into a mypy
pred_mypy = MyPyTable(pred_header, all_predictions)
voted_predictions = []
for i in range(len(all_predictions[0])): #loop through every x_test, create a column of predictions, pick the pred by majority rule
pred_col = pred_mypy.get_column(i)
vals, counts = get_freq_str(pred_col)
j = counts.index(max(counts))
y_predict = vals[j]
voted_predictions.append(y_predict)
# forest_accuracy = get_accuracy(y_test, voted_predictions)
return voted_predictions
def get_occurences_given_columns(table, column_names):
"""Gets the occurence from each column in a given columns list.
Args:
table(MyPyTable): given object of MyPyTable
column_names(list): List of string column names
Returns:
count(list): list of frequencies in each correct index matching with the given columns list"""
column = []
count = []
for i in range(len(column_names)):
count.append(0)
for col in column_names:
attributes = MyPyTable.get_column(table, col, False)
column.append(attributes)
for i in range(len(column)):
for j in column[i]:
if j == 1.0:
count[i] = count[i] + 1
return count
def get_frequencies(MyPyTable, col_name):
"""Gets the frequency and count of a column by name
Args:
MyPyTable(MyPyTable): self of MyPyTable
col_name(str): name of the column
Returns:
values, counts (string, int): name of value and its frequency"""
col = MyPyTable.get_column(col_name)
values = []
counts = []
for value in col:
if value not in values:
# haven't seen this value before
values.append(value)
counts.append(1)
elif value in values:
index = values.index(value)
counts[index] += 1
return values, counts
from mysklearn.mypytable import MyPyTable
from mysklearn.myclassifiers import MyDecisionTreeClassifier, MyNaiveBayesClassifier
import mysklearn.myevaluation as myevaluation
import mysklearn.myutils as myutils
import os
# "pickle" an object (AKA object serialization)
# save a Python object to a binary file
# "unpickle" an object (AKA object de-serialization)
# load a Python object from a binary file (back into memory)
# Get data from csv file
table = MyPyTable().load_from_file(
os.path.join("input_files", "winequality-red.csv"))
y_col = table.get_column("quality", False)
x_cols = table.drop_col("quality")
# Use Naive Bayes to classify
testcase = MyNaiveBayesClassifier()
#Returns x INDEXES
X_train, X_test = myevaluation.stratified_kfold_cross_validation(x_cols,
y_col,
n_splits=10)
X_train, X_test, y_train, y_test = myutils.getInstances(
X_train, X_test, x_cols, y_col)
for i, fold in enumerate(X_train):
train, test = myutils.normalize_values(X_train[i], X_test[i])
testcase.fit(train, y_train[i])
from mysklearn.myclassifiers import MyKNeighborsClassifier
import os
from mysklearn.mypytable import MyPyTable
import mysklearn.myevaluation as myeval
import mysklearn.myutils as myutils
import pickle
# Importing the data and table and cols
movies_fname = os.path.join("input_data", "movies.csv")
# movie_data = MyPyTable().load_from_file_no_encode(movies_fname)
movies_table = MyPyTable().load_from_file(movies_fname, encode='cp1252')
# Getting profit
gross_profit = [
movies_table.get_column('gross')[i] - movies_table.get_column('budget')[i]
for i in range(len(movies_table.data))
]
profitted = [0 if gross < 0 else 1 for gross in gross_profit]
movies_table.add_column(profitted, 'profitted')
# fit the KNN algorithm to the movies data
kn_class = MyKNeighborsClassifier()
feature_cols = [
'budget', 'votes', 'genre', 'rating', 'score', 'star', 'director', 'writer'
]
features = movies_table.get_key_columns(feature_cols)
outcomes = profitted
kn_class.fit(features, outcomes)
packaged_object = kn_class
#heavily based on app from class
import pickle
from mysklearn.myclassifiers import MyRandomForestClassifier
from mysklearn.mypytable import MyPyTable
import os
fname = os.path.join("input_data", "tracks_data_backup.txt")
tracks = MyPyTable().load_from_file(fname)
Danceability = tracks.get_column('danceability')
Energy = tracks.get_column('energy')
Acousticness = tracks.get_column('acousticness')
Valence = tracks.get_column('valence')
y_train = Acousticness
x_train = [[Danceability[i], Energy[i], Valence[i]]
for i in range(len(y_train))]
rf = MyRandomForestClassifier()
rf.fit(x_train, y_train, 20, 7, 2)
rf = MyRandomForestClassifier()
rf.fit(x_train, y_train, 30, 4, 2)
# serialize to file (pickle)
outfile = open("trees.p", "wb")
pickle.dump(rf.trees, outfile)
outfile.close()
# deserialize to object (unpickle)
infile = open("trees.p", "rb")
trees2 = pickle.load(infile)
infile.close()
from mysklearn.myclassifiers import MyNaiveBayesClassifier
import os
from mysklearn.mypytable import MyPyTable
import mysklearn.myevaluation as myevaluation
import mysklearn.myutils as myutils
import pickle
fname = os.path.join("input_data", "collisions.csv")
collisions_data = MyPyTable().load_from_file(fname)
weather = collisions_data.get_column('WEATHER')
road_condition = collisions_data.get_column('ROADCOND')
light_condition = collisions_data.get_column('LIGHTCOND')
junction_type = collisions_data.get_column('JUNCTIONTYPE')
severity = collisions_data.get_column('SEVERITYDESC')
X_train = [[
weather[i], road_condition[i], light_condition[i], junction_type[i],
severity[i]
] for i in range(len(weather))]
y_train = collisions_data.get_column('COLLISIONTYPE')
for i, val in enumerate(y_train):
if val == 'Unknown':
del y_train[i]
del X_train[i]
strattrain_folds, strattest_folds = myevaluation.stratified_kfold_cross_validation(
X_train, y_train, 10)
strat_xtrain, strat_ytrain, strat_xtest, strat_ytest = myutils.get_from_folds(
X_train, y_train, strattrain_folds, strattest_folds)