def __init__(self,
w_size=100,
input_size=12,
layers=1,
n_itr=50,
learn=0.05,
AutoEncoder=False,
adaption=0.1):
self.layers = list()
#setup layers HIGHLY TENTETIVE AND SUBJECT TO CHANGE
for i in range(layers):
layer = mlp.Layer('Rectifier', units=input_size)
self.layers.append(layer)
self.layers.append(mlp.Layer('Softmax'))
self.learner = mlp.Classifier(self.layers,
learning_rate=learn,
n_iter=n_itr)
self.input_size = input_size
self.w_size = self.input_size * w_size
self.data = list()
self.returns = list()
self.labels = list()
self.tstep = 0
self.sharpeA = 1
self.sharpeB = 1
self.adaption = adaption
self.std = 1
return
else:
params.append(values[:1])
# Build the classifiers for all possible combinations of parameters.
names = []
classifiers = []
for (activation, alpha, dropout, iterations, output, regularize, rule,
units) in itertools.product(*params):
params = {'pieces': 2} if activation == "Maxout" else {}
classifiers.append(
mlp.Classifier(layers=[
mlp.Layer(activation, units=units, **params),
mlp.Layer(output)
],
random_state=1,
n_iter=iterations,
n_stable=iterations,
regularize=regularize,
dropout_rate=dropout,
learning_rule=rule,
learning_rate=alpha), )
t = []
for k, v in zip(sorted(PARAMETERS), [
activation, alpha, dropout, iterations, output, regularize, rule,
units
]):
if k in args.params:
t.append(str(v))
names.append(','.join(t))
def create_estimator(estimator_name, class_weight):
estimator = None
param_grid = None
support_class_weight = False
if estimator_name == "logistic_regression":
from sklearn import linear_model
estimator = linear_model.LogisticRegression(class_weight=class_weight)
param_grid = {"C": np.logspace(-3, 4, 20)}
support_class_weight = True
elif estimator_name == "random_forest":
estimator = ensemble.RandomForestClassifier(class_weight=class_weight)
param_grid = {
"n_estimators": list(range(10, 110, 10)),
"max_features": ("auto", 0.5, 0.8, None)
# "max_features": np.arange(int(np.sqrt(n_features)), n_features, step=4)
}
support_class_weight = True
# support_class_weight = False
elif estimator_name == "gradient_boosting":
"""
import xgboost.sklearn as xgb
estimator = xgb.XGBClassifier(learning_rate=0.1)
param_grid = {
# "n_estimators": list(range(150, 250, 10)),
# "max_depth": list(range(3, 8))
}
"""
# for some unknown reason, XGBoost does not perform well on my machine and hangs sometimes
# fallback to use the less efficient implementation in sklearn.
estimator = ensemble.GradientBoostingClassifier(learning_rate=0.1,
warm_start=True)
param_grid = {
"n_estimators": list(range(150, 250, 10)),
"max_depth": list(range(3, 8))
}
elif estimator_name == "adaboost":
estimator = ensemble.AdaBoostClassifier()
param_grid = {
"n_estimators": list(range(30, 150, 10)),
"learning_rate": np.logspace(-1, 0, 2)
}
elif estimator_name.startswith("svc_"):
subtype = estimator_name[4:]
from sklearn import svm
if subtype == "linear": # linear SVC uses liblinear insteaed of libsvm internally, which is more efficient
param_grid = {
"C": np.logspace(-6, 2, 50),
}
estimator = svm.LinearSVC(
dual=
False, # dual=False when n_samples > n_features according to the API doc.
class_weight=class_weight)
else:
estimator = svm.SVC(
shrinking=False,
cache_size=2048,
verbose=False,
probability=False, # use True when predict_proba() is needed
class_weight=class_weight)
if subtype == "rbf":
estimator.set_params(kernel="rbf")
param_grid = {
"C": np.logspace(-2, 2, 20),
"gamma": np.logspace(-2, -1, 3)
}
else: # poly
estimator.set_params(kernel="poly")
param_grid = {"degree": [2], "C": np.logspace(-3, 1, 20)}
support_class_weight = True
elif estimator_name == "mlp1" or estimator_name == "mlp2": # multiple layer perceptron neural network
from sknn import mlp
param_grid = {
"learning_rate": [0.0001],
"regularize": ["l2"], # , "dropout"],
"weight_decay": np.logspace(-6, -5,
2), # parameter for L2 regularizer
"hidden0__type": ["Tanh"] # "Rectifier", "Sigmoid"
}
layers = [mlp.Layer(type="Tanh", name="hidden0")]
# add the second hidden layer as needed
if estimator_name == "mlp2": # 2 hidden layer
layers.append(mlp.Layer(type="Tanh", name="hidden1"))
param_grid["hidden0__units"] = list(range(2, 5, 1))
param_grid["hidden1__units"] = list(range(2, 5, 1))
param_grid["hidden1__type"] = ["Tanh"] # "Rectifier", "Sigmoid"
else:
param_grid["hidden0__units"] = list(range(5, 26, 1))
# add the output layer
layers.append(mlp.Layer("Softmax"))
estimator = mlp.Classifier(layers=layers, batch_size=150)
return estimator, param_grid, support_class_weight
print(" - %i unsure images.\n" % len(unknown))
ds = Dataset()
ds.store(negative, 0, times=1)
ds.store(positive, 1, times=1)
ds.store(unknown, 2, times=2)
X, y = ds.toarray()
nn = mlp.Classifier(layers=[
mlp.Layer("Rectifier", units=48, dropout=0.3),
mlp.Layer("Rectifier", units=32, dropout=0.1),
mlp.Layer("Rectifier", units=24),
mlp.Layer("Softmax")
],
learning_rate=0.01,
learning_rule='adagrad',
n_iter=10,
n_stable=10,
batch_size=50,
valid_set=(X, y),
verbose=1)
try:
nn.fit(X, y)
except KeyboardInterrupt:
pass
score = nn.score(X, y)
print('SCORE:', score * 100.0)
for index, layer in enumerate(weights):
if layer[2] == "output":
#print("deleting", weights[index])
del weights[index]
glob_rf = None
glob_rf = mlp.Classifier(
layers=[
mlp.Native(lasagne.DenseLayer, num_units=1024, nonlinearity=nl.very_leaky_rectify),
mlp.Native(lasagne.DenseLayer, num_units=512, nonlinearity=nl.very_leaky_rectify),
mlp.Native(lasagne.DenseLayer, num_units=256, nonlinearity=nl.very_leaky_rectify),
mlp.Layer("Softmax")],
learning_rate=.005,
n_iter=1,
learning_rule="adadelta",
callback={'on_epoch_finish': store_stats},
loss_type='mcc',
regularize="L1", # possibly L1, to instead filter out useless inputs. L1 gave 5+ in results?
weight_decay=.00001, # default .0001 increase to combat overfitting.
dropout_rate=0, # keep 80% of neurons/inputs at .2, anti overfit
verbose=True,
valid_set=(testX, testY),
batch_size=1) # TRIED NON-1, DIDN'T WORK AT ALL TODO this block needs to be an exact clone of the one above
glob_rf.set_parameters(weights)
glob_rf.valid_set = (testX, testY)
#glob_rf.hidden0 = None
#glob_rf.hidden1 = None
#glob_rf.hidden2 = None
errors = []
# )
nn = None
layers = [
mlp.Convolution("Rectifier", channels=10, kernel_shape=(2, 2)),
mlp.Layer("Rectifier", units=1000),
mlp.Layer("Softmax", units=10)
]
if nn is None:
trainFrame = dc.loadTrainData(describe=False)
trainData = dc.convertPandasDataFrameToNumpyArray(trainFrame)
nn = mlp.Classifier(layers=layers,
learning_rate=0.00001,
valid_size=0,
random_state=0,
n_iter=50,
verbose=True,
batch_size=1000,
learning_rule="nesterov")
nn.fit(trainData[:, 1:], trainData[:, 0])
print("Model fitting complete")
pickle.dump(nn, open("simplenn.pkl", "wb"))
print("Model saved")
testFrame = dc.loadTestData()
testData = dc.convertPandasDataFrameToNumpyArray(testFrame)
testX = testData[:, 0:]
yPred = nn.predict(testX)
data_train = data_train.astype('float') / 255.
labels_train = labels_train
data_test = dataset3['data'].astype('float') / 255.
labels_test = np.array(dataset3['labels'])
n_feat = data_train.shape[1]
n_targets = labels_train.max() + 1
from sknn import mlp
nn = mlp.Classifier(layers=[
mlp.Layer("Tanh", units=n_feat * 2 / 3),
mlp.Layer("Sigmoid", units=n_feat * 1 / 3),
mlp.Layer("Softmax", units=n_targets)
],
n_iter=50,
n_stable=10,
learning_rate=0.001,
valid_size=0.5,
verbose=1)
if PRETRAIN:
from sknn import ae
ae = ae.AutoEncoder(layers=[
ae.Layer("Tanh", units=n_feat * 2 / 3),
ae.Layer("Sigmoid", units=n_feat * 2 / 3)
],
learning_rate=0.002,
n_iter=10,
verbose=1)
ae.fit(data_train)
data_train = data_train.astype('float') / 255.
labels_train = labels_train
data_test = dataset0['data'].astype('float') / 255.
labels_test = np.array(dataset0['labels'])
n_feat = data_train.shape[1]
n_targets = labels_train.max() + 1
from sknn import mlp
nn = mlp.Classifier(layers=[
mlp.Layer("Tanh", units=n_feat / 8),
mlp.Layer("Sigmoid", units=n_feat / 16),
mlp.Layer("Softmax", units=n_targets)
],
n_iter=50,
n_stable=10,
learning_rate=0.002,
learning_rule="momentum",
valid_size=0.1,
verbose=1)
if PRETRAIN:
from sknn import ae
ae = ae.AutoEncoder(layers=[
ae.Layer("Tanh", units=n_feat / 8),
ae.Layer("Sigmoid", units=n_feat / 16)
],
learning_rate=0.002,
n_iter=10,
verbose=1)
clf = DBN([X_train.shape[1], 300, 10],
learn_rates=0.3,
learn_rate_decays=0.9,
epochs=10,
verbose=1)
classifiers.append(('nolearn.dbn', clf))
if 'sknn' in sys.argv:
from sknn import mlp
clf = mlp.Classifier(
layers=[mlp.Layer("Rectifier", units=300),
mlp.Layer("Softmax")],
learning_rate=0.02,
learning_rule='momentum',
batch_size=25,
valid_size=0.0,
n_stable=10,
n_iter=10,
verbose=1,
)
classifiers.append(('sknn.mlp', clf))
if 'lasagne' in sys.argv:
from nolearn.lasagne import NeuralNet
from lasagne.layers import InputLayer, DenseLayer
from lasagne.nonlinearities import softmax
from lasagne.updates import nesterov_momentum
clf = NeuralNet(layers=[
('input', InputLayer),
def fineTuneModel(_all_country_data_with_algos):
print(
"\n \n Fine Tuning Parameters for the top 3 predictive algorithms for each country for each sub dataset split by Mentality/Business Cycle "
)
all_country_data_with_algos = copy.deepcopy(_all_country_data_with_algos)
algos_dict = {
"LDA":
LinearDiscriminantAnalysis(),
"RC":
linear_model.RidgeClassifier(),
"LogR":
linear_model.LogisticRegression(),
"KNN":
neighbors.KNeighborsClassifier(),
"SVM":
svm.LinearSVC(),
"RF":
ensemble.RandomForestClassifier(verbose=0),
"GBC":
ensemble.GradientBoostingClassifier(verbose=0),
"NN":
mlp.Classifier(
layers=[mlp.Layer('Rectifier', units=7),
mlp.Layer("Softmax", )]),
"PAC":
linear_model.PassiveAggressiveClassifier(),
"SGD":
linear_model.SGDClassifier()
}
cv_folds = 3
n_jobs_count = np.arange(1, 2)
results = {}
for country in all_country_data_with_algos.keys():
for _bus_cycle in all_country_data_with_algos[country]:
X = all_country_data_with_algos[country][_bus_cycle].get("X")
Y = all_country_data_with_algos[country][_bus_cycle].get("Y")
all_country_data_with_algos[country][_bus_cycle].update(
{"trained algos": []})
for _algo in all_country_data_with_algos[country][_bus_cycle].get(
"algos"):
#Possible parameters for each var Parameters
_parameters = {}
if _algo == "LDA":
lda_n_components = np.arange(2, 8, 1)
shrinkage = ['auto']
lda_solver = ['lsqr', 'eigen']
_parameters.update({
'n_components': lda_n_components,
'solver': lda_solver,
'shrinkage': shrinkage
})
if _algo == "RC":
rc_class_weight = ['balanced']
rc_solver = ['saga', 'sparse_cg', 'svd']
alpha = np.arange(0.5, 4.5, 0.5)
_parameters.update({
'class_weight': rc_class_weight,
'solver': rc_solver,
'alpha': alpha
})
if _algo == "LogR":
lr_penalty = ['l1', 'l2']
lr_class_weight = ['balanced']
lr_solver = ['liblinear']
_parameters.update({
'penalty': lr_penalty,
'class_weight': lr_class_weight,
'solver': lr_solver,
'random_state': [seed]
})
if _algo == "KNN":
knn_neighbors = np.arange(2, 13, 1)
knn_weights = ['uniform', 'distance']
knn_leaf_size = np.arange(10, 30, 2)
_parameters.update({
'n_neighbors': knn_neighbors,
'weights': knn_weights,
'leaf_size': knn_leaf_size
})
if _algo == "SVM":
##put change of kernel in after
svm_weights = ['balanced']
dual = [False]
_parameters.update({
'class_weight': svm_weights,
'dual': dual,
'random_state': [seed]
})
if _algo == "RF":
rf_max_depth = np.arange(1, 5, 1)
n_estimators = np.asarray([200])
min_samples_leaf = np.arange(3, 6, 1)
min_samples_split = np.arange(3, 5, 1)
max_features = ["sqrt"]
_parameters.update({
'max_depth': rf_max_depth,
'n_estimators': n_estimators,
'min_samples_leaf': min_samples_leaf,
'min_samples_split': min_samples_split,
'max_features': max_features,
'random_state': [seed]
})
if _algo == "GBC":
gb_loss = ['deviance']
gb_max_depth = np.arange(1, 5, 1)
n_estimators = np.asarray([200])
min_samples_leaf = np.arange(3, 6, 1)
min_samples_leaf = np.arange(3, 6, 1)
min_samples_split = np.arange(3, 6, 1)
max_features = ["sqrt"]
_parameters.update({
'loss': gb_loss,
'max_depth': gb_max_depth,
'min_samples_leaf': min_samples_leaf,
'n_estimators': n_estimators,
'min_samples_leaf': min_samples_leaf,
'max_features': max_features,
'random_state': [seed]
})
if _algo == "NN":
layer_1 = [
mlp.Layer(type="Sigmoid", units=7, name="layer1"),
mlp.Layer(type="Softmax", name="layer2")
]
#mlp.Layer('Rectifier',units=5)
nn_layers = [layer_1]
nn_regularize = ['L1']
learning_rate = [0.01]
n_iter = [1000]
weight_decay = [0.01]
learning_rule = ['adadelta']
momentum = [0.90]
n_stable = np.arange(150, 151, 2)
f_stable = [0.001]
dropout_rate = np.asarray([0, 0.25, 0.5])
random_state = [seed]
nn_params = {
'layers': nn_layers,
'regularize': nn_regularize,
'learning_rate': learning_rate,
'n_iter': n_iter,
'learning_rule': learning_rule,
'n_iter': n_iter,
'weight_decay': weight_decay,
'learning_momentum': momentum,
'n_stable': n_stable,
'random_state': random_state
} #hidden layer size should be average of input layer and output layer
_parameters.update(nn_params)
if _algo == "PAC":
class_weight = ['balanced']
max_iter = np.arange(1000, 10001, 1)
_parameters.update({
'class_weight': class_weight,
'max_iter': max_iter,
'random_state': [seed]
})
if _algo == "SGD":
loss = ['squared_hinge', 'hinge']
class_weight = ['balanced']
penalty = ['l2', 'l1', 'elasticnet']
_parameters.update({
'loss': loss,
'class_weight': class_weight,
'max_iter': [1000],
'penalty': penalty,
'random_state': [seed]
})
_grid = model_selection.GridSearchCV(algos_dict.get(_algo),
param_grid=_parameters,
cv=cv_folds,
n_jobs=1)
_grid.fit(np.array(X), Y.as_matrix().flatten())
trained_algo = _grid.best_estimator_
all_country_data_with_algos[country][_bus_cycle][
"trained algos"].append(trained_algo)
return all_country_data_with_algos
def testingAlgoTypes(_all_country_data, MP4, verbose=0):
print(
"\n \n \n Testing various untrained classification algorithms on each country's seperate sub datasets "
)
all_country_data_with_algos = copy.deepcopy(_all_country_data)
##parameters for NeuralNet
nn_layers = [
mlp.Layer('Sigmoid', units=7, name="Layer1"),
mlp.Layer("Softmax", )
]
nn_params = {
'layers': nn_layers,
'learning_momentum': 0.9,
'n_stable': 10,
'f_stable': 0.01,
'learning_rate': 0.001,
'learning_rule': 'adadelta',
'random_state': seed,
'n_iter': 8,
'batch_size': 100,
'warning': None,
'verbose': None,
'debug': False
}
max_iter_params = {'max_iter': 1000}
classifiers = [
LinearDiscriminantAnalysis(solver='eigen', shrinkage='auto'),
linear_model.RidgeClassifier(random_state=seed),
linear_model.LogisticRegression(solver='saga',
penalty='l2',
class_weight='balanced',
random_state=seed),
neighbors.KNeighborsClassifier(n_neighbors=9,
weights='distance',
leaf_size=20),
svm.LinearSVC(class_weight='balanced', random_state=seed, dual=False),
ensemble.RandomForestClassifier(n_estimators=200,
min_samples_split=5,
min_samples_leaf=3,
max_depth=3,
random_state=seed),
ensemble.GradientBoostingClassifier(random_state=seed,
n_estimators=200,
min_samples_split=5,
max_features='sqrt'),
mlp.Classifier(**nn_params),
linear_model.PassiveAggressiveClassifier(max_iter=1000,
random_state=seed,
class_weight="balanced"),
linear_model.SGDClassifier(max_iter=1000,
random_state=seed,
class_weight='balanced',
penalty='l2')
]
headers = [
'LDA', 'RC', 'LogR', 'KNN', 'SVM', 'RF', 'GBC', 'NN', 'PAC', 'SGD'
]
for country in all_country_data_with_algos.keys():
df_cv_results = pd.DataFrame(columns=headers)
for _bus_cycle in all_country_data_with_algos[country].keys(
): #iterating through the different business cycles
means_vars_for_clf = []
result_all_clf = []
Y_target = all_country_data_with_algos[country][_bus_cycle].get(
"Y")
X_features = all_country_data_with_algos[country][_bus_cycle].get(
"X")
for _clf in classifiers:
##Creating Pipelines
#standardizer = ('standardize',preprocessing.StandardScaler())
algo = ('clf', _clf)
steps = []
#steps.append(standardizer)
steps.append(algo)
pipeline_clf = pipeline.Pipeline(steps)
kfold = model_selection.KFold(n_splits=2,
random_state=seed,
shuffle=True)
result_clf = model_selection.cross_val_score(
pipeline_clf,
np.array(X_features),
Y_target.values.ravel(),
cv=kfold,
n_jobs=1)
result_all_clf = result_all_clf + [
result_clf.mean()
] ##used to find top 3 methods
means_vars_for_clf = means_vars_for_clf + [
"{0:.3g}".format(result_clf.mean())
] ##used for excel sheet
df_cv_results.loc[
"{}-{}".format(country, _bus_cycle), :] = means_vars_for_clf
##gathering names of top three algos to be inserted into all_country_data dictionary
top3 = sorted(result_all_clf, reverse=True)[:3]
indexes_of_top_3 = [result_all_clf.index(x) for x in top3]
top_3_algos_by_mean = [headers[x] for x in indexes_of_top_3
] ##stored as 3 letter abbreviation of algo
all_country_data_with_algos[country][_bus_cycle].update(
{"algos": top_3_algos_by_mean})
if MP4 == True:
df_cv_results.to_excel(
'../Reserach/Classifier Cross Validation Scores For All Countries/All/'
+ country + '.xlsx',
index=False)
if MP4 == "Only":
df_cv_results.to_excel(
'../Reserach/Classifier Cross Validation Scores For All Countries/Only/'
+ country + '.xlsx',
index=False)
if MP4 == False:
df_cv_results.to_excel(
'../Reserach/Classifier Cross Validation Scores For All Countries/Excl/'
+ country + '.xlsx',
index=False)
if verbose > 0:
print(df_cv_results)
print("\n")
saveTopThreeAlgos(all_country_data_with_algos)
return all_country_data_with_algos
# ------------------------------------------------------------------------------
if __name__ == '__main__':
# Load dataset
featureVecs, labels, numFeatures, numLabelTypes = loadArffDataset(
'data/faces_vegetables_dataset.arff', normalise=True, displayData=True)
# Construct all classifiers we wish to test, with 'standard' parameters
classifiers = {
'SVM':
svm.SVC(kernel='linear', C=1),
'Decision Tree':
tree.DecisionTreeClassifier(criterion='gini', splitter='best'),
'Feed-Forward Neural Network (Sigmoid)':
mlp.Classifier(layers=[
mlp.Layer('Sigmoid', units=numFeatures),
mlp.Layer('Sigmoid', units=numLabelTypes),
],
n_iter=100),
'Gaussian Naive Bayes':
naive_bayes.GaussianNB(),
'Multi-Nomial Naive Bayes':
naive_bayes.MultinomialNB(),
'Bernoulli Naive Bayes':
naive_bayes.BernoulliNB(),
}
# Test classifiers and compute their mean scores
results = evaluateClassifiers(classifiers, featureVecs, labels, 10)
scores = computeOverallScores(results)
# Output scores in tabular format
def run_neural_net(training_features, training_labels, test_features,
test_labels):
"""
Classifies the data using pybrain's neural net
Parameters
----------
training_data: data used to train the classifier. For each row, item 0 assumed to be the label
test_data: data used to test the classifier. For each row, item 0 assumed to be the label
hidden_units: sets the hidden unit count for the neural net
training_epochs: sets the training epochs for the neural net
training_iterations: # of training loops
Returns
-------
prediction: predicted labels of the test data
accuracy: percent of test data labels accurately predicted
"""
time_1 = time.time()
#set the number of classes in the data
number_of_outputs = training_labels.astype(int).max() + 1
number_of_inputs = training_features.shape[1]
#determine optimal hidden nodes based on Huang et al. (2003)
first_layer_nodes = int(
math.sqrt((number_of_outputs + 2) * number_of_inputs) +
2 * math.sqrt(number_of_inputs / (number_of_outputs + 2)))
second_layer_nodes = int(number_of_outputs *
math.sqrt(number_of_inputs /
(number_of_outputs + 2)))
#set up the layers
input_layer = mlp_nn.Layer("Linear", units=number_of_inputs)
hidden_layer1 = mlp_nn.Layer("Sigmoid", units=first_layer_nodes)
hidden_layer2 = mlp_nn.Layer("Sigmoid", units=second_layer_nodes)
output_layer = mlp_nn.Layer("Softmax", units=number_of_outputs)
layers = [input_layer, hidden_layer1, hidden_layer2, output_layer]
#set up the classifier
neural_net = mlp_nn.Classifier(layers=layers, learning_rate=0.02, n_iter=5)
#set up tuning parameters
parameters = {"learning_rate": [0.02], "n_iter": [1, 5, 10, 25, 50]}
#create cross validation iterator
cv = ShuffleSplit(training_features.shape[0],
n_iter=5,
test_size=0.2,
random_state=0)
#set up tuning algorithm
classifier = GridSearchCV(estimator=neural_net,
cv=cv,
param_grid=parameters)
classifier.fit(training_features, training_labels)
test_prediction = classifier.predict(test_features)
test_accuracy = classifier.score(test_features, test_labels)
time_2 = time.time()
graph_title = "Learning Curves \n(Neural Net, learning rate=%f)" % classifier.best_estimator_.learning_rate
plot_learning_curve_iter(classifier, graph_title)
pylab.savefig(
os.path.join(results_location, 'Validator Curves - Neural Net.png'))
time_3 = time.time()
#output time stats
#time 1 -> time 2 is optimization time
#time 2 -> time 3 is run for just one case
print("Neural Net Time Stats")
print("Optimization Time -> %f" % (time_2 - time_1))
print("Single Run Time -> %f" % (time_3 - time_2))
#output classification report and confusion matrix
print('\n\n----------------------------')
print('Classification Report')
print('----------------------------\n')
print(classification_report(y_true=test_labels, y_pred=test_prediction))
print('\n\n----------------------------')
print('Confusion Matrix')
print('----------------------------\n')
print(confusion_matrix(y_true=test_labels, y_pred=test_prediction))
return test_prediction, test_accuracy