def kfold_validate(self, splits, repeat):
"""
Returns predictions from a repeated k-fold split
Used for statistical tests
"""
model_names = ModelFactory.get_models_list()
result = pd.DataFrame(columns=["actual"] + model_names)
for i in range(0, repeat):
kFold = KFold(n_splits=splits, shuffle=True, random_state=None)
for train_index, test_index in kFold.split(self.data):
train_data = self.data.iloc[train_index]
test_data = self.data.iloc[test_index]
y_train = train_data['non-information']
y_test = test_data['non-information']
features_test = self.extract_features(test_data)
features_train = self.extract_features(train_data)
features_test = self.combine_features(features_test,
comments_only=True)
features_train = self.combine_features(features_train,
comments_only=True)
data = {'actual': y_test.tolist()}
for model_name in model_names:
y_pred = self.execute_model_data(model_name,
features_train, y_train,
features_test)
data[model_name] = y_pred.tolist()
df = pd.DataFrame(data=data)
result = result.append(df)
return result
def compare_models(self, features_train, features_test, y_train, y_test):
"""
Executes models of all types implemented in this project and prints their results
"""
x_train = features_train
model_names = ModelFactory.get_models_list()
score_df = pd.DataFrame(
columns=['name', 'accuracy', 'precision', 'recall', 'f1'])
if self.imbalance_sampling:
x_train, y_train = ImbalanceSampling.get_sampled_data(
self.imbalance_sampling, x_train, y_train)
for name in model_names:
model = ModelFactory.get_model(name, optimised=False)
model.fit_model(x_train, y_train)
y_pred = model.predict(features_test)
score = ScoreMetrics.get_scores(name, y_test, y_pred)
print('-------')
print(name)
ScoreMetrics.print_scores(y_test, y_pred)
score_df = score_df.append(score)
return score_df
import pandas as pd
from models.model_exec import ModelExec
exec = ModelExec(include_comments=False, include_long_code=True)
from mlxtend.evaluate import paired_ttest_5x2cv
from statsmodels.stats.contingency_tables import mcnemar,cochrans_q
from models.model_factory import ModelFactory
# executes McNemar test to check the variance
# between results from k-fold validation of different models
model_names = ModelFactory.get_models_list()
result = exec.kfold_validate(2, 10)
data = []
for i in range(0, len(model_names)):
name1 = model_names[i]
df = pd.DataFrame()
for j in range(0, len(model_names)):
name2 = model_names[j]
if name1 != name2:
yes_yes = 0
no_no = 0
yes_no = 0
no_yes = 0
for index, row in result.iterrows():
r1 = row[name1]
r2 = row[name2]
actual = row['actual']
def kfold_split(self, folds_split, w1, w2, data):
"""
Executes k-fold procedure on models of all types and prints final averaged results for all models.
"""
i = 1
kFold = KFold(n_splits=folds_split, shuffle=True, random_state=None)
model_list = ModelFactory.get_models_list()
model_list.append('ensemble')
vals = [[name, [], [], [], [], [], [], []] for name in model_list]
result = pd.DataFrame(columns=[
'name', 'accuracy', 'precision', 'recall', 'f1',
'matthews_corrcoef', 'balanced_accuracy', 'confusion_matrix'
],
data=vals)
for train_index, test_index in kFold.split(self.data):
train_data = data.iloc[train_index]
test_data = data.iloc[test_index]
features_test = self.extract_features(test_data)
features_train = self.extract_features(train_data)
comment_vectorised = self.vectorise_comment_data(
features_train[4], features_test[4])
comments_train = comment_vectorised[0]
comments_test = comment_vectorised[1]
features_test = self.combine_features(features_test,
comments_only=False)
features_train = self.combine_features(features_train,
comments_only=False)
y_train = train_data['non-information']
y_test = test_data['non-information']
ensemble_result = self.ensemble_model(features_train,
features_test,
comments_train,
comments_test, y_train,
y_test, w1, w2)
res = self.compare_models(comments_train, comments_test,
train_data['non-information'],
test_data['non-information'])
counter = 0
for index, row in res.iterrows():
result.iloc[counter]['accuracy'].append(row['accuracy'])
result.iloc[counter]['precision'].append(row['precision'])
result.iloc[counter]['recall'].append(row['recall'])
result.iloc[counter]['f1'].append(row['f1'])
result.iloc[counter]['matthews_corrcoef'].append(
row['matthews_corrcoef'])
result.iloc[counter]['balanced_accuracy'].append(
row['balanced_accuracy'])
result.iloc[counter]['confusion_matrix'].append(
row['confusion_matrix'])
counter += 1
acc = ensemble_result['accuracy'][0]
result.iloc[counter]['accuracy'].append(acc)
result.iloc[counter]['precision'].append(
ensemble_result['precision'][0])
result.iloc[counter]['recall'].append(ensemble_result['recall'][0])
result.iloc[counter]['f1'].append(ensemble_result['f1'][0])
result.iloc[counter]['matthews_corrcoef'].append(
ensemble_result['matthews_corrcoef'][0])
result.iloc[counter]['balanced_accuracy'].append(
ensemble_result['balanced_accuracy'][0])
result.iloc[counter]['confusion_matrix'].append(
ensemble_result['confusion_matrix'][0])
print('K fold')
self.print_kfold_results(result)
return result