def define_all_possible_ensembles(data, n_estimators):
alg = gen_members(data.shape)
all_ensembles = []
for i, classifiers in enumerate(combinations(estimators(alg),
n_estimators)):
all_ensembles.append([classifiers])
return all_ensembles
def compare_results(data, target, n_estimators, outputfile, stop_time):
accuracy, f1, precision, recall, auc = 0, 0, 0, 0, 0
total_accuracy, total_f1, total_precision, total_recall, total_auc = [], [], [], [], []
alg = gen_members(data.shape)
with open(outputfile, "w") as text_file:
text_file.write('*' * 60)
text_file.write(' Brute Force Ensemble Classifier ')
text_file.write('*' * 60)
text_file.write('\n\nn_estimators = %i' % (n_estimators))
text_file.write('\nstop_time = %i' % (stop_time))
sum_total_iter_time = []
for i in range(0, 10):
fit_time_aux = int(round(time.time() * 1000))
csv_file = 'bfec_seq_results_iter_' + str(i) + '_' + time.strftime(
"%H_%M_%S", time.localtime(time.time())) + '.csv'
ensemble_classifier = BruteForceEnsembleClassifier(
algorithms=alg,
stop_time=stop_time,
n_estimators=int(n_estimators),
random_state=i * 10)
print('\n\nIteration = ', i)
text_file.write("\n\nIteration = %i" % (i))
X_train, X_test, y_train, y_test = train_test_split(
data, target, test_size=0.2, random_state=i * 10)
ensemble, best_accuracy_classifiers = ensemble_classifier.fit(
X_train, y_train, csv_file)
ensemble_classifier.fit_ensemble(X_train, y_train, ensemble,
best_accuracy_classifiers)
fit_total_time = (int(round(time.time() * 1000)) - fit_time_aux)
text_file.write("\n\nBFEC fit done in %i" % (fit_total_time))
text_file.write(" ms")
predict_aux = int(round(time.time() * 1000))
y_pred = ensemble_classifier.predict(X_test)
predict_total_time = (int(round(time.time() * 1000)) - predict_aux)
text_file.write("\n\nBFEC predict done in %i" %
(predict_total_time))
text_file.write(" ms")
accuracy = accuracy_score(y_test, y_pred)
total_accuracy.append(accuracy)
try:
f1 = f1_score(y_test, y_pred)
total_f1.append(f1)
except:
pass
try:
precision = precision_score(y_test, y_pred)
total_precision.append(precision)
except:
pass
try:
recall = recall_score(y_test, y_pred)
total_recall.append(recall)
except:
pass
try:
auc = roc_auc_score(y_test, y_pred)
total_auc.append(auc)
except:
pass
text_file.write("\n\nAccuracy = %f\n" % (accuracy))
if f1 > 0:
text_file.write("F1-score = %f\n" % (f1))
if precision > 0:
text_file.write("Precision = %f\n" % (precision))
if recall > 0:
text_file.write("Recall = %f\n" % (recall))
if auc > 0:
text_file.write("ROC AUC = %f\n" % (auc))
memory.clear(warn=False)
shutil.rmtree(cachedir)
total_iter_time = (int(round(time.time() * 1000)) - fit_time_aux)
text_file.write("\nIteration done in %i" % (total_iter_time))
text_file.write(" ms")
sum_total_iter_time.append(total_iter_time)
text_file.write("\n\nAverage Accuracy = %f\n" %
(statistics.mean(total_accuracy)))
text_file.write("Standard Deviation of Accuracy = %f\n" %
(statistics.stdev(total_accuracy)))
if sum(total_f1) > 0:
text_file.write("\nAverage F1-score = %f\n" %
(statistics.mean(total_f1)))
text_file.write("Standard Deviation of F1-score = %f\n" %
(statistics.stdev(total_f1)))
if sum(total_precision) > 0:
text_file.write("\nAverage Precision = %f\n" %
(statistics.mean(total_precision)))
text_file.write("Standard Deviation of Precision = %f\n" %
(statistics.stdev(total_precision)))
if sum(total_recall) > 0:
text_file.write("\nAverage Recall = %f\n" %
(statistics.mean(total_recall)))
text_file.write("Standard Deviation of Recall = %f\n" %
(statistics.stdev(total_recall)))
if sum(total_auc) > 0:
text_file.write("\nAverage ROC AUC = %f\n" %
(statistics.mean(total_auc)))
text_file.write("Standard Deviation of ROC AUC = %f\n" %
(statistics.stdev(total_auc)))
text_file.write("\n\nAverage duration of iterations = %i" %
statistics.mean(sum_total_iter_time))
text_file.write(" ms")
text_file.write("\nStandard deviation of iterations duration = %i" %
statistics.stdev(sum_total_iter_time))
text_file.write(" ms\n")