def test_pandas_confusion_binary_cm_inverse():
y_true = [True, True, False, False, False, True, False, True, True,
False, True, False, False, False, False, False, True, False,
True, True, True, True, False, False, False, True, False,
True, False, False, False, False, True, True, False, False,
False, True, True, True, True, False, False, False, False,
True, False, False, False, False, False, False, False, False,
False, True, True, False, True, False, True, True, True,
False, False, True, False, True, False, False, True, False,
False, False, False, False, False, False, False, True, False,
True, True, True, True, False, False, True, False, True,
True, False, True, False, True, False, False, True, True,
False, False, True, True, False, False, False, False, False,
False, True, True, False]
y_pred = [False, False, False, False, False, True, False, False, True,
False, True, False, False, False, False, False, False, False,
True, True, True, True, False, False, False, False, False,
False, False, False, False, False, True, False, False, False,
False, True, False, False, False, False, False, False, False,
True, False, False, False, False, False, False, False, False,
False, True, False, False, False, False, False, False, False,
False, False, True, False, False, False, False, True, False,
False, False, False, False, False, False, False, True, False,
False, True, False, False, False, False, True, False, True,
True, False, False, False, True, False, False, True, True,
False, False, True, True, False, False, False, False, False,
False, True, False, False]
binary_cm = BinaryConfusionMatrix(y_true, y_pred)
bcm_sum = binary_cm.sum()
binary_cm_r = binary_cm.inverse() # reverse not in place
assert bcm_sum == binary_cm_r.sum()
def test_pandas_confusion_binary_cm():
y_true = [True, True, False, False, False, True, False, True, True,
False, True, False, False, False, False, False, True, False,
True, True, True, True, False, False, False, True, False,
True, False, False, False, False, True, True, False, False,
False, True, True, True, True, False, False, False, False,
True, False, False, False, False, False, False, False, False,
False, True, True, False, True, False, True, True, True,
False, False, True, False, True, False, False, True, False,
False, False, False, False, False, False, False, True, False,
True, True, True, True, False, False, True, False, True,
True, False, True, False, True, False, False, True, True,
False, False, True, True, False, False, False, False, False,
False, True, True, False]
y_pred = [False, False, False, False, False, True, False, False, True,
False, True, False, False, False, False, False, False, False,
True, True, True, True, False, False, False, False, False,
False, False, False, False, False, True, False, False, False,
False, True, False, False, False, False, False, False, False,
True, False, False, False, False, False, False, False, False,
False, True, False, False, False, False, False, False, False,
False, False, True, False, False, False, False, True, False,
False, False, False, False, False, False, False, True, False,
False, True, False, False, False, False, True, False, True,
True, False, False, False, True, False, False, True, True,
False, False, True, True, False, False, False, False, False,
False, True, False, False]
binary_cm = BinaryConfusionMatrix(y_true, y_pred)
print("Binary confusion matrix:\n%s" % binary_cm)
asserts(y_true, y_pred, binary_cm)
from numpy import *
import decisiontreeweighted
import bagging
from pandas_confusion import BinaryConfusionMatrix
import randomforest
fn = ""
tree = decisiontreeweighted.dtree()
bagger = bagging.bagger()
test,classes,features = tree.read_data(fn)
w = ones((shape(test)[0]),dtype = float)/shape(test)[0]
t=tree.make_tree(test,w,classes,features,1)
y_actu = classes
y_pred = tree.classifyAll(t,test)
print("\nTree Stump Prediction")
print(tree.classifyAll(t,test))
print("\nTrue Classes")
print(classes)
c=bagger.bag(test,classes,features,20)
print("\nBagged Results ")
print(bagger.bagclass(c,test))
binary_confusion_matrix = BinaryConfusionMatrix(y_actu, y_pred)
print(" \nBinary confusion matrix:\n%s" % binary_confusion_matrix)
binary_confusion_matrix.print_stats()
rf = randomforest.main(fn)
def main(save, show):
basepath = os.path.dirname(__file__)
y_true = np.array([True, True, False, False, False, True, False, True, True,
False, True, False, False, False, False, False, True, False,
True, True, True, True, False, False, False, True, False,
True, False, False, False, False, True, True, False, False,
False, True, True, True, True, False, False, False, False,
True, False, False, False, False, False, False, False, False,
False, True, True, False, True, False, True, True, True,
False, False, True, False, True, False, False, True, False,
False, False, False, False, False, False, False, True, False,
True, True, True, True, False, False, True, False, True,
True, False, True, False, True, False, False, True, True,
False, False, True, True, False, False, False, False, False,
False, True, True, False])
y_pred = np.array([False, False, False, False, False, True, False, False, True,
False, True, False, False, False, False, False, False, False,
True, True, True, True, False, False, False, False, False,
False, False, False, False, False, True, False, False, False,
False, True, False, False, False, False, False, False, False,
True, False, False, False, False, False, False, False, False,
False, True, False, False, False, False, False, False, False,
False, False, True, False, False, False, False, True, False,
False, False, False, False, False, False, False, True, False,
False, True, False, False, False, False, True, False, True,
True, False, False, False, True, False, False, True, True,
False, False, True, True, False, False, False, False, False,
False, True, False, False])
# y_true = ~y_true
# y_pred = ~y_pred
binary_cm = BinaryConfusionMatrix(y_true, y_pred)
print("Binary confusion matrix:\n%s" % binary_cm)
print("")
# attributes = ['TP', 'TN', 'FP', 'FN', 'TPR', 'TNR', 'PPV', 'NPV', 'FPR', 'FDR',
# 'FNR', 'ACC', 'F1_score', 'MCC', 'informedness', 'markedness']
# binary_cm.print_stats(attributes)
binary_cm.print_stats()
# stats = binary_cm.stats(attributes)
# for key, val in stats.items():
# print("%s: %f" % (key, val))
print("sklearn confusion_matrix:\n%s" % confusion_matrix(y_true, y_pred))
f1score = f1_score(y_true, y_pred)
print("f1_score: %f" % f1score)
print("sklearn confusion_matrix_of_rev:\n%s" % confusion_matrix(~y_true, ~y_pred))
f1score_r = f1_score(~y_true, ~y_pred)
print("f1_score_of_rev: %f" % f1score_r)
print(classification_report(y_true, y_pred))
np.testing.assert_almost_equal(binary_cm.F1_score, f1score)
binary_cm.plot()
filename = 'binary_cm.png'
if save:
plt.savefig(os.path.join(basepath, '..', 'screenshots', filename))
if show:
plt.show()
binary_cm.plot(normalized=True)
filename = 'binary_cm_norm.png'
if save:
plt.savefig(os.path.join(basepath, '..', 'screenshots', filename))
if show:
plt.show()
# import seaborn as sns
# binary_cm.plot(normalized=True, backend=Backend.Seaborn)
# sns.plt.show()
print("FP+TP= %f" % (binary_cm.FP + binary_cm.TP)) # Positive
print("")
binary_cm_r = binary_cm.inverse()
print("Reversed binary confusion matrix:\n%s" % binary_cm_r)
binary_cm_r.print_stats()
np.testing.assert_almost_equal(binary_cm_r.F1_score, f1score_r)
print(binary_cm.classification_report)
print("sklearn confusion_matrix:\n%s" % confusion_matrix(y_true, y_pred))
print(classification_report(y_true, y_pred))
y_true = np.array(["a", "a", "b", "b", "b", "a", "b", "a", "a",
"b", "a", "b", "b", "b", "b", "b", "a", "b",
"a", "a", "a", "a", "b", "b", "b", "a", "b",
"a", "b", "b", "b", "b", "a", "a", "b", "b",
"b", "a", "a", "a", "a", "b", "b", "b", "b",
"a", "b", "b", "b", "b", "b", "b", "b", "b",
"b", "a", "a", "b", "a", "b", "a", "a", "a",
"b", "b", "a", "b", "a", "b", "b", "a", "b",
"b", "b", "b", "b", "b", "b", "b", "a", "b",
"a", "a", "a", "a", "b", "b", "a", "b", "a",
"a", "b", "a", "b", "a", "b", "b", "a", "a",
"b", "b", "a", "a", "b", "b", "b", "b", "b",
"b", "a", "a", "b"])
y_pred = np.array(["b", "b", "b", "b", "b", "a", "b", "b", "a",
"b", "a", "b", "b", "b", "b", "b", "b", "b",
"a", "a", "a", "a", "b", "b", "b", "b", "b",
"b", "b", "b", "b", "b", "a", "b", "b", "b",
"b", "a", "b", "b", "b", "b", "b", "b", "b",
"a", "b", "b", "b", "b", "b", "b", "b", "b",
"b", "a", "b", "b", "b", "b", "b", "b", "b",
"b", "b", "a", "b", "b", "b", "b", "a", "b",
"b", "b", "b", "b", "b", "b", "b", "a", "b",
"b", "a", "b", "b", "b", "b", "a", "b", "a",
"a", "b", "b", "b", "a", "b", "b", "a", "a",
"b", "b", "a", "a", "b", "b", "b", "b", "b",
"b", "a", "b", "b"])
binary_cm = BinaryConfusionMatrix(y_true, y_pred)
# binary_cm = BinaryConfusionMatrix(y_true, y_pred, labels=["a", "b"])
# binary_cm = BinaryConfusionMatrix(y_true, y_pred, labels=["b", "a"])
print(binary_cm)
print("")
binary_cm.print_stats()
print(binary_cm.classification_report)
print("sklearn confusion_matrix with string as input:\n%s" % confusion_matrix(y_true, y_pred))
print(classification_report(y_true, y_pred))
# ToFix
# "b" is considered as True
# "a" is considered as False
# but it should be "a" as True and "b" as False
# d = binary_cm.dict_class()
# print(d)
y_true_bool = binary_cm.y_true(to_bool=True)
y_pred_bool = binary_cm.y_pred(to_bool=True)
f1score = f1_score(y_true_bool, y_pred_bool)
np.testing.assert_almost_equal(binary_cm.F1_score, f1score)
print("F1_score: %f" % f1score)
def heatmap_confusion_matrix(actual_label, pred_label, output_folder,
plot_title):
""" Plot Confusion Matrix using Seaborn's Heatmap.
This plot contains not just confusion matrix but also
Accuracy, TPR(Recall, Sensitivity), Precision, TNR(specificity) and F1-Score
Columns are predicted labels and Rows are actual labels
Parameters
----------
actual_label : 1D array
actual labels of the data
pred_label : 1D array
predicted labels
output_folder : path
path to output folder where output plot will be saved
plot_title : string
plot title which may conclude data information and model description
Returns
-------
result : Confusion matrix plot with test result statistics
Saved plot file to output_folder
"""
# Create confusion matrix
binary_confusion_matrix = BinaryConfusionMatrix(actual_label, pred_label)
# Result statistics from the confusion matrix
stats = binary_confusion_matrix.stats()
pos_real = stats['P']
neg_real = stats['N']
pos_pred = stats['PositiveTest']
neg_pred = stats['NegativeTest']
TP = stats['TP']
TN = stats['TN']
FP = stats['FP']
FN = stats['FN']
TPR = round(stats['TPR'],
2) #sensitivity, recall: TP/(TP+FN) = TP/pos_real
TNR = round(stats['TNR'], 2) #specificity
PPV = round(stats['PPV'], 2) #precision : TP/(TP+FP) = TP/pos_pred
F1_score = round(stats['F1_score'],
2) #harmonic mean of recall and precision
ACC = round(stats['ACC'], 2)
# Confusion matrix for display
cm = np.array([[TN, FP], [FN, TP]])
"""
TN FP
FN TP
"""
df_cm = df(
cm,
index=['{} \nDecoy'.format(neg_real),
'%d \nActive' % (pos_real)],
columns=['Decoy\n%d' % (neg_pred),
'Active\n%d' % (pos_pred)])
plot = plt.figure(figsize=(6, 6))
plt.rcParams['font.size'] = 10
plt.title(
"Accuracy : {:.2f} TPR : {:.2f}\nPrecision : {:.2f} TNR : {:.2f} F1-Score : {:.2f}"
.format(ACC, TPR, PPV, TNR, F1_score),
loc='left',
fontsize=12)
plt.suptitle(plot_title, y=0.95, fontsize=14)
plt.xlabel('Predicted Label')
plt.ylabel('Actual Label')
plt.subplots_adjust(top=0.8)
# plot heatmap
heatmap(df_cm, annot=True, fmt='d', annot_kws={"size": 14})
# save plot and display it
plot.savefig('{}/test_result_confusion_matrix.png'.format(output_folder))
plt.show()
plt.close()
def main(save, show):
basepath = os.path.dirname(__file__)
y_true = np.array([
True, True, False, False, False, True, False, True, True, False, True,
False, False, False, False, False, True, False, True, True, True, True,
False, False, False, True, False, True, False, False, False, False,
True, True, False, False, False, True, True, True, True, False, False,
False, False, True, False, False, False, False, False, False, False,
False, False, True, True, False, True, False, True, True, True, False,
False, True, False, True, False, False, True, False, False, False,
False, False, False, False, False, True, False, True, True, True, True,
False, False, True, False, True, True, False, True, False, True, False,
False, True, True, False, False, True, True, False, False, False,
False, False, False, True, True, False
])
y_pred = np.array([
False, False, False, False, False, True, False, False, True, False,
True, False, False, False, False, False, False, False, True, True,
True, True, False, False, False, False, False, False, False, False,
False, False, True, False, False, False, False, True, False, False,
False, False, False, False, False, True, False, False, False, False,
False, False, False, False, False, True, False, False, False, False,
False, False, False, False, False, True, False, False, False, False,
True, False, False, False, False, False, False, False, False, True,
False, False, True, False, False, False, False, True, False, True,
True, False, False, False, True, False, False, True, True, False,
False, True, True, False, False, False, False, False, False, True,
False, False
])
# y_true = ~y_true
# y_pred = ~y_pred
binary_cm = BinaryConfusionMatrix(y_true, y_pred)
print("Binary confusion matrix:\n%s" % binary_cm)
print("")
# attributes = ['TP', 'TN', 'FP', 'FN', 'TPR', 'TNR', 'PPV', 'NPV', 'FPR', 'FDR',
# 'FNR', 'ACC', 'F1_score', 'MCC', 'informedness', 'markedness']
# binary_cm.print_stats(attributes)
binary_cm.print_stats()
# stats = binary_cm.stats(attributes)
# for key, val in stats.items():
# print("%s: %f" % (key, val))
print("sklearn confusion_matrix:\n%s" % confusion_matrix(y_true, y_pred))
f1score = f1_score(y_true, y_pred)
print("f1_score: %f" % f1score)
print("sklearn confusion_matrix_of_rev:\n%s" %
confusion_matrix(~y_true, ~y_pred))
f1score_r = f1_score(~y_true, ~y_pred)
print("f1_score_of_rev: %f" % f1score_r)
print(classification_report(y_true, y_pred))
np.testing.assert_almost_equal(binary_cm.F1_score, f1score)
binary_cm.plot()
filename = 'binary_cm.png'
if save:
plt.savefig(os.path.join(basepath, '..', 'screenshots', filename))
if show:
plt.show()
binary_cm.plot(normalized=True)
filename = 'binary_cm_norm.png'
if save:
plt.savefig(os.path.join(basepath, '..', 'screenshots', filename))
if show:
plt.show()
# import seaborn as sns
# binary_cm.plot(normalized=True, backend=Backend.Seaborn)
# sns.plt.show()
print("FP+TP= %f" % (binary_cm.FP + binary_cm.TP)) # Positive
print("")
binary_cm_r = binary_cm.inverse()
print("Reversed binary confusion matrix:\n%s" % binary_cm_r)
binary_cm_r.print_stats()
np.testing.assert_almost_equal(binary_cm_r.F1_score, f1score_r)
print(binary_cm.classification_report)
print("sklearn confusion_matrix:\n%s" % confusion_matrix(y_true, y_pred))
print(classification_report(y_true, y_pred))
y_true = np.array([
"a", "a", "b", "b", "b", "a", "b", "a", "a", "b", "a", "b", "b", "b",
"b", "b", "a", "b", "a", "a", "a", "a", "b", "b", "b", "a", "b", "a",
"b", "b", "b", "b", "a", "a", "b", "b", "b", "a", "a", "a", "a", "b",
"b", "b", "b", "a", "b", "b", "b", "b", "b", "b", "b", "b", "b", "a",
"a", "b", "a", "b", "a", "a", "a", "b", "b", "a", "b", "a", "b", "b",
"a", "b", "b", "b", "b", "b", "b", "b", "b", "a", "b", "a", "a", "a",
"a", "b", "b", "a", "b", "a", "a", "b", "a", "b", "a", "b", "b", "a",
"a", "b", "b", "a", "a", "b", "b", "b", "b", "b", "b", "a", "a", "b"
])
y_pred = np.array([
"b", "b", "b", "b", "b", "a", "b", "b", "a", "b", "a", "b", "b", "b",
"b", "b", "b", "b", "a", "a", "a", "a", "b", "b", "b", "b", "b", "b",
"b", "b", "b", "b", "a", "b", "b", "b", "b", "a", "b", "b", "b", "b",
"b", "b", "b", "a", "b", "b", "b", "b", "b", "b", "b", "b", "b", "a",
"b", "b", "b", "b", "b", "b", "b", "b", "b", "a", "b", "b", "b", "b",
"a", "b", "b", "b", "b", "b", "b", "b", "b", "a", "b", "b", "a", "b",
"b", "b", "b", "a", "b", "a", "a", "b", "b", "b", "a", "b", "b", "a",
"a", "b", "b", "a", "a", "b", "b", "b", "b", "b", "b", "a", "b", "b"
])
binary_cm = BinaryConfusionMatrix(y_true, y_pred)
# binary_cm = BinaryConfusionMatrix(y_true, y_pred, labels=["a", "b"])
# binary_cm = BinaryConfusionMatrix(y_true, y_pred, labels=["b", "a"])
print(binary_cm)
print("")
binary_cm.print_stats()
print(binary_cm.classification_report)
print("sklearn confusion_matrix with string as input:\n%s" %
confusion_matrix(y_true, y_pred))
print(classification_report(y_true, y_pred))
# ToFix
# "b" is considered as True
# "a" is considered as False
# but it should be "a" as True and "b" as False
# d = binary_cm.dict_class()
# print(d)
y_true_bool = binary_cm.y_true(to_bool=True)
y_pred_bool = binary_cm.y_pred(to_bool=True)
f1score = f1_score(y_true_bool, y_pred_bool)
np.testing.assert_almost_equal(binary_cm.F1_score, f1score)
print("F1_score: %f" % f1score)
