def evaluate_roc(y_true, y_pred, method, plot=True):
'''A quick helper for ad-hoc ROC analysis, more seasoned comparison later in R.'''
fpr, tpr, thresholds = roc_curve(y_true, y_pred)
roc_auc = auc(fpr, tpr)
# best point on the ROC curve --> Youden's J
J = tpr - fpr
best_ind = np.argmax(J)
best_threshold = thresholds[best_ind]
print(f'Best threshold: < {np.round(best_threshold,3)} --> negative')
# compute precision and recall at that threshold
binarized = (y_pred >= best_threshold).astype(int)
recall = recall_score(y_true, binarized)
precision = precision_score(y_true, binarized)
print(
f'Recall = {np.round(recall,3)}, Precision = {np.round(precision,3)}')
if plot:
viz = RocCurveDisplay(fpr=fpr,
tpr=tpr,
roc_auc=roc_auc,
estimator_name=method)
viz.plot()
plt.show()
print(f'AUC: {np.round(roc_auc,3)}')
return best_threshold
class RocAucCurve(CurveFabric):
def __init__(self, col_score, col_target, name=None, **kwargs):
super().__init__(col_score, col_target, name=name)
self.fpr = None
self.tpr = None
self.roc_auc = None
def fit(self, df):
self.fpr, self.tpr, _ = roc_curve(df[self.col_target],
df[self.col_score])
self.roc_auc = auc(self.fpr, self.tpr)
return self
def plot(self, ax=None, title=None, **kwargs):
if ax is None:
fig, ax = plt.subplots()
self.ax = ax
self.viz = RocCurveDisplay(fpr=self.fpr,
tpr=self.tpr,
roc_auc=self.roc_auc * 100,
estimator_name=self.name)
if title:
ax.set_title(title, fontsize=14, fontweight='bold')
self.viz.plot(ax=ax, name=self.name, **kwargs)
return self
def evaluate(self, x_test, y_test):
y_pred = self.model.predict(x_test)
y_pred = [1 * (x[0] >= 0.5) for x in y_pred]
print('MLP performance on test for', self.feature_name)
print('Accuracy:', accuracy_score(y_test, y_pred), 'Precision:',
precision_score(y_test, y_pred), 'Recall:',
recall_score(y_test, y_pred))
# Confusion matrix
cm = confusion_matrix(y_test, y_pred)
cm_display = ConfusionMatrixDisplay(cm)
# Precision recall
precision, recall, _ = precision_recall_curve(y_test, y_pred)
pr_display = PrecisionRecallDisplay(precision=precision, recall=recall)
# Roc
fpr, tpr, _ = roc_curve(y_test, y_pred)
roc_display = RocCurveDisplay(fpr=fpr, tpr=tpr)
# Figure
figure: Figure = plt.figure(1, figsize=(15, 6))
figure.suptitle('MLP on {}'.format(self.feature_name), fontsize=20)
(ax1, ax2, ax3) = figure.subplots(1, 3)
ax1.set_title('Confusion matrix')
cm_display.plot(ax=ax1)
ax2.set_title('Precision recall')
pr_display.plot(ax=ax2)
ax3.set_title('Roc curve')
roc_display.plot(ax=ax3)
file_name = '{}-mlp.png'.format(self.feature_name)
figure.savefig(
os.path.join(get_folder_path_from_root('images'), file_name))
plt.show()
def plot_roc_curves(train_roc_auc, split_roc_auc, valid_roc_auc):
fig, ax = plt.subplots()
for name, (roc_auc, auc) in [
('train', train_roc_auc),
('split', split_roc_auc),
('valid', valid_roc_auc),
]:
viz = RocCurveDisplay(fpr=[x[0] for x in auc],
tpr=[x[1] for x in auc],
roc_auc=roc_auc,
estimator_name=name,
pos_label=1.0)
viz.plot(ax=ax, name=name)
return fig
def get_roc_curve(self, gt_index=0, pred_index=1, display=True, model_name="autopilot-model") :
y = self._y()
yh = self._yh()
fpr, tpr, thresholds = roc_curve(y, yh)
roc_auc = auc(fpr, tpr)
viz = RocCurveDisplay(fpr=fpr, tpr=tpr, roc_auc=roc_auc, estimator_name=model_name)
if display :
viz.plot()
return viz, roc_auc, fpr, tpr, thresholds
def plot_roc_curve(pred: torch.Tensor, label: torch.Tensor, name="example estimator"):
pred, label = pred.detach().cpu().numpy().flatten(), label.detach().cpu().numpy().flatten()
fpr, tpr, thresholds = roc_curve(label, pred)
roc_auc = auc(fpr, tpr)
display = RocCurveDisplay(fpr=fpr, tpr=tpr, roc_auc=roc_auc,
estimator_name=name)
# calc optimal threshold
idx = np.arange(len(tpr))
roc = pd.DataFrame({'tf': pd.Series(tpr - (1 - fpr), index=idx), 'threshold': pd.Series(thresholds, index=idx)})
roc_t = roc.iloc[(roc.tf - 0).abs().argsort()[:1]]
display.plot()
plt.show()
return list(roc_t['threshold'])[0]
def loo_roc_curve_plot(clf, x, y):
from functools import partial
def loo_proba(i, x, y, clf):
idx = list(range(len(y)))
idx.pop(i)
clf.fit(x[idx, :], y[idx])
return clf.predict_proba(x[[i], :])[0, 1]
func_ = partial(loo_proba, x=x, y=y, clf=clf)
y_proba = [func_(i) for i in range(len(y))]
fpr, tpr, _ = roc_curve(y, y_proba)
roc_auc = auc(fpr, tpr)
name = clf.__class__.__name__
ax = plt.figure().gca()
ax.plot([0, 1], [0, 1],
linestyle='--',
lw=2,
color='r',
label='Chance',
alpha=.8)
viz = RocCurveDisplay(fpr=fpr,
tpr=tpr,
roc_auc=roc_auc,
estimator_name=name)
return viz.plot(name=name, ax=ax)
def plot(self, ax=None, figsize=(10, 5)):
if ax is None:
fig, ax = plt.subplots(1, 1, figsize=figsize)
ax.set_title("ROC Curve")
possible_colors = GeneralUtils.shuffled_colors()
for class_index, label in enumerate(self.labels):
fpr, tpr = self._roc_curve[label]['fpr'], self._roc_curve[label][
'tpr']
roc_auc = self.auc[label]
viz = RocCurveDisplay(fpr=fpr,
tpr=tpr,
roc_auc=roc_auc,
estimator_name='Classifier')
viz.plot(ax=ax, name=label, color=possible_colors[class_index])
plt.draw()
def roc_curve_plot(clf, x, y):
y_prob = clf.predict_proba(x)[:, 1]
fpr, tpr, _ = roc_curve(y, y_prob)
roc_auc = auc(fpr, tpr)
name = clf.__class__.__name__
viz = RocCurveDisplay(fpr=fpr,
tpr=tpr,
roc_auc=roc_auc,
estimator_name=name)
return viz.plot(name=name)
def roc_curve(self, test_label=None, plot_type='test'):
if test_label is not None:
self.test_label = test_label
if plot_type == 'test':
predict = [self.y_pred]
label = [self.test_label]
elif plot_type == 'train':
predict = [self.y_oof]
label = [self.y_train]
method_name = ['lgb']
fig = plt.figure(figsize=(6, 6))
ax = fig.add_subplot(1, 1, 1)
for pred, label, method_name in zip(predict, label, method_name):
fpr, tpr, thresholds = metrics.roc_curve(label, pred)
auc = metrics.auc(fpr, tpr)
roc_display = RocCurveDisplay(fpr=fpr,
tpr=tpr,
roc_auc=auc,
estimator_name=method_name)
roc_display.plot(ax=ax)
ax.set_title('ROC curve : LightGBM', fontsize=16)
plt.show()
from sklearn.metrics import confusion_matrix
from sklearn.metrics import ConfusionMatrixDisplay
for model_name, model in models.items():
print("Model: ", model_name)
# ROC Curve
if model_name == "elasticnetlinear":
fpr, tpr, thresholds = roc_curve(model[1], y_test)
roc_auc = auc(fpr, tpr)
svc_disp = RocCurveDisplay(fpr=fpr,
tpr=tpr,
roc_auc=roc_auc,
estimator_name='Elastic Net Linear')
svc_disp.plot()
plt.title(
f"ROC Curve of {PREDICTION_PHENO} by {model_name} (Scaled)")
plt.savefig(
f"finalplots/scaledroc_{PREDICTION_PHENO}_{model_name}.png",
dpi=600,
transparent=True,
bbox_inches="tight",
pad_inches=0.3)
else:
if model_name == "rbfsvmapprox":
svc_disp = plot_roc_curve(
model, feature_map_nystroem.fit_transform(X_test), y_test)
else:
svc_disp = plot_roc_curve(model, X_test, y_test)
fraction_of_positives, mean_predicted_value = calibration_curve(
np.array(y_test), y_pred_proba, strategy='uniform', n_bins=20)
plt.figure()
plt.plot(mean_predicted_value, fraction_of_positives, "sr-")
plt.title("Calibration")
plt.xlabel("mean_predicted_value")
plt.ylabel("fraction_of_positives")
fpr, tpr, _ = roc_curve(y_test, y_pred_proba)
roc_auc = auc(fpr, tpr)
display = RocCurveDisplay(fpr=fpr,
tpr=tpr,
roc_auc=roc_auc,
estimator_name=None)
display.plot()
plt.title("ROC")
range_class = np.linspace(np.min(y_pred_proba), np.max(y_pred_proba), 100)
range_class = np.delete(range_class, 0)
range_class = np.delete(range_class, -1)
PPV = np.zeros(len(range_class))
NPV = np.zeros(len(range_class))
j = 0
for i in range_class:
PPV[j] = precision_score(y_test, y_pred_proba > i, pos_label=1)
NPV[j] = precision_score(y_test, y_pred_proba > i, pos_label=0)
j += 1
plt.figure()
plt.plot(range_class, PPV, label='PPV')
plt.plot(range_class, NPV, label='NPV')
fpr, tpr, _ = roc_curve(y_test, y_score, pos_label=clf.classes_[1]) roc_display = RocCurveDisplay(fpr=fpr, tpr=tpr).plot() # %% # Create :class:`PrecisionRecallDisplay` ############################################################################## # Similarly, the precision recall curve can be plotted using `y_score` from # the prevision sections. from sklearn.metrics import precision_recall_curve from sklearn.metrics import PrecisionRecallDisplay prec, recall, _ = precision_recall_curve(y_test, y_score, pos_label=clf.classes_[1]) pr_display = PrecisionRecallDisplay(precision=prec, recall=recall).plot() # %% # Combining the display objects into a single plot ############################################################################## # The display objects store the computed values that were passed as arguments. # This allows for the visualizations to be easliy combined using matplotlib's # API. In the following example, we place the displays next to each other in a # row. # sphinx_gallery_thumbnail_number = 4 import matplotlib.pyplot as plt fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 8)) roc_display.plot(ax=ax1) pr_display.plot(ax=ax2) plt.show()
def predict(x_test, y_test, modal_name):
from sklearn.metrics import confusion_matrix, roc_curve, auc, RocCurveDisplay
import pandas as pd
import seaborn as sn
modal_name = modal_name[0] + '_' + modal_name[1] + '_' + modal_name[2]
if mulmonet:
# split modals
x_test1 = x_test[:, :, :, :3]
x_test2 = x_test[:, :, :, 3:6]
x_test3 = x_test[:, :, :, 6:9]
else:
# stack
x_test1 = np.zeros_like(x_test[:, :, :, :3])
x_test1[:, :, :, 0] = x_test[:, :, :, 0]
x_test1[:, :, :, 1] = x_test[:, :, :, 3]
x_test1[:, :, :, 2] = x_test[:, :, :, 6]
# load model and weights
model = make_mulmoNet_vgg16(IMAGE_SIZE, IMAGE_SIZE, 3, 1)
model.summary()
model.load_weights(PATH + EXPORT_FOLDER + modal_name +
'/weight_checkpoint.hdf5')
# prediction
y_pred = model.predict([x_test1, x_test2, x_test3], BATCH_SIZE)
y_pred_binary = np.rint(y_pred).astype(int)
# confusion matrix
confusion_matrix = confusion_matrix(y_test, y_pred_binary)
print(confusion_matrix)
df_cm = pd.DataFrame(confusion_matrix, range(2), range(2))
sn.set(font_scale=1.4) # for label size
sn.heatmap(df_cm, annot=True, annot_kws={"size": 16}) # font size
plt.savefig(PATH + EXPORT_FOLDER + modal_name + '/confusion_matrix.png')
plt.clf()
fpr, tpr, thresholds = roc_curve(y_test, y_pred)
roc_auc = auc(fpr, tpr)
display = RocCurveDisplay(fpr=fpr,
tpr=tpr,
roc_auc=roc_auc,
estimator_name=modal_name)
display.plot()
plt.savefig(PATH + EXPORT_FOLDER + modal_name + '/ruc_curve.png')
y_test = np.concatenate(y_test)
with open(PATH + EXPORT_FOLDER + modal_name + '/results.txt', 'w') as f:
f.write('TP: ' + str(confusion_matrix[1, 1]))
f.write('\n')
f.write('FP: ' + str(confusion_matrix[0, 1]))
f.write('\n')
f.write('FN: ' + str(confusion_matrix[1, 0]))
f.write('\n')
f.write('TN: ' + str(confusion_matrix[0, 0]))
try:
os.makedirs(PATH + EXPORT_FOLDER + modal_name + '/results')
except (FileExistsError):
print('folders exist')
# GRAD CAM
for i, y in enumerate(y_pred):
img1 = x_test1[i]
img2 = x_test1[i, :, :, 1]
img3 = x_test1[i, :, :, 2]
image_array1 = np.expand_dims(img1, axis=0)
image_array2 = np.expand_dims(img2, axis=0)
image_array3 = np.expand_dims(img3, axis=0)
cam1 = make_gradcam_heatmap([image_array1, image_array2, image_array3],
1, model, 'block5_conv3_m1', 0)
cam2 = make_gradcam_heatmap([image_array1, image_array2, image_array3],
2, model, 'block5_conv3_m2', 0)
cam3 = make_gradcam_heatmap([image_array1, image_array2, image_array3],
3, model, 'block5_conv3_m3', 0)
# f, axarr = plt.subplots(3, 2)
# axarr[0, 0].imshow(utils.denormalize_x(img1), cmap='gray', vmin=0, vmax=255)
# axarr[0, 1].imshow(cam1)
# axarr[1, 0].imshow(utils.denormalize_x(img2), cmap='gray', vmin=0, vmax=255)
# axarr[1, 1].imshow(cam2)
# axarr[2, 0].imshow(utils.denormalize_x(img3), cmap='gray', vmin=0, vmax=255)
# axarr[2, 1].imshow(cam3)
# plt.title(test_names[i])
# plt.show()
correct = 1 if y_pred_binary[i] == y_test[i] else 0
img1 = (img1 + 1) * 127.5
img2 = (img2 + 1) * 127.5
img3 = (img3 + 1) * 127.5
# save_image = np.concatenate((np.repeat(np.expand_dims(img1[:, :, 0], axis=-1), 3, axis=-1),
# np.repeat(np.expand_dims(img1[:, :, 1], axis=-1), 3, axis=-1),
# np.repeat(np.expand_dims(img1[:, :, 2], axis=-1), 3, axis=-1),
# cam1), axis=1)
save_image1 = np.concatenate((img1, cam1), axis=1)
save_image2 = np.concatenate((img2, cam2), axis=1)
save_image3 = np.concatenate((img3, cam3), axis=1)
save_image = np.concatenate((save_image1, save_image2), axis=0)
save_image = np.concatenate((save_image, save_image3), axis=0)
# save_image = np.concatenate((cam1, cam2, cam3), axis=1)
cv2.imwrite(
PATH + EXPORT_FOLDER + modal_name + '/results/' + str(i) + '_' +
str(correct) + str(np.round(y_pred[i], 2)) + '.png', save_image)
losses = []
for trace in testSet:
reconstructed = modelS(trace)
loss = loss_function(reconstructed, trace)
losses.append(loss.item())
m = mean(losses)
std = stdev(losses)
r = []
with open("input/results_30_activities_10k_0.005_description", "r") as f:
for line in f:
r.append(int(line.split(",")[1]))
outliers = []
threshold = sorted(losses)[-len(r)]
for i, x in enumerate(losses):
if x >= threshold:
outliers.append(i)
#create the roc_curve
from sklearn.metrics import RocCurveDisplay, auc, roc_curve
losses_normalized = [(float(i) - min(losses)) / (max(losses) - min(losses))
for i in losses]
true_outliers = [1 if i in r else 0 for i in range(len(losses))]
fprS, tprS, thresholds = roc_curve(true_outliers, losses_normalized)
roc_auc = auc(fprS, tprS)
displayS = RocCurveDisplay(fpr=fprS,
tpr=tprS,
roc_auc=roc_auc,
estimator_name='Denoising autoencoder')
displayS.plot()
plt.plot(sorted(losses))