def analyse_fpr(self, cms, thresholds):
# tn, fp, fn, tp
# fpr = fp / (fp + tn)
div = lambda n, d: n/d if d else 0
thrs = np.asarray(thresholds)
fprs = np.array([div(i[1], i[1]+i[0]) for cm in cms for i in cm]).reshape((len(self.learn_size), 50))
fprs_mean = fprs.mean(axis=0)
fprs_std = fprs.std(axis=0)
fprs_ci = (fprs_std / sqrt(fprs.shape[0])) * 1.96
sens = np.array([div(i[3], i[3]+i[2]) for cm in cms for i in cm]).reshape((len(self.learn_size), 50))
sens_mean = sens.mean(axis=0)
sens_std = sens.std(axis=0)
sens_ci = (sens_std / sqrt(sens.shape[0])) * 1.96
spec = np.array([div(i[0], i[0]+i[1]) for cm in cms for i in cm]).reshape((len(self.learn_size), 50))
spec_mean = spec.mean(axis=0)
spec_std = spec.std(axis=0)
spec_ci = (spec_std / sqrt(spec.shape[0])) * 1.96
auc_mean = auc_calc(fprs_mean, sens_mean)
plt.close('all')
fig, ax = plt.subplots()
ax.errorbar(thrs[0], fprs_mean, yerr=fprs_ci, ecolor='k', label='fpr')
ax.set_xlabel('Probability Threshold')
ax.set_ylabel('False Positive Rate')
ax.set_title('Mean false positive rate per threshold.\nErrorbar = 95% confidence interval')
fig.savefig(self.save_path + '_XGB_False_positive_rate.png')
fig, ax = plt.subplots()
ax.errorbar(thrs[0], sens_mean, yerr=sens_ci, color='b', ecolor='k', label='Sensitivity')
ax.errorbar(thrs[0], spec_mean, yerr=spec_ci, color='r', ecolor='k', label='Specificity')
ax.legend()
ax.set_xlabel('Threshold')
ax.set_ylabel('Sensitiviy[TPR] / Specificity [TNR]')
ax.set_title('Mean sensitivity and specitivity.\nErrorbar = 95% confidence interval')
fig.savefig(self.save_path + '_XGB_sensitivity_vs_specificity.png')
fig, ax = plt.subplots()
ax.step(fprs_mean, sens_mean, color='b')
ax.plot([0, 1], [0, 1], color='k')
ax.set_title('Average ROC curve\n AUC: {:.3f}'.format(auc_mean))
ax.set_xlabel('Fall-Out [FPR]')
ax.set_ylabel('Sensitivity [TPR]')
fig.savefig(self.save_path + '_XGB_average_roc.png')
def analyse_fpr(self, cms, thresholds):
# tn, fp, fn, tp
# fpr = fp / (fp + tn)
div = lambda n, d: n / d if d else 0
thrs = np.asarray(thresholds)
fprs = np.array([div(i[1], i[1] + i[0]) for cm in cms for i in cm]) \
.reshape((len(self.learn_size), 50))
fprs_mean = fprs.mean(axis=0)
fprs_std = fprs.std(axis=0)
fprs_ci = (fprs_std / sqrt(fprs.shape[0])) * 1.96
sens = np.array([div(i[3], i[3] + i[2]) for cm in cms for i in cm]) \
.reshape((len(self.learn_size), 50))
sens_mean = sens.mean(axis=0)
sens_std = sens.std(axis=0)
sens_ci = (sens_std / sqrt(sens.shape[0])) * 1.96
spec = np.array([div(i[0], i[0] + i[1]) for cm in cms for i in cm]) \
.reshape((len(self.learn_size), 50))
spec_mean = spec.mean(axis=0)
spec_std = spec.std(axis=0)
spec_ci = (spec_std / sqrt(spec.shape[0])) * 1.96
auc_mean = auc_calc(fprs_mean, sens_mean)
if self.evaluation_args['plot_analyse_fpr']:
fig, ax = plt.subplots()
ax.plot(thrs[0], fprs_mean, label='fpr')
ax.fill_between(thrs[0],
fprs_mean - fprs_ci,
fprs_mean + fprs_ci,
color='b',
alpha=.1)
ax.set_xlabel('Classification Threshold')
ax.set_ylabel('False Positive Rate')
ax.set_title(
'Mean false positive rate per threshold.\nErrorbar = 95% confidence interval'
)
fig.tight_layout()
fig.savefig(self.save_path + '_False_positive_rate.png',
figsize=self.fig_size,
dpi=self.fig_dpi)
fig, ax = plt.subplots()
ax.plot(thrs[0], sens_mean, color='b', label='Sensitivity')
ax.plot(thrs[0], spec_mean, color='r', label='Specificity')
ax.fill_between(thrs[0],
sens_mean - sens_ci,
sens_mean + sens_ci,
color='b',
alpha=.1)
ax.fill_between(thrs[0],
spec_mean - spec_ci,
spec_mean + spec_ci,
color='r',
alpha=.1)
ax.legend(bbox_to_anchor=(1, 0.5))
ax.set_xlabel('Classification Threshold')
ax.set_ylabel('Sensitiviy (TPR) / Specificity (TNR)')
ax.set_title(
'Mean sensitivity and specitivity.\nErrorbar = 95% confidence interval'
)
fig.tight_layout()
fig.savefig(self.save_path + '_sensitivity_vs_specificity.png',
figsize=self.fig_size,
dpi=self.fig_dpi)
fig, ax = plt.subplots()
ax.step(fprs_mean, sens_mean, color='b')
ax.plot([0, 1], [0, 1], color='k')
ax.set_title('Average ROC curve\nAUC: {:.3f}'.format(auc_mean))
ax.set_xlabel(
'1 - Specificity (FPR)') # Also Fall-Out / False Positive Rate
ax.set_ylabel('Sensitivity (TPR)')
fig.tight_layout()
fig.savefig(self.save_path + '_average_roc.png',
figsize=self.fig_size,
dpi=self.fig_dpi)