def demo_mean_roc():
"""
Demonstrate to aggregate multiple ROC curves.
"""
pos_label = True
n_datasets = 20
rocs = []
for i in range(n_datasets):
x, y = sample_data(n1=300,
mu1=0.0,
std1=0.5,
n2=300,
mu2=1.0,
std2=0.7,
seed=i)
roc = compute_roc(X=x, y=y, pos_label=pos_label)
rocs.append(roc)
# Compute mean ROC curve for a list of ROC containers.
roc_mean = compute_mean_roc(rocs)
# One can plot also that mean ROC curve.
_, ax1 = plt.subplots()
plot_roc(roc_mean, label="My Mean ROC", color="blue", ax=ax1)
# Alternatively, use plot_mean_roc() to also visualize confidence
# and tolerance intervals, CI and TI, respectively.
plot_mean_roc(rocs, show_ci=True, show_ti=True, ax=ax1)
ax1.set_title("Mean ROC with CI and TI")
# One can also show all ROC curves individually.
_, ax2 = plt.subplots()
plot_mean_roc(rocs, show_ci=False, show_ti=False, show_all=True, ax=ax2)
ax2.set_title("Mean ROC and sample ROCs")
def demo_auto_flip():
# If for some stupid reason, the feature x predicts "the other" label,
# one can use the argument auto_flip to adjust for this.
pos_label = True
x, y = sample_data(n1=300, mu1=0.0, std1=0.5, n2=300, mu2=1.0, std2=0.7)
_, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 4.8))
roc1 = compute_roc(X=x, y=y, pos_label=pos_label)
roc2 = compute_roc(X=x, y=y, pos_label=not pos_label)
plot_roc(roc1, show_opt=True, label="Original", color="green", ax=ax1)
plot_roc(roc2, show_opt=True, label="Flipped", color="red", ax=ax1)
ax1.set_title("Flipped label with wrong AUC")
ax1.legend(loc="center")
roc1 = compute_roc(X=x, y=y, pos_label=pos_label)
roc2 = compute_roc(X=x, y=y, pos_label=not pos_label, auto_flip=True)
plot_roc(roc1, show_opt=True, label="Original", color="green", ax=ax2)
plot_roc(roc2, show_opt=True, label="Flipped", color="red", ax=ax2)
ax2.set_title("Fixed, with auto-flip")
ax2.legend(loc="center")
def demo_objectives():
# Note that multiple objective functions can be computed at the same time.
pos_label = True
x, y = sample_data(n1=300, mu1=0.0, std1=0.5, n2=300, mu2=1.0, std2=0.7)
roc = compute_roc(X=x,
y=y,
pos_label=pos_label,
objective=[
"minopt", "minoptsym", "youden", "cost",
"concordance", "lr+", "lr-", "dor", "chi2", "acc",
"cohen"
])
print()
print("Comparison of different objectives:")
for key, val in roc.opd.items():
print("%15s thr=% .3f, J=%7.3f" % (key + ":", val.opt, val.opo))
_, ax1 = plt.subplots()
plot_roc(roc, show_opt=True, ax=ax1)
ax1.legend(loc="upper left", bbox_to_anchor=(1.05, 1), borderaxespad=0.)
ax1.set_title("Visualization of different optimal points")
def test_sklearn_consistentcy(self):
X = self.X.iloc[:, 0]
y = self.y
ret = roc.compute_roc(X=X, y=y, pos_label=self.pos_label)
auc_sklearn = roc_auc_score(y_true=self.y, y_score=X)
np.testing.assert_almost_equal(ret.auc, auc_sklearn)
def demo_basic_usage():
"""
Demonstrate basic usage of roc_utils.
"""
# Construct binary classification problem
x1, y1 = sample_data(n1=300, mu1=0.0, std1=0.5, n2=300, mu2=1.0, std2=0.7)
x2, y2 = sample_data(n1=300, mu1=0.2, std1=0.6, n2=300, mu2=0.8, std2=0.7)
# Show data
_, (ax1, ax2) = plt.subplots(2, 1)
ax1.hist(x1[~y1],
bins=20,
density=True,
color="red",
alpha=0.4,
label="Class 1")
ax1.hist(x1[y1],
bins=20,
density=True,
color="blue",
alpha=0.4,
label="Class 2")
ax1.legend()
ax1.set_xlabel("x")
ax1.set_ylabel("density")
ax1.set_title("Data")
ax2.hist(x2[~y2],
bins=20,
density=True,
color="red",
alpha=0.4,
label="Class 1")
ax2.hist(x2[y2],
bins=20,
density=True,
color="blue",
alpha=0.4,
label="Class 2")
ax2.legend()
ax2.set_xlabel("x")
ax2.set_ylabel("density")
# Compute ROC.
pos_label = True
roc1 = compute_roc(X=x1, y=y1, pos_label=pos_label)
roc2 = compute_roc(X=x2, y=y2, pos_label=pos_label)
# Access details of ROC object.
print("Data 1: AUC=%.3f" % (roc1.auc))
print("Data 2: AUC=%.3f" % (roc2.auc))
print()
print("Available ROC data:")
print(list(roc1.keys()))
# Plot ROC curve.
_, ax3 = plt.subplots()
plot_roc(roc1, label="Data A", color="red", ax=ax3)
plot_roc(roc2, label="Data B", color="green", ax=ax3)
# Place the legend outside.
ax3.legend(loc="upper left", bbox_to_anchor=(1.05, 1), borderaxespad=0.)
ax3.set_title("ROC curves")