def compare(self, others, scores, dtype=np.float16, plot=False):
result0 = self.compute(scores, dtype=dtype)
if not isiterable(others):
others = [others]
result_grid = []
for other in others:
result1 = other.compute(scores, dtype=dtype)
if plot:
from matplotlib import pyplot as plt
from palettable import colorbrewer
colors = colorbrewer.get_map("Set1", "qualitative", 9).mpl_colors
result_row = {}
for score_name, scores0 in result0.iteritems():
scores1 = result1[score_name]
auc_score = dist_auc(scores0, scores1)
result_row[score_name] = auc_score
if plot:
scores0p = [x for x in scores0 if not np.isnan(x)]
scores1p = [x for x in scores1 if not np.isnan(x)]
hmin0, hmax0 = minmaxr(scores0p)
hmin1, hmax1 = minmaxr(scores1p)
bins = np.linspace(min(hmin0, hmin1), max(hmax0, hmax1), 50)
plt.hist(scores0p, bins, alpha=0.5, label="0", color=colors[0], edgecolor="none")
plt.hist(scores1p, bins, alpha=0.5, label="1", color=colors[1], edgecolor="none")
plt.legend(loc="upper right")
plt.title("%s: AUC=%.4f" % (score_name, auc_score))
plt.show()
result_grid.append(result_row)
return result_grid
def test_scores_5():
"""NaN -> 45 (score should be a lot worse)
"""
scores0 = [10, 20, 30, 45]
scores1 = [25, 40, 50, 60]
auc = dist_auc(scores0, scores1)
assert_almost_equal(auc, 0.8125, 4)
def test_scores_1():
"""Complete separation
"""
scores0 = [10, 20, 30, 33]
scores1 = [36, 40, 50, 60]
auc = dist_auc(scores0, scores1)
assert_almost_equal(auc, 1.0, 4)
def test_scores_4():
"""NaN -> 45 (score should be worse)
"""
nan = float('nan')
scores0 = [10, 20, 30, 45]
scores1 = [nan, 40, 50, 60]
auc = dist_auc(scores0, scores1)
assert_almost_equal(auc, 0.8646, 4)
def test_scores_3():
"""Complete separation (excluding NaNs)
"""
nan = float('nan')
scores0 = [10, 20, 30, nan]
scores1 = [nan, 40, 50, 60]
auc = dist_auc(scores0, scores1)
assert_almost_equal(auc, 0.875, 4)
