def setUp(self):
self.data = BinaryClassifierData([\
(0.1, 0),
(0.2, 0),
(0.3, 0),
(0.4, 1),
(0.5, 0),
(0.6, 1),
(0.7, 1),
(0.8, 1),
(0.9, 1)
])
def data(self, data):
"""Sets the data points from which we generate the curve."""
if isinstance(data, BinaryClassifierData):
self._data = data
else:
self._data = BinaryClassifierData(data)
self._calculate_points()
def run_tests(self):
"""Runs pairwise significance tests on the datasets found in
``self.data``."""
data = self.data
expected = data["__class__"]
keys = sorted(data.keys())
keys.remove("__class__")
for key in keys:
self.log.info("Preparing dataset for %s..." % key)
data[key] = BinaryClassifierData(zip(data[key], expected), title=key)
self.log.info("Running significance tests...")
significance_test = PairedPermutationTest(self.curve_class)
for key1, key2 in itertools.product(keys, keys):
if key1 >= key2:
continue
diff, p_value = significance_test.test(data[key1], data[key2])
if p_value < 0.01:
stars = "***"
elif p_value < 0.05:
stars = "**"
elif p_value < 0.1:
stars = "*"
else:
stars = ""
self.log.info("%3s d=%8.3g p=%8.3g %s vs %s" %
(stars, diff, p_value, key1, key2))
class BinaryClassifierDataTest(unittest.TestCase):
def setUp(self):
self.data = BinaryClassifierData([\
(0.1, 0),
(0.2, 0),
(0.3, 0),
(0.4, 1),
(0.5, 0),
(0.6, 1),
(0.7, 1),
(0.8, 1),
(0.9, 1)
])
def test_get_confusion_matrix(self):
mat = self.data.get_confusion_matrix(0.2)
self.assertEqual(repr(mat), "BinaryConfusionMatrix(tp=5, fp=3, fn=0, tn=1)")
mat = self.data.get_confusion_matrix(0.5)
self.assertEqual(repr(mat), "BinaryConfusionMatrix(tp=4, fp=1, fn=1, tn=3)")
mat = self.data.get_confusion_matrix(0.75)
self.assertEqual(repr(mat), "BinaryConfusionMatrix(tp=2, fp=0, fn=3, tn=4)")
mat = self.data.get_confusion_matrix(1.0)
self.assertEqual(repr(mat), "BinaryConfusionMatrix(tp=0, fp=0, fn=5, tn=4)")
def test_iter_confusion_matrices(self):
expected = """\
tp=5, fp=4, fn=0, tn=0
tp=5, fp=3, fn=0, tn=1
tp=5, fp=2, fn=0, tn=2
tp=5, fp=1, fn=0, tn=3
tp=4, fp=1, fn=1, tn=3
tp=4, fp=0, fn=1, tn=4
tp=3, fp=0, fn=2, tn=4
tp=2, fp=0, fn=3, tn=4
tp=1, fp=0, fn=4, tn=4
tp=0, fp=0, fn=5, tn=4"""
expected = ["BinaryConfusionMatrix(%s)" % line \
for line in dedent(expected).split("\n")]
for (threshold, matrix), expected in \
izip_longest(self.data.iter_confusion_matrices(), expected):
self.assertEqual(repr(matrix), expected)
self.assertEqual(matrix, self.data.get_confusion_matrix(threshold))
expected = """\
tp=5, fp=4, fn=0, tn=0
tp=5, fp=2, fn=0, tn=2
tp=4, fp=1, fn=1, tn=3
tp=2, fp=0, fn=3, tn=4
tp=0, fp=0, fn=5, tn=4"""
expected = ["BinaryConfusionMatrix(%s)" % line \
for line in dedent(expected).split("\n")]
for (threshold, matrix), expected in \
izip_longest(self.data.iter_confusion_matrices(4), expected):
self.assertEqual(repr(matrix), expected)
self.assertEqual(matrix, self.data.get_confusion_matrix(threshold))
def print_scores_for_curve(self, curve_class):
"""Calculates AUC scores for curves given by `curve_class` for all
the data in `self.data`.
`curve_class` is a subclass of `BinaryClassifierPerformanceCurve`.
`self.data` must be a dict of lists, and the ``__class__`` key of
`self.data` must map to the expected classes of elements.
"""
data = self.data
expected = data["__class__"]
keys = sorted(data.keys())
keys.remove("__class__")
print("Calculating AUCs for %s..." % curve_class.get_friendly_name())
for key in keys:
observed = data[key]
bc_data = BinaryClassifierData(zip(observed, expected), title=key)
auc = curve_class(bc_data).auc()
print(" AUC[%s] = %.4f" % (key, auc))
print("")
def get_figure_for_curves(self, curve_class):
"""Plots curves given by `curve_class` for all the data in `self.data`.
`curve_class` is a subclass of `BinaryClassifierPerformanceCurve`.
`self.data` must be a dict of lists, and the ``__class__`` key of
`self.data` must map to the expected classes of elements. Returns an
instance of `matplotlib.figure.Figure`."""
fig, axes = None, None
data = self.data
expected = data["__class__"]
keys = sorted(data.keys())
keys.remove("__class__")
styles = ["r-", "b-", "g-", "c-", "m-", "y-", "k-", \
"r--", "b--", "g--", "c--", "m--", "y--", "k--"]
# Plot the curves
line_handles, labels, aucs = [], [], []
for key, style in izip(keys, cycle(styles)):
self.log.info("Calculating %s for %s..." %
(curve_class.get_friendly_name(), key))
observed = data[key]
bc_data = BinaryClassifierData(zip(observed, expected), title=key)
curve = curve_class(bc_data)
if self.options.resampling:
curve.resample(x / 2000. for x in xrange(2001))
if self.options.show_auc:
aucs.append(curve.auc())
labels.append("%s, AUC=%.4f" % (key, aucs[-1]))
else:
labels.append(key)
if not fig:
dpi = self.options.dpi
fig = curve.get_empty_figure(dpi=dpi,
figsize=parse_size(
self.options.size, dpi=dpi))
axes = fig.get_axes()[0]
line_handle = curve.plot_on_axes(axes, style=style, legend=False)
line_handles.append(line_handle)
if aucs:
# Sort the labels of the legend in decreasing order of AUC
indices = sorted(xrange(len(aucs)),
key=aucs.__getitem__,
reverse=True)
line_handles = [line_handles[i] for i in indices]
labels = [labels[i] for i in indices]
aucs = [aucs[i] for i in indices]
if axes:
legend_pos = "best"
# Set logarithmic axes if needed
if "x" in self.options.log_scale:
axes.set_xscale("log")
legend_pos = "upper left"
if "y" in self.options.log_scale:
axes.set_yscale("log")
# Plot the legend
axes.legend(line_handles, labels, loc=legend_pos)
return fig
