def extractor_stat_plot(dataset_name, img_name):
"""Plot the distributions of per-document precision, recall & F1 score """
# np.seterr(all='raise')
fig = plt.figure()
# get results and repackage the data
txt_results = TextBasedResults()
txt_results.load(dataset_name)
txt_results.print_results()
elist = extractor_list_filter(txt_results.text_eval_results.keys())
for ex_index, extractor_cls in enumerate(elist):
# repackage results
extractor_results = txt_results.filtered_results(extractor_cls.SLUG)
results_list_prec = [r.precision for r in extractor_results]
results_list_rec = [r.recall for r in extractor_results]
results_list_f1 = [r.f1_score for r in extractor_results]
width = 0.05 # the width of the bars
ind = np.arange(0, 1, width)
n = len(ind)
print extractor_cls.NAME
eq_count_prec = equidistant_count(0, 1, width, results_list_prec)
print len(results_list_prec)
print sum(eq_count_prec)
eq_count_rec = equidistant_count(0, 1, width, results_list_rec)
print len(results_list_rec)
print sum(eq_count_rec)
eq_count_f1 = equidistant_count(0, 1, width, results_list_f1)
print len(results_list_f1)
print sum(eq_count_f1)
# plotting
ax = fig.add_subplot(6, 3, ex_index + 1, projection="3d")
ax.bar3d(
ind,
np.array([0] * n),
np.array([0] * n),
dx=width,
dy=width * 2,
dz=eq_count_prec,
color="b",
linewidth=0.3,
alpha=0.4,
)
ax.bar3d(
ind,
np.array([1] * n),
np.array([0] * n),
dx=width,
dy=width * 2,
dz=eq_count_rec,
color="c",
linewidth=0.3,
alpha=0.5,
)
ax.bar3d(
ind,
np.array([2] * n),
np.array([0] * n),
dx=width,
dy=width * 2,
dz=eq_count_f1,
color="m",
linewidth=0.3,
alpha=0.8,
)
ax.set_title(extractor_cls.NAME, size="small")
# ax.set_xlabel('\nlimits',size = 'x-small', linespacing=2)
ax.set_zlabel("\nnum. of instances", size="x-small", linespacing=1)
ax.yaxis.set_ticks([])
resize_axis_tick_labels(ax.xaxis)
resize_axis_tick_labels(ax.zaxis)
ax.grid(True, alpha=0.7)
# with 3d plotting we need to use proxy artist because legends
# are not supported
blue = plt.Rectangle((0, 0), 1, 1, fc="b") # proxys
cyan = plt.Rectangle((0, 0), 1, 1, fc="c")
mag = plt.Rectangle((0, 0), 1, 1, fc="m")
fig.legend((blue, cyan, mag), ("precision", "recall", "f1 score"), fancybox=True, prop=dict(size="x-small"))
w, h = fig.get_size_inches()
fig.set_size_inches(w * 1.5, h * 2.5)
fig.subplots_adjust(wspace=0.025, hspace=0.15)
# save plot
out_path = os.path.join(settings.PATH_LOCAL_DATA, "plot-output", img_name)
fig.savefig(out_path, bbox_inches="tight")
def extractor_stat_plot(dataset_name, img_name):
'''Plot the distributions of per-document precision, recall & F1 score '''
#np.seterr(all='raise')
fig = plt.figure()
# get results and repackage the data
txt_results = TextBasedResults()
txt_results.load(dataset_name)
txt_results.print_results()
elist = extractor_list_filter(txt_results.text_eval_results.keys())
for ex_index, extractor_cls in enumerate(elist):
# repackage results
extractor_results = txt_results.filtered_results(extractor_cls.SLUG)
results_list_prec = [r.precision for r in extractor_results]
results_list_rec = [r.recall for r in extractor_results]
results_list_f1 = [r.f1_score for r in extractor_results]
width = 0.05 # the width of the bars
ind = np.arange(0, 1, width)
n = len(ind)
print extractor_cls.NAME
eq_count_prec = equidistant_count(0, 1, width, results_list_prec)
print len(results_list_prec)
print sum(eq_count_prec)
eq_count_rec = equidistant_count(0, 1, width, results_list_rec)
print len(results_list_rec)
print sum(eq_count_rec)
eq_count_f1 = equidistant_count(0, 1, width, results_list_f1)
print len(results_list_f1)
print sum(eq_count_f1)
# plotting
ax = fig.add_subplot(6, 3, ex_index + 1, projection='3d')
ax.bar3d(ind,
np.array([0] * n),
np.array([0] * n),
dx=width,
dy=width * 2,
dz=eq_count_prec,
color='b',
linewidth=0.3,
alpha=0.4)
ax.bar3d(ind,
np.array([1] * n),
np.array([0] * n),
dx=width,
dy=width * 2,
dz=eq_count_rec,
color='c',
linewidth=0.3,
alpha=0.5)
ax.bar3d(ind,
np.array([2] * n),
np.array([0] * n),
dx=width,
dy=width * 2,
dz=eq_count_f1,
color='m',
linewidth=0.3,
alpha=0.8)
ax.set_title(extractor_cls.NAME, size='small')
#ax.set_xlabel('\nlimits',size = 'x-small', linespacing=2)
ax.set_zlabel('\nnum. of instances', size='x-small', linespacing=1)
ax.yaxis.set_ticks([])
resize_axis_tick_labels(ax.xaxis)
resize_axis_tick_labels(ax.zaxis)
ax.grid(True, alpha=0.7)
# with 3d plotting we need to use proxy artist because legends
# are not supported
blue = plt.Rectangle((0, 0), 1, 1, fc='b') # proxys
cyan = plt.Rectangle((0, 0), 1, 1, fc='c')
mag = plt.Rectangle((0, 0), 1, 1, fc='m')
fig.legend((blue, cyan, mag), ('precision', 'recall', 'f1 score'),
fancybox=True,
prop=dict(size='x-small'))
w, h = fig.get_size_inches()
fig.set_size_inches(w * 1.5, h * 2.5)
fig.subplots_adjust(wspace=0.025, hspace=0.15)
# save plot
out_path = os.path.join(settings.PATH_LOCAL_DATA, 'plot-output', img_name)
fig.savefig(out_path, bbox_inches='tight')
class TestTextBasedResults(unittest2.TestCase):
def setUp(self):
self.results = TextBasedResults('e1')
# Result(precision, recall, f1_score, id)
self.results.add_result(Result(0, 0, float('inf'), None))
self.results.add_result(Result(float('inf'), 0, float('nan'), None))
self.results.add_result(Result(float('inf'), 0, float('nan'), None))
self.results.add_result(Result(0, float('inf'), float('nan'), None))
self.results.add_result(Result(0, float('inf'), float('nan'), None))
self.results.add_result(
Result(float('inf'), float('inf'), float('nan'), None))
self.results.add_result(Result(0.2, 0.2, 0.2, None))
self.results.add_result(Result(0.2, 0.2, 0.2, None))
self.results.add_result(Result(0.2, 0.2, 0.2, None))
self.results.add_result(Result(0.2, 0.2, 0.2, None))
self.results.dataset_len = 12
def tearDown(self):
self.results.text_eval_results['e1'] = []
def test_results_contents(self):
contents = self.results.result_contents('e1')
self.assertEqual(contents.fail, 2)
self.assertEqual(contents.succ, 4)
self.assertEqual(contents.rel_empty, 2)
self.assertEqual(contents.ret_empty, 2)
self.assertEqual(contents.rel_ret_empty, 1)
self.assertEqual(contents.missmatch, 1)
def test_result_filter(self):
fr = self.results.filtered_results('e1')
self.assertEqual(len(fr), 4)
def test_precision_statistics(self):
avg, std = self.results.precision_statistics('e1')
self.assertEqual(avg, 0.2)
self.assertEqual(std, 0.)
def test_recall_statistics(self):
avg, std = self.results.recall_statistics('e1')
self.assertEqual(avg, 0.2)
self.assertEqual(std, 0.)
def test_f1score_statistics(self):
avg, std = self.results.f1score_statistics('e1')
self.assertEqual(avg, 0.2)
self.assertEqual(std, 0.)
def test_add_bad_result(self):
r = TextBasedResults('e2')
with self.assertRaises(AssertionError):
r.add_result(Result(2, 1, 1, None))
with self.assertRaises(AssertionError):
r.add_result(Result(float('inf'), float('inf'), 1, None))
with self.assertRaises(AssertionError):
r.add_result(Result(float('inf'), 0, 1, None))
with self.assertRaises(AssertionError):
r.add_result(Result(0, 0, 1, None))
def test_add_good_result(self):
r = TextBasedResults('e3')
try:
r.add_result(Result(0.2, 0.2, 0.2, None))
except AssertionError:
self.fail()
class TestTextBasedResults(unittest2.TestCase):
def setUp(self):
self.results = TextBasedResults('e1')
# Result(precision, recall, f1_score, id)
self.results.add_result(Result(0,0,float('inf'),None))
self.results.add_result(Result(float('inf'),0,float('nan'),None))
self.results.add_result(Result(float('inf'),0,float('nan'),None))
self.results.add_result(Result(0,float('inf'),float('nan'),None))
self.results.add_result(Result(0,float('inf'),float('nan'),None))
self.results.add_result(Result(float('inf'),float('inf'),float('nan'),None))
self.results.add_result(Result(0.2,0.2,0.2,None))
self.results.add_result(Result(0.2,0.2,0.2,None))
self.results.add_result(Result(0.2,0.2,0.2,None))
self.results.add_result(Result(0.2,0.2,0.2,None))
self.results.dataset_len = 12
def tearDown(self):
self.results.text_eval_results['e1'] = []
def test_results_contents(self):
contents = self.results.result_contents('e1')
self.assertEqual(contents.fail, 2)
self.assertEqual(contents.succ, 4)
self.assertEqual(contents.rel_empty, 2)
self.assertEqual(contents.ret_empty, 2)
self.assertEqual(contents.rel_ret_empty, 1)
self.assertEqual(contents.missmatch, 1)
def test_result_filter(self):
fr = self.results.filtered_results('e1')
self.assertEqual(len(fr), 4)
def test_precision_statistics(self):
avg, std = self.results.precision_statistics('e1')
self.assertEqual(avg, 0.2)
self.assertEqual(std, 0.)
def test_recall_statistics(self):
avg, std = self.results.recall_statistics('e1')
self.assertEqual(avg, 0.2)
self.assertEqual(std, 0.)
def test_f1score_statistics(self):
avg, std = self.results.f1score_statistics('e1')
self.assertEqual(avg, 0.2)
self.assertEqual(std, 0.)
def test_add_bad_result(self):
r = TextBasedResults('e2')
with self.assertRaises(AssertionError):
r.add_result(Result(2,1,1,None))
with self.assertRaises(AssertionError):
r.add_result(Result(float('inf'),float('inf'),1,None))
with self.assertRaises(AssertionError):
r.add_result(Result(float('inf'),0,1,None))
with self.assertRaises(AssertionError):
r.add_result(Result(0,0,1,None))
def test_add_good_result(self):
r = TextBasedResults('e3')
try:
r.add_result(Result(0.2,0.2,0.2,None))
except AssertionError:
self.fail()
