def dataset_contents_print_latex(dataset_name):
"""Print the error case analysis in latex"""
# get results
txt_results = TextBasedResults()
txt_results.load(dataset_name)
# package data
elist = extractor_list_filter(txt_results.text_eval_results.keys())
for e in elist:
print "\\texttt{%s} & %d & %d & %d & %d & %d & %d \\\\ \\hline" % (
e.NAME,
txt_results.result_contents(e.SLUG).rel_empty,
txt_results.result_contents(e.SLUG).rel_ret_empty,
txt_results.result_contents(e.SLUG).ret_empty,
txt_results.result_contents(e.SLUG).missmatch,
txt_results.result_contents(e.SLUG).fail,
txt_results.result_contents(e.SLUG).succ,
)
def dataset_contents_print_latex(dataset_name):
'''Print the error case analysis in latex'''
# get results
txt_results = TextBasedResults()
txt_results.load(dataset_name)
# package data
elist = extractor_list_filter(txt_results.text_eval_results.keys())
for e in elist:
print '\\texttt{%s} & %d & %d & %d & %d & %d & %d \\\\ \\hline' % \
(
e.NAME,
txt_results.result_contents(e.SLUG).rel_empty,
txt_results.result_contents(e.SLUG).rel_ret_empty,
txt_results.result_contents(e.SLUG).ret_empty,
txt_results.result_contents(e.SLUG).missmatch,
txt_results.result_contents(e.SLUG).fail,
txt_results.result_contents(e.SLUG).succ,
)
def dataset_contents_plot(dataset_name, img_name):
"""Plot the error case analysis."""
# get results
txt_results = TextBasedResults()
txt_results.load(dataset_name)
txt_results.print_results()
# package data
elist = extractor_list_filter(txt_results.text_eval_results.keys())
extractor_slugs = tuple([e.SLUG for e in elist])
package = [
("|rel| = 0", "#9DFADE", [txt_results.result_contents(ex).rel_empty for ex in extractor_slugs]),
(
"|rel intersect ret| = 0",
"#3C70A3",
[txt_results.result_contents(ex).rel_ret_empty for ex in extractor_slugs],
),
("|ret| = 0", "#5CCBED", [txt_results.result_contents(ex).ret_empty for ex in extractor_slugs]),
("mismatch", "#A76CF5", [txt_results.result_contents(ex).missmatch for ex in extractor_slugs]),
("failed", "#C43156", [txt_results.result_contents(ex).fail for ex in extractor_slugs]),
("successful", "#31C460", [txt_results.result_contents(ex).succ for ex in extractor_slugs]),
]
num_of_extractors = len(extractor_slugs)
ind = np.arange(num_of_extractors) # the x locations for the groups
width = 0.6
fig = plt.gcf()
fig.legend(
[plt.Rectangle((0, 0), 1, 1, fc=p[1]) for p in package],
[p[0] for p in package],
fancybox=True,
prop=dict(size="x-small"),
)
# with successful instances
ax1 = plt.subplot(121)
bottom_y = np.zeros(num_of_extractors)
for pdata in package:
ax1.bar(ind, pdata[2], width, bottom=bottom_y, color=pdata[1], ecolor="g", linewidth=0.2, alpha=0.95)
bottom_y += pdata[2]
ax2 = plt.subplot(122)
bottom_y = np.zeros(num_of_extractors)
del package[-1]
for pdata in package:
ax2.bar(ind, pdata[2], width, bottom=bottom_y, color=pdata[1], ecolor="g", linewidth=0.2, alpha=0.95)
bottom_y += pdata[2]
# xticks labels
extractor_names = [get_extractor_cls(e).NAME for e in extractor_slugs]
ax1.set_xticks(ind + width / 2.0)
ax1.set_xticklabels(extractor_names, size="xx-small", rotation="vertical")
ax2.set_xticks(ind + width / 2.0)
ax2.set_xticklabels(extractor_names, size="xx-small", rotation="vertical")
# grid settings
fig.suptitle("Boundary cases")
ax1.grid(True, alpha=0.5)
ax2.grid(True, alpha=0.5)
# adjustment
w, h = fig.get_size_inches()
fig.set_size_inches(w * 1.5, h * 1.5)
fig.subplots_adjust(bottom=0.2)
# output
out_path = os.path.join(settings.PATH_LOCAL_DATA, "plot-output", img_name)
fig.savefig(out_path, bbox_inches="tight")
class TestTextBasedResults(unittest2.TestCase):
def setUp(self):
self.results = TextBasedResults('e1')
# Result(precision, recall, f1_score, id)
self.results.add_result(Result(0, 0, float('inf'), None))
self.results.add_result(Result(float('inf'), 0, float('nan'), None))
self.results.add_result(Result(float('inf'), 0, float('nan'), None))
self.results.add_result(Result(0, float('inf'), float('nan'), None))
self.results.add_result(Result(0, float('inf'), float('nan'), None))
self.results.add_result(
Result(float('inf'), float('inf'), float('nan'), None))
self.results.add_result(Result(0.2, 0.2, 0.2, None))
self.results.add_result(Result(0.2, 0.2, 0.2, None))
self.results.add_result(Result(0.2, 0.2, 0.2, None))
self.results.add_result(Result(0.2, 0.2, 0.2, None))
self.results.dataset_len = 12
def tearDown(self):
self.results.text_eval_results['e1'] = []
def test_results_contents(self):
contents = self.results.result_contents('e1')
self.assertEqual(contents.fail, 2)
self.assertEqual(contents.succ, 4)
self.assertEqual(contents.rel_empty, 2)
self.assertEqual(contents.ret_empty, 2)
self.assertEqual(contents.rel_ret_empty, 1)
self.assertEqual(contents.missmatch, 1)
def test_result_filter(self):
fr = self.results.filtered_results('e1')
self.assertEqual(len(fr), 4)
def test_precision_statistics(self):
avg, std = self.results.precision_statistics('e1')
self.assertEqual(avg, 0.2)
self.assertEqual(std, 0.)
def test_recall_statistics(self):
avg, std = self.results.recall_statistics('e1')
self.assertEqual(avg, 0.2)
self.assertEqual(std, 0.)
def test_f1score_statistics(self):
avg, std = self.results.f1score_statistics('e1')
self.assertEqual(avg, 0.2)
self.assertEqual(std, 0.)
def test_add_bad_result(self):
r = TextBasedResults('e2')
with self.assertRaises(AssertionError):
r.add_result(Result(2, 1, 1, None))
with self.assertRaises(AssertionError):
r.add_result(Result(float('inf'), float('inf'), 1, None))
with self.assertRaises(AssertionError):
r.add_result(Result(float('inf'), 0, 1, None))
with self.assertRaises(AssertionError):
r.add_result(Result(0, 0, 1, None))
def test_add_good_result(self):
r = TextBasedResults('e3')
try:
r.add_result(Result(0.2, 0.2, 0.2, None))
except AssertionError:
self.fail()
def dataset_contents_plot(dataset_name, img_name):
'''Plot the error case analysis.'''
# get results
txt_results = TextBasedResults()
txt_results.load(dataset_name)
txt_results.print_results()
# package data
elist = extractor_list_filter(txt_results.text_eval_results.keys())
extractor_slugs = tuple([e.SLUG for e in elist])
package = [
('|rel| = 0', '#9DFADE',
[txt_results.result_contents(ex).rel_empty
for ex in extractor_slugs]),
('|rel intersect ret| = 0', '#3C70A3', [
txt_results.result_contents(ex).rel_ret_empty
for ex in extractor_slugs
]),
('|ret| = 0', '#5CCBED',
[txt_results.result_contents(ex).ret_empty
for ex in extractor_slugs]),
('mismatch', '#A76CF5',
[txt_results.result_contents(ex).missmatch
for ex in extractor_slugs]),
('failed', '#C43156',
[txt_results.result_contents(ex).fail for ex in extractor_slugs]),
('successful', '#31C460',
[txt_results.result_contents(ex).succ for ex in extractor_slugs]),
]
num_of_extractors = len(extractor_slugs)
ind = np.arange(num_of_extractors) # the x locations for the groups
width = 0.6
fig = plt.gcf()
fig.legend(
[plt.Rectangle((0, 0), 1, 1, fc=p[1]) for p in package],
[p[0] for p in package],
fancybox=True,
prop=dict(size='x-small'),
)
# with successful instances
ax1 = plt.subplot(121)
bottom_y = np.zeros(num_of_extractors)
for pdata in package:
ax1.bar(ind,
pdata[2],
width,
bottom=bottom_y,
color=pdata[1],
ecolor='g',
linewidth=0.2,
alpha=0.95)
bottom_y += pdata[2]
ax2 = plt.subplot(122)
bottom_y = np.zeros(num_of_extractors)
del package[-1]
for pdata in package:
ax2.bar(ind,
pdata[2],
width,
bottom=bottom_y,
color=pdata[1],
ecolor='g',
linewidth=0.2,
alpha=0.95)
bottom_y += pdata[2]
# xticks labels
extractor_names = [get_extractor_cls(e).NAME for e in extractor_slugs]
ax1.set_xticks(ind + width / 2.)
ax1.set_xticklabels(extractor_names, size='xx-small', rotation='vertical')
ax2.set_xticks(ind + width / 2.)
ax2.set_xticklabels(extractor_names, size='xx-small', rotation='vertical')
# grid settings
fig.suptitle('Boundary cases')
ax1.grid(True, alpha=0.5)
ax2.grid(True, alpha=0.5)
# adjustment
w, h = fig.get_size_inches()
fig.set_size_inches(w * 1.5, h * 1.5)
fig.subplots_adjust(bottom=0.2)
# output
out_path = os.path.join(settings.PATH_LOCAL_DATA, 'plot-output', img_name)
fig.savefig(out_path, bbox_inches='tight')
class TestTextBasedResults(unittest2.TestCase):
def setUp(self):
self.results = TextBasedResults('e1')
# Result(precision, recall, f1_score, id)
self.results.add_result(Result(0,0,float('inf'),None))
self.results.add_result(Result(float('inf'),0,float('nan'),None))
self.results.add_result(Result(float('inf'),0,float('nan'),None))
self.results.add_result(Result(0,float('inf'),float('nan'),None))
self.results.add_result(Result(0,float('inf'),float('nan'),None))
self.results.add_result(Result(float('inf'),float('inf'),float('nan'),None))
self.results.add_result(Result(0.2,0.2,0.2,None))
self.results.add_result(Result(0.2,0.2,0.2,None))
self.results.add_result(Result(0.2,0.2,0.2,None))
self.results.add_result(Result(0.2,0.2,0.2,None))
self.results.dataset_len = 12
def tearDown(self):
self.results.text_eval_results['e1'] = []
def test_results_contents(self):
contents = self.results.result_contents('e1')
self.assertEqual(contents.fail, 2)
self.assertEqual(contents.succ, 4)
self.assertEqual(contents.rel_empty, 2)
self.assertEqual(contents.ret_empty, 2)
self.assertEqual(contents.rel_ret_empty, 1)
self.assertEqual(contents.missmatch, 1)
def test_result_filter(self):
fr = self.results.filtered_results('e1')
self.assertEqual(len(fr), 4)
def test_precision_statistics(self):
avg, std = self.results.precision_statistics('e1')
self.assertEqual(avg, 0.2)
self.assertEqual(std, 0.)
def test_recall_statistics(self):
avg, std = self.results.recall_statistics('e1')
self.assertEqual(avg, 0.2)
self.assertEqual(std, 0.)
def test_f1score_statistics(self):
avg, std = self.results.f1score_statistics('e1')
self.assertEqual(avg, 0.2)
self.assertEqual(std, 0.)
def test_add_bad_result(self):
r = TextBasedResults('e2')
with self.assertRaises(AssertionError):
r.add_result(Result(2,1,1,None))
with self.assertRaises(AssertionError):
r.add_result(Result(float('inf'),float('inf'),1,None))
with self.assertRaises(AssertionError):
r.add_result(Result(float('inf'),0,1,None))
with self.assertRaises(AssertionError):
r.add_result(Result(0,0,1,None))
def test_add_good_result(self):
r = TextBasedResults('e3')
try:
r.add_result(Result(0.2,0.2,0.2,None))
except AssertionError:
self.fail()
