def dataset_contents_print_latex(dataset_name): """Print the error case analysis in latex""" # get results txt_results = TextBasedResults() txt_results.load(dataset_name) # package data elist = extractor_list_filter(txt_results.text_eval_results.keys()) for e in elist: print "\\texttt{%s} & %d & %d & %d & %d & %d & %d \\\\ \\hline" % ( e.NAME, txt_results.result_contents(e.SLUG).rel_empty, txt_results.result_contents(e.SLUG).rel_ret_empty, txt_results.result_contents(e.SLUG).ret_empty, txt_results.result_contents(e.SLUG).missmatch, txt_results.result_contents(e.SLUG).fail, txt_results.result_contents(e.SLUG).succ, )
def dataset_contents_print_latex(dataset_name): '''Print the error case analysis in latex''' # get results txt_results = TextBasedResults() txt_results.load(dataset_name) # package data elist = extractor_list_filter(txt_results.text_eval_results.keys()) for e in elist: print '\\texttt{%s} & %d & %d & %d & %d & %d & %d \\\\ \\hline' % \ ( e.NAME, txt_results.result_contents(e.SLUG).rel_empty, txt_results.result_contents(e.SLUG).rel_ret_empty, txt_results.result_contents(e.SLUG).ret_empty, txt_results.result_contents(e.SLUG).missmatch, txt_results.result_contents(e.SLUG).fail, txt_results.result_contents(e.SLUG).succ, )
def dataset_contents_plot(dataset_name, img_name): """Plot the error case analysis.""" # get results txt_results = TextBasedResults() txt_results.load(dataset_name) txt_results.print_results() # package data elist = extractor_list_filter(txt_results.text_eval_results.keys()) extractor_slugs = tuple([e.SLUG for e in elist]) package = [ ("|rel| = 0", "#9DFADE", [txt_results.result_contents(ex).rel_empty for ex in extractor_slugs]), ( "|rel intersect ret| = 0", "#3C70A3", [txt_results.result_contents(ex).rel_ret_empty for ex in extractor_slugs], ), ("|ret| = 0", "#5CCBED", [txt_results.result_contents(ex).ret_empty for ex in extractor_slugs]), ("mismatch", "#A76CF5", [txt_results.result_contents(ex).missmatch for ex in extractor_slugs]), ("failed", "#C43156", [txt_results.result_contents(ex).fail for ex in extractor_slugs]), ("successful", "#31C460", [txt_results.result_contents(ex).succ for ex in extractor_slugs]), ] num_of_extractors = len(extractor_slugs) ind = np.arange(num_of_extractors) # the x locations for the groups width = 0.6 fig = plt.gcf() fig.legend( [plt.Rectangle((0, 0), 1, 1, fc=p[1]) for p in package], [p[0] for p in package], fancybox=True, prop=dict(size="x-small"), ) # with successful instances ax1 = plt.subplot(121) bottom_y = np.zeros(num_of_extractors) for pdata in package: ax1.bar(ind, pdata[2], width, bottom=bottom_y, color=pdata[1], ecolor="g", linewidth=0.2, alpha=0.95) bottom_y += pdata[2] ax2 = plt.subplot(122) bottom_y = np.zeros(num_of_extractors) del package[-1] for pdata in package: ax2.bar(ind, pdata[2], width, bottom=bottom_y, color=pdata[1], ecolor="g", linewidth=0.2, alpha=0.95) bottom_y += pdata[2] # xticks labels extractor_names = [get_extractor_cls(e).NAME for e in extractor_slugs] ax1.set_xticks(ind + width / 2.0) ax1.set_xticklabels(extractor_names, size="xx-small", rotation="vertical") ax2.set_xticks(ind + width / 2.0) ax2.set_xticklabels(extractor_names, size="xx-small", rotation="vertical") # grid settings fig.suptitle("Boundary cases") ax1.grid(True, alpha=0.5) ax2.grid(True, alpha=0.5) # adjustment w, h = fig.get_size_inches() fig.set_size_inches(w * 1.5, h * 1.5) fig.subplots_adjust(bottom=0.2) # output out_path = os.path.join(settings.PATH_LOCAL_DATA, "plot-output", img_name) fig.savefig(out_path, bbox_inches="tight")
class TestTextBasedResults(unittest2.TestCase): def setUp(self): self.results = TextBasedResults('e1') # Result(precision, recall, f1_score, id) self.results.add_result(Result(0, 0, float('inf'), None)) self.results.add_result(Result(float('inf'), 0, float('nan'), None)) self.results.add_result(Result(float('inf'), 0, float('nan'), None)) self.results.add_result(Result(0, float('inf'), float('nan'), None)) self.results.add_result(Result(0, float('inf'), float('nan'), None)) self.results.add_result( Result(float('inf'), float('inf'), float('nan'), None)) self.results.add_result(Result(0.2, 0.2, 0.2, None)) self.results.add_result(Result(0.2, 0.2, 0.2, None)) self.results.add_result(Result(0.2, 0.2, 0.2, None)) self.results.add_result(Result(0.2, 0.2, 0.2, None)) self.results.dataset_len = 12 def tearDown(self): self.results.text_eval_results['e1'] = [] def test_results_contents(self): contents = self.results.result_contents('e1') self.assertEqual(contents.fail, 2) self.assertEqual(contents.succ, 4) self.assertEqual(contents.rel_empty, 2) self.assertEqual(contents.ret_empty, 2) self.assertEqual(contents.rel_ret_empty, 1) self.assertEqual(contents.missmatch, 1) def test_result_filter(self): fr = self.results.filtered_results('e1') self.assertEqual(len(fr), 4) def test_precision_statistics(self): avg, std = self.results.precision_statistics('e1') self.assertEqual(avg, 0.2) self.assertEqual(std, 0.) def test_recall_statistics(self): avg, std = self.results.recall_statistics('e1') self.assertEqual(avg, 0.2) self.assertEqual(std, 0.) def test_f1score_statistics(self): avg, std = self.results.f1score_statistics('e1') self.assertEqual(avg, 0.2) self.assertEqual(std, 0.) def test_add_bad_result(self): r = TextBasedResults('e2') with self.assertRaises(AssertionError): r.add_result(Result(2, 1, 1, None)) with self.assertRaises(AssertionError): r.add_result(Result(float('inf'), float('inf'), 1, None)) with self.assertRaises(AssertionError): r.add_result(Result(float('inf'), 0, 1, None)) with self.assertRaises(AssertionError): r.add_result(Result(0, 0, 1, None)) def test_add_good_result(self): r = TextBasedResults('e3') try: r.add_result(Result(0.2, 0.2, 0.2, None)) except AssertionError: self.fail()
def dataset_contents_plot(dataset_name, img_name): '''Plot the error case analysis.''' # get results txt_results = TextBasedResults() txt_results.load(dataset_name) txt_results.print_results() # package data elist = extractor_list_filter(txt_results.text_eval_results.keys()) extractor_slugs = tuple([e.SLUG for e in elist]) package = [ ('|rel| = 0', '#9DFADE', [txt_results.result_contents(ex).rel_empty for ex in extractor_slugs]), ('|rel intersect ret| = 0', '#3C70A3', [ txt_results.result_contents(ex).rel_ret_empty for ex in extractor_slugs ]), ('|ret| = 0', '#5CCBED', [txt_results.result_contents(ex).ret_empty for ex in extractor_slugs]), ('mismatch', '#A76CF5', [txt_results.result_contents(ex).missmatch for ex in extractor_slugs]), ('failed', '#C43156', [txt_results.result_contents(ex).fail for ex in extractor_slugs]), ('successful', '#31C460', [txt_results.result_contents(ex).succ for ex in extractor_slugs]), ] num_of_extractors = len(extractor_slugs) ind = np.arange(num_of_extractors) # the x locations for the groups width = 0.6 fig = plt.gcf() fig.legend( [plt.Rectangle((0, 0), 1, 1, fc=p[1]) for p in package], [p[0] for p in package], fancybox=True, prop=dict(size='x-small'), ) # with successful instances ax1 = plt.subplot(121) bottom_y = np.zeros(num_of_extractors) for pdata in package: ax1.bar(ind, pdata[2], width, bottom=bottom_y, color=pdata[1], ecolor='g', linewidth=0.2, alpha=0.95) bottom_y += pdata[2] ax2 = plt.subplot(122) bottom_y = np.zeros(num_of_extractors) del package[-1] for pdata in package: ax2.bar(ind, pdata[2], width, bottom=bottom_y, color=pdata[1], ecolor='g', linewidth=0.2, alpha=0.95) bottom_y += pdata[2] # xticks labels extractor_names = [get_extractor_cls(e).NAME for e in extractor_slugs] ax1.set_xticks(ind + width / 2.) ax1.set_xticklabels(extractor_names, size='xx-small', rotation='vertical') ax2.set_xticks(ind + width / 2.) ax2.set_xticklabels(extractor_names, size='xx-small', rotation='vertical') # grid settings fig.suptitle('Boundary cases') ax1.grid(True, alpha=0.5) ax2.grid(True, alpha=0.5) # adjustment w, h = fig.get_size_inches() fig.set_size_inches(w * 1.5, h * 1.5) fig.subplots_adjust(bottom=0.2) # output out_path = os.path.join(settings.PATH_LOCAL_DATA, 'plot-output', img_name) fig.savefig(out_path, bbox_inches='tight')
class TestTextBasedResults(unittest2.TestCase): def setUp(self): self.results = TextBasedResults('e1') # Result(precision, recall, f1_score, id) self.results.add_result(Result(0,0,float('inf'),None)) self.results.add_result(Result(float('inf'),0,float('nan'),None)) self.results.add_result(Result(float('inf'),0,float('nan'),None)) self.results.add_result(Result(0,float('inf'),float('nan'),None)) self.results.add_result(Result(0,float('inf'),float('nan'),None)) self.results.add_result(Result(float('inf'),float('inf'),float('nan'),None)) self.results.add_result(Result(0.2,0.2,0.2,None)) self.results.add_result(Result(0.2,0.2,0.2,None)) self.results.add_result(Result(0.2,0.2,0.2,None)) self.results.add_result(Result(0.2,0.2,0.2,None)) self.results.dataset_len = 12 def tearDown(self): self.results.text_eval_results['e1'] = [] def test_results_contents(self): contents = self.results.result_contents('e1') self.assertEqual(contents.fail, 2) self.assertEqual(contents.succ, 4) self.assertEqual(contents.rel_empty, 2) self.assertEqual(contents.ret_empty, 2) self.assertEqual(contents.rel_ret_empty, 1) self.assertEqual(contents.missmatch, 1) def test_result_filter(self): fr = self.results.filtered_results('e1') self.assertEqual(len(fr), 4) def test_precision_statistics(self): avg, std = self.results.precision_statistics('e1') self.assertEqual(avg, 0.2) self.assertEqual(std, 0.) def test_recall_statistics(self): avg, std = self.results.recall_statistics('e1') self.assertEqual(avg, 0.2) self.assertEqual(std, 0.) def test_f1score_statistics(self): avg, std = self.results.f1score_statistics('e1') self.assertEqual(avg, 0.2) self.assertEqual(std, 0.) def test_add_bad_result(self): r = TextBasedResults('e2') with self.assertRaises(AssertionError): r.add_result(Result(2,1,1,None)) with self.assertRaises(AssertionError): r.add_result(Result(float('inf'),float('inf'),1,None)) with self.assertRaises(AssertionError): r.add_result(Result(float('inf'),0,1,None)) with self.assertRaises(AssertionError): r.add_result(Result(0,0,1,None)) def test_add_good_result(self): r = TextBasedResults('e3') try: r.add_result(Result(0.2,0.2,0.2,None)) except AssertionError: self.fail()