def test_grpbasic4_1(): bdmslist = compare_dirs("./testfiles/grpbasic4/test1/gold","./testfiles/grpbasic4/test1/estimated") # gold: groups g1, g2, g3 # est: groups e1, e2 # ov: e1-g1: 0.66, e1-g2: 0.85, e1-g3: 0.66 # ov: e2-g1: 0.00, e2-g2: 0.66, e2-g3: 0.40 # e1-g1, e2-g2 is right, so both links are at 0.66 comlist = get_mention_mappings(bdmslist, 0.51) resmap = compare_groups(comlist, 0.6) assert len(resmap)==2 assert resmap['Equiv'].tp==2 assert resmap['Equiv'].fp==0 assert resmap['Equiv'].fn==1 assert resmap['ALL'].tp==2 assert resmap['ALL'].fp==0 assert resmap['ALL'].fn==1 # e1-g1, e2-g2 comlist = get_mention_mappings(bdmslist, 0.51) resmap = compare_groups(comlist, 0.3) assert len(resmap)==2 assert resmap['Equiv'].tp==2 assert resmap['Equiv'].fp==0 assert resmap['Equiv'].fn==1 assert resmap['ALL'].tp==2 assert resmap['ALL'].fp==0 assert resmap['ALL'].fn==1 # e1-g2, e2-nil comlist = get_mention_mappings(bdmslist, 0.51) resmap = compare_groups(comlist, 0.68) assert len(resmap)==2 assert resmap['Equiv'].tp==1 assert resmap['Equiv'].fp==1 assert resmap['Equiv'].fn==2 assert resmap['ALL'].tp==1 assert resmap['ALL'].fp==1 assert resmap['ALL'].fn==2 # e1-nill, e2-nil comlist = get_mention_mappings(bdmslist, 0.51) resmap = compare_groups(comlist, 0.88) assert len(resmap)==2 assert resmap['Equiv'].tp==0 assert resmap['Equiv'].fp==2 assert resmap['Equiv'].fn==3 assert resmap['ALL'].tp==0 assert resmap['ALL'].fp==2 assert resmap['ALL'].fn==3
def main(): parser = argparse.ArgumentParser( description='Evaluate TAC 2018 Systematic Review Methods') parser.add_argument('--gold_dir', metavar='GOLD', required=True, type=str, help='path to directory containing human curated data') parser.add_argument( '--guess_dir', metavar='GUESS', required=True, type=str, help='path to directory containing model generated data') parser.add_argument('--res_file', metavar='RESFILE', required=True, type=str, help='filenane to write results to') parser.add_argument('--mention_threshold', metavar='MENTION_THRESHOLD', required=True, type=float, help='threshold for mention classification (0-1)') parser.add_argument('--group_threshold', metavar='GROUP_THRESHOLD', required=False, type=float, help='threshold for group classification (0-1)') args = parser.parse_args() with open(args.res_file, 'w') as out_file: out_file.write('Results\n') out_file.write("Gold directory " + args.gold_dir + "\n") out_file.write("Guess directory " + args.guess_dir + "\n") out_file.write("Mention threshold " + str(args.mention_threshold) + "\n") if args.group_threshold is None: args.group_threshold = args.mention_threshold out_file.write("Group threshold " + str(args.group_threshold) + "\n") bdmslist = compare_dirs(args.gold_dir, args.guess_dir) mresmap = compare_mentions(bdmslist, args.mention_threshold) mtypes = sorted(list(mresmap.keys())) for mtype in mtypes: out_file.write("Mention results for type " + str(mtype) + " is\t\t" + str(mresmap[mtype]) + "\n") comlist = get_mention_mappings(bdmslist, args.mention_threshold) gresmap = compare_groups(comlist, args.group_threshold) mtypes = sorted(list(gresmap.keys())) for mtype in mtypes: out_file.write("Group results for type " + str(mtype) + " is\t\t" + str(gresmap[mtype]) + "\n")
def test_grpbasic3_0(): bdmslist = compare_dirs("./testfiles/grpbasic3/test0/gold","./testfiles/grpbasic3/test0/estimated") comlist = get_mention_mappings(bdmslist, 0.51) resmap = compare_groups(comlist, 0.51) assert len(resmap)==2 assert resmap['Equiv'].tp==0 assert resmap['Equiv'].fp==1 assert resmap['Equiv'].fn==0 assert resmap['ALL'].tp==0 assert resmap['ALL'].fp==1 assert resmap['ALL'].fn==0
def test_grpbasic3(): bdmslist = compare_dirs("./testfiles/grpbasic/test3/gold","./testfiles/grpbasic/test3/estimated") comlist = get_mention_mappings(bdmslist, 0.5) resmap = compare_groups(comlist, 0.75) assert resmap['TestArticleGroup'].tp==1 assert resmap['TestArticleGroup'].fp==0 assert resmap['TestArticleGroup'].fn==2 assert resmap['Animal'].tp==1 assert resmap['Animal'].fp==0 assert resmap['Animal'].fn==0 assert resmap['Equiv'].tp==0 assert resmap['Equiv'].fp==0 assert resmap['Equiv'].fn==2 assert resmap['ALL'].tp==2 assert resmap['ALL'].fp==0 assert resmap['ALL'].fn==4
def test_grpbasic22(): bdmslist = compare_dirs("./testfiles/grpbasic/test2/gold","./testfiles/grpbasic/test2/estimated") comlist = get_mention_mappings(bdmslist, 0.5) resmap = compare_groups(comlist, 0.75) # animal goes away but testarticle still matches assert resmap['TestArticleGroup'].tp==0 assert resmap['TestArticleGroup'].fp==1 assert resmap['TestArticleGroup'].fn==3 assert resmap['Animal'].tp==0 assert resmap['Animal'].fp==1 assert resmap['Animal'].fn==1 assert resmap['Equiv'].tp==0 assert resmap['Equiv'].fp==0 assert resmap['Equiv'].fn==2 assert resmap['ALL'].tp==0 assert resmap['ALL'].fp==2 assert resmap['ALL'].fn==6
def test_grpbasic_mention_match(): bdmslist = compare_dirs("./testfiles/grpbasic/testmatch/gold","./testfiles/grpbasic/testmatch/estimated") comlist = get_mention_mappings(bdmslist, 0.9) assert len(comlist)==1 assert len(comlist[0])==3 assert len(comlist[0][2])==9 assert "T11" not in comlist[0][2] assert "T12" not in comlist[0][2] assert "T13" not in comlist[0][2] assert comlist[0][2]["T35"] == "T351" assert comlist[0][2]["T36"] == "T361" assert comlist[0][2]["T37"] == "T371" assert comlist[0][2]["T38"] == "T381" assert comlist[0][2]["T39"] == "T391" assert "T140" not in comlist[0][2] assert comlist[0][2]["T50"] == "T501" assert "T151" not in comlist[0][2] assert comlist[0][2]["T54"] == "T541" assert comlist[0][2]["T56"] == "T561" assert "T157" not in comlist[0][2] assert comlist[0][2]["T58"] == "T581"