def test_grpbasic4_1():
bdmslist = compare_dirs("./testfiles/grpbasic4/test1/gold","./testfiles/grpbasic4/test1/estimated")
# gold: groups g1, g2, g3
# est: groups e1, e2
# ov: e1-g1: 0.66, e1-g2: 0.85, e1-g3: 0.66
# ov: e2-g1: 0.00, e2-g2: 0.66, e2-g3: 0.40
# e1-g1, e2-g2 is right, so both links are at 0.66
comlist = get_mention_mappings(bdmslist, 0.51)
resmap = compare_groups(comlist, 0.6)
assert len(resmap)==2
assert resmap['Equiv'].tp==2
assert resmap['Equiv'].fp==0
assert resmap['Equiv'].fn==1
assert resmap['ALL'].tp==2
assert resmap['ALL'].fp==0
assert resmap['ALL'].fn==1
# e1-g1, e2-g2
comlist = get_mention_mappings(bdmslist, 0.51)
resmap = compare_groups(comlist, 0.3)
assert len(resmap)==2
assert resmap['Equiv'].tp==2
assert resmap['Equiv'].fp==0
assert resmap['Equiv'].fn==1
assert resmap['ALL'].tp==2
assert resmap['ALL'].fp==0
assert resmap['ALL'].fn==1
# e1-g2, e2-nil
comlist = get_mention_mappings(bdmslist, 0.51)
resmap = compare_groups(comlist, 0.68)
assert len(resmap)==2
assert resmap['Equiv'].tp==1
assert resmap['Equiv'].fp==1
assert resmap['Equiv'].fn==2
assert resmap['ALL'].tp==1
assert resmap['ALL'].fp==1
assert resmap['ALL'].fn==2
# e1-nill, e2-nil
comlist = get_mention_mappings(bdmslist, 0.51)
resmap = compare_groups(comlist, 0.88)
assert len(resmap)==2
assert resmap['Equiv'].tp==0
assert resmap['Equiv'].fp==2
assert resmap['Equiv'].fn==3
assert resmap['ALL'].tp==0
assert resmap['ALL'].fp==2
assert resmap['ALL'].fn==3
def main():
parser = argparse.ArgumentParser(
description='Evaluate TAC 2018 Systematic Review Methods')
parser.add_argument('--gold_dir',
metavar='GOLD',
required=True,
type=str,
help='path to directory containing human curated data')
parser.add_argument(
'--guess_dir',
metavar='GUESS',
required=True,
type=str,
help='path to directory containing model generated data')
parser.add_argument('--res_file',
metavar='RESFILE',
required=True,
type=str,
help='filenane to write results to')
parser.add_argument('--mention_threshold',
metavar='MENTION_THRESHOLD',
required=True,
type=float,
help='threshold for mention classification (0-1)')
parser.add_argument('--group_threshold',
metavar='GROUP_THRESHOLD',
required=False,
type=float,
help='threshold for group classification (0-1)')
args = parser.parse_args()
with open(args.res_file, 'w') as out_file:
out_file.write('Results\n')
out_file.write("Gold directory " + args.gold_dir + "\n")
out_file.write("Guess directory " + args.guess_dir + "\n")
out_file.write("Mention threshold " + str(args.mention_threshold) +
"\n")
if args.group_threshold is None:
args.group_threshold = args.mention_threshold
out_file.write("Group threshold " + str(args.group_threshold) + "\n")
bdmslist = compare_dirs(args.gold_dir, args.guess_dir)
mresmap = compare_mentions(bdmslist, args.mention_threshold)
mtypes = sorted(list(mresmap.keys()))
for mtype in mtypes:
out_file.write("Mention results for type " + str(mtype) +
" is\t\t" + str(mresmap[mtype]) + "\n")
comlist = get_mention_mappings(bdmslist, args.mention_threshold)
gresmap = compare_groups(comlist, args.group_threshold)
mtypes = sorted(list(gresmap.keys()))
for mtype in mtypes:
out_file.write("Group results for type " + str(mtype) + " is\t\t" +
str(gresmap[mtype]) + "\n")
def test_grpbasic3_0():
bdmslist = compare_dirs("./testfiles/grpbasic3/test0/gold","./testfiles/grpbasic3/test0/estimated")
comlist = get_mention_mappings(bdmslist, 0.51)
resmap = compare_groups(comlist, 0.51)
assert len(resmap)==2
assert resmap['Equiv'].tp==0
assert resmap['Equiv'].fp==1
assert resmap['Equiv'].fn==0
assert resmap['ALL'].tp==0
assert resmap['ALL'].fp==1
assert resmap['ALL'].fn==0
def test_grpbasic3():
bdmslist = compare_dirs("./testfiles/grpbasic/test3/gold","./testfiles/grpbasic/test3/estimated")
comlist = get_mention_mappings(bdmslist, 0.5)
resmap = compare_groups(comlist, 0.75)
assert resmap['TestArticleGroup'].tp==1
assert resmap['TestArticleGroup'].fp==0
assert resmap['TestArticleGroup'].fn==2
assert resmap['Animal'].tp==1
assert resmap['Animal'].fp==0
assert resmap['Animal'].fn==0
assert resmap['Equiv'].tp==0
assert resmap['Equiv'].fp==0
assert resmap['Equiv'].fn==2
assert resmap['ALL'].tp==2
assert resmap['ALL'].fp==0
assert resmap['ALL'].fn==4
def test_grpbasic22():
bdmslist = compare_dirs("./testfiles/grpbasic/test2/gold","./testfiles/grpbasic/test2/estimated")
comlist = get_mention_mappings(bdmslist, 0.5)
resmap = compare_groups(comlist, 0.75) # animal goes away but testarticle still matches
assert resmap['TestArticleGroup'].tp==0
assert resmap['TestArticleGroup'].fp==1
assert resmap['TestArticleGroup'].fn==3
assert resmap['Animal'].tp==0
assert resmap['Animal'].fp==1
assert resmap['Animal'].fn==1
assert resmap['Equiv'].tp==0
assert resmap['Equiv'].fp==0
assert resmap['Equiv'].fn==2
assert resmap['ALL'].tp==0
assert resmap['ALL'].fp==2
assert resmap['ALL'].fn==6