def hilight_disagreement(test_amrs,
gold_amr,
iter_num,
aligner=default_aligner,
gold_aligned_fh=None):
"""
Input:
gold_amr: gold AMR object
test_amrs: list of AMRs to compare to
Returns list of disagreement graphs for each gold-test AMR pair.
"""
amr_graphs = []
smatchgraphs = []
gold_label = u'b'
gold_amr.rename_node(gold_label)
(gold_inst, gold_rel1, gold_rel2) = gold_amr.get_triples2()
(gold_inst_t, gold_rel1_t,
gold_rel2_t) = smatch_graph.amr2dict(gold_inst, gold_rel1, gold_rel2)
for a in test_amrs:
aligner.set_amrs(a, gold_amr)
test_label = u'a'
a.rename_node(test_label)
(test_inst, test_rel1, test_rel2) = a.get_triples2()
if gold_aligned_fh:
best_match = get_next_gold_alignments(gold_aligned_fh)
best_match_num = -1.0
else:
(best_match, best_match_num) = smatch.get_fh(
test_inst,
test_rel1,
test_rel2,
gold_inst,
gold_rel1,
gold_rel2,
test_label,
gold_label,
node_weight_fn=aligner.node_weight_fn,
edge_weight_fn=aligner.edge_weight_fn,
iter_num=iter_num)
disagreement = SmatchGraph(test_inst, test_rel1, test_rel2, \
gold_inst_t, gold_rel1_t, gold_rel2_t, \
best_match, const_map_fn=aligner.const_map_fn)
amr_graphs.append(
(disagreement.smatch2graph(node_weight_fn=aligner.node_weight_fn,
edge_weight_fn=aligner.edge_weight_fn),
best_match_num))
smatchgraphs.append(disagreement)
return (amr_graphs, smatchgraphs)
def hilight_disagreement(test_amrs,
gold_amr,
iter_num,
aligner=default_aligner,
gold_aligned_fh=None):
"""
Input:
test_amrs: list of AMRs to compare to
gold_amr: gold AMR object
iter_num: Number of random restarts to use in smatch algorithm.
Returns list of disagreement graphs for each gold-test AMR pair.
"""
smatchgraphs = []
gold_label = u'b'
gold_amr.rename_node(gold_label)
(gold_inst, gold_rel1, gold_rel2) = gold_amr.get_triples2()
(gold_inst_t, gold_rel1_t,
gold_rel2_t) = smatch_graph.amr2dict(gold_inst, gold_rel1, gold_rel2)
# TODO Also compute the weight score if we read gold alignments in from file
# TODO This would require me to handle constants when we read from file
for a in test_amrs:
aligner.set_amrs(a, gold_amr)
test_label = u'a'
a.rename_node(test_label)
(test_inst, test_rel1, test_rel2) = a.get_triples2()
if gold_aligned_fh:
best_match = get_next_gold_alignments(gold_aligned_fh)
best_match_num = -1.0
else:
(best_match, best_match_num) = smatch.get_fh(
test_inst,
test_rel1,
test_rel2,
gold_inst,
gold_rel1,
gold_rel2,
test_label,
gold_label,
node_weight_fn=aligner.node_weight_fn,
edge_weight_fn=aligner.edge_weight_fn,
iter_num=iter_num)
disagreement = SmatchGraph(test_inst, test_rel1, test_rel2, \
gold_inst_t, gold_rel1_t, gold_rel2_t, \
best_match, const_map_fn=aligner.const_map_fn)
smatchgraphs.append((disagreement, best_match_num))
return smatchgraphs
def hilight_disagreement(test_amrs, gold_amr, iter_num, aligner=default_aligner, gold_aligned_fh=None):
"""
Input:
gold_amr: gold AMR object
test_amrs: list of AMRs to compare to
Returns list of disagreement graphs for each gold-test AMR pair.
"""
amr_graphs = []
smatchgraphs = []
gold_label=u'b'
gold_amr.rename_node(gold_label)
(gold_inst, gold_rel1, gold_rel2) = gold_amr.get_triples2()
(gold_inst_t, gold_rel1_t, gold_rel2_t) = smatch_graph.amr2dict(gold_inst, gold_rel1, gold_rel2)
for a in test_amrs:
aligner.set_amrs(a, gold_amr)
test_label=u'a'
a.rename_node(test_label)
(test_inst, test_rel1, test_rel2) = a.get_triples2()
if gold_aligned_fh:
best_match = get_next_gold_alignments(gold_aligned_fh)
best_match_num = -1.0
else:
(best_match, best_match_num) = smatch.get_fh(test_inst, test_rel1, test_rel2,
gold_inst, gold_rel1, gold_rel2,
test_label, gold_label,
node_weight_fn=aligner.node_weight_fn, edge_weight_fn=aligner.edge_weight_fn,
iter_num=iter_num)
disagreement = SmatchGraph(test_inst, test_rel1, test_rel2, \
gold_inst_t, gold_rel1_t, gold_rel2_t, \
best_match, const_map_fn=aligner.const_map_fn)
amr_graphs.append((disagreement.smatch2graph(node_weight_fn=aligner.node_weight_fn,
edge_weight_fn=aligner.edge_weight_fn),
best_match_num))
smatchgraphs.append(disagreement)
return (amr_graphs, smatchgraphs)
def hilight_disagreement(test_amrs, gold_amr, iter_num, aligner=default_aligner, gold_aligned_fh=None):
"""
Input:
test_amrs: list of AMRs to compare to
gold_amr: gold AMR object
iter_num: Number of random restarts to use in smatch algorithm.
Returns list of disagreement graphs for each gold-test AMR pair.
"""
smatchgraphs = []
gold_label=u'b'
gold_amr.rename_node(gold_label)
(gold_inst, gold_rel1, gold_rel2) = gold_amr.get_triples2()
(gold_inst_t, gold_rel1_t, gold_rel2_t) = smatch_graph.amr2dict(gold_inst, gold_rel1, gold_rel2)
# TODO Also compute the weight score if we read gold alignments in from file
# TODO This would require me to handle constants when we read from file
for a in test_amrs:
aligner.set_amrs(a, gold_amr)
test_label=u'a'
a.rename_node(test_label)
(test_inst, test_rel1, test_rel2) = a.get_triples2()
if gold_aligned_fh:
best_match = get_next_gold_alignments(gold_aligned_fh)
best_match_num = -1.0
else:
(best_match, best_match_num) = smatch.get_fh(test_inst, test_rel1, test_rel2,
gold_inst, gold_rel1, gold_rel2,
test_label, gold_label,
node_weight_fn=aligner.node_weight_fn, edge_weight_fn=aligner.edge_weight_fn,
iter_num=iter_num)
disagreement = SmatchGraph(test_inst, test_rel1, test_rel2, \
gold_inst_t, gold_rel1_t, gold_rel2_t, \
best_match, const_map_fn=aligner.const_map_fn)
smatchgraphs.append((disagreement, best_match_num))
return smatchgraphs
def main(args):
"""Main function of the smatch calculation program"""
global verbose
global iter_num
global single_score
global pr_flag
global match_num_dict
# set the restart number
iter_num = args.r + 1
verbose = False
if args.ms:
single_score = False
if args.v:
verbose = True
if args.pr:
pr_flag = True
total_match_num = 0
total_test_num = 0
total_gold_num = 0
sent_num = 1
prev_amr1 = ""
outfile = open(args.outfile, 'w')
if not single_score:
outfile.write("Sentence\tText")
if pr_flag:
outfile.write("\tPrecision\tRecall")
outfile.write("\tSmatch\n")
while True:
cur_amr1 = smatch.get_amr_line(args.f[0])
(cur_amr2, comments) = amr_metadata.get_amr_line(args.f[1])
if cur_amr1 == "" and cur_amr2 == "":
break
if(cur_amr1 == ""):
# GULLY CHANGED THIS.
# IF WE RUN OUT OF AVAILABLE AMRS FROM FILE 1,
# REUSE THE LAST AVAILABLE AMR
cur_amr1 = prev_amr1
#print >> sys.stderr, "Error: File 1 has less AMRs than file 2"
#print >> sys.stderr, "Ignoring remaining AMRs"
#break
# print >> sys.stderr, "AMR 1 is empty"
# continue
if(cur_amr2 == ""):
print >> sys.stderr, "Error: File 2 has less AMRs than file 1"
print >> sys.stderr, "Ignoring remaining AMRs"
break
# print >> sys.stderr, "AMR 2 is empty"
# continue
prev_amr1 = cur_amr1
amr1 = amr.AMR.parse_AMR_line(cur_amr1)
amr2 = amr.AMR.parse_AMR_line(cur_amr2)
# We were getting screwy SMATCH scores from
# using the amr_metadata construct
meta_enabled_amr = amr_metadata.AmrMeta.from_parse(cur_amr2, comments)
test_label = "a"
gold_label = "b"
amr1.rename_node(test_label)
amr2.rename_node(gold_label)
(test_inst, test_rel1, test_rel2) = amr1.get_triples2()
(gold_inst, gold_rel1, gold_rel2) = amr2.get_triples2()
if verbose:
print "AMR pair", sent_num
print >> sys.stderr, "Instance triples of AMR 1:", len(test_inst)
print >> sys.stderr, test_inst
# print >> sys.stderr,"Relation triples of AMR 1:",len(test_rel)
print >> sys.stderr, "Relation triples of AMR 1:", len(test_rel1) + len(test_rel2)
print >>sys.stderr, test_rel1
print >> sys.stderr, test_rel2
# print >> sys.stderr, test_rel
print >> sys.stderr, "Instance triples of AMR 2:", len(gold_inst)
print >> sys.stderr, gold_inst
# print >> sys.stderr,"Relation triples of file 2:",len(gold_rel)
print >> sys.stderr, "Relation triples of AMR 2:", len(
gold_rel1) + len(gold_rel2)
#print >> sys.stderr,"Relation triples of file 2:",len(gold_rel1)+len(gold_rel2)
print >> sys.stderr, gold_rel1
print >> sys.stderr, gold_rel2
# print >> sys.stderr, gold_rel
if len(test_inst) < len(gold_inst):
(best_match,
best_match_num) = smatch.get_fh(test_inst,
test_rel1,
test_rel2,
gold_inst,
gold_rel1,
gold_rel2,
test_label,
gold_label)
if verbose:
print >> sys.stderr, "AMR pair ", sent_num
print >> sys.stderr, "best match number", best_match_num
print >> sys.stderr, "best match", best_match
else:
(best_match,
best_match_num) = smatch.get_fh(gold_inst,
gold_rel1,
gold_rel2,
test_inst,
test_rel1,
test_rel2,
gold_label,
test_label)
if verbose:
print >> sys.stderr, "Sent ", sent_num
print >> sys.stderr, "best match number", best_match_num
print >> sys.stderr, "best match", best_match
if not single_score:
#(precision,
# recall,
# best_f_score) = smatch.compute_f(best_match_num,
# len(test_rel1) + len(test_inst) + len(test_rel2),
# len(gold_rel1) + len(gold_inst) + len(gold_rel2))
outfile.write( str(meta_enabled_amr.metadata.get("tok", None)) )
#if pr_flag:
# outfile.write( "\t%.2f" % precision )
# outfile.write( "\t%.2f" % recall )
#outfile.write( "\t%.2f" % best_f_score )
print sent_num
outfile.write( "\n" )
total_match_num += best_match_num
total_test_num += len(test_rel1) + len(test_rel2) + len(test_inst)
total_gold_num += len(gold_rel1) + len(gold_rel2) + len(gold_inst)
match_num_dict.clear()
sent_num += 1 # print "F-score:",best_f_score
if verbose:
print >> sys.stderr, "Total match num"
print >> sys.stderr, total_match_num, total_test_num, total_gold_num
if single_score:
(precision, recall, best_f_score) = smatch.compute_f(
total_match_num, total_test_num, total_gold_num)
if pr_flag:
print "Precision: %.2f" % precision
print "Recall: %.2f" % recall
print "Document F-score: %.2f" % best_f_score
args.f[0].close()
args.f[1].close()
outfile.close()