def evaluate_rel_arg_whole_rel(relation_pairs, partial_match_cutoff):
total_correct = 0.0
total_gold = 0.0
total_predicted = 0.0
for g_relation, p_relation in relation_pairs:
assert g_relation is not None or p_relation is not None
if g_relation is None:
total_predicted += 1
elif p_relation is None:
total_gold += 1
else:
g_arg1 = g_relation['Arg1']['TokenIndexSet'] \
if g_relation is not None else set([])
p_arg1 = p_relation['Arg1']['TokenIndexSet'] \
if p_relation is not None else set([])
arg1_f1_score = aligner.compute_f1_span(g_arg1, p_arg1)
g_arg2 = g_relation['Arg2']['TokenIndexSet'] \
if g_relation is not None else set([])
p_arg2 = p_relation['Arg2']['TokenIndexSet'] \
if p_relation is not None else set([])
arg2_f1_score = aligner.compute_f1_span(g_arg2, p_arg2)
if arg1_f1_score >= partial_match_cutoff and \
arg2_f1_score >= partial_match_cutoff:
total_correct += 1
total_predicted += 1
total_gold += 1
return compute_prf(total_gold, total_predicted, total_correct)
def evaluate_rel_arg_whole_rel(relation_pairs, partial_match_cutoff):
total_correct = 0.0
total_gold = 0.0
total_predicted = 0.0
for g_relation, p_relation in relation_pairs:
assert g_relation is not None or p_relation is not None
if g_relation is None:
total_predicted += 1
elif p_relation is None:
total_gold += 1
else:
g_arg1 = g_relation['Arg1']['TokenIndexSet'] \
if g_relation is not None else set([])
p_arg1 = p_relation['Arg1']['TokenIndexSet'] \
if p_relation is not None else set([])
arg1_f1_score = aligner.compute_f1_span(g_arg1, p_arg1)
g_arg2 = g_relation['Arg2']['TokenIndexSet'] \
if g_relation is not None else set([])
p_arg2 = p_relation['Arg2']['TokenIndexSet'] \
if p_relation is not None else set([])
arg2_f1_score = aligner.compute_f1_span(g_arg2, p_arg2)
if arg1_f1_score >= partial_match_cutoff and \
arg2_f1_score >= partial_match_cutoff:
total_correct += 1
total_predicted += 1
total_gold += 1
return compute_prf(total_gold, total_predicted, total_correct)
def evaluate_arg_partial_match(relation_pairs, position, partial_match_cutoff):
"""Evaluate the argument based on partial matching criterion
We evaluate the argument as a whole.
"""
assert position == 1 or position == 2
total_correct = 0.0
total_gold = 0.0
total_predicted = 0.0
for g_relation, p_relation in relation_pairs:
assert g_relation is not None or p_relation is not None
if g_relation is None:
total_predicted += 1
elif p_relation is None:
total_gold += 1
else:
g_arg = g_relation['Arg%s' % position]['TokenIndexSet']
p_arg = p_relation['Arg%s' % position]['TokenIndexSet']
f1_score = aligner.compute_f1_span(g_arg, p_arg)
if f1_score >= partial_match_cutoff:
total_correct += 1
total_predicted += 1
total_gold += 1
return total_gold, total_predicted, total_correct
def evaluate_arg_partial_match(relation_pairs, position, partial_match_cutoff):
"""Evaluate the argument based on partial matching criterion
We evaluate the argument as a whole.
"""
assert position == 1 or position == 2
total_correct = 0.0
total_gold = 0.0
total_predicted = 0.0
for g_relation, p_relation in relation_pairs:
assert g_relation is not None or p_relation is not None
if g_relation is None:
total_predicted += 1
elif p_relation is None:
total_gold += 1
else:
g_arg = g_relation["Arg%s" % position]["TokenIndexSet"]
p_arg = p_relation["Arg%s" % position]["TokenIndexSet"]
f1_score = aligner.compute_f1_span(g_arg, p_arg)
if f1_score >= partial_match_cutoff:
total_correct += 1
total_predicted += 1
total_gold += 1
return total_gold, total_predicted, total_correct
