def interannotator(manual_file1, manual_file2, parses, structure,
show_mistakes, mistake_type):
annotator1 = read_in_hand_alignments(manual_file1, parses)
annotator2 = read_in_hand_alignments(manual_file2, parses)
evaluate(annotator2,
annotator1,
structure,
verbose=show_mistakes,
mistake_type=mistake_type)
def oracle_evaluate(manual, parses, show_mistakes, mistake_type):
manual_alignments = read_in_hand_alignments(manual, parses)
for s_id in manual_alignments:
manual_alignments[s_id]['alignments'][
'paths'] = collections.defaultdict(set)
oracle = get_full_alignment_oracle(manual_alignments)
evaluate(oracle,
read_in_hand_alignments(manual, parses),
'paths',
verbose=show_mistakes,
mistake_type=mistake_type)
def jamr_on_my(jamr_alignments,
manual_file,
parses,
mistake_type='both',
show_mistakes=False):
jamr = read_in_jamr_alignments(jamr_alignments)
gold = read_in_hand_alignments(manual_file, parses)
evaluate(jamr, gold, 'nodes', show_mistakes, mistake_type)
def isi_on_my(isi_alignments,
manual_file,
parses,
mistake_type='both',
show_mistakes=False):
gold = read_in_hand_alignments(manual_file, parses)
isi = read_in_isi_alignments(isi_alignments, parses)
evaluate(isi, gold, 'nodes', show_mistakes, mistake_type)
def my_on_my(manual_file, parses, structure, show_mistakes, mistake_type):
manual = read_in_hand_alignments(manual_file, parses)
neg_dict = read_neg_polarity_items('neg-polarity.txt')
automatic = get_full_alignment(unpickle(parses), neg_dict)
# automatic = get_node_alignment(unpickle(parses), neg_dict)
evaluate(automatic,
manual,
structure,
verbose=show_mistakes,
mistake_type=mistake_type)
def isi_as_oracle(isi_alignments,
manual_file,
parses,
structure='both',
mistake_type='both',
show_mistakes=False):
gold = read_in_hand_alignments(manual_file, parses)
isi = read_in_isi_alignments(isi_alignments, parses)
sent_dict = unpickle(parses)
for s_id in sent_dict:
sent_dict[s_id]['alignments'] = {
'nodes': isi[s_id]['alignments']['nodes'],
'paths': collections.defaultdict(set)
}
oracle = get_full_alignment_oracle(sent_dict)
evaluate(oracle, gold, 'paths', show_mistakes, mistake_type)
def jamr_as_oracle(jamr_alignments,
manual_file,
parses,
mistake_type='both',
show_mistakes=False):
gold = read_in_hand_alignments(manual_file, parses)
jamr = read_in_jamr_alignments(jamr_alignments)
sent_dict = unpickle(parses)
for i in range(196, 201):
sent_dict[i - 1] = sent_dict[i]
del sent_dict[200]
for s_id in sent_dict:
sent_dict[s_id]['alignments'] = {
'nodes': jamr[s_id]['alignments']['nodes'],
'paths': collections.defaultdict(set)
}
oracle = get_full_alignment_oracle(sent_dict)
evaluate(oracle, gold, 'paths', show_mistakes, mistake_type)
target_amr=None):
examples = find_examples(parser_eval, target_dep, normalize_dep,
target_amr)
freq = defaultdict(int)
for s_dict in examples.values():
for edge, info, dep in s_dict['examples']:
amr_rel = info[2][1] if len(info) > 2 else info[0][0][1]
freq[amr_rel] += 1
return freq
amr_data = parse_verbose_smatch("../parser_evaluation/smatch_verbose1.txt")
extracted_misses = extract_missed_triples(amr_data,
unlabelled=True,
mislabeled=False)
manual = read_in_hand_alignments('../alignments/all_alignments.txt',
'parsed_datasets/parsed.p')
eval_alignments = find_alignments_for_misses(manual, extracted_misses)
# #print_out(eval_alignments, "../parser_evaluation/missed_subgraphs.txt")
# chosen_subgraphs = prepare_subgraphs_for_drawing(eval_alignments)
# draw_graphs(eval_alignments, '../parser_evaluation/graphs', special_subgraphs=chosen_subgraphs)
# auto_amrs = read("../parser_evaluation/aligned_amrs.txt.parsed1")
# draw_graphs(auto_amrs, '../parser_evaluation/graphs', want_dep=False)
#
# # print("Dependency edges present in subgraphs corresponding to missed AMR subgraphs")
# # print("\n\tProportion of all dependencies with that label:\n")
# relative_miss_freq = rel_freq_of_misses(eval_alignments)
# print_dict_ordered_by_value(relative_miss_freq)
# print("Dependency edges corresponding to missed AMR edges\n\tcases of single UD edge responsible for the mistake\n")
# miss_freq_single_edge = rel_freq_of_misses(eval_alignments, composite=True, alignments=manual)
# print_dict_ordered_by_value(miss_freq_single_edge)