def load_data3():
amr_file = '../data/split/train.txt'
ccg_dependency_file = '../data/ccg/ccgbank_dependencies.gold.txt'
ccgbank_file = '../data/ccg/ccgbank_parses.gold.txt'
ids_file = '../data/ccg/ids_map_train.tsv'
reader = AMR_Reader()
amrs = reader.load(amr_file, remove_wiki=True)
add_nlp_data(amrs, amr_file)
# predicted data
align_file = amr_file.replace('.txt', '') + '.subgraph_alignments.json'
subgraph_alignments = reader.load_alignments_from_json(align_file, amrs)
align_file = amr_file.replace('.txt', '') + '.relation_alignments.json'
relation_alignments = reader.load_alignments_from_json(align_file, amrs)
align_file = amr_file.replace('.txt', '') + '.reentrancy_alignments.json'
reentrancy_alignments = reader.load_alignments_from_json(align_file, amrs)
ids, dependencies, ccg_lex, ccg_trees = load_gold_ccgs(
ids_file, ccg_dependency_file, ccgbank_file)
amrs = {amr.id: amr for amr in amrs}
amrs2 = []
for id in ids:
amrs2.append(amrs[id])
amrs = amrs2
return amrs, subgraph_alignments, relation_alignments, reentrancy_alignments, dependencies, ccg_lex, ccg_trees
def main():
amr_file = sys.argv[1]
align_file = sys.argv[2]
gold_file = sys.argv[3]
reader = AMR_Reader()
amrs = reader.load(amr_file, remove_wiki=True)
add_nlp_data(amrs, amr_file)
alignments = reader.load_alignments_from_json(align_file, amrs)
gold_alignments = reader.load_alignments_from_json(gold_file, amrs)
pred_subgraph_alignments = reader.load_alignments_from_json(
align_file.replace('relation_', 'subgraph_'), amrs)
gold_subgraph_alignments = reader.load_alignments_from_json(
gold_file.replace('relation_', 'subgraph_'), amrs)
# Display.style([amr for amr in amrs if amr.id in gold_alignments],
# gold_file.replace('.json', '') + f'.html',
# gold_alignments)
if len(amrs) != len(alignments):
amrs = [
amr for amr in amrs
if amr.id in alignments and amr.id in gold_alignments
]
evaluate(amrs, alignments, gold_alignments, mode='edges')
def main():
amr_file = args.train
reader = AMR_Reader()
amrs = reader.load(amr_file, remove_wiki=True)
add_nlp_data(amrs, amr_file)
eval_amr_file, eval_amrs, gold_eval_alignments = None, None, None
if args.test:
eval_amr_file, eval_align_file = args.test
eval_amrs = reader.load(eval_amr_file, remove_wiki=True)
add_nlp_data(eval_amrs, eval_amr_file)
gold_eval_alignments = load_from_json(eval_align_file,
eval_amrs,
unanonymize=True)
eval_amr_ids = {amr.id for amr in eval_amrs}
amrs = [amr for amr in amrs if amr.id not in eval_amr_ids]
# amrs = amrs[:1000]
if args.load_model:
print('Loading model from:', args.load_model)
align_model = Subgraph_Model.load_model(args.load_model)
else:
align_model = Subgraph_Model(amrs, align_duplicates=True)
iters = args.iter
alignments = None
for i in range(iters):
print(f'Epoch {i}: Training data')
alignments = align_model.align_all(amrs)
align_model.update_parameters(amrs, alignments)
perplexity(align_model, amrs, alignments)
report_progress(amr_file, alignments, reader, epoch=i)
print()
if eval_amrs:
print(f'Epoch {i}: Evaluation data')
eval_alignments = align_model.align_all(eval_amrs)
perplexity(align_model, eval_amrs, eval_alignments)
evaluate(eval_amrs, eval_alignments, gold_eval_alignments)
evaluate_duplicates(eval_amrs, eval_alignments,
gold_eval_alignments)
report_progress(eval_amr_file, eval_alignments, reader, epoch=i)
print()
report_progress(amr_file, alignments, reader)
if args.save_model:
align_model.save_model(args.save_model)
print('Saving model to:', args.save_model)
def main():
file = '../data/split/train.txt'
file2 = '../data/train.sents.txt'
reader = AMR_Reader()
amrs = reader.load(file, remove_wiki=True)
add_nlp_data(amrs, file)
# amrs2 = reader.load('../data/split/test.txt', remove_wiki=True)
# add_nlp_data(amrs2, '../data/split/test.txt')
# amrs = amrs+amrs2
with open(file2, 'w+', encoding='utf8') as f:
for amr in amrs:
for token, pos in zip(amr.tokens, amr.pos):
f.write(f'{token}|{pos} ')
f.write('\n')
def load_data2():
amr_file1 = '../data/split/dev.txt'
amr_file2 = '../data/split/test.txt'
ccg_dependency_file = '../data/test.ccg_dependencies.tsv'
ccgbank_file = '../data/test.ccg_parse.txt'
reader = AMR_Reader()
amrs = reader.load(amr_file1, remove_wiki=True)
add_nlp_data(amrs, amr_file1)
amrs2 = reader.load(amr_file2, remove_wiki=True)
add_nlp_data(amrs2, amr_file2)
amrs += amrs2
# gold data
align_file = amr_file1.replace('.txt',
'') + '.subgraph_alignments.gold.json'
subgraph_alignments = reader.load_alignments_from_json(align_file, amrs)
align_file = amr_file1.replace('.txt',
'') + '.relation_alignments.gold.json'
relation_alignments = reader.load_alignments_from_json(align_file, amrs)
align_file = amr_file1.replace('.txt',
'') + '.reentrancy_alignments.gold.json'
reentrancy_alignments = reader.load_alignments_from_json(align_file, amrs)
align_file = amr_file2.replace('.txt',
'') + '.subgraph_alignments.gold.json'
subgraph_alignments.update(
reader.load_alignments_from_json(align_file, amrs))
align_file = amr_file2.replace('.txt',
'') + '.relation_alignments.gold.json'
relation_alignments.update(
reader.load_alignments_from_json(align_file, amrs))
align_file = amr_file2.replace('.txt',
'') + '.reentrancy_alignments.gold.json'
reentrancy_alignments.update(
reader.load_alignments_from_json(align_file, amrs))
sentences = [amr.tokens for amr in amrs]
_, dependencies = align_dependencies_to_sentences(
load_dependencies(ccg_dependency_file, flavor='easysrl'), sentences)
_, ccg_lex, ccg_trees = align_ccgbank_to_sentences(
load_ccgbank(ccgbank_file), sentences)
return amrs, subgraph_alignments, relation_alignments, reentrancy_alignments, dependencies, ccg_lex, ccg_trees
def main():
unaligned_amr_file = args.test
reader = AMR_Reader()
eval_amrs = reader.load(unaligned_amr_file, remove_wiki=True)
add_nlp_data(eval_amrs, unaligned_amr_file)
# subgraphs
print(f'Loading model: {args.subgraph_model}')
subgraph_model = Subgraph_Model.load_model(args.subgraph_model)
sub_alignments = subgraph_model.align_all(eval_amrs)
align_file = unaligned_amr_file.replace('.txt', '') + f'.subgraph_alignments.json'
print(f'Writing subgraph alignments to: {align_file}')
reader.save_alignments_to_json(align_file, sub_alignments)
# relations
print(f'Loading model: {args.relation_model}')
rel_model = Relation_Model.load_model(args.relation_model)
rel_model.subgraph_alignments = sub_alignments
rel_alignments = rel_model.align_all(eval_amrs)
align_file = unaligned_amr_file.replace('.txt', '') + f'.relation_alignments.json'
print(f'Writing relation alignments to: {align_file}')
reader.save_alignments_to_json(align_file, rel_alignments)
# reentrancies
print(f'Loading model: {args.reentrancy_model}')
reent_model = Reentrancy_Model.load_model(args.reentrancy_model)
reent_model.subgraph_alignments = sub_alignments
reent_model.relation_alignments = rel_alignments
reent_alignments = reent_model.align_all(eval_amrs)
align_file = unaligned_amr_file.replace('.txt', '') + f'.reentrancy_alignments.json'
print(f'Writing reentrancy alignments to: {align_file}')
reader.save_alignments_to_json(align_file, reent_alignments)
def load_data1():
amr_file = '../data/split/train.txt'
ccg_dependency_file = '../data/train.ccg_dependencies.tsv'
ccgbank_file = '../data/train.ccg_parse.txt'
reader = AMR_Reader()
amrs = reader.load(amr_file, remove_wiki=True)
add_nlp_data(amrs, amr_file)
# predicted data
align_file = amr_file.replace('.txt', '') + '.subgraph_alignments.json'
subgraph_alignments = reader.load_alignments_from_json(align_file, amrs)
align_file = amr_file.replace('.txt', '') + '.relation_alignments.json'
relation_alignments = reader.load_alignments_from_json(align_file, amrs)
align_file = amr_file.replace('.txt', '') + '.reentrancy_alignments.json'
reentrancy_alignments = reader.load_alignments_from_json(align_file, amrs)
sentences = [amr.tokens for amr in amrs]
_, dependencies = align_dependencies_to_sentences(
load_dependencies(ccg_dependency_file, flavor='easysrl'), sentences)
_, ccg_lex, ccg_trees = align_ccgbank_to_sentences(
load_ccgbank(ccgbank_file), sentences)
return amrs, subgraph_alignments, relation_alignments, reentrancy_alignments, dependencies, ccg_lex, ccg_trees
def main():
amr_file = args.train
reader = AMR_Reader()
amrs = reader.load(amr_file, remove_wiki=True)
add_nlp_data(amrs, amr_file)
align_file = amr_file.replace('.txt', '') + '.subgraph_alignments.json'
subgraph_alignments = reader.load_alignments_from_json(align_file, amrs)
# amrs = amrs[:1000]
eval_amr_file, eval_amrs, gold_eval_alignments = None, None, None
if args.test:
eval_amr_file, eval_align_file = args.test
eval_amrs = reader.load(eval_amr_file, remove_wiki=True)
add_nlp_data(eval_amrs, eval_amr_file)
gold_eval_alignments = reader.load_alignments_from_json(
eval_align_file, eval_amrs)
eval_amr_ids = {amr.id for amr in eval_amrs}
amrs = [amr for amr in amrs if amr.id not in eval_amr_ids]
align_file = eval_amr_file.replace(
'.txt', '') + '.subgraph_alignments.gold.json'
gold_subgraph_alignments = reader.load_alignments_from_json(
align_file, eval_amrs)
align_file = eval_amr_file.replace('.txt',
'') + '.subgraph_alignments.json'
pred_subgraph_alignments = reader.load_alignments_from_json(
align_file, eval_amrs)
if USE_GOLD_SUBGRAPHS:
pred_subgraph_alignments = gold_subgraph_alignments
for amr_id in pred_subgraph_alignments:
subgraph_alignments[amr_id] = pred_subgraph_alignments[amr_id]
for amr in eval_amrs:
spans = [
align.tokens for align in pred_subgraph_alignments[amr.id]
if align.type == 'subgraph'
]
amr.spans = spans
if args.load_model:
print('Loading model from:', args.load_model)
align_model = Relation_Model.load_model(args.load_model)
else:
align_model = Relation_Model(amrs, subgraph_alignments)
iters = args.iter
alignments = None
for i in range(iters):
print(f'Epoch {i}: Training data')
alignments = align_model.align_all(amrs)
align_model.update_parameters(amrs, alignments)
report_progress(amr_file, alignments, reader, epoch=i)
perplexity(align_model, amrs, alignments)
print()
if eval_amrs:
print(f'Epoch {i}: Evaluation data')
eval_alignments = align_model.align_all(eval_amrs)
perplexity(align_model, eval_amrs, eval_alignments)
evaluate_relations(eval_amrs, eval_alignments,
gold_eval_alignments, pred_subgraph_alignments,
gold_subgraph_alignments)
report_progress(eval_amr_file, eval_alignments, reader, epoch=i)
print()
report_progress(amr_file, alignments, reader)
if args.save_model:
align_model.save_model(args.save_model)
print('Saving model to:', args.save_model)