def load_data3(): amr_file = '../data/split/train.txt' ccg_dependency_file = '../data/ccg/ccgbank_dependencies.gold.txt' ccgbank_file = '../data/ccg/ccgbank_parses.gold.txt' ids_file = '../data/ccg/ids_map_train.tsv' reader = AMR_Reader() amrs = reader.load(amr_file, remove_wiki=True) add_nlp_data(amrs, amr_file) # predicted data align_file = amr_file.replace('.txt', '') + '.subgraph_alignments.json' subgraph_alignments = reader.load_alignments_from_json(align_file, amrs) align_file = amr_file.replace('.txt', '') + '.relation_alignments.json' relation_alignments = reader.load_alignments_from_json(align_file, amrs) align_file = amr_file.replace('.txt', '') + '.reentrancy_alignments.json' reentrancy_alignments = reader.load_alignments_from_json(align_file, amrs) ids, dependencies, ccg_lex, ccg_trees = load_gold_ccgs( ids_file, ccg_dependency_file, ccgbank_file) amrs = {amr.id: amr for amr in amrs} amrs2 = [] for id in ids: amrs2.append(amrs[id]) amrs = amrs2 return amrs, subgraph_alignments, relation_alignments, reentrancy_alignments, dependencies, ccg_lex, ccg_trees
def main(): amr_file = sys.argv[1] align_file = sys.argv[2] gold_file = sys.argv[3] reader = AMR_Reader() amrs = reader.load(amr_file, remove_wiki=True) add_nlp_data(amrs, amr_file) alignments = reader.load_alignments_from_json(align_file, amrs) gold_alignments = reader.load_alignments_from_json(gold_file, amrs) pred_subgraph_alignments = reader.load_alignments_from_json( align_file.replace('relation_', 'subgraph_'), amrs) gold_subgraph_alignments = reader.load_alignments_from_json( gold_file.replace('relation_', 'subgraph_'), amrs) # Display.style([amr for amr in amrs if amr.id in gold_alignments], # gold_file.replace('.json', '') + f'.html', # gold_alignments) if len(amrs) != len(alignments): amrs = [ amr for amr in amrs if amr.id in alignments and amr.id in gold_alignments ] evaluate(amrs, alignments, gold_alignments, mode='edges')
def main(): amr_file = args.train reader = AMR_Reader() amrs = reader.load(amr_file, remove_wiki=True) add_nlp_data(amrs, amr_file) eval_amr_file, eval_amrs, gold_eval_alignments = None, None, None if args.test: eval_amr_file, eval_align_file = args.test eval_amrs = reader.load(eval_amr_file, remove_wiki=True) add_nlp_data(eval_amrs, eval_amr_file) gold_eval_alignments = load_from_json(eval_align_file, eval_amrs, unanonymize=True) eval_amr_ids = {amr.id for amr in eval_amrs} amrs = [amr for amr in amrs if amr.id not in eval_amr_ids] # amrs = amrs[:1000] if args.load_model: print('Loading model from:', args.load_model) align_model = Subgraph_Model.load_model(args.load_model) else: align_model = Subgraph_Model(amrs, align_duplicates=True) iters = args.iter alignments = None for i in range(iters): print(f'Epoch {i}: Training data') alignments = align_model.align_all(amrs) align_model.update_parameters(amrs, alignments) perplexity(align_model, amrs, alignments) report_progress(amr_file, alignments, reader, epoch=i) print() if eval_amrs: print(f'Epoch {i}: Evaluation data') eval_alignments = align_model.align_all(eval_amrs) perplexity(align_model, eval_amrs, eval_alignments) evaluate(eval_amrs, eval_alignments, gold_eval_alignments) evaluate_duplicates(eval_amrs, eval_alignments, gold_eval_alignments) report_progress(eval_amr_file, eval_alignments, reader, epoch=i) print() report_progress(amr_file, alignments, reader) if args.save_model: align_model.save_model(args.save_model) print('Saving model to:', args.save_model)
def main(): file = '../data/split/train.txt' file2 = '../data/train.sents.txt' reader = AMR_Reader() amrs = reader.load(file, remove_wiki=True) add_nlp_data(amrs, file) # amrs2 = reader.load('../data/split/test.txt', remove_wiki=True) # add_nlp_data(amrs2, '../data/split/test.txt') # amrs = amrs+amrs2 with open(file2, 'w+', encoding='utf8') as f: for amr in amrs: for token, pos in zip(amr.tokens, amr.pos): f.write(f'{token}|{pos} ') f.write('\n')
def load_data2(): amr_file1 = '../data/split/dev.txt' amr_file2 = '../data/split/test.txt' ccg_dependency_file = '../data/test.ccg_dependencies.tsv' ccgbank_file = '../data/test.ccg_parse.txt' reader = AMR_Reader() amrs = reader.load(amr_file1, remove_wiki=True) add_nlp_data(amrs, amr_file1) amrs2 = reader.load(amr_file2, remove_wiki=True) add_nlp_data(amrs2, amr_file2) amrs += amrs2 # gold data align_file = amr_file1.replace('.txt', '') + '.subgraph_alignments.gold.json' subgraph_alignments = reader.load_alignments_from_json(align_file, amrs) align_file = amr_file1.replace('.txt', '') + '.relation_alignments.gold.json' relation_alignments = reader.load_alignments_from_json(align_file, amrs) align_file = amr_file1.replace('.txt', '') + '.reentrancy_alignments.gold.json' reentrancy_alignments = reader.load_alignments_from_json(align_file, amrs) align_file = amr_file2.replace('.txt', '') + '.subgraph_alignments.gold.json' subgraph_alignments.update( reader.load_alignments_from_json(align_file, amrs)) align_file = amr_file2.replace('.txt', '') + '.relation_alignments.gold.json' relation_alignments.update( reader.load_alignments_from_json(align_file, amrs)) align_file = amr_file2.replace('.txt', '') + '.reentrancy_alignments.gold.json' reentrancy_alignments.update( reader.load_alignments_from_json(align_file, amrs)) sentences = [amr.tokens for amr in amrs] _, dependencies = align_dependencies_to_sentences( load_dependencies(ccg_dependency_file, flavor='easysrl'), sentences) _, ccg_lex, ccg_trees = align_ccgbank_to_sentences( load_ccgbank(ccgbank_file), sentences) return amrs, subgraph_alignments, relation_alignments, reentrancy_alignments, dependencies, ccg_lex, ccg_trees
def main(): unaligned_amr_file = args.test reader = AMR_Reader() eval_amrs = reader.load(unaligned_amr_file, remove_wiki=True) add_nlp_data(eval_amrs, unaligned_amr_file) # subgraphs print(f'Loading model: {args.subgraph_model}') subgraph_model = Subgraph_Model.load_model(args.subgraph_model) sub_alignments = subgraph_model.align_all(eval_amrs) align_file = unaligned_amr_file.replace('.txt', '') + f'.subgraph_alignments.json' print(f'Writing subgraph alignments to: {align_file}') reader.save_alignments_to_json(align_file, sub_alignments) # relations print(f'Loading model: {args.relation_model}') rel_model = Relation_Model.load_model(args.relation_model) rel_model.subgraph_alignments = sub_alignments rel_alignments = rel_model.align_all(eval_amrs) align_file = unaligned_amr_file.replace('.txt', '') + f'.relation_alignments.json' print(f'Writing relation alignments to: {align_file}') reader.save_alignments_to_json(align_file, rel_alignments) # reentrancies print(f'Loading model: {args.reentrancy_model}') reent_model = Reentrancy_Model.load_model(args.reentrancy_model) reent_model.subgraph_alignments = sub_alignments reent_model.relation_alignments = rel_alignments reent_alignments = reent_model.align_all(eval_amrs) align_file = unaligned_amr_file.replace('.txt', '') + f'.reentrancy_alignments.json' print(f'Writing reentrancy alignments to: {align_file}') reader.save_alignments_to_json(align_file, reent_alignments)
def load_data1(): amr_file = '../data/split/train.txt' ccg_dependency_file = '../data/train.ccg_dependencies.tsv' ccgbank_file = '../data/train.ccg_parse.txt' reader = AMR_Reader() amrs = reader.load(amr_file, remove_wiki=True) add_nlp_data(amrs, amr_file) # predicted data align_file = amr_file.replace('.txt', '') + '.subgraph_alignments.json' subgraph_alignments = reader.load_alignments_from_json(align_file, amrs) align_file = amr_file.replace('.txt', '') + '.relation_alignments.json' relation_alignments = reader.load_alignments_from_json(align_file, amrs) align_file = amr_file.replace('.txt', '') + '.reentrancy_alignments.json' reentrancy_alignments = reader.load_alignments_from_json(align_file, amrs) sentences = [amr.tokens for amr in amrs] _, dependencies = align_dependencies_to_sentences( load_dependencies(ccg_dependency_file, flavor='easysrl'), sentences) _, ccg_lex, ccg_trees = align_ccgbank_to_sentences( load_ccgbank(ccgbank_file), sentences) return amrs, subgraph_alignments, relation_alignments, reentrancy_alignments, dependencies, ccg_lex, ccg_trees
def main(): amr_file = args.train reader = AMR_Reader() amrs = reader.load(amr_file, remove_wiki=True) add_nlp_data(amrs, amr_file) align_file = amr_file.replace('.txt', '') + '.subgraph_alignments.json' subgraph_alignments = reader.load_alignments_from_json(align_file, amrs) # amrs = amrs[:1000] eval_amr_file, eval_amrs, gold_eval_alignments = None, None, None if args.test: eval_amr_file, eval_align_file = args.test eval_amrs = reader.load(eval_amr_file, remove_wiki=True) add_nlp_data(eval_amrs, eval_amr_file) gold_eval_alignments = reader.load_alignments_from_json( eval_align_file, eval_amrs) eval_amr_ids = {amr.id for amr in eval_amrs} amrs = [amr for amr in amrs if amr.id not in eval_amr_ids] align_file = eval_amr_file.replace( '.txt', '') + '.subgraph_alignments.gold.json' gold_subgraph_alignments = reader.load_alignments_from_json( align_file, eval_amrs) align_file = eval_amr_file.replace('.txt', '') + '.subgraph_alignments.json' pred_subgraph_alignments = reader.load_alignments_from_json( align_file, eval_amrs) if USE_GOLD_SUBGRAPHS: pred_subgraph_alignments = gold_subgraph_alignments for amr_id in pred_subgraph_alignments: subgraph_alignments[amr_id] = pred_subgraph_alignments[amr_id] for amr in eval_amrs: spans = [ align.tokens for align in pred_subgraph_alignments[amr.id] if align.type == 'subgraph' ] amr.spans = spans if args.load_model: print('Loading model from:', args.load_model) align_model = Relation_Model.load_model(args.load_model) else: align_model = Relation_Model(amrs, subgraph_alignments) iters = args.iter alignments = None for i in range(iters): print(f'Epoch {i}: Training data') alignments = align_model.align_all(amrs) align_model.update_parameters(amrs, alignments) report_progress(amr_file, alignments, reader, epoch=i) perplexity(align_model, amrs, alignments) print() if eval_amrs: print(f'Epoch {i}: Evaluation data') eval_alignments = align_model.align_all(eval_amrs) perplexity(align_model, eval_amrs, eval_alignments) evaluate_relations(eval_amrs, eval_alignments, gold_eval_alignments, pred_subgraph_alignments, gold_subgraph_alignments) report_progress(eval_amr_file, eval_alignments, reader, epoch=i) print() report_progress(amr_file, alignments, reader) if args.save_model: align_model.save_model(args.save_model) print('Saving model to:', args.save_model)