def do_train(opt): # training print "Parser Config:" print "Incorporate Coref Information: %s" % constants.FLAG_COREF print "Incorporate SRL Information: %s" % constants.FLAG_PROP print "Substitue the normal name entity tag with rich name entity tag: %s" % constants.FLAG_RNE print "Using verbalization list: %s" % constants.FLAG_VERB print "Using charniak parser trained on ontonotes: %s" % constants.FLAG_ONTO print "Dependency parser used: %s" % constants.FLAG_DEPPARSER train_instances = preprocess(opt.amr_file, start_corenlp=True, input_format=opt.amrfmt, prp_format=opt.prpfmt) if opt.add: train_instances += preprocess(opt.add, start_corenlp=True, input_format=opt.amrfmt, prp_format=opt.prpfmt) if opt.dev: dev_instances = preprocess(opt.dev, start_corenlp=True, input_format=opt.amrfmt, prp_format=opt.prpfmt) if opt.section != 'all': print "Choosing corpus section: %s" % opt.section tcr = constants.get_corpus_range(opt.section, 'train') train_instances = train_instances[tcr[0]:tcr[1]] if args.dev: dcr = constants.get_corpus_range(opt.section, 'dev') dev_instances = dev_instances[dcr[0]:dcr[1]] else: print "Choosing all sections!" feat_template = opt.feat if opt.feat else None model = Model(elog=experiment_log) parser = Parser(model=model, oracle_type=DET_T2G_ORACLE_ABT, action_type=opt.actionset, verbose=opt.verbose, elog=experiment_log) model.setup(action_type=opt.actionset, instances=train_instances, parser=parser, feature_templates_file=feat_template) print >> experiment_log, "BEGIN TRAINING!" best_f_score, best_p_score, best_r_score = 0., 0., 0. best_model = None best_epoch = 1 for epoch in xrange(1, opt.iterations + 1): print >> experiment_log, "shuffling training instances" random.shuffle(train_instances) print >> experiment_log, "Iteration:", epoch begin_updates = parser.perceptron.get_num_updates() parser.parse_corpus_train(train_instances) parser.perceptron.average_weight() if opt.dev: print >> experiment_log, "Result on develop set:" _, parsed_amr = parser.parse_corpus_test(dev_instances) parsed_suffix = opt.section + '.' + opt.model.split( '.')[-1] + '.' + str(epoch) + '.parsed' write_parsed_amr(parsed_amr, dev_instances, opt.dev, parsed_suffix) if opt.smatcheval: gold_dataset = codecs.open( opt.dev, 'r', encoding='utf-8').read().strip().split('\n\n') parsed_filename = opt.dev + '.' + parsed_suffix parsed_dataset = codecs.open( parsed_filename, 'r', encoding='utf-8').read().strip().split('\n\n') scorer = SmatchScorer() parsed_dataset = [ data for data in parsed_dataset if '# ::id' in data ] gold_dataset = [ data for data in gold_dataset if '# ::id' in data ] for parsed_data, gold_data in zip(parsed_dataset, gold_dataset): parsed_data = ' '.join([ l.strip() for l in parsed_data.splitlines() if not l.startswith('#') ]) gold_data = ' '.join([ l.strip() for l in gold_data.splitlines() if not l.startswith('#') ]) try: scorer.update(gold_data, parsed_data) except: continue pscore, rscore, fscore = scorer.score() print >> experiment_log, "F-score: ", fscore if fscore > best_f_score: best_model = model best_epoch = epoch best_f_score, best_p_score, best_r_score = fscore, pscore, rscore print >> experiment_log, "New best achieved, saved!" if best_model is not None: print >> experiment_log, "Best result on iteration %d:\n Precision: %f\n Recall: %f\n F-score: %f" % ( best_epoch, best_p_score, best_r_score, best_f_score) best_model.save_model(opt.model + '.m') print >> experiment_log, "DONE TRAINING!"
def main(): arg_parser = argparse.ArgumentParser( description="Brandeis transition-based AMR parser 1.0") build_opts(arg_parser) args = arg_parser.parse_args() amr_file = args.amr_file instances = None train_instance = None constants.FLAG_COREF = args.coref constants.FLAG_PROP = args.prop constants.FLAG_RNE = args.rne constants.FLAG_VERB = args.verblist constants.FLAG_ONTO = args.onto constants.FLAG_DEPPARSER = args.depparser if args.mode == 'preprocess': # using corenlp to preprocess the sentences do_preproces(args) elif args.mode == 'test_gold_graph': # preprocess the JAMR aligned amr do_test_gold_graph(args) elif args.mode == 'align': # do alignment if args.input_file: instances = pickle.load(open(args.input_file, 'rb')) else: raise ValueError( "Missing data file! specify it using --input or using preprocessing!" ) gold_instances_file = args.input_file.split('.')[0] + '_gold.p' print >> log, "Doing alignment..." if LOGGED: saveerr = sys.stderr sys.stderr = open('./log/alignment.log', 'w') amr_aligner = Aligner(verbose=args.verbose) ref_graphs = [] begin = args.begin counter = 1 for i in range(len(instances)): snt = instances[i].text amr = instances[i].amr if args.verbose > 1: print >> log, counter print >> log, "Sentence:" print >> log, snt + '\n' print >> log, "AMR:" print >> log, amr.to_amr_string() alresult = amr_aligner.apply_align(snt, amr) ref_amr_graph = SpanGraph.init_ref_graph(amr, alresult) instances[i].addGoldGraph(ref_amr_graph) if args.verbose > 1: print >> log, amr_aligner.print_align_result(alresult, amr) counter += 1 pickle.dump(instances, open(gold_instances_file, 'wb'), pickle.HIGHEST_PROTOCOL) if LOGGED: sys.stderr.close() sys.stderr = saveerr print >> log, "Done alignment and gold graph generation." sys.exit() elif args.mode == 'userGuide': # test user guide actions print 'Read in training instances...' train_instances = preprocess(amr_file, False) sentID = int(raw_input("Input the sent ID:")) amr_parser = Parser() amr_parser.testUserGuide(train_instances[sentID]) sys.exit() elif args.mode == 'oracleGuide': # test deterministic oracle train_instances = preprocess(amr_file, start_corenlp=False, input_format=args.amrfmt, prp_format=args.prpfmt) try: hand_alignments = load_hand_alignments(amr_file + str('.hand_aligned')) except IOError: hand_alignments = [] start_step = args.start_step begin = args.begin amr_parser = Parser(oracle_type=DET_T2G_ORACLE_ABT, verbose=args.verbose) #ref_graphs = pickle.load(open('./data/ref_graph.p','rb')) n_correct_total = .0 n_parsed_total = .0 n_gold_total = .0 pseudo_gold_amr = [] n_correct_tag_total = .0 n_parsed_tag_total = 0. n_gold_tag_total = .0 gold_amr = [] aligned_instances = [] for instance in train_instances[begin:]: if hand_alignments and instance.comment[ 'id'] not in hand_alignments: continue state = amr_parser.testOracleGuide(instance, start_step) n_correct_arc, n1, n_parsed_arc, n_gold_arc, n_correct_tag, n_parsed_tag, n_gold_tag = state.evaluate( ) if n_correct_arc != n1: import pdb pdb.set_trace() n_correct_total += n_correct_arc n_parsed_total += n_parsed_arc n_gold_total += n_gold_arc p = n_correct_arc / n_parsed_arc if n_parsed_arc else .0 r = n_correct_arc / n_gold_arc if n_gold_arc else .0 indicator = 'PROBLEM!' if p < 0.5 else '' if args.verbose > 2: print >> sys.stderr, "Precision: %s Recall: %s %s\n" % ( p, r, indicator) n_correct_tag_total += n_correct_tag n_parsed_tag_total += n_parsed_tag n_gold_tag_total += n_gold_tag p1 = n_correct_tag / n_parsed_tag if n_parsed_tag else .0 r1 = n_correct_tag / n_gold_tag if n_gold_tag else .0 if args.verbose > 2: print >> sys.stderr, "Tagging Precision:%s Recall:%s" % (p1, r1) instance.comment['alignments'] +=\ ''.join(' %s-%s|%s' % (idx-1, idx, instance.amr.get_pid(state.A.abt_node_table[idx])) for idx in state.A.abt_node_table if isinstance(idx,int)) aligned_instances.append(instance) pseudo_gold_amr.append(GraphState.get_parsed_amr(state.A)) pt = n_correct_total / n_parsed_total if n_parsed_total != .0 else .0 rt = n_correct_total / n_gold_total if n_gold_total != .0 else .0 ft = 2 * pt * rt / (pt + rt) if pt + rt != .0 else .0 write_parsed_amr(pseudo_gold_amr, aligned_instances, amr_file, 'pseudo-gold', hand_alignments) print "Total Accuracy: %s, Recall: %s, F-1: %s" % (pt, rt, ft) tp = n_correct_tag_total / n_parsed_tag_total if n_parsed_tag_total != .0 else .0 tr = n_correct_tag_total / n_gold_tag_total if n_gold_tag_total != .0 else .0 print "Tagging Precision:%s Recall:%s" % (tp, tr) elif args.mode == 'train': do_train(args) elif args.mode == 'parse': # actual parsing test_instances = preprocess(amr_file, start_corenlp=False, input_format=args.amrfmt, prp_format=args.prpfmt) if args.section != 'all': print "Choosing corpus section: %s" % (args.section) tcr = constants.get_corpus_range(args.section, 'test') test_instances = test_instances[tcr[0]:tcr[1]] #random.shuffle(test_instances) print >> experiment_log, "Loading model: ", args.model model = Model.load_model(args.model) parser = Parser(model=model, oracle_type=DET_T2G_ORACLE_ABT, action_type=args.actionset, verbose=args.verbose, elog=experiment_log) print >> experiment_log, "BEGIN PARSING" span_graph_pairs, results = parser.parse_corpus_test(test_instances) parsed_suffix = '%s.%s.parsed' % (args.section, args.model.split('.')[-2]) write_parsed_amr(results, test_instances, amr_file, suffix=parsed_suffix) print >> experiment_log, "DONE PARSING" if args.smatcheval: smatch_path = "./smatch_2.0.2/smatch.py" python_path = 'python' options = '--pr -f' parsed_filename = amr_file + '.' + parsed_suffix command = '%s %s %s %s %s' % (python_path, smatch_path, options, parsed_filename, amr_file) print 'Evaluation using command: ' + (command) print subprocess.check_output(command, stderr=subprocess.STDOUT, shell=True) elif args.mode == 'eval': '''break down error analysis''' # TODO: here use pickled file, replace it with parsed AMR and gold AMR span_graph_pairs = pickle.load(open(args.eval[0], 'rb')) instances = pickle.load(open(args.eval[1], 'rb')) amr_parser = Parser(oracle_type=DET_T2G_ORACLE_ABT, verbose=args.verbose) error_stat = defaultdict( lambda: defaultdict(lambda: defaultdict(list))) for spg_pair, instance in zip(span_graph_pairs, instances): amr_parser.errorAnalyze(spg_pair[0], spg_pair[1], instance, error_stat) else: arg_parser.print_help()
def main(): ''' usage = "Usage:%prog [options] amr_file" opt = OptionParser(usage=usage) opt.add_option("-v",action="store",dest="verbose",type='int', default=0,help="set up verbose level") opt.add_option("-a",action="store_true",dest="align", default=False,help="do alignment between sentence and amr") opt.add_option("-b",action="store",dest="begin",type='int', default=0,help="for debugging" "When do alignment, where the alignment begins" "When test oracle, where to begin") opt.add_option("-s",action="store",dest="start_step",type='int', default=0,help="where the step begins,for testing oracle") opt.add_option("-o",action="store",dest="sentfilep", help="output sentences to file and parse the sentence into dependency graph") opt.add_option("-i",action="store",dest="parsedfilep", help="read parsed dependency graph from file") opt.add_option("-g",action="store",dest="userActfile", help="read user input action sequences as guide") opt.add_option("-d",action="store",dest="oracle",type='int',default=0,\ help="test the output actions of deterministic oracle: " "1: tree oracle 2: list-based oracle") ''' arg_parser = argparse.ArgumentParser(description="Brandeis transition-based AMR parser 1.0") arg_parser.add_argument('-v','--verbose',type=int,default=0,help='set up verbose level for debug') arg_parser.add_argument('-b','--begin',type=int,default=0,help='specify which sentence to begin the alignment or oracle testing for debug') arg_parser.add_argument('-s','--start_step',type=int,default=0,help='specify which step to begin oracle testing;for debug') #arg_parser.add_argument('-i','--input_file',help='the input: preprocessed data instances file for aligner or training') arg_parser.add_argument('-d','--dev',help='development file') arg_parser.add_argument('-as','--actionset',choices=['basic'],default='basic',help='choose different action set') arg_parser.add_argument('-m','--mode',choices=['preprocess','test_gold_graph','align','userGuide','oracleGuide','train','parse','eval'],help="preprocess:generate pos tag, dependency tree, ner\n" "align:do alignment between AMR graph and sentence string") arg_parser.add_argument('-dp','--depparser',choices=['stanford','stanfordConvert','stdconv+charniak','clear','mate','turbo'],default='stdconv+charniak',help='choose the dependency parser') arg_parser.add_argument('--coref',action='store_true',help='flag to enable coreference information') arg_parser.add_argument('--prop',action='store_true',help='flag to enable semantic role labeling information') arg_parser.add_argument('--model',help='specify the model file') arg_parser.add_argument('--feat',help='feature template file') arg_parser.add_argument('-iter','--iterations',default=1,type=int,help='training iterations') arg_parser.add_argument('amr_file',nargs='?',help='amr annotation file/input sentence file for parsing') arg_parser.add_argument('--amrfmt',action='store_true',help='specifying the input file is AMR annotation file') arg_parser.add_argument('-e','--eval',nargs=2,help='Error Analysis: give parsed AMR file and gold AMR file') arg_parser.add_argument('--section',choices=['proxy','all'],default='all',help='choose section of the corpus. Only works for LDC2014T12 dataset.') args = arg_parser.parse_args() amr_file = args.amr_file instances = None train_instance = None constants.FLAG_COREF=args.coref constants.FLAG_PROP=args.prop constants.FLAG_DEPPARSER=args.depparser # using corenlp to preprocess the sentences if args.mode == 'preprocess': instances = preprocess(amr_file,START_SNLP=True,INPUT_AMR=args.amrfmt) print "Done preprocessing!" # preprocess the JAMR aligned amr elif args.mode == 'test_gold_graph': instances = preprocess(amr_file,False) #instances = pickle.load(open('data/gold_edge_graph.pkl','rb')) gold_amr = [] for inst in instances: GraphState.sent = inst.tokens gold_amr.append(GraphState.get_parsed_amr(inst.gold_graph)) #pseudo_gold_amr = [GraphState.get_parsed_amr(inst.gold_graph) for inst in instances] write_parsed_amr(gold_amr,instances,amr_file,'abt.gold') #instances = preprocess_aligned(amr_file) print "Done output AMR!" # do alignment elif args.mode == 'align': if args.input_file: instances = pickle.load(open(args.input_file,'rb')) else: raise ValueError("Missing data file! specify it using --input or using preprocessing!") gold_instances_file = args.input_file.split('.')[0]+'_gold.p' print >> log, "Doing alignment..." if LOGGED: saveerr = sys.stderr sys.stderr = open('./log/alignment.log','w') amr_aligner = Aligner(verbose=args.verbose) ref_graphs = [] begin = args.begin counter = 1 #for snt, amr in zip(snts[begin:],amrs[begin:]): for i in range(len(instances)): snt = instances[i].text amr = instances[i].amr if args.verbose > 1: print >> log, counter print >> log, "Sentence:" print >> log, snt+'\n' print >> log, "AMR:" print >> log, amr.to_amr_string() alresult = amr_aligner.apply_align(snt,amr) ref_amr_graph = SpanGraph.init_ref_graph(amr,alresult) #ref_graphs.append(ref_amr_graph) instances[i].addGoldGraph(ref_amr_graph) if args.verbose > 1: #print >> log, "Reference tuples:" #print >> log, ref_depGraph.print_tuples() print >> log, amr_aligner.print_align_result(alresult,amr) #raw_input('ENTER to continue') counter += 1 pickle.dump(instances,open(gold_instances_file,'wb'),pickle.HIGHEST_PROTOCOL) #pickle.dump(ref_graphs,open('./data/ref_graph.p','wb'),pickle.HIGHEST_PROTOCOL) if LOGGED: sys.stderr.close() sys.stderr = saveerr print >> log, "Done alignment and gold graph generation." sys.exit() # test user guide actions elif args.mode == 'userGuide': print 'Read in training instances...' train_instances = preprocess(amr_file,False) sentID = int(raw_input("Input the sent ID:")) amr_parser = Parser() amr_parser.testUserGuide(train_instances[sentID]) sys.exit() # test deterministic oracle elif args.mode == 'oracleGuide': train_instances = preprocess(amr_file,START_SNLP=False) try: hand_alignments = load_hand_alignments(amr_file+str('.hand_aligned')) except IOError: hand_alignments = [] start_step = args.start_step begin = args.begin amr_parser = Parser(oracle_type=DET_T2G_ORACLE_ABT,verbose=args.verbose) #ref_graphs = pickle.load(open('./data/ref_graph.p','rb')) n_correct_total = .0 n_parsed_total = .0 n_gold_total = .0 pseudo_gold_amr = [] n_correct_tag_total = .0 n_parsed_tag_total = 0. n_gold_tag_total = .0 gold_amr = [] aligned_instances = [] #print "shuffling training instances" #random.shuffle(train_instances) for instance in train_instances[begin:]: if hand_alignments and instance.comment['id'] not in hand_alignments: continue state = amr_parser.testOracleGuide(instance,start_step) n_correct_arc,n1,n_parsed_arc, n_gold_arc,n_correct_tag,n_parsed_tag,n_gold_tag = state.evaluate() #assert n_correct_arc == n1 if n_correct_arc != n1: import pdb pdb.set_trace() n_correct_total += n_correct_arc n_parsed_total += n_parsed_arc n_gold_total += n_gold_arc p = n_correct_arc/n_parsed_arc if n_parsed_arc else .0 r = n_correct_arc/n_gold_arc if n_parsed_arc else .0 indicator = 'PROBLEM!' if p < 0.5 else '' if args.verbose > 2: print >> sys.stderr, "Precision: %s Recall: %s %s\n" % (p,r,indicator) n_correct_tag_total += n_correct_tag n_parsed_tag_total += n_parsed_tag n_gold_tag_total += n_gold_tag p1 = n_correct_tag/n_parsed_tag if n_parsed_tag else .0 r1 = n_correct_tag/n_gold_tag if n_parsed_tag else .0 if args.verbose > 2: print >> sys.stderr,"Tagging Precision:%s Recall:%s" % (p1,r1) instance.comment['alignments'] += ''.join(' %s-%s|%s'%(idx-1,idx,instance.amr.get_pid(state.A.abt_node_table[idx])) for idx in state.A.abt_node_table if isinstance(idx,int)) aligned_instances.append(instance) pseudo_gold_amr.append(GraphState.get_parsed_amr(state.A)) #gold_amr.append(instance.amr) #assert set(state.A.tuples()) == set(instance.gold_graph.tuples()) pt = n_correct_total/n_parsed_total if n_parsed_total != .0 else .0 rt = n_correct_total/n_gold_total if n_gold_total !=.0 else .0 ft = 2*pt*rt/(pt+rt) if pt+rt != .0 else .0 write_parsed_amr(pseudo_gold_amr,aligned_instances,amr_file,'pseudo-gold',hand_alignments) print "Total Accuracy: %s, Recall: %s, F-1: %s" % (pt,rt,ft) tp = n_correct_tag_total/n_parsed_tag_total if n_parsed_tag_total != .0 else .0 tr = n_correct_tag_total/n_gold_tag_total if n_gold_tag_total != .0 else .0 print "Tagging Precision:%s Recall:%s" % (tp,tr) #amr_parser.record_actions('data/action_set.txt') elif args.mode == 'train': # training print "Parser Config:" print "Incorporate Coref Information: %s"%(constants.FLAG_COREF) print "Incorporate SRL Information: %s"%(constants.FLAG_PROP) print "Dependency parser used: %s"%(constants.FLAG_DEPPARSER) train_instances = preprocess(amr_file,START_SNLP=False) if args.dev: dev_instances = preprocess(args.dev,START_SNLP=False) if args.section != 'all': print "Choosing corpus section: %s"%(args.section) tcr = constants.get_corpus_range(args.section,'train') train_instances = train_instances[tcr[0]:tcr[1]] if args.dev: dcr = constants.get_corpus_range(args.section,'dev') dev_instances = dev_instances[dcr[0]:dcr[1]] feat_template = args.feat if args.feat else None model = Model(elog=experiment_log) #model.output_feature_generator() parser = Parser(model=model,oracle_type=DET_T2G_ORACLE_ABT,action_type=args.actionset,verbose=args.verbose,elog=experiment_log) model.setup(action_type=args.actionset,instances=train_instances,parser=parser,feature_templates_file=feat_template) print >> experiment_log, "BEGIN TRAINING!" for iter in xrange(1,args.iterations+1): print >> experiment_log, "shuffling training instances" random.shuffle(train_instances) print >> experiment_log, "Iteration:",iter begin_updates = parser.perceptron.get_num_updates() parser.parse_corpus_train(train_instances) parser.perceptron.average_weight() #model.save_model(args.model+'-iter'+str(iter)+'-'+str(int(time.time()))+'.m') model.save_model(args.model+'-iter'+str(iter)+'.m') if args.dev: print >> experiment_log ,"Result on develop set:" _,parsed_amr = parser.parse_corpus_test(dev_instances) write_parsed_amr(parsed_amr,dev_instances,args.dev,args.section+'.'+str(iter)+'.parsed') print >> experiment_log ,"DONE TRAINING!" elif args.mode == 'parse': # actual parsing test_instances = preprocess(amr_file,START_SNLP=False,INPUT_AMR=False) if args.section != 'all': print "Choosing corpus section: %s"%(args.section) tcr = constants.get_corpus_range(args.section,'test') test_instances = test_instances[tcr[0]:tcr[1]] #random.shuffle(test_instances) print >> experiment_log, "Loading model: ", args.model model = Model.load_model(args.model) parser = Parser(model=model,oracle_type=DET_T2G_ORACLE_ABT,action_type=args.actionset,verbose=args.verbose,elog=experiment_log) print >> experiment_log ,"BEGIN PARSING" span_graph_pairs,results = parser.parse_corpus_test(test_instances) write_parsed_amr(results,test_instances,amr_file,suffix='%s.parsed'%(args.section)) #write_span_graph(span_graph_pairs,test_instances,amr_file,suffix='spg.50') ################ # for eval # ################ #pickle.dump(span_graph_pairs,open('data/eval/%s_spg_pair.pkl'%(amr_file),'wb'),pickle.HIGHEST_PROTOCOL) #pickle.dump(test_instances,open('data/eval/%s_instances.pkl'%(amr_file),'wb'),pickle.HIGHEST_PROTOCOL) print >> experiment_log ,"DONE PARSING" #plt.hist(results) #plt.savefig('result.png') elif args.mode == 'eval': '''break down error analysis''' # TODO: here use pickled file, replace it with parsed AMR and gold AMR span_graph_pairs = pickle.load(open(args.eval[0],'rb')) instances = pickle.load(open(args.eval[1],'rb')) amr_parser = Parser(oracle_type=DET_T2G_ORACLE_ABT,verbose=args.verbose) error_stat = defaultdict(lambda:defaultdict(lambda:defaultdict(list))) for spg_pair,instance in zip(span_graph_pairs,instances): amr_parser.errorAnalyze(spg_pair[0],spg_pair[1],instance,error_stat) else: arg_parser.print_help()