def main(): model_name = 'sprp_onmt_copy_512' base_path = '/home/nlp/aharonr6' moses_path = base_path + '/git/mosesdecoder' test_dirs_path_prefix = base_path + '/git/Split-and-Rephrase/evaluation-directories/validation/' # the file containing the ids of the test sentences test_sent_ids_path = base_path + '/git/Split-and-Rephrase/benchmark/complex-sents/validation.id' # a directory that will hold single sentence files for the hypotheses test_hypothesis_sents_dir = base_path + '/git/phrasing/models/{}/validation_complex_output_sents/'.format( model_name) if not os.path.exists(test_hypothesis_sents_dir): os.mkdir(test_hypothesis_sents_dir) test_target = base_path + '/git/phrasing/models/{}/validation.complex.unique.output'.format( model_name) print 'starting multi-ref evaluation...' avg_bleu, avg_tokens_per_sent, avg_simple_sents_per_complex = evaluate.evaluate_avg_concat_bleu( moses_path, test_sent_ids_path, test_hypothesis_sents_dir, test_target, test_dirs_path_prefix, splitter='. ') print 'avg BLEU:{} avg tokens/sent: {} avg split: {}'.format( avg_bleu, avg_tokens_per_sent, avg_simple_sents_per_complex) return
def main(): model_name = 'sprp_onmt_baseline_512_relations_split' base_path = '/home/nlp/aharonr6' moses_path = base_path + '/git/mosesdecoder' model_files = filter( os.path.isfile, glob.glob(base_path + '/git/phrasing/models/{}/*.pt'.format(model_name))) model_files.sort(key=lambda x: os.path.getmtime(x)) test_dirs_path_prefix = base_path + '/git/Split-and-Rephrase/evaluation-directories-RDFs-relations/validation/' # the file containing the ids of the test sentences test_sent_ids_path = base_path + '/git/Split-and-Rephrase/benchmark/complex-sents/validation-rdfs-relations.id' # a directory that will hold single sentence files for the hypotheses test_hypothesis_sents_dir = base_path + '/git/phrasing/models/{}/validation_complex_output_sents/'.format( model_name) if not os.path.exists(test_hypothesis_sents_dir): os.mkdir(test_hypothesis_sents_dir) results_file_path = base_path + '/git/phrasing/models/{}/overtime_dev_sprp_eval.txt'.format( model_name) results_file = codecs.open(results_file_path, 'w', 'utf8') for model in model_files: command = 'python {}/git/forks/OpenNMT-py/translate.py \ -gpu 1 \ -batch_size 1 \ -model {} \ -src {}/git/Split-and-Rephrase/baseline-seq2seq-RDFs-relations/validation.complex.unique \ -output {}/git/phrasing/models/{}/validation.complex.unique.output \ -beam_size 12 \ -verbose \ -attn_debug \ -replace_unk' os.system( command.format(base_path, model, base_path, base_path, model_name)) test_target = base_path + '/git/phrasing/models/{}/validation.complex.unique.output'.format( model_name) print 'starting multi-ref evaluation...' avg_bleu, avg_tokens_per_sent, avg_simple_sents_per_complex = evaluate.evaluate_avg_concat_bleu( moses_path, test_sent_ids_path, test_hypothesis_sents_dir, test_target, test_dirs_path_prefix) results_file.write('{}\t{}\t{}\t{}\n'.format( model, avg_bleu, avg_tokens_per_sent, avg_simple_sents_per_complex)) return
def main(): # run dev evaluation on all checkpoints to choose the best model for test model_name = 'sprp_onmt_copy_512' base_path = '/home/nlp/aharonr6' moses_path = base_path + '/git/mosesdecoder' models = [ 'sprp_copy_attn_acc_85.43_ppl_2.02_e1.pt', 'sprp_copy_attn_acc_85.83_ppl_1.96_e2.pt', 'sprp_copy_attn_acc_86.47_ppl_1.85_e3.pt', 'sprp_copy_attn_acc_85.95_ppl_2.02_e4.pt', 'sprp_copy_attn_acc_87.00_ppl_2.11_e5.pt', 'sprp_copy_attn_acc_86.98_ppl_2.16_e6.pt', 'sprp_copy_attn_acc_87.23_ppl_2.17_e7.pt', 'sprp_copy_attn_acc_87.33_ppl_2.20_e8.pt', 'sprp_copy_attn_acc_87.38_ppl_2.21_e9.pt', 'sprp_copy_attn_acc_87.35_ppl_2.22_e10.pt', 'sprp_copy_attn_acc_87.35_ppl_2.24_e11.pt', 'sprp_copy_attn_acc_87.34_ppl_2.23_e12.pt', 'sprp_copy_attn_acc_87.34_ppl_2.24_e13.pt', 'sprp_copy_attn_acc_87.36_ppl_2.24_e14.pt', 'sprp_copy_attn_acc_87.35_ppl_2.24_e15.pt' ] test_dirs_path_prefix = base_path + '/git/Split-and-Rephrase/evaluation-directories/validation/' # the file containing the ids of the test sentences test_sent_ids_path = base_path + '/git/Split-and-Rephrase/benchmark/complex-sents/validation.id' # a directory that will hold single sentence files for the hypotheses test_hypothesis_sents_dir = base_path + '/git/phrasing/models/{}/validation_complex_output_sents/'.format( model_name) if not os.path.exists(test_hypothesis_sents_dir): os.mkdir(test_hypothesis_sents_dir) results_file_path = base_path + '/git/phrasing/models/{}/overtime_dev_sprp_eval.txt'.format( model_name) results_file = codecs.open(results_file_path, 'w', 'utf8') for model in models: command = 'python {}/git/forks/OpenNMT-py/translate.py \ -gpu 1 \ -batch_size 1 \ -model {}/git/phrasing/models/{}/{} \ -src {}/git/Split-and-Rephrase/baseline-seq2seq/validation.complex.unique \ -output {}/git/phrasing/models/{}/validation.complex.unique.output \ -beam_size 12 \ -verbose \ -attn_debug \ -replace_unk' os.system( command.format(base_path, base_path, model_name, model, base_path, base_path, model_name)) test_target = base_path + '/git/phrasing/models/{}/validation.complex.unique.output'.format( model_name) print 'starting multi-ref evaluation...' avg_bleu, avg_tokens_per_sent, avg_simple_sents_per_complex = evaluate.evaluate_avg_concat_bleu( moses_path, test_sent_ids_path, test_hypothesis_sents_dir, test_target, test_dirs_path_prefix, splitter='. ') results_file.write('{}\t{}\t{}\t{}\n'.format( model, avg_bleu, avg_tokens_per_sent, avg_simple_sents_per_complex)) return