def eval_rouge(dec_dir, ref_dir): """ evaluate by original Perl implementation""" # silence pyrouge logging assert _ROUGE_PATH is not None assert os.path.exists(_ROUGE_PATH) print(_ROUGE_PATH) log.get_global_console_logger().setLevel(logging.WARNING) dec_pattern = '(\d+).dec' ref_pattern = '#ID#.ref' cmd = '-c 95 -r 1000 -n 2 -m' with tempfile.TemporaryDirectory() as tmp_dir: Rouge155.convert_summaries_to_rouge_format( dec_dir, join(tmp_dir, 'dec')) Rouge155.convert_summaries_to_rouge_format( ref_dir, join(tmp_dir, 'ref')) Rouge155.write_config_static( join(tmp_dir, 'dec'), dec_pattern, join(tmp_dir, 'ref'), ref_pattern, join(tmp_dir, 'settings.xml'), system_id=1 ) cmd = (join(_ROUGE_PATH, 'ROUGE-1.5.5.pl') + ' -e {} '.format(join(_ROUGE_PATH, 'data')) + cmd + ' -a {}'.format(join(tmp_dir, 'settings.xml'))) print("cmd:{}".format(cmd)) output = sp.check_output(cmd.split(' '), universal_newlines=True) return output
def get_rouge(path, dec): log.get_global_console_logger().setLevel(logging.WARNING) dec_pattern = '(\d+).dec' ref_pattern = '#ID#.ref' dec_dir = join(path, 'decode') ref_dir = join(path, 'reference') with open(join(dec_dir, '0.dec'), 'w') as f: for sentence in dec: print(sentence, file=f) cmd = '-c 95 -r 1000 -n 2 -m' with tempfile.TemporaryDirectory() as tmp_dir: Rouge155.convert_summaries_to_rouge_format(dec_dir, join(tmp_dir, 'dec')) Rouge155.convert_summaries_to_rouge_format(ref_dir, join(tmp_dir, 'ref')) Rouge155.write_config_static(join(tmp_dir, 'dec'), dec_pattern, join(tmp_dir, 'ref'), ref_pattern, join(tmp_dir, 'settings.xml'), system_id=1) cmd = (join(_ROUGE_PATH, 'ROUGE-1.5.5.pl') + ' -e {} '.format(join(_ROUGE_PATH, 'data')) + cmd + ' -a {}'.format(join(tmp_dir, 'settings.xml'))) output = sp.check_output(cmd.split(' '), universal_newlines=True) line = output.split('\n') rouge1 = float(line[3].split(' ')[3]) rouge2 = float(line[7].split(' ')[3]) rougel = float(line[11].split(' ')[3]) return (rouge1 + rouge2 + rougel) / 3
def eval_rouge(dec_dir, ref_dir): assert _ROUGE_PATH is not None log.get_global_console_logger().setLevel(logging.WARNING) dec_pattern = '(\d+).dec' ref_pattern = '#ID#.ref' cmd = '-c 95 -r 1000 -n 2 -m' with tempfile.TemporaryDirectory() as tmp_dir: Rouge155.convert_summaries_to_rouge_format(dec_dir, join(tmp_dir, 'dec')) Rouge155.convert_summaries_to_rouge_format(ref_dir, join(tmp_dir, 'ref')) Rouge155.write_config_static(join(tmp_dir, 'dec'), dec_pattern, join(tmp_dir, 'ref'), ref_pattern, join(tmp_dir, 'settings.xml'), system_id=1) cmd = (join(_ROUGE_PATH, 'ROUGE-1.5.5.pl') + ' -e {} '.format(join(_ROUGE_PATH, 'data')) + cmd + ' -a {}'.format(join(tmp_dir, 'settings.xml'))) output = sp.check_output(cmd.split(' '), universal_newlines=True) R_1 = float(output.split('\n')[3].split(' ')[3]) R_2 = float(output.split('\n')[7].split(' ')[3]) R_L = float(output.split('\n')[11].split(' ')[3]) print(output) return R_1, R_2, R_L
def eval_rouge(dec_pattern, dec_dir, ref_pattern, ref_dir, cmd='-c 95 -r 1000 -n 2 -m', system_id=1): """ evaluate by original Perl implementation""" # silence pyrouge logging assert _ROUGE_PATH is not None log.get_global_console_logger().setLevel(logging.WARNING) with tempfile.TemporaryDirectory() as tmp_dir: try: Rouge155.convert_summaries_to_rouge_format( dec_dir, join(tmp_dir, 'dec')) except: print("part1:") #pdb.set_trace() try: Rouge155.convert_summaries_to_rouge_format( ref_dir, join(tmp_dir, 'ref')) except: print("part2:") #pdb.set_trace() try: Rouge155.write_config_static( join(tmp_dir, 'dec'), dec_pattern, join(tmp_dir, 'ref'), ref_pattern, join(tmp_dir, 'settings.xml'), system_id ) except: print("part3:") #pdb.set_trace() cmd = (join(_ROUGE_PATH, 'ROUGE-1.5.5.pl') + ' -e {} '.format(join(_ROUGE_PATH, 'data')) + cmd + ' -a {}'.format(join(tmp_dir, 'settings.xml'))) output = sp.check_output(cmd.split(' '), universal_newlines=True) return output
def eval_rouge(dec_pattern, dec_dir, ref_pattern, ref_dir, cmd='-c 95 -r 1000 -n 2 -m', system_id=1): """ evaluate by original Perl implementation""" # silence pyrouge logging try: _ROUGE_PATH = '/home/yhj/ROUGE/RELEASE-1.5.5' except KeyError: print('Warning: ROUGE is not configured') _ROUGE_PATH = None assert _ROUGE_PATH is not None log.get_global_console_logger().setLevel(logging.WARNING) with tempfile.TemporaryDirectory() as tmp_dir: Rouge155.convert_summaries_to_rouge_format( dec_dir, os.path.join(tmp_dir, 'dec')) Rouge155.convert_summaries_to_rouge_format( ref_dir, os.path.join(tmp_dir, 'ref')) Rouge155.write_config_static( os.path.join(tmp_dir, 'dec'), dec_pattern, os.path.join(tmp_dir, 'ref'), ref_pattern, os.path.join(tmp_dir, 'settings.xml'), system_id ) cmd = (os.path.join(_ROUGE_PATH, 'ROUGE-1.5.5.pl') + ' -e {} '.format(os.path.join(_ROUGE_PATH, 'data')) + cmd + ' -a {}'.format(os.path.join(tmp_dir, 'settings.xml'))) output = sp.check_output(cmd.split(' '), universal_newlines=True) return output
def eval_rouge(dec_pattern, dec_dir, ref_pattern, ref_dir, dir_name, cmd='-c 95 -r 1000 -n 2 -m', system_id=1): """ evaluate by original Perl implementation""" # silence pyrouge logging assert _ROUGE_PATH is not None log.get_global_console_logger().setLevel(logging.WARNING) # with tempfile.TemporaryDirectory() as tmp_dir: tmp_dir = '/mnt/e/Work/Ahmed/Summarization/SummRuNNer/output/{}/temp/'.format( dir_name) if os.path.exists(tmp_dir): shutil.rmtree(tmp_dir) os.mkdir(tmp_dir) Rouge155.convert_summaries_to_rouge_format(dec_dir, join(tmp_dir, 'dec')) Rouge155.convert_summaries_to_rouge_format(ref_dir, join(tmp_dir, 'ref')) Rouge155.write_config_static(join(tmp_dir, 'dec'), dec_pattern, join(tmp_dir, 'ref'), ref_pattern, join(tmp_dir, 'settings.xml'), system_id) cmd = ('sudo perl ' + _ROUGE_PATH + '/ROUGE-1.5.5.pl' + ' -e {} '.format(join(_ROUGE_PATH, 'data')) + cmd + ' -a {}'.format(join(tmp_dir, 'settings.xml'))) output = sp.check_output(cmd, universal_newlines=True, shell=True) return output
def eval_rouge(dec_pattern, dec_dir, ref_pattern, ref_dir, cmd='-c 95 -r 1000 -n 2 -m', system_id=1, force=False): """ evaluate by original Perl implementation""" assert _ROUGE_PATH is not None # silence pyrouge logging log.get_global_console_logger().setLevel(logging.WARNING) rouge_dec = join(dec_dir, '../rouge_dec') if not os.path.exists(rouge_dec) or force: Rouge155.convert_summaries_to_rouge_format(dec_dir, rouge_dec) rouge_ref = join(ref_dir, '../rouge_{}_ref'.format(basename(normpath(ref_dir)))) if not os.path.exists(rouge_ref) or force: Rouge155.convert_summaries_to_rouge_format(ref_dir, rouge_ref) rouge_settings = join(dec_dir, '../rouge_settings.xml') if not os.path.exists(rouge_settings) or force: Rouge155.write_config_static(rouge_dec, dec_pattern, rouge_ref, ref_pattern, rouge_settings, system_id) cmd = (join(_ROUGE_PATH, 'ROUGE-1.5.5.pl') + ' -e {} '.format(join(_ROUGE_PATH, 'data')) + cmd + ' -a {}'.format(rouge_settings)) output = sp.check_output(cmd.split(' '), universal_newlines=True) return output
def eval_rouge(dec_pattern, dec_dir, ref_pattern, ref_dir, cmd='-c 95 -r 1000 -n 2 -m', system_id=1): """ evaluate by original Perl implementation""" # silence pyrouge logging with tempfile.TemporaryDirectory() as tmp_dir: Rouge155.convert_summaries_to_rouge_format( dec_dir, os.path.join(tmp_dir, 'dec')) Rouge155.convert_summaries_to_rouge_format( ref_dir, os.path.join(tmp_dir, 'ref')) Rouge155.write_config_static( os.path.join(tmp_dir, 'dec'), dec_pattern, os.path.join(tmp_dir, 'ref'), ref_pattern, os.path.join(tmp_dir, 'settings.xml'), system_id ) cmd = (os.path.join('/home/yhj/ROUGE/RELEASE-1.5.5', 'ROUGE-1.5.5.pl') + ' -e {} '.format(os.path.join('/home/yhj/ROUGE/RELEASE-1.5.5', 'data')) + cmd + ' -a {}'.format(os.path.join(tmp_dir, 'settings.xml'))) output = sp.check_output(cmd.split(' '), universal_newlines=True) return output
def eval_rouge(dec_pattern, dec_dir, ref_pattern, ref_dir, cmd='-c 95 -r 1000 -n 2 -m -d', system_id=1): print('evaluate') """ evaluate by original Perl implementation""" # silence pyrouge logging assert _ROUGE_PATH is not None log.get_global_console_logger().setLevel(logging.WARNING) with tempfile.TemporaryDirectory() as tmp_dir: Rouge155.convert_summaries_to_rouge_format( dec_dir, join(tmp_dir, 'dec')) Rouge155.convert_summaries_to_rouge_format( ref_dir, join(tmp_dir, 'ref')) Rouge155.write_config_static( join(tmp_dir, 'dec'), dec_pattern, join(tmp_dir, 'ref'), ref_pattern, join(tmp_dir, 'settings.xml'), system_id ) cmd = (join(_ROUGE_PATH, 'ROUGE-1.5.5.pl') + ' -e {} '.format(join(_ROUGE_PATH, 'data')) + cmd + ' -a {}'.format(join(tmp_dir, 'settings.xml'))) output = sp.check_output(cmd.split(' '), universal_newlines=True) rouge_1 = [] rouge_2 = [] rouge_l = [] for line in output.split('\n'): if 'ROUGE-1 Eval' in line: rouge_1.append(line.split()[-1][2:]) if 'ROUGE-2 Eval' in line: rouge_2.append(line.split()[-1][2:]) if 'ROUGE-L Eval' in line: rouge_l.append(line.split()[-1][2:]) rouge_1 = '\n'.join(rouge_1) rouge_2 = '\n'.join(rouge_2) rouge_l = '\n'.join(rouge_l) return rouge_1, rouge_2, rouge_l
def eval_rouge(dec_pattern, dec_dir, ref_pattern, ref_dir, cmd='-c 95 -r 1000 -n 2 -m', system_id=1): """ evaluate by original Perl implementation""" # silence pyrouge logging assert _ROUGE_PATH is not None log.get_global_console_logger().setLevel(logging.WARNING) with tempfile.TemporaryDirectory() as tmp_dir: Rouge155.convert_summaries_to_rouge_format( dec_dir, join(tmp_dir, 'dec')) Rouge155.convert_summaries_to_rouge_format( ref_dir, join(tmp_dir, 'ref')) Rouge155.write_config_static( join(tmp_dir, 'dec'), dec_pattern, join(tmp_dir, 'ref'), ref_pattern, join(tmp_dir, 'settings.xml'), system_id ) cmd = (join(_ROUGE_PATH, 'ROUGE-1.5.5.pl') + ' -e {} '.format(join(_ROUGE_PATH, 'data')) + cmd + ' -a {}'.format(join(tmp_dir, 'settings.xml'))) output = sp.check_output(cmd.split(' '), universal_newlines=True) return output
def rouge_eval(ref_dir, dec_dir, dec_pattern='(\d+)_decoded.txt', ref_pattern='#ID#_reference.txt', cmd="-c 95 -r 1000 -n 2 -m", system_id=1): # only print rouge 1 2 L assert _ROUGE_PATH is not None log.get_global_console_logger().setLevel(logging.WARNING) with tempfile.TemporaryDirectory() as tmp_dir: tmp_dec_dir = os.path.join(tmp_dir, 'dec') tmp_ref_dir = os.path.join(tmp_dir, 'ref') Rouge155.convert_summaries_to_rouge_format(dec_dir, tmp_dec_dir) Rouge155.convert_summaries_to_rouge_format(ref_dir, tmp_ref_dir) Rouge155.write_config_static(tmp_dec_dir, dec_pattern, tmp_ref_dir, ref_pattern, os.path.join(tmp_dir, 'settings.xml'), system_id) cmd = (os.path.join(_ROUGE_PATH, 'ROUGE-1.5.5.pl') + ' -e {} '.format(os.path.join(_ROUGE_PATH, 'data')) + cmd + ' -a {}'.format(os.path.join(tmp_dir, 'settings.xml'))) output = sp.check_output(cmd.split(' '), universal_newlines=True) return output
def main(): args = get_args() Rouge155.write_config_static( args.system_dir, args.system_filename_pattern, args.model_dir, args.model_filename_pattern, args.config_file_path, args.system_id)