def eval_rouge(dec_dir, ref_dir):
assert _ROUGE_PATH is not None
log.get_global_console_logger().setLevel(logging.WARNING)
dec_pattern = '(\d+).dec'
ref_pattern = '#ID#.ref'
cmd = '-c 95 -r 1000 -n 2 -m'
with tempfile.TemporaryDirectory() as tmp_dir:
Rouge155.convert_summaries_to_rouge_format(dec_dir,
join(tmp_dir, 'dec'))
Rouge155.convert_summaries_to_rouge_format(ref_dir,
join(tmp_dir, 'ref'))
Rouge155.write_config_static(join(tmp_dir, 'dec'),
dec_pattern,
join(tmp_dir, 'ref'),
ref_pattern,
join(tmp_dir, 'settings.xml'),
system_id=1)
cmd = (join(_ROUGE_PATH, 'ROUGE-1.5.5.pl') +
' -e {} '.format(join(_ROUGE_PATH, 'data')) + cmd +
' -a {}'.format(join(tmp_dir, 'settings.xml')))
output = sp.check_output(cmd.split(' '), universal_newlines=True)
R_1 = float(output.split('\n')[3].split(' ')[3])
R_2 = float(output.split('\n')[7].split(' ')[3])
R_L = float(output.split('\n')[11].split(' ')[3])
print(output)
return R_1, R_2, R_L
def eval_rouge(dec_dir, ref_dir):
""" evaluate by original Perl implementation"""
# silence pyrouge logging
assert _ROUGE_PATH is not None
assert os.path.exists(_ROUGE_PATH)
print(_ROUGE_PATH)
log.get_global_console_logger().setLevel(logging.WARNING)
dec_pattern = '(\d+).dec'
ref_pattern = '#ID#.ref'
cmd = '-c 95 -r 1000 -n 2 -m'
with tempfile.TemporaryDirectory() as tmp_dir:
Rouge155.convert_summaries_to_rouge_format(
dec_dir, join(tmp_dir, 'dec'))
Rouge155.convert_summaries_to_rouge_format(
ref_dir, join(tmp_dir, 'ref'))
Rouge155.write_config_static(
join(tmp_dir, 'dec'), dec_pattern,
join(tmp_dir, 'ref'), ref_pattern,
join(tmp_dir, 'settings.xml'), system_id=1
)
cmd = (join(_ROUGE_PATH, 'ROUGE-1.5.5.pl')
+ ' -e {} '.format(join(_ROUGE_PATH, 'data'))
+ cmd
+ ' -a {}'.format(join(tmp_dir, 'settings.xml')))
print("cmd:{}".format(cmd))
output = sp.check_output(cmd.split(' '), universal_newlines=True)
return output
def get_rouge(path, dec):
log.get_global_console_logger().setLevel(logging.WARNING)
dec_pattern = '(\d+).dec'
ref_pattern = '#ID#.ref'
dec_dir = join(path, 'decode')
ref_dir = join(path, 'reference')
with open(join(dec_dir, '0.dec'), 'w') as f:
for sentence in dec:
print(sentence, file=f)
cmd = '-c 95 -r 1000 -n 2 -m'
with tempfile.TemporaryDirectory() as tmp_dir:
Rouge155.convert_summaries_to_rouge_format(dec_dir,
join(tmp_dir, 'dec'))
Rouge155.convert_summaries_to_rouge_format(ref_dir,
join(tmp_dir, 'ref'))
Rouge155.write_config_static(join(tmp_dir, 'dec'),
dec_pattern,
join(tmp_dir, 'ref'),
ref_pattern,
join(tmp_dir, 'settings.xml'),
system_id=1)
cmd = (join(_ROUGE_PATH, 'ROUGE-1.5.5.pl') +
' -e {} '.format(join(_ROUGE_PATH, 'data')) + cmd +
' -a {}'.format(join(tmp_dir, 'settings.xml')))
output = sp.check_output(cmd.split(' '), universal_newlines=True)
line = output.split('\n')
rouge1 = float(line[3].split(' ')[3])
rouge2 = float(line[7].split(' ')[3])
rougel = float(line[11].split(' ')[3])
return (rouge1 + rouge2 + rougel) / 3
def eval_rouge(dec_pattern,
dec_dir,
ref_pattern,
ref_dir,
dir_name,
cmd='-c 95 -r 1000 -n 2 -m',
system_id=1):
""" evaluate by original Perl implementation"""
# silence pyrouge logging
assert _ROUGE_PATH is not None
log.get_global_console_logger().setLevel(logging.WARNING)
# with tempfile.TemporaryDirectory() as tmp_dir:
tmp_dir = '/mnt/e/Work/Ahmed/Summarization/SummRuNNer/output/{}/temp/'.format(
dir_name)
if os.path.exists(tmp_dir):
shutil.rmtree(tmp_dir)
os.mkdir(tmp_dir)
Rouge155.convert_summaries_to_rouge_format(dec_dir, join(tmp_dir, 'dec'))
Rouge155.convert_summaries_to_rouge_format(ref_dir, join(tmp_dir, 'ref'))
Rouge155.write_config_static(join(tmp_dir, 'dec'), dec_pattern,
join(tmp_dir, 'ref'), ref_pattern,
join(tmp_dir, 'settings.xml'), system_id)
cmd = ('sudo perl ' + _ROUGE_PATH + '/ROUGE-1.5.5.pl' +
' -e {} '.format(join(_ROUGE_PATH, 'data')) + cmd +
' -a {}'.format(join(tmp_dir, 'settings.xml')))
output = sp.check_output(cmd, universal_newlines=True, shell=True)
return output
def eval_rouge(dec_pattern, dec_dir, ref_pattern, ref_dir,
cmd='-c 95 -r 1000 -n 2 -m', system_id=1):
""" evaluate by original Perl implementation"""
# silence pyrouge logging
try:
_ROUGE_PATH = '/home/yhj/ROUGE/RELEASE-1.5.5'
except KeyError:
print('Warning: ROUGE is not configured')
_ROUGE_PATH = None
assert _ROUGE_PATH is not None
log.get_global_console_logger().setLevel(logging.WARNING)
with tempfile.TemporaryDirectory() as tmp_dir:
Rouge155.convert_summaries_to_rouge_format(
dec_dir, os.path.join(tmp_dir, 'dec'))
Rouge155.convert_summaries_to_rouge_format(
ref_dir, os.path.join(tmp_dir, 'ref'))
Rouge155.write_config_static(
os.path.join(tmp_dir, 'dec'), dec_pattern,
os.path.join(tmp_dir, 'ref'), ref_pattern,
os.path.join(tmp_dir, 'settings.xml'), system_id
)
cmd = (os.path.join(_ROUGE_PATH, 'ROUGE-1.5.5.pl')
+ ' -e {} '.format(os.path.join(_ROUGE_PATH, 'data'))
+ cmd
+ ' -a {}'.format(os.path.join(tmp_dir, 'settings.xml')))
output = sp.check_output(cmd.split(' '), universal_newlines=True)
return output
def eval_rouge(dec_pattern, dec_dir, ref_pattern, ref_dir,
cmd='-c 95 -r 1000 -n 2 -m', system_id=1):
""" evaluate by original Perl implementation"""
# silence pyrouge logging
assert _ROUGE_PATH is not None
log.get_global_console_logger().setLevel(logging.WARNING)
with tempfile.TemporaryDirectory() as tmp_dir:
try:
Rouge155.convert_summaries_to_rouge_format(
dec_dir, join(tmp_dir, 'dec'))
except:
print("part1:")
#pdb.set_trace()
try:
Rouge155.convert_summaries_to_rouge_format(
ref_dir, join(tmp_dir, 'ref'))
except:
print("part2:")
#pdb.set_trace()
try:
Rouge155.write_config_static(
join(tmp_dir, 'dec'), dec_pattern,
join(tmp_dir, 'ref'), ref_pattern,
join(tmp_dir, 'settings.xml'), system_id
)
except:
print("part3:")
#pdb.set_trace()
cmd = (join(_ROUGE_PATH, 'ROUGE-1.5.5.pl')
+ ' -e {} '.format(join(_ROUGE_PATH, 'data'))
+ cmd
+ ' -a {}'.format(join(tmp_dir, 'settings.xml')))
output = sp.check_output(cmd.split(' '), universal_newlines=True)
return output
def eval_rouge(dec_pattern,
dec_dir,
ref_pattern,
ref_dir,
cmd='-c 95 -r 1000 -n 2 -m',
system_id=1,
force=False):
""" evaluate by original Perl implementation"""
assert _ROUGE_PATH is not None
# silence pyrouge logging
log.get_global_console_logger().setLevel(logging.WARNING)
rouge_dec = join(dec_dir, '../rouge_dec')
if not os.path.exists(rouge_dec) or force:
Rouge155.convert_summaries_to_rouge_format(dec_dir, rouge_dec)
rouge_ref = join(ref_dir,
'../rouge_{}_ref'.format(basename(normpath(ref_dir))))
if not os.path.exists(rouge_ref) or force:
Rouge155.convert_summaries_to_rouge_format(ref_dir, rouge_ref)
rouge_settings = join(dec_dir, '../rouge_settings.xml')
if not os.path.exists(rouge_settings) or force:
Rouge155.write_config_static(rouge_dec, dec_pattern, rouge_ref,
ref_pattern, rouge_settings, system_id)
cmd = (join(_ROUGE_PATH, 'ROUGE-1.5.5.pl') +
' -e {} '.format(join(_ROUGE_PATH, 'data')) + cmd +
' -a {}'.format(rouge_settings))
output = sp.check_output(cmd.split(' '), universal_newlines=True)
return output
def main():
args = get_args()
if args.split_sents:
from pyrouge.utils.sentence_splitter import PunktSentenceSplitter
tmp = mkdtemp()
PunktSentenceSplitter.split_files(args.input_dir, tmp)
args.input_dir = tmp
Rouge155.convert_summaries_to_rouge_format(args.input_dir, args.output_dir)
def main(): args = get_args() if args.split_sents: from pyrouge.utils.sentence_splitter import PunktSentenceSplitter tmp = mkdtemp() PunktSentenceSplitter.split_files(args.input_dir, tmp) args.input_dir = tmp Rouge155.convert_summaries_to_rouge_format(args.input_dir, args.output_dir)
def convert2rouge_format():
model_input_dir = 'doc/model/04model'
model_output_dir = 'doc/model/04model_rouge'
system_input_dir = 'doc/systems/04systems'
system_output_dir = 'doc/systems/04systems_rouge'
Rouge155.convert_summaries_to_rouge_format(model_input_dir,
model_output_dir)
Rouge155.convert_summaries_to_rouge_format(system_input_dir,
system_output_dir)
def eval_rouge(dec_pattern, dec_dir, ref_pattern, ref_dir,
cmd='-c 95 -r 1000 -n 2 -m', system_id=1):
""" evaluate by original Perl implementation"""
# silence pyrouge logging
with tempfile.TemporaryDirectory() as tmp_dir:
Rouge155.convert_summaries_to_rouge_format(
dec_dir, os.path.join(tmp_dir, 'dec'))
Rouge155.convert_summaries_to_rouge_format(
ref_dir, os.path.join(tmp_dir, 'ref'))
Rouge155.write_config_static(
os.path.join(tmp_dir, 'dec'), dec_pattern,
os.path.join(tmp_dir, 'ref'), ref_pattern,
os.path.join(tmp_dir, 'settings.xml'), system_id
)
cmd = (os.path.join('/home/yhj/ROUGE/RELEASE-1.5.5', 'ROUGE-1.5.5.pl')
+ ' -e {} '.format(os.path.join('/home/yhj/ROUGE/RELEASE-1.5.5', 'data'))
+ cmd
+ ' -a {}'.format(os.path.join(tmp_dir, 'settings.xml')))
output = sp.check_output(cmd.split(' '), universal_newlines=True)
return output
def eval_rouge(dec_pattern, dec_dir, ref_pattern, ref_dir,
cmd='-c 95 -r 1000 -n 2 -m -d', system_id=1):
print('evaluate')
""" evaluate by original Perl implementation"""
# silence pyrouge logging
assert _ROUGE_PATH is not None
log.get_global_console_logger().setLevel(logging.WARNING)
with tempfile.TemporaryDirectory() as tmp_dir:
Rouge155.convert_summaries_to_rouge_format(
dec_dir, join(tmp_dir, 'dec'))
Rouge155.convert_summaries_to_rouge_format(
ref_dir, join(tmp_dir, 'ref'))
Rouge155.write_config_static(
join(tmp_dir, 'dec'), dec_pattern,
join(tmp_dir, 'ref'), ref_pattern,
join(tmp_dir, 'settings.xml'), system_id
)
cmd = (join(_ROUGE_PATH, 'ROUGE-1.5.5.pl')
+ ' -e {} '.format(join(_ROUGE_PATH, 'data'))
+ cmd
+ ' -a {}'.format(join(tmp_dir, 'settings.xml')))
output = sp.check_output(cmd.split(' '), universal_newlines=True)
rouge_1 = []
rouge_2 = []
rouge_l = []
for line in output.split('\n'):
if 'ROUGE-1 Eval' in line:
rouge_1.append(line.split()[-1][2:])
if 'ROUGE-2 Eval' in line:
rouge_2.append(line.split()[-1][2:])
if 'ROUGE-L Eval' in line:
rouge_l.append(line.split()[-1][2:])
rouge_1 = '\n'.join(rouge_1)
rouge_2 = '\n'.join(rouge_2)
rouge_l = '\n'.join(rouge_l)
return rouge_1, rouge_2, rouge_l
def eval_rouge(dec_pattern, dec_dir, ref_pattern, ref_dir,
cmd='-c 95 -r 1000 -n 2 -m', system_id=1):
""" evaluate by original Perl implementation"""
# silence pyrouge logging
assert _ROUGE_PATH is not None
log.get_global_console_logger().setLevel(logging.WARNING)
with tempfile.TemporaryDirectory() as tmp_dir:
Rouge155.convert_summaries_to_rouge_format(
dec_dir, join(tmp_dir, 'dec'))
Rouge155.convert_summaries_to_rouge_format(
ref_dir, join(tmp_dir, 'ref'))
Rouge155.write_config_static(
join(tmp_dir, 'dec'), dec_pattern,
join(tmp_dir, 'ref'), ref_pattern,
join(tmp_dir, 'settings.xml'), system_id
)
cmd = (join(_ROUGE_PATH, 'ROUGE-1.5.5.pl')
+ ' -e {} '.format(join(_ROUGE_PATH, 'data'))
+ cmd
+ ' -a {}'.format(join(tmp_dir, 'settings.xml')))
output = sp.check_output(cmd.split(' '), universal_newlines=True)
return output
def rouge_eval(ref_dir,
dec_dir,
dec_pattern='(\d+)_decoded.txt',
ref_pattern='#ID#_reference.txt',
cmd="-c 95 -r 1000 -n 2 -m",
system_id=1):
# only print rouge 1 2 L
assert _ROUGE_PATH is not None
log.get_global_console_logger().setLevel(logging.WARNING)
with tempfile.TemporaryDirectory() as tmp_dir:
tmp_dec_dir = os.path.join(tmp_dir, 'dec')
tmp_ref_dir = os.path.join(tmp_dir, 'ref')
Rouge155.convert_summaries_to_rouge_format(dec_dir, tmp_dec_dir)
Rouge155.convert_summaries_to_rouge_format(ref_dir, tmp_ref_dir)
Rouge155.write_config_static(tmp_dec_dir, dec_pattern, tmp_ref_dir,
ref_pattern,
os.path.join(tmp_dir,
'settings.xml'), system_id)
cmd = (os.path.join(_ROUGE_PATH, 'ROUGE-1.5.5.pl') +
' -e {} '.format(os.path.join(_ROUGE_PATH, 'data')) + cmd +
' -a {}'.format(os.path.join(tmp_dir, 'settings.xml')))
output = sp.check_output(cmd.split(' '), universal_newlines=True)
return output
#-*- encoding: utf-8 -*- from pyrouge import Rouge155 r = Rouge155() #r.system_dir = '/home/chenbjin/SearchJob/DUC2002_Summarization_Documents/system' #r.model_dir = '/home/chenbjin/SearchJob/DUC2002_Summarization_Documents/model' r.system_dir = '/home/chenbjin/SearchJob/DUC2002_Summarization_Documents/wordnet.system.summary/' r.model_dir = '/home/chenbjin/SearchJob/DUC2002_Summarization_Documents/model.summary/' r.system_filename_pattern = "DUC2002.(\d+).txt" r.model_filename_pattern = "DUC2002.[A-Z].#ID#.txt" output = r.convert_and_evaluate() print(output) output_dict = r.output_to_dict(output) ''' system_input_dir = '/home/chenbjin/SearchJob/DUC2002_Summarization_Documents/system.summary' system_output_dir = '/home/chenbjin/SearchJob/DUC2002_Summarization_Documents/system.output' Rouge155.convert_summaries_to_rouge_format(system_input_dir, system_output_dir) '''
