def run(summ_path,
ref_path,
rouge_args=None,
verbose=False,
saveto=None,
eos=".",
ignore_empty=False,
stemming=False,
to_json=False):
s = settings.Settings()
s._load()
stime = time()
dirpath = tempfile.mkdtemp()
sys_root, model_root = [os.path.join(dirpath, _)
for _ in ["system", "model"]]
# print("Preparing documents...", end=" ")
utils.mkdirs([sys_root, model_root])
ignored = utils.split_files(model_file=ref_path,
system_file=summ_path,
model_dir=model_root,
system_dir=sys_root,
eos=eos,
ignore_empty=ignore_empty)
# print("%d line(s) ignored" % len(ignored))
# print("Running ROUGE...")
log_level = logging.ERROR if not verbose else None
r = pyrouge.Rouge155(rouge_dir=os.path.dirname(s.data['ROUGE_path']),
log_level=log_level)
r.system_dir = sys_root
r.model_dir = model_root
r.system_filename_pattern = r's.(\d+).txt'
r.model_filename_pattern = 'm.[A-Z].#ID#.txt'
# logger = logging.getLogger(r.__name__)
# logger.setLevel(logging.ERROR)
data_arg = "-e %s" % s.data['ROUGE_data']
if not rouge_args:
rouge_args = [
'-c', 95,
'-r', 1000,
'-n', 2,
'-a']
if stemming:
rouge_args.append("-m")
rouge_args_str = " ".join([str(_) for _ in rouge_args])
else:
rouge_args_str = rouge_args
rouge_args_str = "%s %s" % (data_arg, rouge_args_str)
output = r.convert_and_evaluate(rouge_args=rouge_args_str)
if saveto is not None:
saveto = open(saveto, 'w')
utils.tee(saveto, output)
# print("Elapsed time: %.3f seconds" % (time() - stime))
if to_json:
return output_to_json(output)
return output
def _calc_rouge(self, args):
summ_path = args['summ_path']
ref_path = args['ref_path']
eos = args['eos']
ignore_empty_reference = args['ignore_empty_reference']
ignore_empty_summary = args['ignore_empty_summary']
stemming = args['stemming']
s = settings.Settings()
s._load()
with tempfile.TemporaryDirectory() as dirpath:
sys_root, model_root = [
os.path.join(dirpath, _) for _ in ["system", "model"]
]
utils.mkdirs([sys_root, model_root])
ignored = utils.split_files(
model_path=ref_path,
system_path=summ_path,
model_dir=model_root,
system_dir=sys_root,
eos=eos,
ignore_empty_reference=ignore_empty_reference,
ignore_empty_summary=ignore_empty_summary)
r = Rouge155(rouge_dir=os.path.dirname(s.data['ROUGE_path']),
log_level=logging.ERROR,
stemming=stemming)
r.system_dir = sys_root
r.model_dir = model_root
r.system_filename_pattern = r's.(\d+).txt'
r.model_filename_pattern = 'm.[A-Z].#ID#.txt'
data_arg = "-e %s" % s.data['ROUGE_data']
rouge_args_str = "%s %s" % (data_arg, self.rouge_args)
output = r.convert_and_evaluate(rouge_args=rouge_args_str)
res = self._get_info(output)
return res
def run(summ_path,
ref_path,
rouge_args=None,
verbose=False,
saveto=None,
eos=".",
ignore_empty_reference=False,
ignore_empty_summary=False,
stemming=True):
s = settings.Settings()
s._load()
stime = time()
with tempfile.TemporaryDirectory() as dirpath:
sys_root, model_root = [
os.path.join(dirpath, _) for _ in ["system", "model"]
]
print("Preparing documents...", end=" ")
utils.mkdirs([sys_root, model_root])
ignored = utils.split_files(
model_path=ref_path,
system_path=summ_path,
model_dir=model_root,
system_dir=sys_root,
eos=eos,
ignore_empty_reference=ignore_empty_reference,
ignore_empty_summary=ignore_empty_summary)
print("%d line(s) ignored" % len(ignored))
print("Running ROUGE...")
log_level = logging.ERROR if not verbose else None
r = pyrouge.Rouge155(rouge_dir=os.path.dirname(s.data['ROUGE_path']),
log_level=log_level,
stemming=stemming)
r.system_dir = sys_root
r.model_dir = model_root
r.system_filename_pattern = r's.(\d+).txt'
r.model_filename_pattern = 'm.[A-Z].#ID#.txt'
data_arg = "-e %s" % s.data['ROUGE_data']
if not rouge_args:
rouge_args = ['-c', 95, '-r', 1000, '-n', 2, '-a']
rouge_args_str = " ".join([str(_) for _ in rouge_args])
else:
rouge_args_str = rouge_args
rouge_args_str = "%s %s" % (data_arg, rouge_args_str)
output = r.convert_and_evaluate(rouge_args=rouge_args_str)
if saveto is not None:
saveto = open(saveto, 'w')
utils.tee(saveto, output)
print("Elapsed time: %.3f seconds" % (time() - stime))
def run(summ_path,
ref_path,
rouge_args=None,
verbose=False,
saveto=None,
eos="."):
if saveto is not None:
saveto = open(saveto, 'w')
s = settings.Settings()
s._load()
stime = time()
dirpath = tempfile.mkdtemp()
sys_root, model_root = [
os.path.join(dirpath, _) for _ in ["system", "model"]
]
print("Preparing documents...")
utils.mkdirs([sys_root, model_root])
utils.split_files(model_file=ref_path,
system_file=summ_path,
model_dir=model_root,
system_dir=sys_root,
eos=eos)
print("Running ROUGE...")
log_level = logging.ERROR if not verbose else None
r = pyrouge.Rouge155(rouge_dir=os.path.dirname(s.data['ROUGE_path']),
log_level=log_level)
r.system_dir = sys_root
r.model_dir = model_root
r.system_filename_pattern = r's.(\d+).txt'
r.model_filename_pattern = 'm.[A-Z].#ID#.txt'
data_arg = "-e %s" % s.data['ROUGE_data']
if not rouge_args:
rouge_args = ['-c', 95, '-r', 1000, '-n', 2, '-a']
rouge_args_str = " ".join([str(_) for _ in rouge_args])
else:
rouge_args_str = rouge_args
rouge_args_str = "%s %s" % (data_arg, rouge_args_str)
output = r.convert_and_evaluate(rouge_args=rouge_args_str)
utils.tee(saveto, output)
print("Elapsed time: %.3f seconds" % (time() - stime))
def main():
parser = argparse.ArgumentParser(description="Calculating ROUGE score between two files (line-by-line)")
parser.add_argument("summary", help="Path of summary file")
parser.add_argument("reference", help="Path of references file")
parser.add_argument('-v', '--verbose', action="store_true",
help="""Prints ROUGE logs""")
parser.add_argument('-a', '--args', help="ROUGE Arguments")
parser.add_argument('-s', '--saveto', dest="saveto", help="File to save scores")
parser.add_argument('-e', '--eos', dest="eos", default='.',
help="End of sentence separator (for multisentence). Default: '.'")
args = parser.parse_args()
ref_path = args.reference
summ_path = args.summary
rouge_args = args.args
verbose = args.verbose
saveto = args.saveto
if saveto is not None:
saveto = open(saveto, 'w')
s = settings.Settings()
s._load()
stime = time()
dirpath = tempfile.mkdtemp()
sys_root, model_root = [os.path.join(dirpath, _)
for _ in ["system", "model"]]
print("Preparing documents...")
utils.mkdirs([sys_root, model_root])
utils.split_files(model_file=ref_path,
system_file=summ_path,
model_dir=model_root,
system_dir=sys_root,
eos=args.eos)
print("Running ROUGE...")
log_level = logging.ERROR if not verbose else None
r = pyrouge.Rouge155(rouge_dir=os.path.dirname(s.data['ROUGE_path']),
log_level=log_level)
r.system_dir = sys_root
r.model_dir = model_root
r.system_filename_pattern = 's.(\d+).txt'
r.model_filename_pattern = 'm.[A-Z].#ID#.txt'
data_arg = "-e %s" % s.data['ROUGE_data']
if not rouge_args:
rouge_args = [
'-c', 95,
'-r', 1000,
'-n', 2,
'-a']
rouge_args_str = " ".join([str(_) for _ in rouge_args])
else:
rouge_args_str = rouge_args
rouge_args_str = "%s %s" % (data_arg, rouge_args_str)
output = r.convert_and_evaluate(rouge_args=rouge_args_str)
utils.tee(saveto, output)
print("Elapsed time: %.3f secondes" % (time() - stime))