def run(self, args):
params = Params.from_file(args.config, args.overrides)
dataset_reader = DatasetReader.from_params(
params.pop('dataset_reader'))
metrics = _load_metrics(params)
instances = dataset_reader.read()
metrics_dicts = score_instances(instances, metrics)
# Save the results to the output file
with JsonlWriter(args.output_jsonl) as out:
for instance_id in sorted(metrics_dicts.keys()):
for summarizer_id in sorted(metrics_dicts[instance_id].keys()):
out.write(metrics_dicts[instance_id][summarizer_id])
def run(self, args):
params = Params.from_file(args.config, args.overrides)
dataset_reader = DatasetReader.from_params(
params.pop('dataset_reader'))
metrics = load_metrics(params)
instances = dataset_reader.read()
summaries = [instance.summary for instance in instances]
macro = MetricsDict()
micro_list = get_initial_micro_list(instances)
for metric in metrics:
# Prepare the extra input arguments
eval_args = []
for field in metric.required_fields:
eval_args.append(
[instance.fields[field] for instance in instances])
# Score all the summaries
this_macro, this_micro_list = metric.evaluate(
summaries, *eval_args)
# Update the global metrics dictionaries
macro.update(this_macro)
for micro, this_micro in zip(micro_list, this_micro_list):
micro.metrics.update(this_micro)
dirname = os.path.dirname(args.macro_output_json)
if dirname:
os.makedirs(dirname, exist_ok=True)
serialized_macro = jsons.dumps({'metrics': macro},
jdkwargs={'indent': 2})
with open(args.macro_output_json, 'w') as out:
out.write(serialized_macro)
if not args.silent:
print(serialized_macro)
with JsonlWriter(args.micro_output_jsonl) as out:
for metrics_dict in micro_list:
out.write(metrics_dict)
def run(self, args):
prepare_global_logging(file_path=args.log_file, silent=args.silent)
import_module_and_submodules('sacrerouge')
include_packages = args.include_packages or []
for package in include_packages:
import_module_and_submodules(package)
params = Params.from_file(args.config, args.overrides)
dataset_reader = DatasetReader.from_params(params.pop('dataset_reader'))
metrics = _load_metrics(params)
input_files = params.pop('input_files')
if isinstance(input_files, str):
input_files = [input_files]
instances = dataset_reader.read(*input_files)
metrics_dicts = score_instances(instances, metrics, args.disable_peer_jackknifing)
save_score_results(metrics_dicts, args.output_jsonl, args.silent)
def run(self, args):
prepare_global_logging(file_path=args.log_file, silent=args.silent)
import_module_and_submodules('sacrerouge')
include_packages = args.include_packages or []
for package in include_packages:
import_module_and_submodules(package)
params = Params.from_file(args.config, args.overrides)
dataset_reader = DatasetReader.from_params(
params.pop('dataset_reader'))
metrics = load_metrics(params)
input_files = params.pop('input_files')
if isinstance(input_files, str):
input_files = [input_files]
instances = dataset_reader.read(*input_files)
macro, micro_list = evaluate_instances(instances, metrics)
save_evaluation_results(macro, micro_list, args.macro_output_json,
args.micro_output_jsonl, args.silent)