def get_eval_callback(eval_dataset):
_, eval_tensors = create_pipeline(dataset_split=eval_dataset)
eval_callback = EvaluatorCallback(
eval_tensors=eval_tensors,
user_iter_callback=lambda x, y: eval_iter_callback(x, y, schema_preprocessor, eval_dataset),
user_epochs_done_callback=lambda x: eval_epochs_done_callback(
x,
args.task_name,
eval_dataset,
args.data_dir,
nf.work_dir,
args.tracker_model,
args.debug_mode,
dialogues_processor,
schema_preprocessor,
args.joint_acc_across_turn,
args.no_fuzzy_match,
),
tb_writer=nf.tb_writer,
eval_step=args.eval_epoch_freq * steps_per_epoch,
)
return eval_callback
# we'll write predictions to file in DSTC8 format during evaluation callback
input_json_files = [
os.path.join(args.data_dir, args.eval_dataset, 'dialogues_{:03d}.json'.format(fid))
for fid in data_processor.FILE_RANGES[args.task_name][args.eval_dataset]
]
schema_json_file = os.path.join(args.data_dir, args.eval_dataset, 'schema.json')
# Write predictions to file in DSTC8 format.
prediction_dir = os.path.join(nf.work_dir, 'predictions', 'pred_res_{}_{}'.format(args.eval_dataset, args.task_name))
output_metric_file = os.path.join(nf.work_dir, 'metrics.txt')
os.makedirs(prediction_dir, exist_ok=True)
eval_callback = EvaluatorCallback(
eval_tensors=eval_tensors,
user_iter_callback=lambda x, y: eval_iter_callback(x, y, schema_preprocessor, args.eval_dataset),
user_epochs_done_callback=lambda x: eval_epochs_done_callback(
x,
input_json_files,
args.eval_dataset,
args.data_dir,
prediction_dir,
output_metric_file,
args.state_tracker,
args.debug_mode,
schema_preprocessor,
args.joint_acc_across_turn,
args.no_fuzzy_match,
),
tb_writer=nf.tb_writer,
eval_step=args.eval_epoch_freq * steps_per_epoch,