train_callback = nemo_core.SimpleLossLoggerCallback(
tensors=[total_loss_train, gate_loss_train, ptr_loss_train],
print_func=lambda x: logging.info(
f'Loss:{str(np.round(x[0].item(), 3))}, '
f'Gate Loss:{str(np.round(x[1].item(), 3))}, '
f'Pointer Loss:{str(np.round(x[2].item(), 3))}'
),
tb_writer=nf.tb_writer,
get_tb_values=lambda x: [["loss", x[0]], ["gate_loss", x[1]], ["pointer_loss", x[2]]],
step_freq=steps_per_epoch_train,
)
eval_callback = nemo_core.EvaluatorCallback(
eval_tensors=tensors_eval,
user_iter_callback=lambda x, y: eval_iter_callback(x, y, data_desc),
user_epochs_done_callback=lambda x: eval_epochs_done_callback(x, data_desc),
tb_writer=nf.tb_writer,
eval_step=steps_per_epoch_train,
)
ckpt_callback = nemo_core.CheckpointCallback(
folder=nf.checkpoint_dir, epoch_freq=args.save_epoch_freq, step_freq=args.save_step_freq
)
if args.lr_policy is not None:
total_steps = args.num_epochs * steps_per_epoch_train
lr_policy_fn = get_lr_policy(
args.lr_policy, total_steps=total_steps, warmup_ratio=args.lr_warmup_proportion, min_lr=args.min_lr
)
else:
lr_policy_fn = None
step_freq=args.train_step_freq,
tb_writer=nf.tb_writer,
)
ckpt_callback = nemo_core.CheckpointCallback(
folder=nf.checkpoint_dir, epoch_freq=args.save_epoch_freq, step_freq=args.save_step_freq
)
callbacks = [train_callback, ckpt_callback]
if "eval" in args.mode:
eval_callback = nemo_core.EvaluatorCallback(
eval_tensors=eval_output,
user_iter_callback=lambda x, y: eval_iter_callback(x, y),
user_epochs_done_callback=lambda x: eval_epochs_done_callback(
x,
eval_data_layer=eval_data_layer,
do_lower_case=args.do_lower_case,
n_best_size=args.n_best_size,
max_answer_length=args.max_answer_length,
version_2_with_negative=args.version_2_with_negative,
null_score_diff_threshold=args.null_score_diff_threshold,
),
tb_writer=nf.tb_writer,
eval_step=args.eval_step_freq,
)
callbacks.append(eval_callback)
optimization_params = {
"lr": args.lr,
"weight_decay": args.weight_decay,
}
if args.max_steps < 0:
total_steps = args.num_epochs * train_steps_per_epoch
token_params = {
'bos_token': None,
'eos_token': '[SEP]',
'pad_token': '[PAD]',
'cls_token': '[CLS]'
}
train_loss, steps_per_epoch, _, _ = create_pipeline()
_, _, eval_data_layer, eval_tensors = create_pipeline(evaluate=True)
callbacks_eval = [
nemo_core.EvaluatorCallback(
eval_tensors=eval_tensors,
user_iter_callback=lambda x, y: eval_iter_callback(x, y),
user_epochs_done_callback=lambda x: eval_epochs_done_callback(
x, args.work_dir, eval_task_names[0]),
tb_writer=nf.tb_writer,
eval_step=steps_per_epoch,
)
]
"""
MNLI task has two dev sets: matched and mismatched
Create additional callback and data layer for MNLI mismatched dev set
"""
if args.task_name == 'mnli':
_, _, eval_data_layer_mm, eval_tensors_mm = create_pipeline(
evaluate=True, processor=task_processors[1])
callbacks_eval.append(
nemo_core.EvaluatorCallback(
eval_tensors=eval_tensors_mm,
user_iter_callback=lambda x, y: eval_iter_callback(x, y),
