train_callback = nemo_core.SimpleLossLoggerCallback( tensors=[total_loss_train, gate_loss_train, ptr_loss_train], print_func=lambda x: logging.info( f'Loss:{str(np.round(x[0].item(), 3))}, ' f'Gate Loss:{str(np.round(x[1].item(), 3))}, ' f'Pointer Loss:{str(np.round(x[2].item(), 3))}' ), tb_writer=nf.tb_writer, get_tb_values=lambda x: [["loss", x[0]], ["gate_loss", x[1]], ["pointer_loss", x[2]]], step_freq=steps_per_epoch_train, ) eval_callback = nemo_core.EvaluatorCallback( eval_tensors=tensors_eval, user_iter_callback=lambda x, y: eval_iter_callback(x, y, data_desc), user_epochs_done_callback=lambda x: eval_epochs_done_callback(x, data_desc), tb_writer=nf.tb_writer, eval_step=steps_per_epoch_train, ) ckpt_callback = nemo_core.CheckpointCallback( folder=nf.checkpoint_dir, epoch_freq=args.save_epoch_freq, step_freq=args.save_step_freq ) if args.lr_policy is not None: total_steps = args.num_epochs * steps_per_epoch_train lr_policy_fn = get_lr_policy( args.lr_policy, total_steps=total_steps, warmup_ratio=args.lr_warmup_proportion, min_lr=args.min_lr ) else: lr_policy_fn = None
step_freq=args.train_step_freq, tb_writer=nf.tb_writer, ) ckpt_callback = nemo_core.CheckpointCallback( folder=nf.checkpoint_dir, epoch_freq=args.save_epoch_freq, step_freq=args.save_step_freq ) callbacks = [train_callback, ckpt_callback] if "eval" in args.mode: eval_callback = nemo_core.EvaluatorCallback( eval_tensors=eval_output, user_iter_callback=lambda x, y: eval_iter_callback(x, y), user_epochs_done_callback=lambda x: eval_epochs_done_callback( x, eval_data_layer=eval_data_layer, do_lower_case=args.do_lower_case, n_best_size=args.n_best_size, max_answer_length=args.max_answer_length, version_2_with_negative=args.version_2_with_negative, null_score_diff_threshold=args.null_score_diff_threshold, ), tb_writer=nf.tb_writer, eval_step=args.eval_step_freq, ) callbacks.append(eval_callback) optimization_params = { "lr": args.lr, "weight_decay": args.weight_decay, } if args.max_steps < 0: total_steps = args.num_epochs * train_steps_per_epoch
token_params = { 'bos_token': None, 'eos_token': '[SEP]', 'pad_token': '[PAD]', 'cls_token': '[CLS]' } train_loss, steps_per_epoch, _, _ = create_pipeline() _, _, eval_data_layer, eval_tensors = create_pipeline(evaluate=True) callbacks_eval = [ nemo_core.EvaluatorCallback( eval_tensors=eval_tensors, user_iter_callback=lambda x, y: eval_iter_callback(x, y), user_epochs_done_callback=lambda x: eval_epochs_done_callback( x, args.work_dir, eval_task_names[0]), tb_writer=nf.tb_writer, eval_step=steps_per_epoch, ) ] """ MNLI task has two dev sets: matched and mismatched Create additional callback and data layer for MNLI mismatched dev set """ if args.task_name == 'mnli': _, _, eval_data_layer_mm, eval_tensors_mm = create_pipeline( evaluate=True, processor=task_processors[1]) callbacks_eval.append( nemo_core.EvaluatorCallback( eval_tensors=eval_tensors_mm, user_iter_callback=lambda x, y: eval_iter_callback(x, y),