batch_idx=batch_idx, batch_size=args.batch_size, epoch_idx=e_idx) print('Frame Accr: {}'.format(train_frame_accr)) print('Inter Cost: {}'.format(train_inter_cost)) print('Intra Cost: {}'.format(train_intra_cost)) status_sw.print_elapsed() status_sw.reset() train_frame_loss_sum += train_frame_loss print('End of Epoch {}'.format(e_idx)) epoch_sw.print_elapsed() print('Saving the network') save_network(network_params=network_params, trainer_params=updater_params, epoch_cnt=e_idx, save_path=args.save_path + '_last_model.pkl') print('Evaluating the network on the validation dataset') eval_sw = StopWatch() valid_frame_loss, valid_fer = eval_network(predict_fn, valid_datastream) test_frame_loss, test_fer = eval_network(predict_fn, test_datastream) eval_sw.print_elapsed() print('Train CE: {}'.format(train_frame_loss_sum / batch_idx)) print('Valid CE: {}, FER: {}'.format(valid_frame_loss, valid_fer)) print('Test CE: {}, FER: {}'.format(test_frame_loss, test_fer)) if valid_fer < best_fer(eval_history): symlink_force('{}_last_model.pkl'.format(args.save_path),
train_frame_loss_sum += train_frame_loss # show results if batch_idx % args.train_disp_freq == 0: show_status(save_path=args.save_path, ce_frame=train_frame_loss, network_grads_norm=train_grads_norm, batch_idx=batch_idx, batch_size=args.batch_size, epoch_idx=e_idx) print('Frame Accr: {}'.format(train_frame_accr)) if batch_idx % 250 == 0: print('Saving the network') save_network(network_params=network_params, trainer_params=updater_params, epoch_cnt=total_batch_cnt, save_path=args.save_path + '_last_model.pkl') if batch_idx % 1000 == 0: print('Evaluating the network on the validation dataset') valid_frame_loss, valid_fer = eval_network( predict_fn, valid_datastream) test_frame_loss, test_fer = eval_network( predict_fn, test_datastream) print('Train CE: {}'.format(train_frame_loss_sum / batch_idx)) print('Valid CE: {}, FER: {}'.format(valid_frame_loss, valid_fer)) print('Test CE: {}, FER: {}'.format(test_frame_loss, test_fer)) train_frame_loss_sum = 0.0
ce_frame_sum, network_grads_norm = train_output total_ce_sum += ce_frame_sum total_frame_count += compressed_batch[-1].sum() if b_idx % args.log_freq == 0: show_status(args.save_path, total_ce_sum / total_frame_count, network_grads_norm, b_idx, args.batch_size, e_idx) status_sw.print_elapsed() status_sw.reset() print('End of Epoch {}'.format(e_idx)) epoch_sw.print_elapsed() print('Saving the network') save_network(network_params, trainer_params, e_idx, args.save_path + '_last_model.pkl') print('Evaluating the network on the validation dataset') eval_sw = StopWatch() #train_ce_frame, train_fer = eval_net(predict_fn, train_ds) valid_ce_frame, valid_fer = eval_net_compress(predict_fn, valid_ds, hmrnn, args.batch_size, args.use_ivector_input) test_ce_frame, test_fer = eval_net_compress(predict_fn, test_ds, hmrnn, args.batch_size, args.use_ivector_input) eval_sw.print_elapsed() avg_train_ce = total_ce_sum / total_frame_count print('Train CE: {}'.format(avg_train_ce)) print('Valid CE: {}, FER: {}'.format(valid_ce_frame, valid_fer))