epoch_sw.print_elapsed() print('Saving the network') save_network(network_params=network_params, trainer_params=updater_params, epoch_cnt=e_idx, save_path=args.save_path + '_last_model.pkl') print('Evaluating the network on the validation dataset') eval_sw = StopWatch() valid_frame_loss, valid_fer = eval_network(predict_fn, valid_datastream) test_frame_loss, test_fer = eval_network(predict_fn, test_datastream) eval_sw.print_elapsed() print('Train CE: {}'.format(train_frame_loss_sum / batch_idx)) print('Valid CE: {}, FER: {}'.format(valid_frame_loss, valid_fer)) print('Test CE: {}, FER: {}'.format(test_frame_loss, test_fer)) if valid_fer < best_fer(eval_history): # symlink_force('{}_last_model.pkl'.format(args.save_path), '{}_best_model.pkl'.format(args.save_path)) save_network(network_params=network_params, trainer_params=updater_params, epoch_cnt=e_idx, save_path=args.save_path + '_best_model.pkl') print('Saving the evaluation history') er = EvalRecord(train_frame_loss_sum / batch_idx, valid_frame_loss, valid_fer, test_frame_loss, test_fer) eval_history.append(er) save_eval_history(eval_history, args.save_path + '_eval_history.pkl')
save_path=args.save_path + '_last_model.pkl') if batch_idx % 1000 == 0: print('Evaluating the network on the validation dataset') valid_frame_loss, valid_fer = eval_network( predict_fn, valid_datastream) test_frame_loss, test_fer = eval_network( predict_fn, test_datastream) print('Train CE: {}'.format(train_frame_loss_sum / batch_idx)) print('Valid CE: {}, FER: {}'.format(valid_frame_loss, valid_fer)) print('Test CE: {}, FER: {}'.format(test_frame_loss, test_fer)) train_frame_loss_sum = 0.0 if valid_fer < best_fer(eval_history): print('Best model saved based on FER from ' + str(best_fer(eval_history)) + ' to ' + str(valid_fer)) save_network(network_params=network_params, trainer_params=updater_params, epoch_cnt=total_batch_cnt, save_path=args.save_path + '_best_model.pkl') print('Saving the evaluation history') er = EvalRecord(train_frame_loss_sum / batch_idx, valid_frame_loss, valid_fer, test_frame_loss, test_fer) eval_history.append(er) save_eval_history(eval_history, args.save_path + '_eval_history.pkl')
print('Frame Accr: {}'.format(train_frame_accr)) if batch_idx % 250 == 0: print('Saving the network') save_network(network_params=network_params, trainer_params=updater_params, epoch_cnt=total_batch_cnt, save_path=args.save_path + '_last_model.pkl') if batch_idx % 1000 == 0: print('Evaluating the network on the validation dataset') valid_frame_loss, valid_fer = eval_network(predict_fn, valid_datastream) test_frame_loss, test_fer = eval_network(predict_fn, test_datastream) print('Train CE: {}'.format(train_frame_loss_sum / batch_idx)) print('Valid CE: {}, FER: {}'.format(valid_frame_loss, valid_fer)) print('Test CE: {}, FER: {}'.format(test_frame_loss, test_fer)) train_frame_loss_sum = 0.0 if valid_fer < best_fer(eval_history): print('Best model saved based on FER from ' + str(best_fer(eval_history)) + ' to ' + str(valid_fer)) save_network(network_params=network_params, trainer_params=updater_params, epoch_cnt=total_batch_cnt, save_path=args.save_path + '_best_model.pkl') print('Saving the evaluation history') er = EvalRecord(train_frame_loss_sum / batch_idx, valid_frame_loss, valid_fer, test_frame_loss, test_fer) eval_history.append(er) save_eval_history(eval_history, args.save_path + '_eval_history.pkl') print('End of Epoch {}'.format(e_idx))