epoch_sw.print_elapsed()
print('Saving the network')
save_network(network_params=network_params,
trainer_params=updater_params,
epoch_cnt=e_idx,
save_path=args.save_path + '_last_model.pkl')
print('Evaluating the network on the validation dataset')
eval_sw = StopWatch()
valid_frame_loss, valid_fer = eval_network(predict_fn,
valid_datastream)
test_frame_loss, test_fer = eval_network(predict_fn, test_datastream)
eval_sw.print_elapsed()
print('Train CE: {}'.format(train_frame_loss_sum / batch_idx))
print('Valid CE: {}, FER: {}'.format(valid_frame_loss, valid_fer))
print('Test CE: {}, FER: {}'.format(test_frame_loss, test_fer))
if valid_fer < best_fer(eval_history):
# symlink_force('{}_last_model.pkl'.format(args.save_path), '{}_best_model.pkl'.format(args.save_path))
save_network(network_params=network_params,
trainer_params=updater_params,
epoch_cnt=e_idx,
save_path=args.save_path + '_best_model.pkl')
print('Saving the evaluation history')
er = EvalRecord(train_frame_loss_sum / batch_idx, valid_frame_loss,
valid_fer, test_frame_loss, test_fer)
eval_history.append(er)
save_eval_history(eval_history, args.save_path + '_eval_history.pkl')
save_path=args.save_path + '_last_model.pkl')
if batch_idx % 1000 == 0:
print('Evaluating the network on the validation dataset')
valid_frame_loss, valid_fer = eval_network(
predict_fn, valid_datastream)
test_frame_loss, test_fer = eval_network(
predict_fn, test_datastream)
print('Train CE: {}'.format(train_frame_loss_sum / batch_idx))
print('Valid CE: {}, FER: {}'.format(valid_frame_loss,
valid_fer))
print('Test CE: {}, FER: {}'.format(test_frame_loss,
test_fer))
train_frame_loss_sum = 0.0
if valid_fer < best_fer(eval_history):
print('Best model saved based on FER from ' +
str(best_fer(eval_history)) + ' to ' +
str(valid_fer))
save_network(network_params=network_params,
trainer_params=updater_params,
epoch_cnt=total_batch_cnt,
save_path=args.save_path + '_best_model.pkl')
print('Saving the evaluation history')
er = EvalRecord(train_frame_loss_sum / batch_idx,
valid_frame_loss, valid_fer, test_frame_loss,
test_fer)
eval_history.append(er)
save_eval_history(eval_history,
args.save_path + '_eval_history.pkl')
print('Frame Accr: {}'.format(train_frame_accr))
if batch_idx % 250 == 0:
print('Saving the network')
save_network(network_params=network_params,
trainer_params=updater_params,
epoch_cnt=total_batch_cnt,
save_path=args.save_path + '_last_model.pkl')
if batch_idx % 1000 == 0:
print('Evaluating the network on the validation dataset')
valid_frame_loss, valid_fer = eval_network(predict_fn, valid_datastream)
test_frame_loss, test_fer = eval_network(predict_fn, test_datastream)
print('Train CE: {}'.format(train_frame_loss_sum / batch_idx))
print('Valid CE: {}, FER: {}'.format(valid_frame_loss, valid_fer))
print('Test CE: {}, FER: {}'.format(test_frame_loss, test_fer))
train_frame_loss_sum = 0.0
if valid_fer < best_fer(eval_history):
print('Best model saved based on FER from ' + str(best_fer(eval_history)) + ' to ' + str(valid_fer))
save_network(network_params=network_params,
trainer_params=updater_params,
epoch_cnt=total_batch_cnt,
save_path=args.save_path + '_best_model.pkl')
print('Saving the evaluation history')
er = EvalRecord(train_frame_loss_sum / batch_idx, valid_frame_loss, valid_fer, test_frame_loss,
test_fer)
eval_history.append(er)
save_eval_history(eval_history, args.save_path + '_eval_history.pkl')
print('End of Epoch {}'.format(e_idx))
