batch_idx=batch_idx,
batch_size=args.batch_size,
epoch_idx=e_idx)
print('Frame Accr: {}'.format(train_frame_accr))
print('Inter Cost: {}'.format(train_inter_cost))
print('Intra Cost: {}'.format(train_intra_cost))
status_sw.print_elapsed()
status_sw.reset()
train_frame_loss_sum += train_frame_loss
print('End of Epoch {}'.format(e_idx))
epoch_sw.print_elapsed()
print('Saving the network')
save_network(network_params=network_params,
trainer_params=updater_params,
epoch_cnt=e_idx,
save_path=args.save_path + '_last_model.pkl')
print('Evaluating the network on the validation dataset')
eval_sw = StopWatch()
valid_frame_loss, valid_fer = eval_network(predict_fn,
valid_datastream)
test_frame_loss, test_fer = eval_network(predict_fn, test_datastream)
eval_sw.print_elapsed()
print('Train CE: {}'.format(train_frame_loss_sum / batch_idx))
print('Valid CE: {}, FER: {}'.format(valid_frame_loss, valid_fer))
print('Test CE: {}, FER: {}'.format(test_frame_loss, test_fer))
if valid_fer < best_fer(eval_history):
symlink_force('{}_last_model.pkl'.format(args.save_path),
train_frame_loss_sum += train_frame_loss
# show results
if batch_idx % args.train_disp_freq == 0:
show_status(save_path=args.save_path,
ce_frame=train_frame_loss,
network_grads_norm=train_grads_norm,
batch_idx=batch_idx,
batch_size=args.batch_size,
epoch_idx=e_idx)
print('Frame Accr: {}'.format(train_frame_accr))
if batch_idx % 250 == 0:
print('Saving the network')
save_network(network_params=network_params,
trainer_params=updater_params,
epoch_cnt=total_batch_cnt,
save_path=args.save_path + '_last_model.pkl')
if batch_idx % 1000 == 0:
print('Evaluating the network on the validation dataset')
valid_frame_loss, valid_fer = eval_network(
predict_fn, valid_datastream)
test_frame_loss, test_fer = eval_network(
predict_fn, test_datastream)
print('Train CE: {}'.format(train_frame_loss_sum / batch_idx))
print('Valid CE: {}, FER: {}'.format(valid_frame_loss,
valid_fer))
print('Test CE: {}, FER: {}'.format(test_frame_loss,
test_fer))
train_frame_loss_sum = 0.0
ce_frame_sum, network_grads_norm = train_output
total_ce_sum += ce_frame_sum
total_frame_count += compressed_batch[-1].sum()
if b_idx % args.log_freq == 0:
show_status(args.save_path, total_ce_sum / total_frame_count,
network_grads_norm, b_idx, args.batch_size, e_idx)
status_sw.print_elapsed()
status_sw.reset()
print('End of Epoch {}'.format(e_idx))
epoch_sw.print_elapsed()
print('Saving the network')
save_network(network_params, trainer_params, e_idx,
args.save_path + '_last_model.pkl')
print('Evaluating the network on the validation dataset')
eval_sw = StopWatch()
#train_ce_frame, train_fer = eval_net(predict_fn, train_ds)
valid_ce_frame, valid_fer = eval_net_compress(predict_fn, valid_ds,
hmrnn, args.batch_size,
args.use_ivector_input)
test_ce_frame, test_fer = eval_net_compress(predict_fn, test_ds, hmrnn,
args.batch_size,
args.use_ivector_input)
eval_sw.print_elapsed()
avg_train_ce = total_ce_sum / total_frame_count
print('Train CE: {}'.format(avg_train_ce))
print('Valid CE: {}, FER: {}'.format(valid_ce_frame, valid_fer))
