len_info = seg_len_info(z_1_3d)
if args.use_ivector_input:
train_output = training_fn(*compressed_batch)
else:
train_output = training_fn(compressed_batch[0],
compressed_batch[1],
compressed_batch[4],
compressed_batch[5])
ce_frame_sum, network_grads_norm = train_output
total_ce_sum += ce_frame_sum
total_frame_count += compressed_batch[-1].sum()
if b_idx % args.log_freq == 0:
show_status(args.save_path, total_ce_sum / total_frame_count,
network_grads_norm, b_idx, args.batch_size, e_idx)
status_sw.print_elapsed()
status_sw.reset()
print('End of Epoch {}'.format(e_idx))
epoch_sw.print_elapsed()
print('Saving the network')
save_network(network_params, trainer_params, e_idx,
args.save_path + '_last_model.pkl')
print('Evaluating the network on the validation dataset')
eval_sw = StopWatch()
#train_ce_frame, train_fer = eval_net(predict_fn, train_ds)
valid_ce_frame, valid_fer = eval_net_compress(predict_fn, valid_ds,
hmrnn, args.batch_size,
# get data
input_data, input_mask, target_data, target_mask = batch_data
# update model
train_output = train_fn(input_data, input_mask, target_data,
target_mask)
train_frame_loss = train_output[0]
train_frame_accr = train_output[1]
train_grads_norm = train_output[2]
train_frame_loss_sum += train_frame_loss
# show results
if batch_idx % args.train_disp_freq == 0:
show_status(save_path=args.save_path,
ce_frame=train_frame_loss,
network_grads_norm=train_grads_norm,
batch_idx=batch_idx,
batch_size=args.batch_size,
epoch_idx=e_idx)
print('Frame Accr: {}'.format(train_frame_accr))
if batch_idx % 250 == 0:
print('Saving the network')
save_network(network_params=network_params,
trainer_params=updater_params,
epoch_cnt=total_batch_cnt,
save_path=args.save_path + '_last_model.pkl')
if batch_idx % 1000 == 0:
print('Evaluating the network on the validation dataset')
valid_frame_loss, valid_fer = eval_network(
predict_fn, valid_datastream)
train_ce_frame_sum = 0.0
status_sw = StopWatch()
for b_idx, data in enumerate(train_ds.get_epoch_iterator(), start=1):
input_data, input_mask, ivector_data, ivector_mask, target_data, target_mask = data
if args.use_ivector_input or args.use_ivector_model:
train_output = training_fn(input_data, input_mask,
ivector_data, target_data,
target_mask)
else:
train_output = training_fn(input_data, input_mask, target_data,
target_mask)
ce_frame, network_grads_norm = train_output
if b_idx % args.train_disp_freq == 0:
show_status(args.save_path, ce_frame, network_grads_norm,
b_idx, args.batch_size, e_idx)
status_sw.print_elapsed()
status_sw.reset()
train_ce_frame_sum += ce_frame
print('End of Epoch {}'.format(e_idx))
epoch_sw.print_elapsed()
print('Saving the network')
save_network(network_params, trainer_params, e_idx,
args.save_path + '_last_model.pkl')
print('Evaluating the network on the validation dataset')
eval_sw = StopWatch()
use_ivectors = args.use_ivector_input or args.use_ivector_model
