len_info = seg_len_info(z_1_3d) if args.use_ivector_input: train_output = training_fn(*compressed_batch) else: train_output = training_fn(compressed_batch[0], compressed_batch[1], compressed_batch[4], compressed_batch[5]) ce_frame_sum, network_grads_norm = train_output total_ce_sum += ce_frame_sum total_frame_count += compressed_batch[-1].sum() if b_idx % args.log_freq == 0: show_status(args.save_path, total_ce_sum / total_frame_count, network_grads_norm, b_idx, args.batch_size, e_idx) status_sw.print_elapsed() status_sw.reset() print('End of Epoch {}'.format(e_idx)) epoch_sw.print_elapsed() print('Saving the network') save_network(network_params, trainer_params, e_idx, args.save_path + '_last_model.pkl') print('Evaluating the network on the validation dataset') eval_sw = StopWatch() #train_ce_frame, train_fer = eval_net(predict_fn, train_ds) valid_ce_frame, valid_fer = eval_net_compress(predict_fn, valid_ds, hmrnn, args.batch_size,
# get data input_data, input_mask, target_data, target_mask = batch_data # update model train_output = train_fn(input_data, input_mask, target_data, target_mask) train_frame_loss = train_output[0] train_frame_accr = train_output[1] train_grads_norm = train_output[2] train_frame_loss_sum += train_frame_loss # show results if batch_idx % args.train_disp_freq == 0: show_status(save_path=args.save_path, ce_frame=train_frame_loss, network_grads_norm=train_grads_norm, batch_idx=batch_idx, batch_size=args.batch_size, epoch_idx=e_idx) print('Frame Accr: {}'.format(train_frame_accr)) if batch_idx % 250 == 0: print('Saving the network') save_network(network_params=network_params, trainer_params=updater_params, epoch_cnt=total_batch_cnt, save_path=args.save_path + '_last_model.pkl') if batch_idx % 1000 == 0: print('Evaluating the network on the validation dataset') valid_frame_loss, valid_fer = eval_network( predict_fn, valid_datastream)
train_ce_frame_sum = 0.0 status_sw = StopWatch() for b_idx, data in enumerate(train_ds.get_epoch_iterator(), start=1): input_data, input_mask, ivector_data, ivector_mask, target_data, target_mask = data if args.use_ivector_input or args.use_ivector_model: train_output = training_fn(input_data, input_mask, ivector_data, target_data, target_mask) else: train_output = training_fn(input_data, input_mask, target_data, target_mask) ce_frame, network_grads_norm = train_output if b_idx % args.train_disp_freq == 0: show_status(args.save_path, ce_frame, network_grads_norm, b_idx, args.batch_size, e_idx) status_sw.print_elapsed() status_sw.reset() train_ce_frame_sum += ce_frame print('End of Epoch {}'.format(e_idx)) epoch_sw.print_elapsed() print('Saving the network') save_network(network_params, trainer_params, e_idx, args.save_path + '_last_model.pkl') print('Evaluating the network on the validation dataset') eval_sw = StopWatch() use_ivectors = args.use_ivector_input or args.use_ivector_model