def validate(dev_loader, model, device, criterion, perceptual_entropy, epoch, args, voc_model): """validate.""" losses = AverageMeter() spec_losses = AverageMeter() if args.perceptual_loss > 0: pe_losses = AverageMeter() if args.n_mels > 0: mel_losses = AverageMeter() mcd_metric = AverageMeter() if args.double_mel_loss: double_mel_losses = AverageMeter() model.eval() log_save_dir = os.path.join(args.model_save_dir, "epoch{}/log_val_figure".format(epoch)) if not os.path.exists(log_save_dir): os.makedirs(log_save_dir) start = time.time() with torch.no_grad(): for (step, data_step) in enumerate(dev_loader, 1): if args.db_joint: ( phone, beat, pitch, spec, real, imag, length, chars, char_len_list, mel, singer_id, ) = data_step singer_id = np.array(singer_id).reshape( np.shape(phone)[0], -1) # [batch size, 1] singer_vec = singer_id.repeat(np.shape(phone)[1], axis=1) # [batch size, length] singer_vec = torch.from_numpy(singer_vec).to(device) else: ( phone, beat, pitch, spec, real, imag, length, chars, char_len_list, mel, ) = data_step phone = phone.to(device) beat = beat.to(device) pitch = pitch.to(device).float() spec = spec.to(device).float() if mel is not None: mel = mel.to(device).float() real = real.to(device).float() imag = imag.to(device).float() length_mask = length.unsqueeze(2) if mel is not None: length_mel_mask = length_mask.repeat(1, 1, mel.shape[2]).float() length_mel_mask = length_mel_mask.to(device) length_mask = length_mask.repeat(1, 1, spec.shape[2]).float() length_mask = length_mask.to(device) length = length.to(device) char_len_list = char_len_list.to(device) if not args.use_asr_post: chars = chars.to(device) char_len_list = char_len_list.to(device) else: phone = phone.float() if args.model_type == "GLU_Transformer": if args.db_joint: output, att, output_mel, output_mel2 = model( chars, phone, pitch, beat, singer_vec, pos_char=char_len_list, pos_spec=length, ) else: output, att, output_mel, output_mel2 = model( chars, phone, pitch, beat, pos_char=char_len_list, pos_spec=length, ) elif args.model_type == "LSTM": if args.db_joint: output, hidden, output_mel, output_mel2 = model( phone, pitch, beat, singer_vec) else: output, hidden, output_mel, output_mel2 = model( phone, pitch, beat) att = None elif args.model_type == "GRU_gs": output, att, output_mel = model(spec, phone, pitch, beat, length, args) att = None elif args.model_type == "PureTransformer": output, att, output_mel, output_mel2 = model( chars, phone, pitch, beat, pos_char=char_len_list, pos_spec=length) elif args.model_type == "Conformer": output, att, output_mel, output_mel2 = model( chars, phone, pitch, beat, pos_char=char_len_list, pos_spec=length) elif args.model_type == "Comformer_full": if args.db_joint: output, att, output_mel, output_mel2 = model( chars, phone, pitch, beat, singer_vec, pos_char=char_len_list, pos_spec=length, ) else: output, att, output_mel, output_mel2 = model( chars, phone, pitch, beat, pos_char=char_len_list, pos_spec=length, ) elif args.model_type == "USTC_DAR": output_mel = model(phone, pitch, beat, length, args) att = None spec_origin = spec.clone() mel_origin = mel.clone() if args.normalize: sepc_normalizer = GlobalMVN(args.stats_file) mel_normalizer = GlobalMVN(args.stats_mel_file) output_mel_normalizer = GlobalMVN(args.stats_mel_file) spec, _ = sepc_normalizer(spec, length) mel, _ = mel_normalizer(mel, length) if args.model_type == "USTC_DAR": spec_loss = 0 else: spec_loss = criterion(output, spec, length_mask) if args.n_mels > 0: mel_loss = criterion(output_mel, mel, length_mel_mask) if args.double_mel_loss: double_mel_loss = criterion(output_mel2, mel, length_mel_mask) else: double_mel_loss = 0 else: mel_loss = 0 double_mel_loss = 0 if args.vocoder_category == "wavernn": dev_loss = mel_loss + double_mel_loss else: dev_loss = mel_loss + double_mel_loss + spec_loss if args.perceptual_loss > 0: pe_loss = perceptual_entropy(output, real, imag) final_loss = (args.perceptual_loss * pe_loss + (1 - args.perceptual_loss) * dev_loss) else: final_loss = dev_loss losses.update(final_loss.item(), phone.size(0)) if args.model_type != "USTC_DAR": spec_losses.update(spec_loss.item(), phone.size(0)) if args.perceptual_loss > 0: # pe_loss = perceptual_entropy(output, real, imag) pe_losses.update(pe_loss.item(), phone.size(0)) if args.n_mels > 0: mel_losses.update(mel_loss.item(), phone.size(0)) if args.double_mel_loss: double_mel_losses.update(double_mel_loss.item(), phone.size(0)) if args.model_type == "USTC_DAR": # normalize inverse stage if args.normalize and args.stats_file: # output_mel, _ = mel_normalizer.inverse(output_mel, length) mel, _ = mel_normalizer.inverse(mel, length) output_mel, _ = output_mel_normalizer.inverse( output_mel, length) mcd_value, length_sum = ( 0, 1, ) # FIX ME! Calculate_melcd_fromMelSpectrum else: # normalize inverse stage if args.normalize and args.stats_file: output, _ = sepc_normalizer.inverse(output, length) # output_mel, _ = mel_normalizer.inverse(output_mel, length) mel, _ = mel_normalizer.inverse(mel, length) output_mel, _ = output_mel_normalizer.inverse( output_mel, length) (mcd_value, length_sum) = Metrics.Calculate_melcd_fromLinearSpectrum( output, spec_origin, length, args) mcd_metric.update(mcd_value, length_sum) if step % args.dev_step_log == 0: if args.model_type == "USTC_DAR": log_figure_mel(step, output_mel, mel_origin, att, length, log_save_dir, args) else: if args.vocoder_category == "wavernn": for i in range(output_mel.shape[0]): one_batch_output_mel = output_mel[i].unsqueeze(0) one_batch_mel = mel[i].unsqueeze(0) log_mel( step, one_batch_output_mel, one_batch_mel, att, length, log_save_dir, args, voc_model, ) else: log_figure(step, output, spec_origin, att, length, log_save_dir, args) out_log = ("step {}: train_loss {:.4f}; " "spec_loss {:.4f}; mcd_value {:.4f};".format( step, losses.avg, spec_losses.avg, mcd_metric.avg)) if args.perceptual_loss > 0: out_log += "pe_loss {:.4f}; ".format(pe_losses.avg) if args.n_mels > 0: out_log += "mel_loss {:.4f}; ".format(mel_losses.avg) if args.double_mel_loss: out_log += "dmel_loss {:.4f}; ".format( double_mel_losses.avg) end = time.time() print("{} -- sum_time: {}s".format(out_log, (end - start))) info = { "loss": losses.avg, "spec_loss": spec_losses.avg, "mcd_value": mcd_metric.avg, } if args.perceptual_loss > 0: info["pe_loss"] = pe_losses.avg if args.n_mels > 0: info["mel_loss"] = mel_losses.avg return info
def infer(args): """infer.""" torch.cuda.set_device(args.gpu_id) logging.info(f"GPU {args.gpu_id} is used") torch.backends.cudnn.deterministic = True torch.backends.cudnn.benchmark = False torch.backends.cudnn.enabled = False device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # prepare model if args.model_type == "GLU_Transformer": model = GLU_TransformerSVS( phone_size=args.phone_size, embed_size=args.embedding_size, hidden_size=args.hidden_size, glu_num_layers=args.glu_num_layers, dropout=args.dropout, output_dim=args.feat_dim, dec_nhead=args.dec_nhead, dec_num_block=args.dec_num_block, n_mels=args.n_mels, double_mel_loss=args.double_mel_loss, local_gaussian=args.local_gaussian, device=device, ) elif args.model_type == "LSTM": model = LSTMSVS( phone_size=args.phone_size, embed_size=args.embedding_size, d_model=args.hidden_size, num_layers=args.num_rnn_layers, dropout=args.dropout, d_output=args.feat_dim, n_mels=args.n_mels, double_mel_loss=args.double_mel_loss, device=device, use_asr_post=args.use_asr_post, ) elif args.model_type == "GRU_gs": model = GRUSVS_gs( phone_size=args.phone_size, embed_size=args.embedding_size, d_model=args.hidden_size, num_layers=args.num_rnn_layers, dropout=args.dropout, d_output=args.feat_dim, n_mels=args.n_mels, double_mel_loss=args.double_mel_loss, device=device, use_asr_post=args.use_asr_post, ) elif args.model_type == "PureTransformer": model = TransformerSVS( phone_size=args.phone_size, embed_size=args.embedding_size, hidden_size=args.hidden_size, glu_num_layers=args.glu_num_layers, dropout=args.dropout, output_dim=args.feat_dim, dec_nhead=args.dec_nhead, dec_num_block=args.dec_num_block, n_mels=args.n_mels, double_mel_loss=args.double_mel_loss, local_gaussian=args.local_gaussian, device=device, ) elif args.model_type == "Conformer": model = ConformerSVS( phone_size=args.phone_size, embed_size=args.embedding_size, enc_attention_dim=args.enc_attention_dim, enc_attention_heads=args.enc_attention_heads, enc_linear_units=args.enc_linear_units, enc_num_blocks=args.enc_num_blocks, enc_dropout_rate=args.enc_dropout_rate, enc_positional_dropout_rate=args.enc_positional_dropout_rate, enc_attention_dropout_rate=args.enc_attention_dropout_rate, enc_input_layer=args.enc_input_layer, enc_normalize_before=args.enc_normalize_before, enc_concat_after=args.enc_concat_after, enc_positionwise_layer_type=args.enc_positionwise_layer_type, enc_positionwise_conv_kernel_size=(args.enc_positionwise_conv_kernel_size), enc_macaron_style=args.enc_macaron_style, enc_pos_enc_layer_type=args.enc_pos_enc_layer_type, enc_selfattention_layer_type=args.enc_selfattention_layer_type, enc_activation_type=args.enc_activation_type, enc_use_cnn_module=args.enc_use_cnn_module, enc_cnn_module_kernel=args.enc_cnn_module_kernel, enc_padding_idx=args.enc_padding_idx, output_dim=args.feat_dim, dec_nhead=args.dec_nhead, dec_num_block=args.dec_num_block, n_mels=args.n_mels, double_mel_loss=args.double_mel_loss, local_gaussian=args.local_gaussian, dec_dropout=args.dec_dropout, device=device, ) elif args.model_type == "Comformer_full": model = ConformerSVS_FULL( phone_size=args.phone_size, embed_size=args.embedding_size, output_dim=args.feat_dim, n_mels=args.n_mels, enc_attention_dim=args.enc_attention_dim, enc_attention_heads=args.enc_attention_heads, enc_linear_units=args.enc_linear_units, enc_num_blocks=args.enc_num_blocks, enc_dropout_rate=args.enc_dropout_rate, enc_positional_dropout_rate=args.enc_positional_dropout_rate, enc_attention_dropout_rate=args.enc_attention_dropout_rate, enc_input_layer=args.enc_input_layer, enc_normalize_before=args.enc_normalize_before, enc_concat_after=args.enc_concat_after, enc_positionwise_layer_type=args.enc_positionwise_layer_type, enc_positionwise_conv_kernel_size=(args.enc_positionwise_conv_kernel_size), enc_macaron_style=args.enc_macaron_style, enc_pos_enc_layer_type=args.enc_pos_enc_layer_type, enc_selfattention_layer_type=args.enc_selfattention_layer_type, enc_activation_type=args.enc_activation_type, enc_use_cnn_module=args.enc_use_cnn_module, enc_cnn_module_kernel=args.enc_cnn_module_kernel, enc_padding_idx=args.enc_padding_idx, dec_attention_dim=args.dec_attention_dim, dec_attention_heads=args.dec_attention_heads, dec_linear_units=args.dec_linear_units, dec_num_blocks=args.dec_num_blocks, dec_dropout_rate=args.dec_dropout_rate, dec_positional_dropout_rate=args.dec_positional_dropout_rate, dec_attention_dropout_rate=args.dec_attention_dropout_rate, dec_input_layer=args.dec_input_layer, dec_normalize_before=args.dec_normalize_before, dec_concat_after=args.dec_concat_after, dec_positionwise_layer_type=args.dec_positionwise_layer_type, dec_positionwise_conv_kernel_size=(args.dec_positionwise_conv_kernel_size), dec_macaron_style=args.dec_macaron_style, dec_pos_enc_layer_type=args.dec_pos_enc_layer_type, dec_selfattention_layer_type=args.dec_selfattention_layer_type, dec_activation_type=args.dec_activation_type, dec_use_cnn_module=args.dec_use_cnn_module, dec_cnn_module_kernel=args.dec_cnn_module_kernel, dec_padding_idx=args.dec_padding_idx, device=device, ) else: raise ValueError("Not Support Model Type %s" % args.model_type) logging.info(f"{model}") logging.info(f"The model has {count_parameters(model):,} trainable parameters") # Load model weights logging.info(f"Loading pretrained weights from {args.model_file}") checkpoint = torch.load(args.model_file, map_location=device) state_dict = checkpoint["state_dict"] model_dict = model.state_dict() state_dict_new = {} para_list = [] for k, v in state_dict.items(): # assert k in model_dict if ( k == "normalizer.mean" or k == "normalizer.std" or k == "mel_normalizer.mean" or k == "mel_normalizer.std" ): continue if model_dict[k].size() == state_dict[k].size(): state_dict_new[k] = v else: para_list.append(k) logging.info( f"Total {len(state_dict)} parameter sets, " f"loaded {len(state_dict_new)} parameter set" ) if len(para_list) > 0: logging.warning(f"Not loading {para_list} because of different sizes") model.load_state_dict(state_dict_new) logging.info(f"Loaded checkpoint {args.model_file}") model = model.to(device) model.eval() # Decode test_set = SVSDataset( align_root_path=args.test_align, pitch_beat_root_path=args.test_pitch, wav_root_path=args.test_wav, char_max_len=args.char_max_len, max_len=args.num_frames, sr=args.sampling_rate, preemphasis=args.preemphasis, nfft=args.nfft, frame_shift=args.frame_shift, frame_length=args.frame_length, n_mels=args.n_mels, power=args.power, max_db=args.max_db, ref_db=args.ref_db, standard=args.standard, sing_quality=args.sing_quality, ) collate_fn_svs = SVSCollator( args.num_frames, args.char_max_len, args.use_asr_post, args.phone_size, ) test_loader = torch.utils.data.DataLoader( dataset=test_set, batch_size=1, shuffle=False, num_workers=args.num_workers, collate_fn=collate_fn_svs, pin_memory=True, ) if args.loss == "l1": criterion = MaskedLoss("l1") elif args.loss == "mse": criterion = MaskedLoss("mse") else: raise ValueError("Not Support Loss Type") losses = AverageMeter() spec_losses = AverageMeter() if args.perceptual_loss > 0: pe_losses = AverageMeter() if args.n_mels > 0: mel_losses = AverageMeter() mcd_metric = AverageMeter() f0_distortion_metric, vuv_error_metric = ( AverageMeter(), AverageMeter(), ) if args.double_mel_loss: double_mel_losses = AverageMeter() model.eval() if not os.path.exists(args.prediction_path): os.makedirs(args.prediction_path) f0_ground_truth_all = np.reshape(np.array([]), (-1, 1)) f0_synthesis_all = np.reshape(np.array([]), (-1, 1)) start_t_test = time.time() # preload vocoder model if args.vocoder_category == "wavernn": voc_model = WaveRNN( rnn_dims=512, fc_dims=512, bits=9, pad=2, upsample_factors=( 5, 5, 11, ), feat_dims=80, compute_dims=128, res_out_dims=128, res_blocks=10, hop_length=275, # 12.5ms - in line with Tacotron 2 paper sample_rate=22050, mode="MOL", ).to(device) voc_model.load("./weights/wavernn/latest_weights.pyt") with torch.no_grad(): for ( step, ( phone, beat, pitch, spec, real, imag, length, chars, char_len_list, mel, ), ) in enumerate(test_loader, 1): # if step >= args.decode_sample: # break phone = phone.to(device) beat = beat.to(device) pitch = pitch.to(device).float() spec = spec.to(device).float() mel = mel.to(device).float() real = real.to(device).float() imag = imag.to(device).float() length_mask = length.unsqueeze(2) length_mel_mask = length_mask.repeat(1, 1, mel.shape[2]).float() length_mask = length_mask.repeat(1, 1, spec.shape[2]).float() length_mask = length_mask.to(device) length_mel_mask = length_mel_mask.to(device) length = length.to(device) char_len_list = char_len_list.to(device) if not args.use_asr_post: chars = chars.to(device) char_len_list = char_len_list.to(device) else: phone = phone.float() if args.model_type == "GLU_Transformer": output, att, output_mel, output_mel2 = model( chars, phone, pitch, beat, pos_char=char_len_list, pos_spec=length, ) elif args.model_type == "LSTM": output, hidden, output_mel, output_mel2 = model(phone, pitch, beat) att = None elif args.model_type == "GRU_gs": output, att, output_mel = model(spec, phone, pitch, beat, length, args) att = None elif args.model_type == "PureTransformer": output, att, output_mel, output_mel2 = model( chars, phone, pitch, beat, pos_char=char_len_list, pos_spec=length, ) elif args.model_type == "Conformer": output, att, output_mel, output_mel2 = model( chars, phone, pitch, beat, pos_char=char_len_list, pos_spec=length, ) elif args.model_type == "Comformer_full": output, att, output_mel, output_mel2 = model( chars, phone, pitch, beat, pos_char=char_len_list, pos_spec=length, ) spec_origin = spec.clone() # spec_origin = spec if args.normalize: sepc_normalizer = GlobalMVN(args.stats_file) mel_normalizer = GlobalMVN(args.stats_mel_file) spec, _ = sepc_normalizer(spec, length) mel, _ = mel_normalizer(mel, length) spec_loss = criterion(output, spec, length_mask) if args.n_mels > 0: mel_loss = criterion(output_mel, mel, length_mel_mask) else: mel_loss = 0 final_loss = mel_loss + spec_loss losses.update(final_loss.item(), phone.size(0)) spec_losses.update(spec_loss.item(), phone.size(0)) if args.n_mels > 0: mel_losses.update(mel_loss.item(), phone.size(0)) # normalize inverse stage if args.normalize and args.stats_file: output, _ = sepc_normalizer.inverse(output, length) # spec,_ = sepc_normalizer.inverse(spec,length) (mcd_value, length_sum,) = Metrics.Calculate_melcd_fromLinearSpectrum( output, spec_origin, length, args ) ( f0_distortion_value, voiced_frame_number_step, vuv_error_value, frame_number_step, f0_ground_truth_step, f0_synthesis_step, ) = Metrics.Calculate_f0RMSE_VUV_CORR_fromWav( output, spec_origin, length, args, "test" ) f0_ground_truth_all = np.concatenate( (f0_ground_truth_all, f0_ground_truth_step), axis=0 ) f0_synthesis_all = np.concatenate( (f0_synthesis_all, f0_synthesis_step), axis=0 ) mcd_metric.update(mcd_value, length_sum) f0_distortion_metric.update(f0_distortion_value, voiced_frame_number_step) vuv_error_metric.update(vuv_error_value, frame_number_step) if step % 1 == 0: if args.vocoder_category == "griffin": log_figure( step, output, spec_origin, att, length, args.prediction_path, args, ) elif args.vocoder_category == "wavernn": log_mel( step, output_mel, spec_origin, att, length, args.prediction_path, args, voc_model, ) out_log = ( "step {}:train_loss{:.4f};" "spec_loss{:.4f};mcd_value{:.4f};".format( step, losses.avg, spec_losses.avg, mcd_metric.avg, ) ) if args.perceptual_loss > 0: out_log += " pe_loss {:.4f}; ".format(pe_losses.avg) if args.n_mels > 0: out_log += " mel_loss {:.4f}; ".format(mel_losses.avg) if args.double_mel_loss: out_log += " dmel_loss {:.4f}; ".format(double_mel_losses.avg) end = time.time() logging.info(f"{out_log} -- sum_time: {(end - start_t_test)}s") end_t_test = time.time() out_log = "Test Stage: " out_log += "spec_loss: {:.4f} ".format(spec_losses.avg) if args.n_mels > 0: out_log += "mel_loss: {:.4f}, ".format(mel_losses.avg) # if args.perceptual_loss > 0: # out_log += 'pe_loss: {:.4f}, '.format(train_info['pe_loss']) f0_corr = Metrics.compute_f0_corr(f0_ground_truth_all, f0_synthesis_all) out_log += "\n\t mcd_value {:.4f} dB ".format(mcd_metric.avg) out_log += ( " f0_rmse_value {:.4f} Hz, " "vuv_error_value {:.4f} %, F0_CORR {:.4f}; ".format( np.sqrt(f0_distortion_metric.avg), vuv_error_metric.avg * 100, f0_corr, ) ) logging.info("{} time: {:.2f}s".format(out_log, end_t_test - start_t_test))