def gen_waveform(labels, acoustic_features, acoustic_out_scaler, binary_dict, continuous_dict, stream_sizes, has_dynamic_features, subphone_features="coarse_coding", log_f0_conditioning=True, pitch_idx=None, num_windows=3, post_filter=True, sample_rate=48000, frame_period=5, relative_f0=True): windows = get_windows(num_windows) # Apply MLPG if necessary if np.any(has_dynamic_features): acoustic_features = multi_stream_mlpg( acoustic_features, acoustic_out_scaler.var_, windows, stream_sizes, has_dynamic_features) static_stream_sizes = get_static_stream_sizes( stream_sizes, has_dynamic_features, len(windows)) else: static_stream_sizes = stream_sizes # Split multi-stream features mgc, target_f0, vuv, bap = split_streams(acoustic_features, static_stream_sizes) # Gen waveform by the WORLD vocodoer fftlen = pyworld.get_cheaptrick_fft_size(sample_rate) alpha = pysptk.util.mcepalpha(sample_rate) if post_filter: mgc = merlin_post_filter(mgc, alpha) spectrogram = pysptk.mc2sp(mgc, fftlen=fftlen, alpha=alpha) aperiodicity = pyworld.decode_aperiodicity(bap.astype(np.float64), sample_rate, fftlen) ### F0 ### if relative_f0: diff_lf0 = target_f0 # need to extract pitch sequence from the musical score linguistic_features = fe.linguistic_features(labels, binary_dict, continuous_dict, add_frame_features=True, subphone_features=subphone_features) f0_score = _midi_to_hz(linguistic_features, pitch_idx, False)[:, None] lf0_score = f0_score.copy() nonzero_indices = np.nonzero(lf0_score) lf0_score[nonzero_indices] = np.log(f0_score[nonzero_indices]) lf0_score = interp1d(lf0_score, kind="slinear") f0 = diff_lf0 + lf0_score f0[vuv < 0.5] = 0 f0[np.nonzero(f0)] = np.exp(f0[np.nonzero(f0)]) else: f0 = target_f0 generated_waveform = pyworld.synthesize(f0.flatten().astype(np.float64), spectrogram.astype(np.float64), aperiodicity.astype(np.float64), sample_rate, frame_period) return generated_waveform
def gen_spsvs_static_features( labels, acoustic_features, binary_dict, numeric_dict, stream_sizes, has_dynamic_features, subphone_features="coarse_coding", pitch_idx=None, num_windows=3, frame_period=5, relative_f0=True, vibrato_scale=1.0, vuv_threshold=0.3, force_fix_vuv=True, ): """Generate static features from predicted acoustic features Args: labels (HTSLabelFile): HTS labels acoustic_features (ndarray): predicted acoustic features binary_dict (dict): binary feature dictionary numeric_dict (dict): numeric feature dictionary stream_sizes (list): stream sizes has_dynamic_features (list): whether each stream has dynamic features subphone_features (str): subphone feature type pitch_idx (int): index of pitch features num_windows (int): number of windows frame_period (float): frame period relative_f0 (bool): whether to use relative f0 vibrato_scale (float): vibrato scale vuv_threshold (float): vuv threshold force_fix_vuv (bool): whether to use post-processing to fix VUV. Returns: tuple: tuple of mgc, lf0, vuv and bap. """ if np.any(has_dynamic_features): static_stream_sizes = get_static_stream_sizes( stream_sizes, has_dynamic_features, num_windows ) else: static_stream_sizes = stream_sizes # Copy here to avoid inplace operations on input acoustic features acoustic_features = acoustic_features.copy() # Split multi-stream features streams = split_streams(acoustic_features, static_stream_sizes) if len(streams) == 4: mgc, target_f0, vuv, bap = streams vib, vib_flags = None, None elif len(streams) == 5: # Assuming diff-based vibrato parameters mgc, target_f0, vuv, bap, vib = streams vib_flags = None elif len(streams) == 6: # Assuming sine-based vibrato parameters mgc, target_f0, vuv, bap, vib, vib_flags = streams else: raise RuntimeError("Not supported streams") linguistic_features = fe.linguistic_features( labels, binary_dict, numeric_dict, add_frame_features=True, subphone_features=subphone_features, ) # Correct V/UV based on special phone flags if force_fix_vuv: vuv = correct_vuv_by_phone(vuv, binary_dict, linguistic_features) # F0 if relative_f0: diff_lf0 = target_f0 f0_score = _midi_to_hz(linguistic_features, pitch_idx, False)[:, None] lf0_score = f0_score.copy() nonzero_indices = np.nonzero(lf0_score) lf0_score[nonzero_indices] = np.log(f0_score[nonzero_indices]) lf0_score = interp1d(lf0_score, kind="slinear") f0 = diff_lf0 + lf0_score f0[vuv < vuv_threshold] = 0 f0[np.nonzero(f0)] = np.exp(f0[np.nonzero(f0)]) else: f0 = target_f0 f0[vuv < vuv_threshold] = 0 f0[np.nonzero(f0)] = np.exp(f0[np.nonzero(f0)]) if vib is not None: if vib_flags is not None: # Generate sine-based vibrato vib_flags = vib_flags.flatten() m_a, m_f = vib[:, 0], vib[:, 1] # Fill zeros for non-vibrato frames m_a[vib_flags < 0.5] = 0 m_f[vib_flags < 0.5] = 0 # Gen vibrato sr_f0 = int(1 / (frame_period * 0.001)) f0 = gen_sine_vibrato(f0.flatten(), sr_f0, m_a, m_f, vibrato_scale) else: # Generate diff-based vibrato f0 = f0.flatten() + vibrato_scale * vib.flatten() # NOTE: Back to log-domain for convenience lf0 = f0.copy() lf0[np.nonzero(lf0)] = np.log(f0[np.nonzero(lf0)]) # NOTE: interpolation is necessary lf0 = interp1d(lf0, kind="slinear") lf0 = lf0[:, None] if len(lf0.shape) == 1 else lf0 vuv = vuv[:, None] if len(vuv.shape) == 1 else vuv return mgc, lf0, vuv, bap
def svs( self, labels, vocoder_type="world", post_filter_type="merlin", trajectory_smoothing=True, trajectory_smoothing_cutoff=50, vuv_threshold=0.1, vibrato_scale=1.0, return_states=False, force_fix_vuv=True, post_filter=None, ): """Synthesize waveform given HTS-style labels Args: labels (nnmnkwii.io.HTSLabelFile): HTS-style labels vocoder_type (str): Vocoder type. world or pwg post_filter_type (str): Post-filter type. merlin or nnsvs. Returns: tuple: (synthesized waveform, sampling rate) """ vocoder_type = vocoder_type.lower() if vocoder_type not in ["world", "pwg"]: raise ValueError(f"Unknown vocoder type: {vocoder_type}") if post_filter_type not in ["merlin", "nnsvs", "gv", "none"]: raise ValueError(f"Unknown post-filter type: {post_filter_type}") if vocoder_type == "pwg" and self.vocoder is None: raise ValueError("""Pre-trained vocodr model is not found. WORLD is only supported for waveform generation""") if post_filter is not None: warn("post_filter is deprecated. Use post_filter_type instead.") post_filter_type = "merlin" if post_filter else "none" # Time-lag lag = predict_timelag( self.device, labels, self.timelag_model, self.timelag_config, self.timelag_in_scaler, self.timelag_out_scaler, self.binary_dict, self.numeric_dict, self.pitch_indices, self.config.log_f0_conditioning, self.config.timelag.allowed_range, self.config.timelag.allowed_range_rest, self.config.timelag.force_clip_input_features, ) # Duration predictions durations = predict_duration( self.device, labels, self.duration_model, self.duration_config, self.duration_in_scaler, self.duration_out_scaler, self.binary_dict, self.numeric_dict, self.pitch_indices, self.config.log_f0_conditioning, self.config.duration.force_clip_input_features, ) # Normalize phoneme durations duration_modified_labels = postprocess_duration(labels, durations, lag) # Predict acoustic features acoustic_features = predict_acoustic( self.device, duration_modified_labels, self.acoustic_model, self.acoustic_config, self.acoustic_in_scaler, self.acoustic_out_scaler, self.binary_dict, self.numeric_dict, self.config.acoustic.subphone_features, self.pitch_indices, self.config.log_f0_conditioning, self.config.acoustic.force_clip_input_features, ) # Apply GV post-filtering if post_filter_type in ["nnsvs", "gv"]: static_stream_sizes = get_static_stream_sizes( self.acoustic_config.stream_sizes, self.acoustic_config.has_dynamic_features, self.acoustic_config.num_windows, ) mgc_end_dim = static_stream_sizes[0] acoustic_features[:, :mgc_end_dim] = variance_scaling( self.postfilter_out_scaler.var_.reshape(-1)[:mgc_end_dim], acoustic_features[:, :mgc_end_dim], offset=2, ) # bap bap_start_dim = sum(static_stream_sizes[:3]) bap_end_dim = sum(static_stream_sizes[:4]) acoustic_features[:, bap_start_dim:bap_end_dim] = variance_scaling( self.postfilter_out_scaler.var_.reshape(-1) [bap_start_dim:bap_end_dim], acoustic_features[:, bap_start_dim:bap_end_dim], offset=0, ) # Learned post-filter using nnsvs if post_filter_type == "nnsvs" and self.postfilter_model is not None: in_feats = torch.from_numpy(acoustic_features).float().unsqueeze(0) in_feats = ( self.postfilter_out_scaler.transform(in_feats).float().to( self.device)) out_feats = self.postfilter_model.inference( in_feats, [in_feats.shape[1]]) acoustic_features = (self.postfilter_out_scaler.inverse_transform( out_feats.cpu()).squeeze(0).numpy()) # Generate WORLD parameters mgc, lf0, vuv, bap = gen_spsvs_static_features( duration_modified_labels, acoustic_features, self.binary_dict, self.numeric_dict, self.acoustic_config.stream_sizes, self.acoustic_config.has_dynamic_features, self.config.acoustic.subphone_features, self.pitch_idx, self.acoustic_config.num_windows, self.config.frame_period, self.config.acoustic.relative_f0, vibrato_scale=vibrato_scale, vuv_threshold=vuv_threshold, force_fix_vuv=force_fix_vuv, ) # NOTE: spectral enhancement based on the Merlin's post-filter implementation if post_filter_type == "merlin": alpha = pysptk.util.mcepalpha(self.config.sample_rate) mgc = merlin_post_filter(mgc, alpha) # Remove high-frequency components of mgc/bap # NOTE: It seems to be effective to suppress artifacts of GAN-based post-filtering if trajectory_smoothing: modfs = int(1 / 0.005) for d in range(mgc.shape[1]): mgc[:, d] = lowpass_filter(mgc[:, d], modfs, cutoff=trajectory_smoothing_cutoff) for d in range(bap.shape[1]): bap[:, d] = lowpass_filter(bap[:, d], modfs, cutoff=trajectory_smoothing_cutoff) # Waveform generation by (1) WORLD or (2) neural vocoder if vocoder_type == "world": f0, spectrogram, aperiodicity = gen_world_params( mgc, lf0, vuv, bap, self.config.sample_rate, vuv_threshold=vuv_threshold) wav = pyworld.synthesize( f0, spectrogram, aperiodicity, self.config.sample_rate, self.config.frame_period, ) elif vocoder_type == "pwg": # NOTE: So far vocoder models are trained on binary V/UV features vuv = (vuv > vuv_threshold).astype(np.float32) voc_inp = (torch.from_numpy( self.vocoder_in_scaler.transform( np.concatenate([mgc, lf0, vuv, bap], axis=-1))).float().to(self.device)) wav = self.vocoder.inference(voc_inp).view(-1).to("cpu").numpy() wav = self.post_process(wav) if return_states: states = { "mgc": mgc, "lf0": lf0, "vuv": vuv, "bap": bap, } if vocoder_type == "world": states.update({ "f0": f0, "spectrogram": spectrogram, "aperiodicity": aperiodicity, }) return wav, self.config.sample_rate, states return wav, self.config.sample_rate
def my_app(config: DictConfig) -> None: # NOTE: set discriminator's in_dim automatically if config.model.netD.in_dim is None: if config.train.adv_use_static_feats_only: stream_sizes = get_static_stream_sizes( config.model.stream_sizes, config.model.has_dynamic_features, config.model.num_windows, ) else: stream_sizes = np.asarray(config.model.stream_sizes) D_in_dim = int((stream_sizes * np.asarray(config.train.adv_streams)).sum()) if config.train.mask_nth_mgc_for_adv_loss > 0: D_in_dim -= config.train.mask_nth_mgc_for_adv_loss config.model.netD.in_dim = D_in_dim if "max_time_frames" in config.data and config.data.max_time_frames > 0: collate_fn = partial( collate_fn_random_segments, max_time_frames=config.data.max_time_frames ) else: collate_fn = collate_fn_default device = torch.device("cuda" if torch.cuda.is_available() else "cpu") ( (netG, optG, schedulerG), (netD, optD, schedulerD), grad_scaler, data_loaders, writer, logger, in_scaler, out_scaler, ) = setup_gan(config, device, collate_fn) check_resf0_config(logger, netG, config, in_scaler, out_scaler) out_scaler = PyTorchStandardScaler( torch.from_numpy(out_scaler.mean_), torch.from_numpy(out_scaler.scale_) ).to(device) use_mlflow = config.mlflow.enabled if use_mlflow: with mlflow.start_run() as run: # NOTE: modify out_dir when running with mlflow config.train.out_dir = f"{config.train.out_dir}/{run.info.run_id}" save_configs(config) log_params_from_omegaconf_dict(config) last_dev_loss = train_loop( config, logger, device, netG, optG, schedulerG, netD, optD, schedulerD, grad_scaler, data_loaders, writer, in_scaler, out_scaler, use_mlflow, ) else: save_configs(config) last_dev_loss = train_loop( config, logger, device, netG, optG, schedulerG, netD, optD, schedulerD, grad_scaler, data_loaders, writer, in_scaler, out_scaler, use_mlflow, ) return last_dev_loss
def gen_waveform(labels, acoustic_features, binary_dict, continuous_dict, stream_sizes, has_dynamic_features, subphone_features="coarse_coding", log_f0_conditioning=True, pitch_idx=None, num_windows=3, post_filter=True, sample_rate=48000, frame_period=5, relative_f0=True): windows = get_windows(num_windows) # Apply MLPG if necessary if np.any(has_dynamic_features): static_stream_sizes = get_static_stream_sizes(stream_sizes, has_dynamic_features, len(windows)) else: static_stream_sizes = stream_sizes # Split multi-stream features mgc, target_f0, vuv, bap = split_streams(acoustic_features, static_stream_sizes) # Gen waveform by the WORLD vocodoer fftlen = pyworld.get_cheaptrick_fft_size(sample_rate) alpha = pysptk.util.mcepalpha(sample_rate) if post_filter: mgc = merlin_post_filter(mgc, alpha) spectrogram = pysptk.mc2sp(mgc, fftlen=fftlen, alpha=alpha) aperiodicity = pyworld.decode_aperiodicity(bap.astype(np.float64), sample_rate, fftlen) # fill aperiodicity with ones for unvoiced regions aperiodicity[vuv.reshape(-1) < 0.5, :] = 1.0 # WORLD fails catastrophically for out of range aperiodicity aperiodicity = np.clip(aperiodicity, 0.0, 1.0) ### F0 ### if relative_f0: diff_lf0 = target_f0 # need to extract pitch sequence from the musical score linguistic_features = fe.linguistic_features( labels, binary_dict, continuous_dict, add_frame_features=True, subphone_features=subphone_features) f0_score = _midi_to_hz(linguistic_features, pitch_idx, False)[:, None] lf0_score = f0_score.copy() nonzero_indices = np.nonzero(lf0_score) lf0_score[nonzero_indices] = np.log(f0_score[nonzero_indices]) lf0_score = interp1d(lf0_score, kind="slinear") f0 = diff_lf0 + lf0_score f0[vuv < 0.5] = 0 f0[np.nonzero(f0)] = np.exp(f0[np.nonzero(f0)]) else: f0 = target_f0 f0[vuv < 0.5] = 0 f0[np.nonzero(f0)] = np.exp(f0[np.nonzero(f0)]) generated_waveform = pyworld.synthesize(f0.flatten().astype(np.float64), spectrogram.astype(np.float64), aperiodicity.astype(np.float64), sample_rate, frame_period) # 音量を小さくする(音割れ防止) # TODO: ここのかける定数をいい感じにする spectrogram *= 0.000000001 sp = pyworld.code_spectral_envelope(spectrogram, sample_rate, 60) return f0, sp, bap, generated_waveform
def eval_spss_model( step, netG, in_feats, out_feats, lengths, model_config, out_scaler, writer, sr, trajectory_smoothing=True, trajectory_smoothing_cutoff=50, ): # make sure to be in eval mode netG.eval() is_autoregressive = (netG.module.is_autoregressive() if isinstance( netG, nn.DataParallel) else netG.is_autoregressive()) prediction_type = (netG.module.prediction_type() if isinstance( netG, nn.DataParallel) else netG.prediction_type()) utt_indices = [-1, -2, -3] utt_indices = utt_indices[:min(3, len(in_feats))] if np.any(model_config.has_dynamic_features): static_stream_sizes = get_static_stream_sizes( model_config.stream_sizes, model_config.has_dynamic_features, model_config.num_windows, ) else: static_stream_sizes = model_config.stream_sizes for utt_idx in utt_indices: out_feats_denorm_ = out_scaler.inverse_transform( out_feats[utt_idx, :lengths[utt_idx]].unsqueeze(0)) mgc, lf0, vuv, bap = get_static_features( out_feats_denorm_, model_config.num_windows, model_config.stream_sizes, model_config.has_dynamic_features, )[:4] mgc = mgc.squeeze(0).cpu().numpy() lf0 = lf0.squeeze(0).cpu().numpy() vuv = vuv.squeeze(0).cpu().numpy() bap = bap.squeeze(0).cpu().numpy() f0, spectrogram, aperiodicity = gen_world_params( mgc, lf0, vuv, bap, sr) wav = pyworld.synthesize(f0, spectrogram, aperiodicity, sr, 5) group = f"utt{np.abs(utt_idx)}_reference" wav = wav / np.abs(wav).max() if np.max(wav) > 1.0 else wav writer.add_audio(group, wav, step, sr) # Run forward if is_autoregressive: outs = netG( in_feats[utt_idx, :lengths[utt_idx]].unsqueeze(0), [lengths[utt_idx]], out_feats[utt_idx, :lengths[utt_idx]].unsqueeze(0), ) else: outs = netG(in_feats[utt_idx, :lengths[utt_idx]].unsqueeze(0), [lengths[utt_idx]]) # ResF0 case if isinstance(outs, tuple) and len(outs) == 2: outs, _ = outs if prediction_type == PredictionType.PROBABILISTIC: pi, sigma, mu = outs pred_out_feats = mdn_get_most_probable_sigma_and_mu(pi, sigma, mu)[1] else: pred_out_feats = outs # NOTE: multiple outputs if isinstance(pred_out_feats, list): pred_out_feats = pred_out_feats[-1] if isinstance(pred_out_feats, tuple): pred_out_feats = pred_out_feats[0] if not isinstance(pred_out_feats, list): pred_out_feats = [pred_out_feats] # Run inference if prediction_type == PredictionType.PROBABILISTIC: inference_out_feats, _ = netG.inference( in_feats[utt_idx, :lengths[utt_idx]].unsqueeze(0), [lengths[utt_idx]]) else: inference_out_feats = netG.inference( in_feats[utt_idx, :lengths[utt_idx]].unsqueeze(0), [lengths[utt_idx]]) pred_out_feats.append(inference_out_feats) # Plot normalized input/output in_feats_ = in_feats[utt_idx, :lengths[utt_idx]].cpu().numpy() out_feats_ = out_feats[utt_idx, :lengths[utt_idx]].cpu().numpy() fig, ax = plt.subplots(3, 1, figsize=(8, 8)) ax[0].set_title("Reference features") ax[1].set_title("Input features") ax[2].set_title("Predicted features") mesh = librosa.display.specshow(out_feats_.T, x_axis="frames", y_axis="frames", ax=ax[0], cmap="viridis") # NOTE: assuming normalized to N(0, 1) mesh.set_clim(-4, 4) fig.colorbar(mesh, ax=ax[0]) mesh = librosa.display.specshow(in_feats_.T, x_axis="frames", y_axis="frames", ax=ax[1], cmap="viridis") mesh.set_clim(-4, 4) fig.colorbar(mesh, ax=ax[1]) mesh = librosa.display.specshow( inference_out_feats.squeeze(0).cpu().numpy().T, x_axis="frames", y_axis="frames", ax=ax[2], cmap="viridis", ) mesh.set_clim(-4, 4) fig.colorbar(mesh, ax=ax[2]) for ax_ in ax: ax_.set_ylabel("Feature") plt.tight_layout() group = f"utt{np.abs(utt_idx)}_inference" writer.add_figure(f"{group}/Input-Output", fig, step) plt.close() assert len(pred_out_feats) == 2 for idx, pred_out_feats_ in enumerate(pred_out_feats): pred_out_feats_ = pred_out_feats_.squeeze(0).cpu().numpy() pred_out_feats_denorm = (out_scaler.inverse_transform( torch.from_numpy(pred_out_feats_).to( in_feats.device)).cpu().numpy()) if np.any(model_config.has_dynamic_features): # (T, D_out) -> (T, static_dim) pred_out_feats_denorm = multi_stream_mlpg( pred_out_feats_denorm, (out_scaler.scale_**2).cpu().numpy(), get_windows(model_config.num_windows), model_config.stream_sizes, model_config.has_dynamic_features, ) pred_mgc, pred_lf0, pred_vuv, pred_bap = split_streams( pred_out_feats_denorm, static_stream_sizes)[:4] # Remove high-frequency components of mgc/bap # NOTE: It seems to be effective to suppress artifacts of GAN-based post-filtering if trajectory_smoothing: modfs = int(1 / 0.005) for d in range(pred_mgc.shape[1]): pred_mgc[:, d] = lowpass_filter( pred_mgc[:, d], modfs, cutoff=trajectory_smoothing_cutoff) for d in range(pred_bap.shape[1]): pred_bap[:, d] = lowpass_filter( pred_bap[:, d], modfs, cutoff=trajectory_smoothing_cutoff) # Generated sample f0, spectrogram, aperiodicity = gen_world_params( pred_mgc, pred_lf0, pred_vuv, pred_bap, sr) wav = pyworld.synthesize(f0, spectrogram, aperiodicity, sr, 5) wav = wav / np.abs(wav).max() if np.max(wav) > 1.0 else wav if idx == 1: group = f"utt{np.abs(utt_idx)}_inference" else: group = f"utt{np.abs(utt_idx)}_forward" writer.add_audio(group, wav, step, sr) plot_spsvs_params( step, writer, mgc, lf0, vuv, bap, pred_mgc, pred_lf0, pred_vuv, pred_bap, group=group, sr=sr, )
def my_app(config: DictConfig) -> None: global logger logger = getLogger(config.verbose) logger.info(OmegaConf.to_yaml(config)) device = torch.device("cuda" if use_cuda else "cpu") utt_list = to_absolute_path(config.utt_list) in_dir = to_absolute_path(config.in_dir) out_dir = to_absolute_path(config.out_dir) utt_ids = load_utt_list(utt_list) os.makedirs(out_dir, exist_ok=True) model_config = OmegaConf.load(to_absolute_path(config.model.model_yaml)) model = hydra.utils.instantiate(model_config.netG).to(device) checkpoint = torch.load( to_absolute_path(config.model.checkpoint), map_location=lambda storage, loc: storage, ) model.load_state_dict(checkpoint["state_dict"]) model.eval() out_scaler = joblib.load(to_absolute_path(config.out_scaler_path)) mean_ = get_static_features( out_scaler.mean_.reshape(1, 1, out_scaler.mean_.shape[-1]), model_config.num_windows, model_config.stream_sizes, model_config.has_dynamic_features, ) mean_ = np.concatenate(mean_, -1).reshape(1, -1) var_ = get_static_features( out_scaler.var_.reshape(1, 1, out_scaler.var_.shape[-1]), model_config.num_windows, model_config.stream_sizes, model_config.has_dynamic_features, ) var_ = np.concatenate(var_, -1).reshape(1, -1) scale_ = get_static_features( out_scaler.scale_.reshape(1, 1, out_scaler.scale_.shape[-1]), model_config.num_windows, model_config.stream_sizes, model_config.has_dynamic_features, ) scale_ = np.concatenate(scale_, -1).reshape(1, -1) static_scaler = StandardScaler(mean_, var_, scale_) static_stream_sizes = get_static_stream_sizes( model_config.stream_sizes, model_config.has_dynamic_features, model_config.num_windows, ) for utt_id in tqdm(utt_ids): in_feats = (torch.from_numpy( np.load(join(in_dir, utt_id + "-feats.npy"))).unsqueeze(0).to(device)) static_feats = _gen_static_features(model, model_config, in_feats, out_scaler) mgc_end_dim = static_stream_sizes[0] bap_start_dim = sum(static_stream_sizes[:3]) bap_end_dim = sum(static_stream_sizes[:4]) if config.gv_postfilter: # mgc static_feats[:, :mgc_end_dim] = variance_scaling( static_scaler.var_.reshape(-1)[:mgc_end_dim], static_feats[:, :mgc_end_dim], offset=config.mgc_offset, ) # bap static_feats[:, bap_start_dim:bap_end_dim] = variance_scaling( static_scaler.var_.reshape(-1)[bap_start_dim:bap_end_dim], static_feats[:, bap_start_dim:bap_end_dim], offset=config.bap_offset, ) if config.normalize: static_feats = static_scaler.transform(static_feats) out_path = join(out_dir, f"{utt_id}-feats.npy") np.save(out_path, static_feats.astype(np.float32), allow_pickle=False)