class Synthesizer(object): def __init__( self, tts_checkpoint: str, tts_config_path: str, tts_speakers_file: str = "", vocoder_checkpoint: str = "", vocoder_config: str = "", encoder_checkpoint: str = "", encoder_config: str = "", use_cuda: bool = False, ) -> None: """General 🐸 TTS interface for inference. It takes a tts and a vocoder model and synthesize speech from the provided text. The text is divided into a list of sentences using `pysbd` and synthesize speech on each sentence separately. If you have certain special characters in your text, you need to handle them before providing the text to Synthesizer. TODO: set the segmenter based on the source language Args: tts_checkpoint (str): path to the tts model file. tts_config_path (str): path to the tts config file. vocoder_checkpoint (str, optional): path to the vocoder model file. Defaults to None. vocoder_config (str, optional): path to the vocoder config file. Defaults to None. encoder_checkpoint (str, optional): path to the speaker encoder model file. Defaults to `""`, encoder_config (str, optional): path to the speaker encoder config file. Defaults to `""`, use_cuda (bool, optional): enable/disable cuda. Defaults to False. """ self.tts_checkpoint = tts_checkpoint self.tts_config_path = tts_config_path self.tts_speakers_file = tts_speakers_file self.vocoder_checkpoint = vocoder_checkpoint self.vocoder_config = vocoder_config self.encoder_checkpoint = encoder_checkpoint self.encoder_config = encoder_config self.use_cuda = use_cuda self.tts_model = None self.vocoder_model = None self.speaker_manager = None self.num_speakers = 0 self.tts_speakers = {} self.d_vector_dim = 0 self.seg = self._get_segmenter("en") self.use_cuda = use_cuda if self.use_cuda: assert torch.cuda.is_available( ), "CUDA is not availabe on this machine." self._load_tts(tts_checkpoint, tts_config_path, use_cuda) self.output_sample_rate = self.tts_config.audio["sample_rate"] if vocoder_checkpoint: self._load_vocoder(vocoder_checkpoint, vocoder_config, use_cuda) self.output_sample_rate = self.vocoder_config.audio["sample_rate"] @staticmethod def _get_segmenter(lang: str): """get the sentence segmenter for the given language. Args: lang (str): target language code. Returns: [type]: [description] """ return pysbd.Segmenter(language=lang, clean=True) def _load_speakers(self, speaker_file: str) -> None: """Load the SpeakerManager to organize multi-speaker TTS. It loads the speakers meta-data and the speaker encoder if it is defined. Args: speaker_file (str): path to the speakers meta-data file. """ print("Loading speakers ...") self.speaker_manager = SpeakerManager( encoder_model_path=self.encoder_checkpoint, encoder_config_path=self.encoder_config, ) self.speaker_manager.load_d_vectors_file( self.tts_config.get("d_vector_file", speaker_file)) self.num_speakers = self.speaker_manager.num_speakers self.d_vector_dim = self.speaker_manager.d_vector_dim def _set_tts_speaker_file(self): """Set the TTS speaker file used by a multi-speaker model.""" # setup if multi-speaker settings are in the global model config if (hasattr(self.tts_config, "use_speaker_embedding") and self.tts_config.use_speaker_embedding is True): if self.tts_config.use_d_vector_file: self.tts_speakers_file = (self.tts_speakers_file if self.tts_speakers_file else self.tts_config["d_vector_file"]) self.tts_config["d_vector_file"] = self.tts_speakers_file else: self.tts_speakers_file = (self.tts_speakers_file if self.tts_speakers_file else self.tts_config["speakers_file"]) # setup if multi-speaker settings are in the model args config if (self.tts_speakers_file is None and hasattr(self.tts_config, "model_args") and hasattr( self.tts_config.model_args, "use_speaker_embedding") and self.tts_config.model_args.use_speaker_embedding): _args = self.tts_config.model_args if _args.use_d_vector_file: self.tts_speakers_file = (self.tts_speakers_file if self.tts_speakers_file else _args["d_vector_file"]) _args["d_vector_file"] = self.tts_speakers_file else: self.tts_speakers_file = (self.tts_speakers_file if self.tts_speakers_file else _args["speakers_file"]) def _load_tts(self, tts_checkpoint: str, tts_config_path: str, use_cuda: bool) -> None: """Load the TTS model. Args: tts_checkpoint (str): path to the model checkpoint. tts_config_path (str): path to the model config file. use_cuda (bool): enable/disable CUDA use. """ # pylint: disable=global-statement self.tts_config = load_config(tts_config_path) # Patch stats_path stats_path = self.tts_config["audio"].get("stats_path", "") if stats_path and (not os.path.isfile(stats_path)): stats_path = os.path.join(os.path.dirname(tts_checkpoint), os.path.split(stats_path)[1]) self.tts_config["audio"]["stats_path"] = stats_path # Patch speakers file speakers_file = self.tts_config.get("model_args", {}).get("speakers_file", "") if speakers_file and (not os.path.isfile(speakers_file)): speakers_file = os.path.join(os.path.dirname(tts_checkpoint), os.path.split(speakers_file)[1]) self.tts_config["model_args"]["speakers_file"] = speakers_file self.use_phonemes = self.tts_config.use_phonemes self.ap = AudioProcessor(verbose=False, **self.tts_config.audio) self.tts_model = setup_tts_model(config=self.tts_config) self.tts_model.load_checkpoint(self.tts_config, tts_checkpoint, eval=True) if use_cuda: self.tts_model.cuda() self._set_tts_speaker_file() def _load_vocoder(self, model_file: str, model_config: str, use_cuda: bool) -> None: """Load the vocoder model. Args: model_file (str): path to the model checkpoint. model_config (str): path to the model config file. use_cuda (bool): enable/disable CUDA use. """ self.vocoder_config = load_config(model_config) # Patch stats_path stats_path = self.vocoder_config["audio"].get("stats_path", "") if stats_path and (not os.path.isfile(stats_path)): stats_path = os.path.join(os.path.dirname(model_file), os.path.split(stats_path)[1]) self.vocoder_config["audio"]["stats_path"] = stats_path self.vocoder_ap = AudioProcessor(verbose=False, **self.vocoder_config.audio) self.vocoder_model = setup_vocoder_model(self.vocoder_config) self.vocoder_model.load_checkpoint(self.vocoder_config, model_file, eval=True) if use_cuda: self.vocoder_model.cuda() def split_into_sentences(self, text) -> List[str]: """Split give text into sentences. Args: text (str): input text in string format. Returns: List[str]: list of sentences. """ return self.seg.segment(text) def save_wav(self, wav: List[int], path: str) -> None: """Save the waveform as a file. Args: wav (List[int]): waveform as a list of values. path (str): output path to save the waveform. """ wav = np.array(wav) self.ap.save_wav(wav, path, self.output_sample_rate) def tts(self, text: str, speaker_idx: str = "", speaker_wav=None, style_wav=None) -> List[int]: """🐸 TTS magic. Run all the models and generate speech. Args: text (str): input text. speaker_idx (str, optional): spekaer id for multi-speaker models. Defaults to "". speaker_wav (): style_wav ([type], optional): style waveform for GST. Defaults to None. Returns: List[int]: [description] """ start_time = time.time() wavs = [] sens = self.split_into_sentences(text) print(" > Text splitted to sentences.") print(sens) # handle multi-speaker speaker_embedding = None speaker_id = None if isinstance(speaker_idx, int): speaker_id = speaker_idx elif self.tts_speakers_file: if speaker_idx and isinstance(speaker_idx, str): if self.tts_config.use_d_vector_file: # get the speaker embedding from the saved d_vectors. speaker_embedding = self.tts_model.speaker_manager.get_d_vectors_by_speaker( speaker_idx)[0] else: # get speaker idx from the speaker name try: speaker_id = self.tts_model.speaker_manager.speaker_ids[ speaker_idx] except KeyError: # Interpet as int speaker_id = int(speaker_idx) elif not speaker_idx and not speaker_wav: raise ValueError( " [!] Look like you use a multi-speaker model. " "You need to define either a `speaker_idx` or a `style_wav` to use a multi-speaker model." ) else: speaker_embedding = None else: if speaker_idx: raise ValueError( f" [!] Missing speaker.json file path for selecting speaker {speaker_idx}." "Define path for speaker.json if it is a multi-speaker model or remove defined speaker idx. " ) # compute a new d_vector from the given clip. if speaker_wav is not None: speaker_embedding = self.tts_model.speaker_manager.compute_d_vector_from_clip( speaker_wav) use_gl = self.vocoder_model is None for sen in sens: # synthesize voice outputs = synthesis( model=self.tts_model, text=sen, CONFIG=self.tts_config, use_cuda=self.use_cuda, ap=self.ap, speaker_id=speaker_id, style_wav=style_wav, enable_eos_bos_chars=self.tts_config.enable_eos_bos_chars, use_griffin_lim=use_gl, d_vector=speaker_embedding, ) waveform = outputs["wav"] mel_postnet_spec = ( outputs["outputs"]["model_outputs"][0].detach().cpu().numpy()) if not use_gl: # denormalize tts output based on tts audio config mel_postnet_spec = self.ap.denormalize(mel_postnet_spec.T).T device_type = "cuda" if self.use_cuda else "cpu" # renormalize spectrogram based on vocoder config vocoder_input = self.vocoder_ap.normalize(mel_postnet_spec.T) # compute scale factor for possible sample rate mismatch scale_factor = [ 1, self.vocoder_config["audio"]["sample_rate"] / self.ap.sample_rate, ] if scale_factor[1] != 1: print(" > interpolating tts model output.") vocoder_input = interpolate_vocoder_input( scale_factor, vocoder_input) else: vocoder_input = torch.tensor(vocoder_input).unsqueeze(0) # pylint: disable=not-callable # run vocoder model # [1, T, C] waveform = self.vocoder_model.inference( vocoder_input.to(device_type)) if self.use_cuda and not use_gl: waveform = waveform.cpu() if not use_gl: waveform = waveform.numpy() waveform = waveform.squeeze() # trim silence waveform = trim_silence(waveform, self.ap) wavs += list(waveform) wavs += [0] * 10000 # compute stats process_time = time.time() - start_time audio_time = len(wavs) / self.tts_config.audio["sample_rate"] print(f" > Processing time: {process_time}") print(f" > Real-time factor: {process_time / audio_time}") return wavs
class TestAudio(unittest.TestCase): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.ap = AudioProcessor(**conf) def test_audio_synthesis(self): """1. load wav 2. set normalization parameters 3. extract mel-spec 4. invert to wav and save the output """ print(" > Sanity check for the process wav -> mel -> wav") def _test(max_norm, signal_norm, symmetric_norm, clip_norm): self.ap.max_norm = max_norm self.ap.signal_norm = signal_norm self.ap.symmetric_norm = symmetric_norm self.ap.clip_norm = clip_norm wav = self.ap.load_wav(WAV_FILE) mel = self.ap.melspectrogram(wav) wav_ = self.ap.inv_melspectrogram(mel) file_name = "/audio_test-melspec_max_norm_{}-signal_norm_{}-symmetric_{}-clip_norm_{}.wav".format( max_norm, signal_norm, symmetric_norm, clip_norm ) print(" | > Creating wav file at : ", file_name) self.ap.save_wav(wav_, OUT_PATH + file_name) # maxnorm = 1.0 _test(1.0, False, False, False) _test(1.0, True, False, False) _test(1.0, True, True, False) _test(1.0, True, False, True) _test(1.0, True, True, True) # maxnorm = 4.0 _test(4.0, False, False, False) _test(4.0, True, False, False) _test(4.0, True, True, False) _test(4.0, True, False, True) _test(4.0, True, True, True) def test_normalize(self): """Check normalization and denormalization for range values and consistency""" print(" > Testing normalization and denormalization.") wav = self.ap.load_wav(WAV_FILE) wav = self.ap.sound_norm(wav) # normalize audio to get abetter normalization range below. self.ap.signal_norm = False x = self.ap.melspectrogram(wav) x_old = x self.ap.signal_norm = True self.ap.symmetric_norm = False self.ap.clip_norm = False self.ap.max_norm = 4.0 x_norm = self.ap.normalize(x) print( f" > MaxNorm: {self.ap.max_norm}, ClipNorm:{self.ap.clip_norm}, SymmetricNorm:{self.ap.symmetric_norm}, SignalNorm:{self.ap.signal_norm} Range-> {x_norm.max()} -- {x_norm.min()}" ) assert (x_old - x).sum() == 0 # check value range assert x_norm.max() <= self.ap.max_norm + 1, x_norm.max() assert x_norm.min() >= 0 - 1, x_norm.min() # check denorm. x_ = self.ap.denormalize(x_norm) assert (x - x_).sum() < 1e-3, (x - x_).mean() self.ap.signal_norm = True self.ap.symmetric_norm = False self.ap.clip_norm = True self.ap.max_norm = 4.0 x_norm = self.ap.normalize(x) print( f" > MaxNorm: {self.ap.max_norm}, ClipNorm:{self.ap.clip_norm}, SymmetricNorm:{self.ap.symmetric_norm}, SignalNorm:{self.ap.signal_norm} Range-> {x_norm.max()} -- {x_norm.min()}" ) assert (x_old - x).sum() == 0 # check value range assert x_norm.max() <= self.ap.max_norm, x_norm.max() assert x_norm.min() >= 0, x_norm.min() # check denorm. x_ = self.ap.denormalize(x_norm) assert (x - x_).sum() < 1e-3, (x - x_).mean() self.ap.signal_norm = True self.ap.symmetric_norm = True self.ap.clip_norm = False self.ap.max_norm = 4.0 x_norm = self.ap.normalize(x) print( f" > MaxNorm: {self.ap.max_norm}, ClipNorm:{self.ap.clip_norm}, SymmetricNorm:{self.ap.symmetric_norm}, SignalNorm:{self.ap.signal_norm} Range-> {x_norm.max()} -- {x_norm.min()}" ) assert (x_old - x).sum() == 0 # check value range assert x_norm.max() <= self.ap.max_norm + 1, x_norm.max() assert x_norm.min() >= -self.ap.max_norm - 2, x_norm.min() # pylint: disable=invalid-unary-operand-type assert x_norm.min() <= 0, x_norm.min() # check denorm. x_ = self.ap.denormalize(x_norm) assert (x - x_).sum() < 1e-3, (x - x_).mean() self.ap.signal_norm = True self.ap.symmetric_norm = True self.ap.clip_norm = True self.ap.max_norm = 4.0 x_norm = self.ap.normalize(x) print( f" > MaxNorm: {self.ap.max_norm}, ClipNorm:{self.ap.clip_norm}, SymmetricNorm:{self.ap.symmetric_norm}, SignalNorm:{self.ap.signal_norm} Range-> {x_norm.max()} -- {x_norm.min()}" ) assert (x_old - x).sum() == 0 # check value range assert x_norm.max() <= self.ap.max_norm, x_norm.max() assert x_norm.min() >= -self.ap.max_norm, x_norm.min() # pylint: disable=invalid-unary-operand-type assert x_norm.min() <= 0, x_norm.min() # check denorm. x_ = self.ap.denormalize(x_norm) assert (x - x_).sum() < 1e-3, (x - x_).mean() self.ap.signal_norm = True self.ap.symmetric_norm = False self.ap.max_norm = 1.0 x_norm = self.ap.normalize(x) print( f" > MaxNorm: {self.ap.max_norm}, ClipNorm:{self.ap.clip_norm}, SymmetricNorm:{self.ap.symmetric_norm}, SignalNorm:{self.ap.signal_norm} Range-> {x_norm.max()} -- {x_norm.min()}" ) assert (x_old - x).sum() == 0 assert x_norm.max() <= self.ap.max_norm, x_norm.max() assert x_norm.min() >= 0, x_norm.min() x_ = self.ap.denormalize(x_norm) assert (x - x_).sum() < 1e-3 self.ap.signal_norm = True self.ap.symmetric_norm = True self.ap.max_norm = 1.0 x_norm = self.ap.normalize(x) print( f" > MaxNorm: {self.ap.max_norm}, ClipNorm:{self.ap.clip_norm}, SymmetricNorm:{self.ap.symmetric_norm}, SignalNorm:{self.ap.signal_norm} Range-> {x_norm.max()} -- {x_norm.min()}" ) assert (x_old - x).sum() == 0 assert x_norm.max() <= self.ap.max_norm, x_norm.max() assert x_norm.min() >= -self.ap.max_norm, x_norm.min() # pylint: disable=invalid-unary-operand-type assert x_norm.min() < 0, x_norm.min() x_ = self.ap.denormalize(x_norm) assert (x - x_).sum() < 1e-3 def test_scaler(self): scaler_stats_path = os.path.join(get_tests_input_path(), "scale_stats.npy") conf.stats_path = scaler_stats_path conf.preemphasis = 0.0 conf.do_trim_silence = True conf.signal_norm = True ap = AudioProcessor(**conf) mel_mean, mel_std, linear_mean, linear_std, _ = ap.load_stats(scaler_stats_path) ap.setup_scaler(mel_mean, mel_std, linear_mean, linear_std) self.ap.signal_norm = False self.ap.preemphasis = 0.0 # test scaler forward and backward transforms wav = self.ap.load_wav(WAV_FILE) mel_reference = self.ap.melspectrogram(wav) mel_norm = ap.melspectrogram(wav) mel_denorm = ap.denormalize(mel_norm) assert abs(mel_reference - mel_denorm).max() < 1e-4 def test_compute_f0(self): # pylint: disable=no-self-use ap = AudioProcessor(**conf) wav = ap.load_wav(WAV_FILE) pitch = ap.compute_f0(wav) mel = ap.melspectrogram(wav) assert pitch.shape[0] == mel.shape[1]
class Synthesizer(object): def __init__( self, tts_checkpoint: str, tts_config_path: str, tts_speakers_file: str = "", tts_languages_file: str = "", vocoder_checkpoint: str = "", vocoder_config: str = "", encoder_checkpoint: str = "", encoder_config: str = "", use_cuda: bool = False, ) -> None: """General 🐸 TTS interface for inference. It takes a tts and a vocoder model and synthesize speech from the provided text. The text is divided into a list of sentences using `pysbd` and synthesize speech on each sentence separately. If you have certain special characters in your text, you need to handle them before providing the text to Synthesizer. TODO: set the segmenter based on the source language Args: tts_checkpoint (str): path to the tts model file. tts_config_path (str): path to the tts config file. vocoder_checkpoint (str, optional): path to the vocoder model file. Defaults to None. vocoder_config (str, optional): path to the vocoder config file. Defaults to None. encoder_checkpoint (str, optional): path to the speaker encoder model file. Defaults to `""`, encoder_config (str, optional): path to the speaker encoder config file. Defaults to `""`, use_cuda (bool, optional): enable/disable cuda. Defaults to False. """ self.tts_checkpoint = tts_checkpoint self.tts_config_path = tts_config_path self.tts_speakers_file = tts_speakers_file self.tts_languages_file = tts_languages_file self.vocoder_checkpoint = vocoder_checkpoint self.vocoder_config = vocoder_config self.encoder_checkpoint = encoder_checkpoint self.encoder_config = encoder_config self.use_cuda = use_cuda self.tts_model = None self.vocoder_model = None self.speaker_manager = None self.num_speakers = 0 self.tts_speakers = {} self.language_manager = None self.num_languages = 0 self.tts_languages = {} self.d_vector_dim = 0 self.seg = self._get_segmenter("en") self.use_cuda = use_cuda if self.use_cuda: assert torch.cuda.is_available( ), "CUDA is not availabe on this machine." self._load_tts(tts_checkpoint, tts_config_path, use_cuda) self.output_sample_rate = self.tts_config.audio["sample_rate"] if vocoder_checkpoint: self._load_vocoder(vocoder_checkpoint, vocoder_config, use_cuda) self.output_sample_rate = self.vocoder_config.audio["sample_rate"] else: print(" > Using Griffin-Lim as no vocoder model defined") @staticmethod def _get_segmenter(lang: str): """get the sentence segmenter for the given language. Args: lang (str): target language code. Returns: [type]: [description] """ return pysbd.Segmenter(language=lang, clean=True) def _load_tts(self, tts_checkpoint: str, tts_config_path: str, use_cuda: bool) -> None: """Load the TTS model. 1. Load the model config. 2. Init the AudioProcessor. 3. Init the model from the config. 4. Move the model to the GPU if CUDA is enabled. 5. Init the speaker manager for the model. Args: tts_checkpoint (str): path to the model checkpoint. tts_config_path (str): path to the model config file. use_cuda (bool): enable/disable CUDA use. """ # pylint: disable=global-statement self.tts_config = load_config(tts_config_path) if self.tts_config[ "use_phonemes"] and self.tts_config["phonemizer"] is None: raise ValueError("Phonemizer is not defined in the TTS config.") self.tts_model = setup_tts_model(config=self.tts_config) if not self.encoder_checkpoint: self._set_speaker_encoder_paths_from_tts_config() self.tts_model.load_checkpoint(self.tts_config, tts_checkpoint, eval=True) if use_cuda: self.tts_model.cuda() if self.encoder_checkpoint and hasattr(self.tts_model, "speaker_manager"): self.tts_model.speaker_manager.init_encoder( self.encoder_checkpoint, self.encoder_config) def _set_speaker_encoder_paths_from_tts_config(self): """Set the encoder paths from the tts model config for models with speaker encoders.""" if hasattr(self.tts_config, "model_args") and hasattr( self.tts_config.model_args, "speaker_encoder_config_path"): self.encoder_checkpoint = self.tts_config.model_args.speaker_encoder_model_path self.encoder_config = self.tts_config.model_args.speaker_encoder_config_path def _load_vocoder(self, model_file: str, model_config: str, use_cuda: bool) -> None: """Load the vocoder model. 1. Load the vocoder config. 2. Init the AudioProcessor for the vocoder. 3. Init the vocoder model from the config. 4. Move the model to the GPU if CUDA is enabled. Args: model_file (str): path to the model checkpoint. model_config (str): path to the model config file. use_cuda (bool): enable/disable CUDA use. """ self.vocoder_config = load_config(model_config) self.vocoder_ap = AudioProcessor(verbose=False, **self.vocoder_config.audio) self.vocoder_model = setup_vocoder_model(self.vocoder_config) self.vocoder_model.load_checkpoint(self.vocoder_config, model_file, eval=True) if use_cuda: self.vocoder_model.cuda() def split_into_sentences(self, text) -> List[str]: """Split give text into sentences. Args: text (str): input text in string format. Returns: List[str]: list of sentences. """ return self.seg.segment(text) def save_wav(self, wav: List[int], path: str) -> None: """Save the waveform as a file. Args: wav (List[int]): waveform as a list of values. path (str): output path to save the waveform. """ wav = np.array(wav) self.tts_model.ap.save_wav(wav, path, self.output_sample_rate) def tts( self, text: str = "", speaker_name: str = "", language_name: str = "", speaker_wav: Union[str, List[str]] = None, style_wav=None, reference_wav=None, reference_speaker_name=None, ) -> List[int]: """🐸 TTS magic. Run all the models and generate speech. Args: text (str): input text. speaker_name (str, optional): spekaer id for multi-speaker models. Defaults to "". language_name (str, optional): language id for multi-language models. Defaults to "". speaker_wav (Union[str, List[str]], optional): path to the speaker wav. Defaults to None. style_wav ([type], optional): style waveform for GST. Defaults to None. reference_wav ([type], optional): reference waveform for voice conversion. Defaults to None. reference_speaker_name ([type], optional): spekaer id of reference waveform. Defaults to None. Returns: List[int]: [description] """ start_time = time.time() wavs = [] if not text and not reference_wav: raise ValueError( "You need to define either `text` (for sythesis) or a `reference_wav` (for voice conversion) to use the Coqui TTS API." ) if text: sens = self.split_into_sentences(text) print(" > Text splitted to sentences.") print(sens) # handle multi-speaker speaker_embedding = None speaker_id = None if self.tts_speakers_file or hasattr(self.tts_model.speaker_manager, "ids"): if speaker_name and isinstance(speaker_name, str): if self.tts_config.use_d_vector_file: # get the average speaker embedding from the saved d_vectors. speaker_embedding = self.tts_model.speaker_manager.get_mean_embedding( speaker_name, num_samples=None, randomize=False) speaker_embedding = np.array(speaker_embedding)[ None, :] # [1 x embedding_dim] else: # get speaker idx from the speaker name speaker_id = self.tts_model.speaker_manager.ids[ speaker_name] elif not speaker_name and not speaker_wav: raise ValueError( " [!] Look like you use a multi-speaker model. " "You need to define either a `speaker_name` or a `speaker_wav` to use a multi-speaker model." ) else: speaker_embedding = None else: if speaker_name: raise ValueError( f" [!] Missing speakers.json file path for selecting speaker {speaker_name}." "Define path for speaker.json if it is a multi-speaker model or remove defined speaker idx. " ) # handle multi-lingaul language_id = None if self.tts_languages_file or ( hasattr(self.tts_model, "language_manager") and self.tts_model.language_manager is not None): if language_name and isinstance(language_name, str): language_id = self.tts_model.language_manager.ids[ language_name] elif not language_name: raise ValueError( " [!] Look like you use a multi-lingual model. " "You need to define either a `language_name` or a `style_wav` to use a multi-lingual model." ) else: raise ValueError( f" [!] Missing language_ids.json file path for selecting language {language_name}." "Define path for language_ids.json if it is a multi-lingual model or remove defined language idx. " ) # compute a new d_vector from the given clip. if speaker_wav is not None: speaker_embedding = self.tts_model.speaker_manager.compute_embedding_from_clip( speaker_wav) use_gl = self.vocoder_model is None if not reference_wav: for sen in sens: # synthesize voice outputs = synthesis( model=self.tts_model, text=sen, CONFIG=self.tts_config, use_cuda=self.use_cuda, speaker_id=speaker_id, language_id=language_id, style_wav=style_wav, use_griffin_lim=use_gl, d_vector=speaker_embedding, ) waveform = outputs["wav"] mel_postnet_spec = outputs["outputs"]["model_outputs"][ 0].detach().cpu().numpy() if not use_gl: # denormalize tts output based on tts audio config mel_postnet_spec = self.tts_model.ap.denormalize( mel_postnet_spec.T).T device_type = "cuda" if self.use_cuda else "cpu" # renormalize spectrogram based on vocoder config vocoder_input = self.vocoder_ap.normalize( mel_postnet_spec.T) # compute scale factor for possible sample rate mismatch scale_factor = [ 1, self.vocoder_config["audio"]["sample_rate"] / self.tts_model.ap.sample_rate, ] if scale_factor[1] != 1: print(" > interpolating tts model output.") vocoder_input = interpolate_vocoder_input( scale_factor, vocoder_input) else: vocoder_input = torch.tensor(vocoder_input).unsqueeze( 0) # pylint: disable=not-callable # run vocoder model # [1, T, C] waveform = self.vocoder_model.inference( vocoder_input.to(device_type)) if self.use_cuda and not use_gl: waveform = waveform.cpu() if not use_gl: waveform = waveform.numpy() waveform = waveform.squeeze() # trim silence if self.tts_config.audio["do_trim_silence"] is True: waveform = trim_silence(waveform, self.tts_model.ap) wavs += list(waveform) wavs += [0] * 10000 else: # get the speaker embedding or speaker id for the reference wav file reference_speaker_embedding = None reference_speaker_id = None if self.tts_speakers_file or hasattr( self.tts_model.speaker_manager, "speaker_ids"): if reference_speaker_name and isinstance( reference_speaker_name, str): if self.tts_config.use_d_vector_file: # get the speaker embedding from the saved d_vectors. reference_speaker_embedding = self.tts_model.speaker_manager.get_embeddings_by_name( reference_speaker_name)[0] reference_speaker_embedding = np.array( reference_speaker_embedding)[ None, :] # [1 x embedding_dim] else: # get speaker idx from the speaker name reference_speaker_id = self.tts_model.speaker_manager.ids[ reference_speaker_name] else: reference_speaker_embedding = self.tts_model.speaker_manager.compute_embedding_from_clip( reference_wav) outputs = transfer_voice( model=self.tts_model, CONFIG=self.tts_config, use_cuda=self.use_cuda, reference_wav=reference_wav, speaker_id=speaker_id, d_vector=speaker_embedding, use_griffin_lim=use_gl, reference_speaker_id=reference_speaker_id, reference_d_vector=reference_speaker_embedding, ) waveform = outputs if not use_gl: mel_postnet_spec = outputs[0].detach().cpu().numpy() # denormalize tts output based on tts audio config mel_postnet_spec = self.tts_model.ap.denormalize( mel_postnet_spec.T).T device_type = "cuda" if self.use_cuda else "cpu" # renormalize spectrogram based on vocoder config vocoder_input = self.vocoder_ap.normalize(mel_postnet_spec.T) # compute scale factor for possible sample rate mismatch scale_factor = [ 1, self.vocoder_config["audio"]["sample_rate"] / self.tts_model.ap.sample_rate, ] if scale_factor[1] != 1: print(" > interpolating tts model output.") vocoder_input = interpolate_vocoder_input( scale_factor, vocoder_input) else: vocoder_input = torch.tensor(vocoder_input).unsqueeze(0) # pylint: disable=not-callable # run vocoder model # [1, T, C] waveform = self.vocoder_model.inference( vocoder_input.to(device_type)) if self.use_cuda: waveform = waveform.cpu() if not use_gl: waveform = waveform.numpy() wavs = waveform.squeeze() # compute stats process_time = time.time() - start_time audio_time = len(wavs) / self.tts_config.audio["sample_rate"] print(f" > Processing time: {process_time}") print(f" > Real-time factor: {process_time / audio_time}") return wavs
class Synthesizer(object): def __init__(self, tts_checkpoint, tts_config, vocoder_checkpoint=None, vocoder_config=None, use_cuda=False): """General 🐸 TTS interface for inference. It takes a tts and a vocoder model and synthesize speech from the provided text. The text is divided into a list of sentences using `pysbd` and synthesize speech on each sentence separately. If you have certain special characters in your text, you need to handle them before providing the text to Synthesizer. TODO: handle multi-speaker and GST inference. Args: tts_checkpoint (str): path to the tts model file. tts_config (str): path to the tts config file. vocoder_checkpoint (str, optional): path to the vocoder model file. Defaults to None. vocoder_config (str, optional): path to the vocoder config file. Defaults to None. use_cuda (bool, optional): enable/disable cuda. Defaults to False. """ self.tts_checkpoint = tts_checkpoint self.tts_config = tts_config self.vocoder_checkpoint = vocoder_checkpoint self.vocoder_config = vocoder_config self.use_cuda = use_cuda self.wavernn = None self.vocoder_model = None self.num_speakers = 0 self.tts_speakers = None self.speaker_embedding_dim = None self.seg = self.get_segmenter("en") self.use_cuda = use_cuda if self.use_cuda: assert torch.cuda.is_available( ), "CUDA is not availabe on this machine." self.load_tts(tts_checkpoint, tts_config, use_cuda) self.output_sample_rate = self.tts_config.audio["sample_rate"] if vocoder_checkpoint: self.load_vocoder(vocoder_checkpoint, vocoder_config, use_cuda) self.output_sample_rate = self.vocoder_config.audio["sample_rate"] @staticmethod def get_segmenter(lang): return pysbd.Segmenter(language=lang, clean=True) def load_speakers(self): # load speakers if self.model_config.use_speaker_embedding is not None: self.tts_speakers = load_speaker_mapping( self.tts_config.tts_speakers_json) self.num_speakers = len(self.tts_speakers) else: self.num_speakers = 0 # set external speaker embedding if self.tts_config.use_external_speaker_embedding_file: speaker_embedding = self.tts_speakers[list( self.tts_speakers.keys())[0]]["embedding"] self.speaker_embedding_dim = len(speaker_embedding) def init_speaker(self, speaker_idx): # load speakers speaker_embedding = None if hasattr(self, "tts_speakers") and speaker_idx is not None: assert speaker_idx < len( self.tts_speakers ), f" [!] speaker_idx is out of the range. {speaker_idx} vs {len(self.tts_speakers)}" if self.tts_config.use_external_speaker_embedding_file: speaker_embedding = self.tts_speakers[speaker_idx]["embedding"] return speaker_embedding def load_tts(self, tts_checkpoint, tts_config, use_cuda): # pylint: disable=global-statement global symbols, phonemes self.tts_config = load_config(tts_config) self.use_phonemes = self.tts_config.use_phonemes self.ap = AudioProcessor(verbose=False, **self.tts_config.audio) if "characters" in self.tts_config.keys(): symbols, phonemes = make_symbols(**self.tts_config.characters) if self.use_phonemes: self.input_size = len(phonemes) else: self.input_size = len(symbols) self.tts_model = setup_model(self.input_size, num_speakers=self.num_speakers, c=self.tts_config) self.tts_model.load_checkpoint(tts_config, tts_checkpoint, eval=True) if use_cuda: self.tts_model.cuda() def load_vocoder(self, model_file, model_config, use_cuda): self.vocoder_config = load_config(model_config) self.vocoder_ap = AudioProcessor(verbose=False, **self.vocoder_config["audio"]) self.vocoder_model = setup_generator(self.vocoder_config) self.vocoder_model.load_checkpoint(self.vocoder_config, model_file, eval=True) if use_cuda: self.vocoder_model.cuda() def save_wav(self, wav, path): wav = np.array(wav) self.ap.save_wav(wav, path, self.output_sample_rate) def split_into_sentences(self, text): return self.seg.segment(text) def tts(self, text, speaker_idx=None): start_time = time.time() wavs = [] sens = self.split_into_sentences(text) print(" > Text splitted to sentences.") print(sens) speaker_embedding = self.init_speaker(speaker_idx) use_gl = self.vocoder_model is None for sen in sens: # synthesize voice waveform, _, _, mel_postnet_spec, _, _ = synthesis( self.tts_model, sen, self.tts_config, self.use_cuda, self.ap, speaker_idx, None, False, self.tts_config.enable_eos_bos_chars, use_gl, speaker_embedding=speaker_embedding, ) if not use_gl: # denormalize tts output based on tts audio config mel_postnet_spec = self.ap.denormalize(mel_postnet_spec.T).T device_type = "cuda" if self.use_cuda else "cpu" # renormalize spectrogram based on vocoder config vocoder_input = self.vocoder_ap.normalize(mel_postnet_spec.T) # compute scale factor for possible sample rate mismatch scale_factor = [ 1, self.vocoder_config["audio"]["sample_rate"] / self.ap.sample_rate ] if scale_factor[1] != 1: print(" > interpolating tts model output.") vocoder_input = interpolate_vocoder_input( scale_factor, vocoder_input) else: vocoder_input = torch.tensor(vocoder_input).unsqueeze(0) # pylint: disable=not-callable # run vocoder model # [1, T, C] waveform = self.vocoder_model.inference( vocoder_input.to(device_type)) if self.use_cuda and not use_gl: waveform = waveform.cpu() if not use_gl: waveform = waveform.numpy() waveform = waveform.squeeze() # trim silence waveform = trim_silence(waveform, self.ap) wavs += list(waveform) wavs += [0] * 10000 # compute stats process_time = time.time() - start_time audio_time = len(wavs) / self.tts_config.audio["sample_rate"] print(f" > Processing time: {process_time}") print(f" > Real-time factor: {process_time / audio_time}") return wavs
class Synthesizer: def load(self, t_checkpoint_path, v_checkpoint_path, t_config_path=None, v_config_path=None, model_name='tacotron'): if t_checkpoint_path.endswith('.pt'): self.model_name = 'nvidia' print('Constructing model: %s' % self.model_name) # set-up params hparams = create_hparams() # load model from checkpoint self.model = Tacotron2(hparams) self.model.load_state_dict(torch.load(t_checkpoint_path, map_location='cpu')['state_dict']) _ = self.model.eval() else: # elif t_checkpoint_path.endswith('.pth.tar'): self.model_name = 'coqui' print('Constructing model: %s' % self.model_name) # load tts config and audio processor self.tts_config = load_config(t_config_path) self.tts_model = setup_tts_model(config=self.tts_config) self.tts_model.load_checkpoint(self.tts_config, t_checkpoint_path, eval=True) self.ap = AudioProcessor(verbose=False, **self.tts_config.audio) # load vocoder config and audio processor vocoder_config = load_config(v_config_path) self.vocoder_ap = AudioProcessor(verbose=False, **vocoder_config.audio) # Load neurips MelGAN for mel2audio synthesis self.vocoder = torch.hub.load('descriptinc/melgan-neurips', 'load_melgan') melgan_ckpt = torch.load(v_checkpoint_path, map_location='cpu') self.vocoder.mel2wav.load_state_dict(melgan_ckpt) def synthesize(self, response_text): # pre cleaning text = self.pre_clean(response_text) if self.model_name == 'nvidia': # TODO choose language? cleaner = ['catalan_cleaners'] # Prepare text input sequence = np.array(text_to_sequence(text, cleaner))[None, :] sequence = torch.from_numpy(sequence).to(device='cpu', dtype=torch.int64) # TODO run within the queue # decode text input mel_outputs, mel_outputs_postnet, _, alignments = self.model.inference(sequence) # TODO run within the queue # Synthesize using neurips Melgan with torch.no_grad(): audio = self.vocoder.inverse(mel_outputs_postnet.float()) audio_numpy = audio[0].data.cpu().numpy() # normalize and convert from float32 to int16 pcm audio_numpy /= np.max(np.abs(audio_numpy)) audio_numpy *= 32768*0.99 waveform = audio_numpy.astype(np.int16) elif self.model_name == 'coqui': wavs = [] sens = self.split_into_sentences(text) for sen in sens: outputs = synthesis(model=self.tts_model, text=text, CONFIG=self.tts_config, use_cuda=False, ap=self.ap, speaker_id=None, style_wav=None, enable_eos_bos_chars=self.tts_config.enable_eos_bos_chars, use_griffin_lim=False, d_vector=None) # extract and normalize the spectogram mel_postnet_spec = outputs["outputs"]["model_outputs"][0].detach().cpu().numpy() mel_postnet_spec = self.ap.denormalize(mel_postnet_spec.T).T vocoder_input = self.vocoder_ap.normalize(mel_postnet_spec.T) vocoder_input = torch.tensor(vocoder_input).unsqueeze(0) # Synthesize using neurips Melgan audio = self.vocoder.inverse(vocoder_input.to('cpu')) audio_numpy = audio[0].data.cpu().numpy() audio_numpy /= np.max(np.abs(audio_numpy)) audio_numpy *= 32768*0.99 waveform = list(audio_numpy.astype(np.int16).squeeze()) wavs += waveform waveform = audio_numpy.astype(np.int16).squeeze() else: raise ValueError('% unknown model name for synthesis'%self.model_name) # out out = io.BytesIO() # save scipy.io.wavfile.write(out, 22050, waveform) return out.getvalue() def pre_clean(self, response_text): if not re.search("[.?!:,;][ ]*$", response_text): return '%s. .'%response_text else: return '%s.'%response_text @staticmethod def split_into_sentences(text): return SEG.segment(text)