def test_no_espeak_phonemes(): # prepare the config config = VitsConfig( batch_size=2, eval_batch_size=2, num_loader_workers=0, num_eval_loader_workers=0, text_cleaner="english_cleaners", use_phonemes=True, use_espeak_phonemes=False, phoneme_language="en-us", phoneme_cache_path="tests/data/ljspeech/phoneme_cache/", run_eval=True, test_delay_epochs=-1, epochs=1, print_step=1, print_eval=True, datasets=[dataset_config_en, dataset_config_pt], ) config.save_json(config_path) # run test run_cli( f'CUDA_VISIBLE_DEVICES="" python TTS/bin/find_unique_phonemes.py --config_path "{config_path}"' )
def test_multispeaker_inference(self): num_speakers = 10 config = VitsConfig(num_speakers=num_speakers, use_speaker_embedding=True) model = Vits(config).to(device) batch_size = 1 input_dummy, *_ = self._create_inputs(config, batch_size=batch_size) speaker_ids = torch.randint(0, num_speakers, (batch_size, )).long().to(device) outputs = model.inference(input_dummy, {"speaker_ids": speaker_ids}) self._check_inference_outputs(config, outputs, input_dummy, batch_size=batch_size) batch_size = 2 input_dummy, input_lengths, *_ = self._create_inputs( config, batch_size=batch_size) speaker_ids = torch.randint(0, num_speakers, (batch_size, )).long().to(device) outputs = model.inference(input_dummy, { "x_lengths": input_lengths, "speaker_ids": speaker_ids }) self._check_inference_outputs(config, outputs, input_dummy, batch_size=batch_size)
def test_multilingual_forward(self): num_speakers = 10 num_langs = 3 batch_size = 2 args = VitsArgs(language_ids_file=LANG_FILE, use_language_embedding=True, spec_segment_size=10) config = VitsConfig(num_speakers=num_speakers, use_speaker_embedding=True, model_args=args) input_dummy, input_lengths, _, spec, spec_lengths, waveform = self._create_inputs( config, batch_size=batch_size) speaker_ids = torch.randint(0, num_speakers, (batch_size, )).long().to(device) lang_ids = torch.randint(0, num_langs, (batch_size, )).long().to(device) model = Vits(config).to(device) output_dict = model.forward( input_dummy, input_lengths, spec, spec_lengths, waveform, aux_input={ "speaker_ids": speaker_ids, "language_ids": lang_ids }, ) self._check_forward_outputs(config, output_dict)
def test_d_vector_inference(self): args = VitsArgs( spec_segment_size=10, num_chars=32, use_d_vector_file=True, d_vector_dim=256, d_vector_file=os.path.join(get_tests_data_path(), "dummy_speakers.json"), ) config = VitsConfig(model_args=args) model = Vits.init_from_config(config, verbose=False).to(device) model.eval() # batch size = 1 input_dummy = torch.randint(0, 24, (1, 128)).long().to(device) d_vectors = torch.randn(1, 256).to(device) outputs = model.inference(input_dummy, aux_input={"d_vectors": d_vectors}) self._check_inference_outputs(config, outputs, input_dummy) # batch size = 2 input_dummy, input_lengths, *_ = self._create_inputs(config) d_vectors = torch.randn(2, 256).to(device) outputs = model.inference(input_dummy, aux_input={ "x_lengths": input_lengths, "d_vectors": d_vectors }) self._check_inference_outputs(config, outputs, input_dummy, batch_size=2)
def test_train_step(self): # setup the model with torch.autograd.set_detect_anomaly(True): config = VitsConfig( model_args=VitsArgs(num_chars=32, spec_segment_size=10)) model = Vits(config).to(device) model.train() # model to train optimizers = model.get_optimizer() criterions = model.get_criterion() criterions = [criterions[0].to(device), criterions[1].to(device)] # reference model to compare model weights model_ref = Vits(config).to(device) # # pass the state to ref model model_ref.load_state_dict(copy.deepcopy(model.state_dict())) count = 0 for param, param_ref in zip(model.parameters(), model_ref.parameters()): assert (param - param_ref).sum() == 0, param count = count + 1 for _ in range(5): batch = self._create_batch(config, 2) for idx in [0, 1]: outputs, loss_dict = model.train_step( batch, criterions, idx) self.assertFalse(not outputs) self.assertFalse(not loss_dict) loss_dict["loss"].backward() optimizers[idx].step() optimizers[idx].zero_grad() # check parameter changes self._check_parameter_changes(model, model_ref)
def test_inference(self): num_speakers = 0 config = VitsConfig(num_speakers=num_speakers, use_speaker_embedding=True) input_dummy = torch.randint(0, 24, (1, 128)).long().to(device) model = Vits(config).to(device) _ = model.inference(input_dummy)
def test_test_run(self): config = VitsConfig(model_args=VitsArgs(num_chars=32)) model = Vits.init_from_config(config, verbose=False).to(device) model.run_data_dep_init = False model.eval() test_figures, test_audios = model.test_run(None) self.assertTrue(test_figures is not None) self.assertTrue(test_audios is not None)
def test_forward(self): num_speakers = 0 config = VitsConfig(num_speakers=num_speakers, use_speaker_embedding=True) config.model_args.spec_segment_size = 10 input_dummy, input_lengths, _, spec, spec_lengths, waveform = self._create_inputs( config) model = Vits(config).to(device) output_dict = model.forward(input_dummy, input_lengths, spec, spec_lengths, waveform) self._check_forward_outputs(config, output_dict)
def test_load_checkpoint(self): chkp_path = os.path.join(get_tests_output_path(), "dummy_glow_tts_checkpoint.pth") config = VitsConfig(VitsArgs(num_chars=32)) model = Vits.init_from_config(config, verbose=False).to(device) chkp = {} chkp["model"] = model.state_dict() torch.save(chkp, chkp_path) model.load_checkpoint(config, chkp_path) self.assertTrue(model.training) model.load_checkpoint(config, chkp_path, eval=True) self.assertFalse(model.training)
def test_multilingual_inference(self): num_speakers = 10 num_langs = 3 args = VitsArgs(language_ids_file=LANG_FILE, use_language_embedding=True, spec_segment_size=10) config = VitsConfig(num_speakers=num_speakers, use_speaker_embedding=True, model_args=args) model = Vits(config).to(device) input_dummy = torch.randint(0, 24, (1, 128)).long().to(device) speaker_ids = torch.randint(0, num_speakers, (1, )).long().to(device) lang_ids = torch.randint(0, num_langs, (1, )).long().to(device) _ = model.inference(input_dummy, { "speaker_ids": speaker_ids, "language_ids": lang_ids }) batch_size = 1 input_dummy, *_ = self._create_inputs(config, batch_size=batch_size) speaker_ids = torch.randint(0, num_speakers, (batch_size, )).long().to(device) lang_ids = torch.randint(0, num_langs, (batch_size, )).long().to(device) outputs = model.inference(input_dummy, { "speaker_ids": speaker_ids, "language_ids": lang_ids }) self._check_inference_outputs(config, outputs, input_dummy, batch_size=batch_size) batch_size = 2 input_dummy, input_lengths, *_ = self._create_inputs( config, batch_size=batch_size) speaker_ids = torch.randint(0, num_speakers, (batch_size, )).long().to(device) lang_ids = torch.randint(0, num_langs, (batch_size, )).long().to(device) outputs = model.inference( input_dummy, { "x_lengths": input_lengths, "speaker_ids": speaker_ids, "language_ids": lang_ids }) self._check_inference_outputs(config, outputs, input_dummy, batch_size=batch_size)
def test_init_from_config(self): config = VitsConfig(model_args=VitsArgs(num_chars=32)) model = Vits.init_from_config(config, verbose=False).to(device) config = VitsConfig(model_args=VitsArgs(num_chars=32, num_speakers=2)) model = Vits.init_from_config(config, verbose=False).to(device) self.assertTrue(not hasattr(model, "emb_g")) config = VitsConfig(model_args=VitsArgs( num_chars=32, num_speakers=2, use_speaker_embedding=True)) model = Vits.init_from_config(config, verbose=False).to(device) self.assertEqual(model.num_speakers, 2) self.assertTrue(hasattr(model, "emb_g")) config = VitsConfig(model_args=VitsArgs( num_chars=32, num_speakers=2, use_speaker_embedding=True, speakers_file=os.path.join(get_tests_data_path(), "ljspeech", "speakers.json"), )) model = Vits.init_from_config(config, verbose=False).to(device) self.assertEqual(model.num_speakers, 10) self.assertTrue(hasattr(model, "emb_g")) config = VitsConfig(model_args=VitsArgs( num_chars=32, use_d_vector_file=True, d_vector_dim=256, d_vector_file=os.path.join(get_tests_data_path(), "dummy_speakers.json"), )) model = Vits.init_from_config(config, verbose=False).to(device) self.assertTrue(model.num_speakers == 1) self.assertTrue(not hasattr(model, "emb_g")) self.assertTrue(model.embedded_speaker_dim == config.d_vector_dim)
def test_multilingual_inference(self): num_speakers = 10 num_langs = 3 args = VitsArgs(language_ids_file=LANG_FILE, use_language_embedding=True, spec_segment_size=10) config = VitsConfig(num_speakers=num_speakers, use_speaker_embedding=True, model_args=args) input_dummy = torch.randint(0, 24, (1, 128)).long().to(device) speaker_ids = torch.randint(0, num_speakers, (1, )).long().to(device) lang_ids = torch.randint(0, num_langs, (1, )).long().to(device) model = Vits(config).to(device) _ = model.inference(input_dummy, { "speaker_ids": speaker_ids, "language_ids": lang_ids })
def test_secl_forward(self): num_speakers = 10 num_langs = 3 batch_size = 2 speaker_encoder_config = load_config(SPEAKER_ENCODER_CONFIG) speaker_encoder_config.model_params["use_torch_spec"] = True speaker_encoder = setup_encoder_model(speaker_encoder_config).to( device) speaker_manager = SpeakerManager() speaker_manager.encoder = speaker_encoder args = VitsArgs( language_ids_file=LANG_FILE, use_language_embedding=True, spec_segment_size=10, use_speaker_encoder_as_loss=True, ) config = VitsConfig(num_speakers=num_speakers, use_speaker_embedding=True, model_args=args) config.audio.sample_rate = 16000 input_dummy, input_lengths, _, spec, spec_lengths, waveform = self._create_inputs( config, batch_size=batch_size) speaker_ids = torch.randint(0, num_speakers, (batch_size, )).long().to(device) lang_ids = torch.randint(0, num_langs, (batch_size, )).long().to(device) model = Vits(config, speaker_manager=speaker_manager).to(device) output_dict = model.forward( input_dummy, input_lengths, spec, spec_lengths, waveform, aux_input={ "speaker_ids": speaker_ids, "language_ids": lang_ids }, ) self._check_forward_outputs(config, output_dict, speaker_encoder_config)
def test_multispeaker_forward(self): num_speakers = 10 config = VitsConfig(num_speakers=num_speakers, use_speaker_embedding=True) config.model_args.spec_segment_size = 10 input_dummy, input_lengths, _, spec, spec_lengths, waveform = self._create_inputs( config) speaker_ids = torch.randint(0, num_speakers, (8, )).long().to(device) model = Vits(config).to(device) output_dict = model.forward(input_dummy, input_lengths, spec, spec_lengths, waveform, aux_input={"speaker_ids": speaker_ids}) self._check_forward_outputs(config, output_dict)
def test_train_eval_log(self): batch_size = 2 config = VitsConfig( model_args=VitsArgs(num_chars=32, spec_segment_size=10)) model = Vits.init_from_config(config, verbose=False).to(device) model.run_data_dep_init = False model.train() batch = self._create_batch(config, batch_size) logger = TensorboardLogger(log_dir=os.path.join( get_tests_output_path(), "dummy_vits_logs"), model_name="vits_test_train_log") criterion = model.get_criterion() criterion = [criterion[0].to(device), criterion[1].to(device)] outputs = [None] * 2 outputs[0], _ = model.train_step(batch, criterion, 0) outputs[1], _ = model.train_step(batch, criterion, 1) model.train_log(batch, outputs, logger, None, 1) model.eval_log(batch, outputs, logger, None, 1) logger.finish()
def test_inference(self): num_speakers = 0 config = VitsConfig(num_speakers=num_speakers, use_speaker_embedding=True) model = Vits(config).to(device) batch_size = 1 input_dummy, *_ = self._create_inputs(config, batch_size=batch_size) outputs = model.inference(input_dummy) self._check_inference_outputs(config, outputs, input_dummy, batch_size=batch_size) batch_size = 2 input_dummy, input_lengths, *_ = self._create_inputs( config, batch_size=batch_size) outputs = model.inference(input_dummy, aux_input={"x_lengths": input_lengths}) self._check_inference_outputs(config, outputs, input_dummy, batch_size=batch_size)
def test_d_vector_forward(self): batch_size = 2 args = VitsArgs( spec_segment_size=10, num_chars=32, use_d_vector_file=True, d_vector_dim=256, d_vector_file=os.path.join(get_tests_data_path(), "dummy_speakers.json"), ) config = VitsConfig(model_args=args) model = Vits.init_from_config(config, verbose=False).to(device) model.train() input_dummy, input_lengths, _, spec, spec_lengths, waveform = self._create_inputs( config, batch_size=batch_size) d_vectors = torch.randn(batch_size, 256).to(device) output_dict = model.forward(input_dummy, input_lengths, spec, spec_lengths, waveform, aux_input={"d_vectors": d_vectors}) self._check_forward_outputs(config, output_dict)
def test_get_criterion(self): config = VitsConfig(VitsArgs(num_chars=32)) model = Vits.init_from_config(config, verbose=False).to(device) criterion = model.get_criterion() self.assertTrue(criterion is not None)
config = VitsConfig( model_args=vitsArgs, audio=audio_config, run_name="vits_vctk", use_speaker_embedding=True, batch_size=32, eval_batch_size=16, batch_group_size=0, num_loader_workers=4, num_eval_loader_workers=4, run_eval=True, test_delay_epochs=-1, epochs=1000, text_cleaner="multilingual_cleaners", use_phonemes=False, phoneme_language="en-us", phoneme_cache_path=os.path.join(output_path, "phoneme_cache"), compute_input_seq_cache=True, print_step=25, use_language_weighted_sampler=True, print_eval=False, mixed_precision=False, sort_by_audio_len=True, min_seq_len=32 * 256 * 4, max_seq_len=160000, output_path=output_path, datasets=dataset_config, characters={ "pad": "_", "eos": "&", "bos": "*", "characters": "!¡'(),-.:;¿?abcdefghijklmnopqrstuvwxyzµßàáâäåæçèéêëìíîïñòóôöùúûüąćęłńœśşźżƒабвгдежзийклмнопрстуфхцчшщъыьэюяёєіїґӧ «°±µ»$%&‘’‚“`”„", "punctuations": "!¡'(),-.:;¿? ", "phonemes": None, "unique": True, }, test_sentences=[ [ "It took me quite a long time to develop a voice, and now that I have it I'm not going to be silent.", "mary_ann", None, "en_US", ], [ "Il m'a fallu beaucoup de temps pour d\u00e9velopper une voix, et maintenant que je l'ai, je ne vais pas me taire.", "ezwa", None, "fr_FR", ], [ "Ich finde, dieses Startup ist wirklich unglaublich.", "eva_k", None, "de_DE" ], [ "Я думаю, что этот стартап действительно удивительный.", "oblomov", None, "ru_RU" ], ], )
) vitsArgs = VitsArgs(use_speaker_embedding=True, ) config = VitsConfig( model_args=vitsArgs, audio=audio_config, run_name="vits_vctk", batch_size=32, eval_batch_size=16, batch_group_size=5, num_loader_workers=4, num_eval_loader_workers=4, run_eval=True, test_delay_epochs=-1, epochs=1000, text_cleaner="english_cleaners", use_phonemes=True, phoneme_language="en", phoneme_cache_path=os.path.join(output_path, "phoneme_cache"), compute_input_seq_cache=True, print_step=25, print_eval=False, mixed_precision=True, max_text_len=325, # change this if you have a larger VRAM than 16GB output_path=output_path, datasets=[dataset_config], ) # INITIALIZE THE AUDIO PROCESSOR # Audio processor is used for feature extraction and audio I/O. # It mainly serves to the dataloader and the training loggers.
signal_norm=False, do_amp_to_db_linear=False, ) config = VitsConfig( audio=audio_config, run_name="vits_ljspeech", batch_size=48, eval_batch_size=16, batch_group_size=5, num_loader_workers=4, num_eval_loader_workers=4, run_eval=True, test_delay_epochs=-1, epochs=1000, text_cleaner="english_cleaners", use_phonemes=True, phoneme_language="en-us", phoneme_cache_path=os.path.join(output_path, "phoneme_cache"), compute_input_seq_cache=True, print_step=25, print_eval=True, mixed_precision=True, max_seq_len=500000, output_path=output_path, datasets=[dataset_config], ) # init audio processor ap = AudioProcessor(**config.audio.to_dict())
meta_file_val="metadata.csv", path="tests/data/ljspeech", language="pt-br", ) config = VitsConfig( batch_size=2, eval_batch_size=2, num_loader_workers=0, num_eval_loader_workers=0, text_cleaner="multilingual_cleaners", use_phonemes=False, phoneme_cache_path="tests/data/ljspeech/phoneme_cache/", run_eval=True, test_delay_epochs=-1, epochs=1, print_step=1, print_eval=True, test_sentences=[ ["Be a voice, not an echo.", "ljspeech-0", None, "en"], ["Be a voice, not an echo.", "ljspeech-1", None, "pt-br"], ], datasets=[ dataset_config_en, dataset_config_en, dataset_config_en, dataset_config_pt ], ) # set audio config config.audio.do_trim_silence = True config.audio.trim_db = 60 # active multilingual mode
from tests import get_device_id, get_tests_output_path, run_cli from TTS.tts.configs.vits_config import VitsConfig config_path = os.path.join(get_tests_output_path(), "test_model_config.json") output_path = os.path.join(get_tests_output_path(), "train_outputs") config = VitsConfig( batch_size=2, eval_batch_size=2, num_loader_workers=0, num_eval_loader_workers=0, text_cleaner="english_cleaners", use_phonemes=True, use_espeak_phonemes=True, phoneme_language="en-us", phoneme_cache_path="tests/data/ljspeech/phoneme_cache/", run_eval=True, test_delay_epochs=-1, epochs=1, print_step=1, print_eval=True, test_sentences=[ ["Be a voice, not an echo.", "ljspeech"], ], ) # set audio config config.audio.do_trim_silence = True config.audio.trim_db = 60 # active multispeaker d-vec mode config.model_args.use_speaker_embedding = True
vitsArgs = VitsArgs(use_speaker_embedding=True, ) config = VitsConfig( model_args=vitsArgs, audio=audio_config, run_name="vits_vctk", batch_size=32, eval_batch_size=16, batch_group_size=5, num_loader_workers=4, num_eval_loader_workers=4, run_eval=True, test_delay_epochs=-1, epochs=1000, text_cleaner="english_cleaners", use_phonemes=True, phoneme_cache_path=os.path.join(output_path, "phoneme_cache"), compute_input_seq_cache=True, print_step=25, print_eval=False, mixed_precision=True, sort_by_audio_len=True, min_seq_len=32 * 256 * 4, max_seq_len=1500000, output_path=output_path, datasets=[dataset_config], ) # init audio processor ap = AudioProcessor(**config.audio.to_dict())