def test_tokenizer(self):
tokenizer = Tokenizer(alphabet=list('ab c'))
self.assertEqual(5, tokenizer.start_token_index)
self.assertEqual(6, tokenizer.end_token_index)
self.assertEqual(7, tokenizer.vocab_size)
seq = tokenizer('a b d')
self.assertEqual([5, 1, 3, 2, 3, 6], seq)
seq = np.array([5, 1, 3, 2, 8, 6])
seq = tf.convert_to_tensor(seq)
text = tokenizer.decode(seq)
self.assertEqual('>a b<', text)
def default_pipeline(cls, language, add_start_end):
if language == 'en':
cleaner = English()
elif language == 'de':
cleaner = German()
else:
raise ValueError(
f'language must be either "en" or "de", not {language}.')
phonemizer = Phonemizer(language=language)
tokenizer = Tokenizer(sorted(list(_phonemes) + list(_punctuations)),
add_start_end=add_start_end)
return cls(cleaner=cleaner, phonemizer=phonemizer, tokenizer=tokenizer)
def default_pipeline(cls, language, add_start_end, with_stress, njobs=1):
if (language == 'en') or (language == 'en-gb') or (language
== 'en-us'):
cleaner = English()
elif language == 'de':
cleaner = German()
else:
raise ValueError(
f'language must be either "en" (en-us, en-gb) or "de", not {language}.'
)
phonemizer = Phonemizer(language=language,
strip=False,
njobs=njobs,
with_stress=with_stress)
tokenizer = Tokenizer(all_phonemes, add_start_end=add_start_end)
return cls(cleaner=cleaner, phonemizer=phonemizer, tokenizer=tokenizer)
from preprocessing.metadata_readers import get_preprocessor_by_name
from preprocessing.datasets import DataReader, AutoregressivePreprocessor, TextMelDataset
from preprocessing.text.tokenizer import Tokenizer
from preprocessing.text.symbols import all_phonemes
from pathlib import Path
ljspeech_folder = '/Volumes/data/datasets/LJSpeech-1.1'
metadata_path = '/Volumes/data/datasets/LJSpeech-1.1/phonemized_metadata.txt'
metadata_reader = get_preprocessor_by_name('ljspeech')
data_reader = DataReader(data_directory=ljspeech_folder,
metadata_path=metadata_path,
metadata_reading_function=metadata_reader,
scan_wavs=True)
mel_dir = Path('/Volumes/data/datasets/LJSpeech-1.1/mels')
tokenizer = Tokenizer(alphabet=all_phonemes)
preprocessor = AutoregressivePreprocessor(mel_channels=80,
mel_start_value=.5,
mel_end_value=-.5,
tokenizer=tokenizer)
dataset_creator = TextMelDataset(data_reader=data_reader,
preprocessor=preprocessor,
mel_directory=mel_dir)
dataset = dataset_creator.get_dataset(
shuffle=True,
drop_remainder=False,
bucket_batch_sizes=[6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 1])
for i in range(10):
mel, text, stop, fname = dataset.next_batch()
pass
from preprocessing.metadata_readers import get_preprocessor_by_name
from preprocessing.datasets import DataReader, AutoregressivePreprocessor, TextMelDataset
from preprocessing.text.tokenizer import Tokenizer
from preprocessing.text.symbols import all_phonemes
from pathlib import Path
ljspeech_folder = '/Volumes/data/datasets/LJSpeech-1.1'
metadata_path = '/Volumes/data/datasets/LJSpeech-1.1/phonemized_metadata.txt'
metadata_reader = get_preprocessor_by_name('ljspeech')
data_reader = DataReader(data_directory=ljspeech_folder,
metadata_path=metadata_path,
metadata_reading_function=metadata_reader,
scan_wavs=True)
mel_dir = Path('/Volumes/data/datasets/LJSpeech-1.1/mels')
tokenizer = Tokenizer()
preprocessor = AutoregressivePreprocessor(mel_channels=80,
mel_start_value=.5,
mel_end_value=-.5,
tokenizer=tokenizer)
dataset_creator = TextMelDataset(data_reader=data_reader,
preprocessor=preprocessor,
mel_directory=mel_dir)
dataset = dataset_creator.get_dataset(
shuffle=True,
drop_remainder=False,
bucket_batch_sizes=[6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 1])
for i in range(10):
mel, text, stop, fname = dataset.next_batch()
pass