def preprocess(experiment: Experiment, args: Mapping) -> None:
""" Preprocesses wavs and text, returns None or raises an Exception. """
# bring data in format usable by Tacotron-2
# for now just copy them over
raw_wav_dir = os.path.join(experiment.paths["acoustic_model"],
"LJSpeech-1.1", "wavs")
fu.ensure_dir(raw_wav_dir)
fu.copy_files(args["wav_dir"], raw_wav_dir)
raw_metadata_file = os.path.join(experiment.paths["acoustic_model"],
"LJSpeech-1.1", "metadata.csv")
fu.copy_file(args["text_file"], raw_metadata_file)
# run Tacotron-2 preprocessing
tacoargs = namedtuple(
"tacoargs",
"base_dir hparams dataset language voice reader merge_books book output n_jobs"
.split())
tacoargs.base_dir = experiment.paths["acoustic_model"]
tacoargs.language = experiment.config["language"]
tacoargs.output = experiment.paths["acoustic_features"]
tacoargs.hparams = ""
tacoargs.n_jobs = multiprocessing.cpu_count()
# for now we always exploit the LJ settings
tacoargs.dataset = "LJSpeech-1.1"
tacoargs.voice = "female"
tacoargs.reader = "LJ"
tacoargs.merge_books = "True"
tacoargs.book = "northandsouth"
modified_hp = tacotron2.hparams.hparams.parse(tacoargs.hparams)
tacotron2.preprocess.run_preprocess(tacoargs, modified_hp)
def download_pretrained(experiment: Experiment, url: str) -> None:
""" Downloads a pretrained model. """
pretrained_dir = _get_pretrained_folder(experiment)
pretrained_zip = os.path.join(pretrained_dir, "wavernnalt_pretrained.zip")
fu.ensure_dir(pretrained_dir)
fu.download_file(url, pretrained_zip)
with zipfile.ZipFile(pretrained_zip, 'r') as zip_ref:
zip_ref.extractall(pretrained_dir)
os.remove(pretrained_zip)
def download_pretrained(experiment: Experiment, url: str) -> None:
""" Downloads a pretrained model. """
pretrained_dir = os.path.join(experiment.paths["acoustic_model"],
"logs-Tacotron")
pretrained_zip = os.path.join(pretrained_dir, "taco_pretrained.zip")
fu.ensure_dir(pretrained_dir)
fu.download_file(url, pretrained_zip)
with zipfile.ZipFile(pretrained_zip, 'r') as zip_ref:
zip_ref.extractall(pretrained_dir)
os.remove(pretrained_zip)
def get_pretrained_wavernn(model_id, targetdir):
""" Downloads a pretrained waveRNN model. """
if model_id not in consts.PRETRAINED_WAVERNN_MODELS:
raise FileNotFoundError(model_id)
(model_url, model_filename) = consts.PRETRAINED_WAVERNN_MODELS[model_id]
model_dir = os.path.join(targetdir, model_id)
model_path = os.path.join(model_dir, model_filename)
if os.path.exists(model_path):
return model_path
fu.ensure_dir(model_dir)
fu.download_file(model_url, model_path)
return model_path
def synthesize(sentences, output_dir):
# Tacotron first
args = namedtuple(
"tacoargs", "mode model checkpoint output_dir mels_dir hparams name".split())
args.mode = "eval"
args.model = "Tacotron-2"
args.checkpoint = "pretrained/"
args.output_dir = "output"
args.mels_dir = "tacotron_output/eval"
args.base_dir = ''
args.input_dir = 'training_data/'
args.hparams = ''
args.name = "Tacotron-2"
args.log_dir = None
taco_checkpoint, _, hparams = prepare_run(args)
taco_checkpoint = os.path.join("tacotron2", taco_checkpoint)
tacotron_synthesize(args, hparams, taco_checkpoint, sentences)
# now WaveRNN
if not os.path.exists(MODEL_PATH):
raise FileNotFoundError(MODEL_PATH)
model = Model(rnn_dims=512, fc_dims=512, bits=bits, pad=2,
upsample_factors=(5, 5, 11), feat_dims=80,
compute_dims=128, res_out_dims=128, res_blocks=10).to(device)
print("Loading WaveRNN model from " + MODEL_PATH)
model.load_state_dict(torch.load(MODEL_PATH))
# mels_paths = [f for f in sorted(
# os.listdir(args.mels_dir)) if f.endswith(".npy")]
map_path = os.path.join(args.mels_dir, 'map.txt')
f = open(map_path)
maps = f.readlines()
f.close()
mels_paths = [x.split('|')[1] for x in maps]
test_mels = [np.load(m).T for m in mels_paths]
fu.ensure_dir(output_dir)
for i, mel in enumerate(test_mels):
print('\nGenerating: %i/%i' % (i+1, len(test_mels)))
model.generate(mel, output_dir + f'/{i}_generated.wav')
def generate(experiment: Experiment,
sentences,
generate_features: bool = True,
generate_waveforms: bool = True) -> None:
"""
Generates from the model.
:param Experiment experiment: The experiment to generate from.
:param bool generate_features: Store acoustic features
:param bool generate_waveforms: Generate a waveform from acoustic features using Griffin-Lim
"""
# python synthesize.py --model Tacotron --tacotron_name Tacotron-2 --mode eval --text_list text_list.txt &> /dev/null
tacoargs = namedtuple(
"tacoargs",
"mode model checkpoint output_dir mels_dir hparams name tacotron_name GTA"
.split())
tacoargs.checkpoint = _get_pretrained_folder(experiment)
tacoargs.hparams = ''
tacoargs.name = "Tacotron"
tacoargs.tacotron_name = "Tacotron"
tacoargs.model = "Tacotron"
#tacoargs.input_dir = 'training_data/'
tacoargs.mels_dir = experiment.paths["acoustic2wavegen_features"]
tacoargs.output_dir = experiment.paths["acoustic2wavegen_features"]
tacoargs.mode = "eval"
tacoargs.GTA = False
#tacoargs.base_dir = ''
#tacoargs.log_dir = None
#taco_checkpoint, _, hparams = tacotron2.synthesize.prepare_run(tacoargs)
modified_hp = tacotron2.hparams.hparams.parse(tacoargs.hparams)
#taco_checkpoint = os.path.join("tacotron2", taco_checkpoint)
tacotron2.tacotron.synthesize.tacotron_synthesize(tacoargs, modified_hp,
tacoargs.checkpoint,
sentences)
output_dir = os.path.join(experiment.paths["synthesized_wavs"],
"griffinlim")
fu.ensure_dir(output_dir)
fu.copy_files(
os.path.join(experiment.paths["acoustic2wavegen_features"],
"logs-eval", "wavs"), output_dir)
def generate(experiment: Experiment, sentences=None) -> None:
"""
Generates waveforms from existing acoustic features.
:param Experiment experiment: The experiment to generate from.
:param sentences: List of feature files to synthesize. If None synthesizes all files in the feature folder.
"""
mels_dir = experiment.paths["acoustic2wavegen_features"]
output_dir = os.path.join(experiment.paths["synthesized_wavs"], "wavernn")
fu.ensure_dir(output_dir)
pretrained_dir = _get_pretrained_folder(experiment)
checkpoint_file = os.path.join(pretrained_dir, "checkpoint.pth")
map_file = os.path.join(mels_dir, "eval", "map.txt")
print(map_file)
map_content = np.genfromtxt(map_file, dtype=None, delimiter='|')
for i, line in enumerate(map_content):
(text, mel_file, _) = line
LOGGER.info("Generating waveform from %s with text: %s" % (line, text))
mel_file = mel_file.decode("utf-8")
mel = np.load(mel_file).T
output_file = os.path.join(output_dir, os.path.basename(mel_file))
wsynth.main(mel, checkpoint_file, output_file)
def convert_training_data(experiment: Experiment, args: Mapping) -> None:
""" Converts output of acoustic model for training, returns None or raises an Exception. """
# TODO: load voice specific hparams
# wpreproc.hp.override_from_dict(otherhparams.values())
dataset_ids = []
map_file = os.path.join(
experiment.paths["acoustic2wavegen_training_features"], "map.txt")
output_dir = experiment.paths["wavegen_features"]
output_wav_dir = os.path.join(output_dir, "wav")
output_mel_dir = os.path.join(output_dir, "mel")
output_test_dir = os.path.join(output_dir, "test")
fu.ensure_dir(output_wav_dir)
fu.ensure_dir(output_mel_dir)
fu.ensure_dir(output_test_dir)
if not os.path.exists(map_file):
# in future we might find the files by some other means
raise FileNotFoundError(map_file)
# read map file to get paths to audio, original mel and GTA mel
gta_map = np.genfromtxt(map_file, dtype=None, delimiter='|')
for i, line in enumerate(gta_map):
(audio_file, mel_file, gta_file, _, _) = line
fileid = "%05d" % (i)
LOGGER.info("Converting %s and %s" % (audio_file, gta_file))
gta_mel = np.load(gta_file.decode("utf-8")).T
#audio, mel = wpreproc.get_wav_mel(audio_file.decode("utf-8"))
audio = wpreproc.get_wav(audio_file.decode("utf-8"))
if i < NUM_TEST_FILES:
np.save(os.path.join(output_test_dir, "test_{}_mel.npy".format(i)),
gta_mel.astype(np.float32))
np.save(os.path.join(output_test_dir, "test_{}_wav.npy".format(i)),
audio)
else:
np.save(os.path.join(output_mel_dir, fileid),
gta_mel.astype(np.float32))
np.save(os.path.join(output_wav_dir, fileid), audio)
dataset_ids.append(fileid)
with open(os.path.join(output_dir, 'dataset_ids.pkl'), 'wb') as f:
pickle.dump(dataset_ids, f)
def create_file_structure(experiment_path):
""" Creates the folder structure for a new experiment. """
fu.ensure_dir(experiment_path)
return _apply_file_structure(experiment_path, _create_subfolder)
def _create_subfolder(experiment_path, folder):
""" Creates a subfolder inside the experiment directory. """
absfolder = os.path.join(experiment_path, folder)
fu.ensure_dir(absfolder)
return absfolder