def main():
from idiaptts.src.model_trainers.vtln.VTLNSpeakerAdaptionModelTrainer import VTLNSpeakerAdaptionModelTrainer
hparams = VTLNSpeakerAdaptionModelTrainer.create_hparams()
hparams.use_gpu = False
hparams.voice = "English"
hparams.model_name = "WarpingLayerTest.nn"
hparams.add_deltas = True
hparams.num_coded_sps = 30
# hparams.num_questions = 505
hparams.num_questions = 425
hparams.out_dir = "experiments/" + hparams.voice + "/VTLNArtificiallyWarped/"
hparams.data_dir = os.path.realpath("database")
hparams.model_name = "warping_layer_test"
hparams.synth_dir = hparams.out_dir
batch_size = 2
dir_world_labels = os.path.join("experiments", hparams.voice, "WORLD")
from idiaptts.src.data_preparation.world.WorldFeatLabelGen import WorldFeatLabelGen
gen_in = WorldFeatLabelGen(dir_world_labels,
add_deltas=hparams.add_deltas,
num_coded_sps=hparams.num_coded_sps)
gen_in.get_normalisation_params(gen_in.dir_labels)
from idiaptts.src.model_trainers.AcousticModelTrainer import AcousticModelTrainer
trainer = AcousticModelTrainer(
"experiments/" + hparams.voice + "/WORLD",
"experiments/" + hparams.voice + "/questions", "ignored",
hparams.num_questions, hparams)
sp_mean = gen_in.norm_params[0][:hparams.num_coded_sps *
(3 if hparams.add_deltas else 1)]
sp_std_dev = gen_in.norm_params[1][:hparams.num_coded_sps *
(3 if hparams.add_deltas else 1)]
wl = WarpingLayer((hparams.num_coded_sps, ), (hparams.num_coded_sps, ),
hparams)
wl.set_norm_params(sp_mean, sp_std_dev)
# id_list = ["dorian/doriangray_16_00199"]
id_list = ["p225/p225_051"]
hparams.num_speakers = 1
t_benchmark = 0
for id_name in id_list:
for idx, alpha in enumerate(np.arange(-0.15, 0.2, 0.05)):
out_dir = hparams.out_dir + "alpha_{0:0.2f}/".format(alpha)
makedirs_safe(out_dir)
sample = WorldFeatLabelGen.load_sample(
id_name,
os.path.join("experiments", hparams.voice, "WORLD"),
add_deltas=True,
num_coded_sps=hparams.num_coded_sps)
sample_pre = gen_in.preprocess_sample(sample)
coded_sps = sample_pre[:, :hparams.num_coded_sps *
(3 if hparams.add_deltas else 1)]
alpha_vec = np.ones((coded_sps.shape[0], 1)) * alpha
coded_sps = coded_sps[:len(alpha_vec), None, ...].repeat(
batch_size, 1) # Copy data in batch dimension.
alpha_vec = alpha_vec[:, None, None].repeat(
batch_size, 1) # Copy data in batch dimension.
t_start = timer()
mfcc_warped, (_, nn_alpha) = wl(torch.from_numpy(coded_sps),
None, (len(coded_sps), ),
(len(coded_sps), ),
alphas=torch.from_numpy(alpha_vec))
mfcc_warped.sum().backward()
t_benchmark += timer() - t_start
assert ((mfcc_warped[:, 0] == mfcc_warped[:, 1]).all()
) # Compare results for cloned coded_sps within batch.
if alpha == 0:
assert ((mfcc_warped == coded_sps).all()
) # Compare results for no warping.
sample_pre[:len(mfcc_warped), :hparams.num_coded_sps * (
3 if hparams.add_deltas else 1)] = mfcc_warped[:, 0].detach()
sample_post = gen_in.postprocess_sample(sample_pre)
# Manually create samples without normalisation but with deltas.
sample_pre = (sample_pre * gen_in.norm_params[1] +
gen_in.norm_params[0]).astype(np.float32)
if np.isnan(sample_pre).any():
raise ValueError(
"Detected nan values in output features for {}.".format(
id_name))
# Save warped features.
makedirs_safe(os.path.dirname(os.path.join(out_dir, id_name)))
sample_pre.tofile(
os.path.join(out_dir, id_name + WorldFeatLabelGen.ext_deltas))
hparams.synth_dir = out_dir
Synthesiser.run_world_synth({id_name: sample_post}, hparams)
print("Process time for {} runs: {}".format(
len(id_list) * idx, timedelta(seconds=t_benchmark)))
def run_wavenet_vocoder(synth_output, hparams):
# Import ModelHandlerPyTorch here to prevent circular dependencies.
from idiaptts.src.neural_networks.pytorch.ModelHandlerPyTorch import ModelHandlerPyTorch
assert hparams.synth_vocoder_path is not None, "Please set path to neural vocoder in hparams.synth_vocoder_path"
# Add identifier to suffix.
old_synth_file_suffix = hparams.synth_file_suffix
hparams.synth_file_suffix += '_' + hparams.synth_vocoder
if not hasattr(hparams, 'bit_depth'):
hparams.add_hparam("bit_depth", 16)
synth_output = copy.copy(synth_output)
input_fs_Hz = 1000.0 / hparams.frame_size_ms
assert hasattr(hparams, "frame_rate_output_Hz") and hparams.frame_rate_output_Hz is not None, \
"hparams.frame_rate_output_Hz has to be set and match the trained WaveNet."
in_to_out_multiplier = hparams.frame_rate_output_Hz / input_fs_Hz
# # dir_world_features = os.path.join(self.OutputGen.dir_labels, self.dir_extracted_acoustic_features)
input_gen = WorldFeatLabelGen(
None,
add_deltas=False,
sampling_fn=partial(sample_linearly,
in_to_out_multiplier=in_to_out_multiplier,
dtype=np.float32))
# Load normalisation parameters for wavenet input.
try:
norm_params_path = os.path.splitext(
hparams.synth_vocoder_path)[0] + "_norm_params.npy"
input_gen.norm_params = np.load(norm_params_path).reshape(2, -1)
except FileNotFoundError:
logging.error(
"Cannot find normalisation parameters for WaveNet input at {}."
"Please save them there with numpy.save().".format(
norm_params_path))
raise
model_handler = ModelHandlerPyTorch()
model_handler.model, *_ = model_handler.load_model(
hparams.synth_vocoder_path, hparams, verbose=False)
for id_name, output in synth_output.items():
logging.info("Synthesise {} with {} vocoder.".format(
id_name, hparams.synth_vocoder_path))
# Any other post-processing could be done here.
# Normalize input.
output = input_gen.preprocess_sample(output)
# output (T x C) --transpose--> (C x T) --unsqueeze(0)--> (B x C x T).
output = output.transpose()[None, ...]
# Wavenet input has to be (B x C x T).
output, _ = model_handler.forward(
output, hparams, batch_seq_lengths=(output.shape[-1], ))
# output, _ = model_handler.forward(output[:, :, :1000], hparams, batch_seq_lengths=(1000,)) # DEBUG
output = output[0].transpose(
) # Remove batch dim and transpose back to (T x C).
out_channels = output.shape[1]
if out_channels > 1: # Check if the output is one-hot (quantized) or 1 (raw).
# Revert mu-law quantization.
output = output.argmax(axis=1)
synth_output[
id_name] = RawWaveformLabelGen.mu_law_companding_reversed(
output, out_channels)
# Save the audio.
wav_file_path = os.path.join(
hparams.synth_dir, "".join(
(os.path.basename(id_name).rsplit('.', 1)[0], "_",
hparams.model_name, hparams.synth_file_suffix, ".",
hparams.synth_ext)))
Synthesiser.raw_to_file(wav_file_path, synth_output[id_name],
hparams.synth_fs, hparams.bit_depth)
# Restore identifier.
hparams.setattr_no_type_check(
"synth_file_suffix",
old_synth_file_suffix) # Can be None, thus no type check.
def main():
"""Create samples with artificial alpha for each phoneme."""
from idiaptts.src.model_trainers.vtln.VTLNSpeakerAdaptionModelTrainer import VTLNSpeakerAdaptionModelTrainer
hparams = VTLNSpeakerAdaptionModelTrainer.create_hparams()
hparams.use_gpu = False
hparams.voice = sys.argv[1]
hparams.model_name = "WarpingLayerTest.nn"
hparams.add_deltas = True
hparams.num_coded_sps = 30
alpha_range = 0.2
num_phonemes = 70
num_random_alphas = 7
# num_random_alphas = 53
# Randomly pick alphas for each phoneme.
np.random.seed(42)
# phonemes_to_alpha_tensor = ((np.random.choice(np.random.rand(num_random_alphas), num_phonemes) - 0.5) * 2 * alpha_range)
phonemes_to_alpha_tensor = ((np.random.rand(num_phonemes) - 0.5) * 2 *
alpha_range)
# hparams.num_questions = 505
hparams.num_questions = 609
# hparams.num_questions = 425
hparams.out_dir = os.path.join("experiments", hparams.voice,
"WORLD_artificially_warped")
hparams.data_dir = os.path.realpath("database")
hparams.model_name = "warping_layer_test"
hparams.synth_dir = hparams.out_dir
dir_world_labels = os.path.join("experiments", hparams.voice, "WORLD")
print(
"Create artificially warped MGCs for {} in {} for {} questions, {} random alphas, and an alpha range of {}."
.format(hparams.voice, hparams.out_dir, hparams.num_questions,
len(np.unique(phonemes_to_alpha_tensor)), alpha_range))
from idiaptts.src.data_preparation.world.WorldFeatLabelGen import WorldFeatLabelGen
gen_in = WorldFeatLabelGen(dir_world_labels,
add_deltas=hparams.add_deltas,
num_coded_sps=hparams.num_coded_sps)
gen_in.get_normalisation_params(gen_in.dir_labels)
from idiaptts.src.model_trainers.AcousticModelTrainer import AcousticModelTrainer
trainer = AcousticModelTrainer(
os.path.join("experiments", hparams.voice, "WORLD"),
os.path.join("experiments", hparams.voice, "questions"), "ignored",
hparams.num_questions, hparams)
hparams.num_speakers = 1
speaker = "p276"
num_synth_files = 5 # Number of files to synthesise to check warping manually.
sp_mean = gen_in.norm_params[0][:hparams.num_coded_sps *
(3 if hparams.add_deltas else 1)]
sp_std_dev = gen_in.norm_params[1][:hparams.num_coded_sps *
(3 if hparams.add_deltas else 1)]
wl = WarpingLayer((hparams.num_coded_sps, ), (hparams.num_coded_sps, ),
hparams)
wl.set_norm_params(sp_mean, sp_std_dev)
def _question_to_phoneme_index(questions):
"""Helper function to convert questions to their current phoneme index."""
if questions.shape[-1] == 505: # German question set.
indices = np.arange(86, 347, 5, dtype=np.int)
elif questions.shape[-1] == 425: # English radio question set.
indices = np.arange(58, 107, dtype=np.int)
elif questions.shape[-1] == 609: # English unilex question set.
indices = np.arange(92, 162, dtype=np.int)
else:
raise NotImplementedError(
"Unknown question set with {} questions.".format(
questions.shape[-1]))
return QuestionLabelGen.questions_to_phoneme_indices(
questions, indices)
# with open(os.path.join(hparams.data_dir, "file_id_list_{}_train.txt".format(hparams.voice))) as f:
with open(
os.path.join(hparams.data_dir, "file_id_list_{}_adapt.txt".format(
hparams.voice))) as f:
id_list = f.readlines()
id_list[:] = [s.strip(' \t\n\r') for s in id_list
if speaker in s] # Trim line endings in-place.
out_dir = hparams.out_dir
makedirs_safe(out_dir)
makedirs_safe(os.path.join(out_dir,
"cmp_mgc" + str(hparams.num_coded_sps)))
t_benchmark = 0
org_to_warped_mcd = 0.0
for idx, id_name in enumerate(id_list):
sample = WorldFeatLabelGen.load_sample(
id_name,
os.path.join("experiments", hparams.voice, "WORLD"),
add_deltas=True,
num_coded_sps=hparams.num_coded_sps)
sample_pre = gen_in.preprocess_sample(sample)
coded_sps = sample_pre[:, :hparams.num_coded_sps *
(3 if hparams.add_deltas else 1)]
questions = QuestionLabelGen.load_sample(
id_name,
os.path.join("experiments", hparams.voice, "questions"),
num_questions=hparams.num_questions)
questions = questions[:len(coded_sps)]
phoneme_indices = _question_to_phoneme_index(questions)
alpha_vec = phonemes_to_alpha_tensor[phoneme_indices %
len(phonemes_to_alpha_tensor),
None]
coded_sps = coded_sps[:len(alpha_vec), None,
...] # Create a batch dimension.
alpha_vec = alpha_vec[:, None,
None] # Create a batch and feature dimension.
t_start = timer()
mfcc_warped, (_, nn_alpha) = wl(torch.from_numpy(coded_sps),
None, (len(coded_sps), ),
(len(coded_sps), ),
alphas=torch.from_numpy(alpha_vec))
t_benchmark += timer() - t_start
sample_pre[:len(mfcc_warped), :hparams.num_coded_sps *
(3 if hparams.add_deltas else 1)] = mfcc_warped[:,
0].detach()
sample_post = gen_in.postprocess_sample(sample_pre)
# Manually create samples without normalisation but with deltas.
sample_pre = (sample_pre * gen_in.norm_params[1] +
gen_in.norm_params[0]).astype(np.float32)
if np.isnan(sample_pre).any():
raise ValueError(
"Detected nan values in output features for {}.".format(
id_name))
# Compute error between warped version and original one.
org_to_warped_mcd += metrics.melcd(
sample[:, 0:hparams.num_coded_sps],
sample_pre[:, 0:hparams.num_coded_sps])
# Save warped features.
sample_pre.tofile(
os.path.join(
out_dir, "cmp_mgc" + str(hparams.num_coded_sps),
os.path.basename(id_name + WorldFeatLabelGen.ext_deltas)))
hparams.synth_dir = out_dir
if idx < num_synth_files: # Only synthesize a few of samples.
trainer.run_world_synth({id_name: sample_post}, hparams)
print("Process time for {} warpings: {}. MCD caused by warping: {:.2f}".
format(len(id_list), timedelta(seconds=t_benchmark),
org_to_warped_mcd / len(id_list)))
# Copy normalisation files which are necessary for training.
for feature in ["_bap", "_lf0", "_mgc{}".format(hparams.num_coded_sps)]:
shutil.copyfile(
os.path.join(
gen_in.dir_labels, gen_in.dir_deltas,
MeanCovarianceExtractor.file_name_appendix + feature + ".bin"),
os.path.join(
out_dir, "cmp_mgc" + str(hparams.num_coded_sps),
MeanCovarianceExtractor.file_name_appendix + feature + ".bin"))
def main():
from idiaptts.src.model_trainers.vtln.VTLNSpeakerAdaptionModelTrainer import VTLNSpeakerAdaptionModelTrainer
hparams = VTLNSpeakerAdaptionModelTrainer.create_hparams()
hparams.use_gpu = False
hparams.voice = "English"
hparams.model_name = "AllPassWarpModelTest.nn"
hparams.add_deltas = True
hparams.num_coded_sps = 30
# hparams.num_questions = 505
hparams.num_questions = 425
hparams.out_dir = os.path.join("experiments", hparams.voice,
"VTLNArtificiallyWarped")
hparams.data_dir = os.path.realpath("database")
hparams.model_name = "all_pass_warp_test"
hparams.synth_dir = hparams.out_dir
batch_size = 2
dir_world_labels = os.path.join("experiments", hparams.voice, "WORLD")
# hparams.add_hparam("warp_matrix_size", hparams.num_coded_sps)
hparams.alpha_ranges = [
0.2,
]
from idiaptts.src.data_preparation.world.WorldFeatLabelGen import WorldFeatLabelGen
gen_in = WorldFeatLabelGen(dir_world_labels,
add_deltas=hparams.add_deltas,
num_coded_sps=hparams.num_coded_sps,
num_bap=hparams.num_bap)
gen_in.get_normalisation_params(gen_in.dir_labels)
from idiaptts.src.model_trainers.AcousticModelTrainer import AcousticModelTrainer
trainer = AcousticModelTrainer(
"experiments/" + hparams.voice + "/WORLD",
"experiments/" + hparams.voice + "/questions", "ignored",
hparams.num_questions, hparams)
sp_mean = gen_in.norm_params[0][:hparams.num_coded_sps *
(3 if hparams.add_deltas else 1)]
sp_std_dev = gen_in.norm_params[1][:hparams.num_coded_sps *
(3 if hparams.add_deltas else 1)]
all_pass_warp_model = AllPassWarpModel((hparams.num_coded_sps, ),
(hparams.num_coded_sps, ), hparams)
all_pass_warp_model.set_norm_params(sp_mean, sp_std_dev)
# id_list = ["dorian/doriangray_16_00199"]
# id_list = ["p225/p225_051", "p277/p277_012", "p278/p278_012", "p279/p279_012"]
id_list = ["p225/p225_051"]
t_benchmark = 0
for id_name in id_list:
sample = WorldFeatLabelGen.load_sample(
id_name,
os.path.join("experiments", hparams.voice, "WORLD"),
add_deltas=True,
num_coded_sps=hparams.num_coded_sps,
num_bap=hparams.num_bap,
sp_type=hparams.sp_type)
sample_pre = gen_in.preprocess_sample(sample)
coded_sps = sample_pre[:, :hparams.num_coded_sps *
(3 if hparams.add_deltas else 1)].copy()
coded_sps = coded_sps[:, None,
...].repeat(batch_size,
1) # Copy data in batch dimension.
for idx, alpha in enumerate(np.arange(-0.2, 0.21, 0.05)):
out_dir = os.path.join(hparams.out_dir,
"alpha_{0:0.2f}".format(alpha))
makedirs_safe(out_dir)
alpha_vec = np.ones((coded_sps.shape[0], 1)) * alpha
alpha_vec = alpha_vec[:, None].repeat(
batch_size, 1) # Copy data in batch dimension.
t_start = timer()
sp_warped, (_, nn_alpha) = all_pass_warp_model(
torch.from_numpy(coded_sps.copy()),
None, (len(coded_sps), ), (len(coded_sps), ),
alphas=torch.tensor(alpha_vec, requires_grad=True))
sp_warped.sum().backward()
t_benchmark += timer() - t_start
# assert((mfcc_warped[:, 0] == mfcc_warped[:, 1]).all()) # Compare results for cloned coded_sps within batch.
if np.isclose(alpha, 0):
assert np.isclose(
sp_warped.detach().cpu().numpy(),
coded_sps).all() # Compare no warping results.
sample_pre[:len(sp_warped), :hparams.num_coded_sps * (
3 if hparams.add_deltas else 1)] = sp_warped[:, 0].detach()
sample_post = gen_in.postprocess_sample(sample_pre,
apply_mlpg=False)
# Manually create samples without normalisation but with deltas.
sample_pre_with_deltas = (sample_pre * gen_in.norm_params[1] +
gen_in.norm_params[0]).astype(np.float32)
if np.isnan(sample_pre_with_deltas).any():
raise ValueError(
"Detected nan values in output features for {}.".format(
id_name))
# Save warped features.
makedirs_safe(os.path.dirname(os.path.join(out_dir, id_name)))
sample_pre_with_deltas.tofile(
os.path.join(out_dir,
id_name + "." + WorldFeatLabelGen.ext_deltas))
hparams.synth_dir = out_dir
# sample_no_deltas = WorldFeatLabelGen.convert_from_world_features(*WorldFeatLabelGen.convert_to_world_features(sample, contains_deltas=hparams.add_deltas, num_coded_sps=hparams.num_coded_sps, num_bap=hparams.num_bap))
Synthesiser.run_world_synth({id_name: sample_post}, hparams)
print("Process time for {} runs: {}, average: {}".format(
len(id_list) * idx, timedelta(seconds=t_benchmark),
timedelta(seconds=t_benchmark) / (len(id_list) * idx)))
