def save_log(sess, global_step, model, plot_dir, wav_dir, hparams, model_name):
log('\nSaving intermediate states at step {}'.format(global_step))
idx = 0
y_hat, y, loss, length, input_mel, upsampled_features = sess.run([model.tower_y_hat_log[0][idx],
model.tower_y_log[0][idx],
model.loss,
model.tower_input_lengths[0][idx],
model.tower_c[0][idx], model.tower_upsampled_local_features[0][idx]])
#mask by length
y_hat[length:] = 0
y[length:] = 0
#Make audio and plot paths
pred_wav_path = os.path.join(wav_dir, 'step-{}-pred.wav'.format(global_step))
target_wav_path = os.path.join(wav_dir, 'step-{}-real.wav'.format(global_step))
plot_path = os.path.join(plot_dir, 'step-{}-waveplot.png'.format(global_step))
mel_path = os.path.join(plot_dir, 'step-{}-reconstruction-mel-spectrogram.png'.format(global_step))
upsampled_path = os.path.join(plot_dir, 'step-{}-upsampled-features.png'.format(global_step))
#Save audio
save_wavenet_wav(y_hat, pred_wav_path, sr=hparams.sample_rate, inv_preemphasize=hparams.preemphasize, k=hparams.preemphasis)
save_wavenet_wav(y, target_wav_path, sr=hparams.sample_rate, inv_preemphasize=hparams.preemphasize, k=hparams.preemphasis)
#Save figure
util.waveplot(plot_path, y_hat, y, hparams, title='{}, {}, step={}, loss={:.5f}'.format(model_name, time_string(), global_step, loss))
#Compare generated wav mel with original input mel to evaluate wavenet audio reconstruction performance
#Both mels should match on low frequency information, wavenet mel should contain more high frequency detail when compared to Tacotron mels.
T2_output_range = (-hparams.max_abs_value, hparams.max_abs_value) if hparams.symmetric_mels else (0, hparams.max_abs_value)
generated_mel = _interp(melspectrogram(y_hat, hparams).T, T2_output_range)
util.plot_spectrogram(generated_mel, mel_path, title='Local Condition vs Reconst. Mel-Spectrogram, step={}, loss={:.5f}'.format(
global_step, loss), target_spectrogram=input_mel.T)
util.plot_spectrogram(upsampled_features.T, upsampled_path, title='Upsampled Local Condition features, step={}, loss={:.5f}'.format(
global_step, loss), auto_aspect=True)
def eval_step(sess, global_step, model, plot_dir, wav_dir, summary_writer, hparams, model_name):
'''Evaluate model during training.
Supposes that model variables are averaged.
'''
start_time = time.time()
y_hat, y_target, loss, input_mel, upsampled_features = sess.run([model.tower_y_hat[0], model.tower_y_target[0],
model.eval_loss, model.tower_eval_c[0],
model.tower_eval_upsampled_local_features[0]])
duration = time.time() - start_time
log('Time Evaluation: Generation of {} audio frames took {:.3f} sec ({:.3f} frames/sec)'.format(
len(y_target), duration, len(y_target) / duration))
# Make audio and plot paths
pred_wav_path = os.path.join(wav_dir, 'step-{}-pred.wav'.format(global_step))
target_wav_path = os.path.join(wav_dir, 'step-{}-real.wav'.format(global_step))
plot_path = os.path.join(plot_dir, 'step-{}-waveplot.png'.format(global_step))
mel_path = os.path.join(plot_dir, 'step-{}-reconstruction-mel-spectrogram.png'.format(global_step))
upsampled_path = os.path.join(plot_dir, 'step-{}-upsampled-features.png'.format(global_step))
# Save figure
util.waveplot(plot_path, y_hat, y_target, model._hparams,
title='{}, {}, step={}, loss={:.5f}'.format(model_name, time_string(), global_step, loss))
log('Eval loss for global step {}: {:.3f}'.format(global_step, loss))
# Compare generated wav mel with original input mel to evaluate wavenet audio reconstruction performance
# Both mels should match on low frequency information, wavenet mel should contain more high frequency detail when compared to Tacotron mels.
T2_output_range = (-hparams.max_abs_value, hparams.max_abs_value) if hparams.symmetric_mels else (
0, hparams.max_abs_value)
generated_mel = _interp(melspectrogram(y_hat, hparams).T, T2_output_range)
util.plot_spectrogram(generated_mel, mel_path,
title='Local Condition vs Reconst. Mel-Spectrogram, step={}, loss={:.5f}'.format(
global_step, loss), target_spectrogram=input_mel.T)
util.plot_spectrogram(upsampled_features.T, upsampled_path,
title='Upsampled Local Condition features, step={}, loss={:.5f}'.format(
global_step, loss), auto_aspect=True)
# Save Audio
save_wavenet_wav(y_hat, pred_wav_path, sr=hparams.sample_rate, inv_preemphasize=hparams.preemphasize,
k=hparams.preemphasis)
save_wavenet_wav(y_target, target_wav_path, sr=hparams.sample_rate, inv_preemphasize=hparams.preemphasize,
k=hparams.preemphasis)
# Write eval summary to tensorboard
log('Writing eval summary!')
add_test_stats(summary_writer, global_step, loss, hparams=hparams)
def synthesize(self, mel_spectrograms, speaker_ids, basenames, out_dir,
log_dir):
hparams = self._hparams
local_cond, global_cond = self._check_conditions()
#Switch mels in case of debug
if self.synth_debug:
assert len(hparams.wavenet_debug_mels) == len(
hparams.wavenet_debug_wavs)
mel_spectrograms = [
np.load(mel_file) for mel_file in hparams.wavenet_debug_mels
]
#Get True length of audio to be synthesized: audio_len = mel_len * hop_size
audio_lengths = [
len(x) * get_hop_size(self._hparams) for x in mel_spectrograms
]
#Prepare local condition batch
maxlen = max([len(x) for x in mel_spectrograms])
#[-max, max] or [0,max]
T2_output_range = (
-self._hparams.max_abs_value,
self._hparams.max_abs_value) if self._hparams.symmetric_mels else (
0, self._hparams.max_abs_value)
if self._hparams.clip_for_wavenet:
mel_spectrograms = [
np.clip(x, T2_output_range[0], T2_output_range[1])
for x in mel_spectrograms
]
c_batch = np.stack([
_pad_inputs(x, maxlen, _pad=T2_output_range[0])
for x in mel_spectrograms
]).astype(np.float32)
if self._hparams.normalize_for_wavenet:
#rerange to [0, 1]
c_batch = _interp(c_batch, T2_output_range).astype(np.float32)
g = None if speaker_ids is None else np.asarray(
speaker_ids, dtype=np.int32).reshape(len(c_batch), 1)
feed_dict = {}
if local_cond:
feed_dict[self.local_conditions] = c_batch
else:
feed_dict[self.synthesis_length] = 100
if global_cond:
feed_dict[self.global_conditions] = g
if self.synth_debug:
debug_wavs = hparams.wavenet_debug_wavs
assert len(debug_wavs) % hparams.wavenet_num_gpus == 0
test_wavs = [
np.load(debug_wav).reshape(-1, 1) for debug_wav in debug_wavs
]
#pad wavs to same length
max_test_len = max([len(x) for x in test_wavs])
test_wavs = np.stack([
_pad_inputs(x, max_test_len) for x in test_wavs
]).astype(np.float32)
assert len(test_wavs) == len(debug_wavs)
feed_dict[self.targets] = test_wavs.reshape(
len(test_wavs), max_test_len, 1)
feed_dict[self.input_lengths] = np.asarray([test_wavs.shape[1]])
#Generate wavs and clip extra padding to select Real speech parts
generated_wavs, upsampled_features = self.session.run(
[
self.model.tower_y_hat,
self.model.tower_synth_upsampled_local_features
],
feed_dict=feed_dict)
#Linearize outputs (n_gpus -> 1D)
generated_wavs = [
wav for gpu_wavs in generated_wavs for wav in gpu_wavs
]
upsampled_features = [
feat for gpu_feats in upsampled_features for feat in gpu_feats
]
generated_wavs = [
generated_wav[:length]
for generated_wav, length in zip(generated_wavs, audio_lengths)
]
upsampled_features = [
upsampled_feature[:, :length] for upsampled_feature, length in zip(
upsampled_features, audio_lengths)
]
audio_filenames = []
for i, (generated_wav, input_mel, upsampled_feature) in enumerate(
zip(generated_wavs, mel_spectrograms, upsampled_features)):
#Save wav to disk
audio_filename = os.path.join(out_dir,
'{}.wav'.format(basenames[i]))
save_wavenet_wav(generated_wav,
audio_filename,
sr=hparams.sample_rate,
inv_preemphasize=hparams.preemphasize,
k=hparams.preemphasis)
audio_filenames.append(audio_filename)
#Compare generated wav mel with original input mel to evaluate wavenet audio reconstruction performance
#Both mels should match on low frequency information, wavenet mel should contain more high frequency detail when compared to Tacotron mels.
generated_mel = melspectrogram(generated_wav, hparams).T
util.plot_spectrogram(
generated_mel,
os.path.join(
log_dir,
'wavenet-mel-spectrogram-{}.png'.format(basenames[i])),
title=
'Local Condition vs Reconstructed Audio Mel-Spectrogram analysis',
target_spectrogram=input_mel)
#Save upsampled features to visualize checkerboard artifacts.
util.plot_spectrogram(
upsampled_feature.T,
os.path.join(
log_dir,
'wavenet-upsampled_features-{}.png'.format(basenames[i])),
title='Upmsampled Local Condition features',
auto_aspect=True)
#Save waveplot to disk
if log_dir is not None:
plot_filename = os.path.join(
log_dir, 'wavenet-waveplot-{}.png'.format(basenames[i]))
util.waveplot(plot_filename,
generated_wav,
None,
hparams,
title='WaveNet generated Waveform.')
return audio_filenames
