def save_log(sess, global_step, model, plot_dir, wav_dir, hparams, model_name):
log('\nSaving intermediate states at step {}'.format(global_step))
idx = 0
y_hat, y, loss, length, input_mel, upsampled_features = sess.run([model.tower_y_hat_log[0][idx],
model.tower_y_log[0][idx],
model.loss,
model.tower_input_lengths[0][idx],
model.tower_c[0][idx], model.tower_upsampled_local_features[0][idx]])
#mask by length
y_hat[length:] = 0
y[length:] = 0
#Make audio and plot paths
pred_wav_path = os.path.join(wav_dir, 'step-{}-pred.wav'.format(global_step))
target_wav_path = os.path.join(wav_dir, 'step-{}-real.wav'.format(global_step))
plot_path = os.path.join(plot_dir, 'step-{}-waveplot.png'.format(global_step))
mel_path = os.path.join(plot_dir, 'step-{}-reconstruction-mel-spectrogram.png'.format(global_step))
upsampled_path = os.path.join(plot_dir, 'step-{}-upsampled-features.png'.format(global_step))
#Save audio
save_wavenet_wav(y_hat, pred_wav_path, sr=hparams.sample_rate, inv_preemphasize=hparams.preemphasize, k=hparams.preemphasis)
save_wavenet_wav(y, target_wav_path, sr=hparams.sample_rate, inv_preemphasize=hparams.preemphasize, k=hparams.preemphasis)
#Save figure
util.waveplot(plot_path, y_hat, y, hparams, title='{}, {}, step={}, loss={:.5f}'.format(model_name, time_string(), global_step, loss))
#Compare generated wav mel with original input mel to evaluate wavenet audio reconstruction performance
#Both mels should match on low frequency information, wavenet mel should contain more high frequency detail when compared to Tacotron mels.
T2_output_range = (-hparams.max_abs_value, hparams.max_abs_value) if hparams.symmetric_mels else (0, hparams.max_abs_value)
generated_mel = _interp(melspectrogram(y_hat, hparams).T, T2_output_range)
util.plot_spectrogram(generated_mel, mel_path, title='Local Condition vs Reconst. Mel-Spectrogram, step={}, loss={:.5f}'.format(
global_step, loss), target_spectrogram=input_mel.T)
util.plot_spectrogram(upsampled_features.T, upsampled_path, title='Upsampled Local Condition features, step={}, loss={:.5f}'.format(
global_step, loss), auto_aspect=True)
def synthesize(self, mel_spectrograms, speaker_ids, basenames, out_dir, log_dir):
hparams = self._hparams
local_cond, global_cond = self._check_conditions()
#Get True length of audio to be synthesized: audio_len = mel_len * hop_size
audio_lengths = [len(x) * get_hop_size(self._hparams) for x in mel_spectrograms]
#Prepare local condition batch
maxlen = max([len(x) for x in mel_spectrograms])
#[-max, max] or [0,max]
T2_output_range = (-self._hparams.max_abs_value, self._hparams.max_abs_value) if self._hparams.symmetric_mels else (0, self._hparams.max_abs_value)
if self._hparams.clip_for_wavenet:
mel_spectrograms = [np.clip(x, T2_output_range[0], T2_output_range[1]) for x in mel_spectrograms]
c_batch = np.stack([_pad_inputs(x, maxlen, _pad=T2_output_range[0]) for x in mel_spectrograms]).astype(np.float32)
if self._hparams.normalize_for_wavenet:
#rerange to [0, 1]
c_batch = np.interp(c_batch, T2_output_range, (0, 1))
g = None if speaker_ids is None else np.asarray(speaker_ids, dtype=np.int32).reshape(len(c_batch), 1)
feed_dict = {}
if local_cond:
feed_dict[self.local_conditions] = c_batch
else:
feed_dict[self.synthesis_length] = 100
if global_cond:
feed_dict[self.global_conditions] = g
#Generate wavs and clip extra padding to select Real speech parts
generated_wavs = self.session.run(self.model.y_hat, feed_dict=feed_dict)
generated_wavs = [generated_wav[:length] for generated_wav, length in zip(generated_wavs, audio_lengths)]
audio_filenames = []
for i, generated_wav in enumerate(generated_wavs):
#Save wav to disk
audio_filename = os.path.join(out_dir, 'wavenet-audio-{}.wav'.format(basenames[i]))
save_wavenet_wav(generated_wav, audio_filename, sr=hparams.sample_rate)
audio_filenames.append(audio_filename)
#Save waveplot to disk
if log_dir is not None:
plot_filename = os.path.join(log_dir, 'wavenet-waveplot-{}.png'.format(basenames[i]))
util.waveplot(plot_filename, generated_wav, None, hparams)
return audio_filenames
def eval_step(sess, global_step, model, plot_dir, wav_dir, summary_writer, hparams, model_name):
'''Evaluate model during training.
Supposes that model variables are averaged.
'''
start_time = time.time()
y_hat, y_target, loss, input_mel, upsampled_features = sess.run([model.tower_y_hat[0], model.tower_y_target[0],
model.eval_loss, model.tower_eval_c[0], model.tower_eval_upsampled_local_features[0]])
duration = time.time() - start_time
log('Time Evaluation: Generation of {} audio frames took {:.3f} sec ({:.3f} frames/sec)'.format(
len(y_target), duration, len(y_target)/duration))
#Make audio and plot paths
pred_wav_path = os.path.join(wav_dir, 'step-{}-pred.wav'.format(global_step))
target_wav_path = os.path.join(wav_dir, 'step-{}-real.wav'.format(global_step))
plot_path = os.path.join(plot_dir, 'step-{}-waveplot.png'.format(global_step))
mel_path = os.path.join(plot_dir, 'step-{}-reconstruction-mel-spectrogram.png'.format(global_step))
upsampled_path = os.path.join(plot_dir, 'step-{}-upsampled-features.png'.format(global_step))
#Save Audio
save_wavenet_wav(y_hat, pred_wav_path, sr=hparams.sample_rate, inv_preemphasize=hparams.preemphasize, k=hparams.preemphasis)
save_wavenet_wav(y_target, target_wav_path, sr=hparams.sample_rate, inv_preemphasize=hparams.preemphasize, k=hparams.preemphasis)
#Save figure
util.waveplot(plot_path, y_hat, y_target, model._hparams, title='{}, {}, step={}, loss={:.5f}'.format(model_name, time_string(), global_step, loss))
log('Eval loss for global step {}: {:.3f}'.format(global_step, loss))
#Compare generated wav mel with original input mel to evaluate wavenet audio reconstruction performance
#Both mels should match on low frequency information, wavenet mel should contain more high frequency detail when compared to Tacotron mels.
T2_output_range = (-hparams.max_abs_value, hparams.max_abs_value) if hparams.symmetric_mels else (0, hparams.max_abs_value)
generated_mel = _interp(melspectrogram(y_hat, hparams).T, T2_output_range)
util.plot_spectrogram(generated_mel, mel_path, title='Local Condition vs Reconst. Mel-Spectrogram, step={}, loss={:.5f}'.format(
global_step, loss), target_spectrogram=input_mel.T)
util.plot_spectrogram(upsampled_features.T, upsampled_path, title='Upsampled Local Condition features, step={}, loss={:.5f}'.format(
global_step, loss), auto_aspect=True)
#Write eval summary to tensorboard
log('Writing eval summary!')
add_test_stats(summary_writer, global_step, loss, hparams=hparams)
def synthesize(self, mel_spectrograms, speaker_ids, basenames, out_dir,
log_dir):
hparams = self._hparams
local_cond, global_cond = self._check_conditions()
#Switch mels in case of debug
# if self.synth_debug:
# assert len(hparams.wavenet_debug_mels) == len(hparams.wavenet_debug_wavs)
# mel_spectrograms = [np.load(mel_file) for mel_file in hparams.wavenet_debug_mels]
#Get True length of audio to be synthesized: audio_len = mel_len * hop_size
audio_lengths = [
len(x) * get_hop_size(self._hparams) for x in mel_spectrograms
]
#Prepare local condition batch
maxlen = max([len(x) for x in mel_spectrograms])
#[-max, max] or [0,max]
T2_output_range = (
-self._hparams.max_abs_value,
self._hparams.max_abs_value) if self._hparams.symmetric_mels else (
0, self._hparams.max_abs_value)
if self._hparams.clip_for_wavenet:
mel_spectrograms = [
np.clip(x, T2_output_range[0], T2_output_range[1])
for x in mel_spectrograms
]
c_batch = np.stack([
_pad_inputs(x, maxlen, _pad=T2_output_range[0])
for x in mel_spectrograms
]).astype(np.float32)
if self._hparams.normalize_for_wavenet:
#rerange to [0, 1]
c_batch = np.interp(c_batch, T2_output_range, (0, 1))
g = None if speaker_ids is None else np.asarray(
speaker_ids, dtype=np.int32).reshape(len(c_batch), 1)
feed_dict = {}
if local_cond:
feed_dict[self.local_conditions] = c_batch
else:
feed_dict[self.synthesis_length] = 100
if global_cond:
feed_dict[self.global_conditions] = g
# if self.synth_debug:
# debug_wavs = hparams.wavenet_debug_wavs
# assert len(debug_wavs) % hparams.wavenet_num_gpus == 0
# test_wavs = [np.load(debug_wav).reshape(-1, 1) for debug_wav in debug_wavs]
#
# #pad wavs to same length
# max_test_len = max([len(x) for x in test_wavs])
# test_wavs = np.stack([_pad_inputs(x, max_test_len) for x in test_wavs]).astype(np.float32)
#
# assert len(test_wavs) == len(debug_wavs)
# feed_dict[self.targets] = test_wavs.reshape(len(test_wavs), max_test_len, 1)
# feed_dict[self.input_lengths] = np.asarray([test_wavs.shape[1]])
#Generate wavs and clip extra padding to select Real speech parts
#generated_wavs, upsampled_features = self.session.run([self.model.tower_y_hat, self.model.tower_synth_upsampled_local_features], feed_dict=feed_dict)
#Linearize outputs (n_gpus -> 1D)
#generated_wavs = [wav for gpu_wavs in generated_wavs for wav in gpu_wavs]
#upsampled_features = [feat for gpu_feats in upsampled_features for feat in gpu_feats]
#generated_wavs = [generated_wav[:length] for generated_wav, length in zip(generated_wavs, audio_lengths)]
#upsampled_features = [upsampled_feature[:, :length] for upsampled_feature, length in zip(upsampled_features, audio_lengths)]
generated_wavs = self.session.run(self.model.y_hat,
feed_dict=feed_dict)
generated_wavs = [
generated_wav[:length]
for generated_wav, length in zip(generated_wavs, audio_lengths)
]
audio_filenames = []
for i, generated_wav in enumerate(generated_wavs):
#Save wav to disk
audio_filename = os.path.join(
out_dir, 'wavenet-audio-{}.wav'.format(basenames[i]))
save_wavenet_wav(generated_wav,
audio_filename,
sr=hparams.sample_rate)
audio_filenames.append(audio_filename)
#Compare generated wav mel with original input mel to evaluate wavenet audio reconstruction performance
#Both mels should match on low frequency information, wavenet mel should contain more high frequency detail when compared to Tacotron mels.
# generated_mel = melspectrogram(generated_wav, hparams).T
# util.plot_spectrogram(generated_mel, os.path.join(log_dir, 'wavenet-mel-spectrogram-{}.png'.format(basenames[i])),
# title='Local Condition vs Reconstructed Audio Mel-Spectrogram analysis', target_spectrogram=input_mel)
# #Save upsampled features to visualize checkerboard artifacts.
# util.plot_spectrogram(upsampled_feature.T, os.path.join(log_dir, 'wavenet-upsampled_features-{}.png'.format(basenames[i])),
# title='Upmsampled Local Condition features', auto_aspect=True)
#Save waveplot to disk
if log_dir is not None:
plot_filename = os.path.join(
log_dir, 'wavenet-waveplot-{}.png'.format(basenames[i]))
util.waveplot(plot_filename,
generated_wav,
None,
hparams,
title='WaveNet generated Waveform.')
return audio_filenames
