def _process_wave(self, wav_file, num_frames):
try:
wav = audio.load_wav(wav_file, sr=audio_hparams.sample_rate)
except FileNotFoundError:
print(
'file {} present in csv metadata is not present in wav folder. skipping!'
.format(wav_file))
if audio_hparams.trim_silence:
wav = audio.trim_silence(wav, audio_hparams)
expect_len = num_frames * audio_hparams.hop_size + audio_hparams.win_size
if len(wav) < expect_len:
wav = np.concatenate([wav] * np.math.ceil(expect_len / len(wav)))
if len(wav) > expect_len:
sp = random.randint(0, len(wav) - expect_len)
wav = wav[sp:sp + expect_len]
wav = audio.preemphasis(wav, audio_hparams.preemphasis,
audio_hparams.preemphasize)
if audio_hparams.rescale:
wav = wav / np.abs(wav).max() * audio_hparams.rescaling_max
mels = audio.melspectrogram(wav, audio_hparams).astype(np.float32).T
return mels
def _process_utterance(mel_dir, linear_dir, wav_dir, index, wav_path, text, hparams):
"""
Preprocesses a single utterance wav/text pair
this writes the mel scale spectogram to disk and return a tuple to write
to the train.txt file
Args:
- mel_dir: the directory to write the mel spectograms into
- linear_dir: the directory to write the linear spectrograms into
- wav_dir: the directory to write the preprocessed wav into
- index: the numeric index to use in the spectogram filename
- wav_path: path to the audio file containing the speech input
- text: text spoken in the input audio file
- hparams: hyper parameters
Returns:
- A tuple: (audio_filename, mel_filename, linear_filename, time_steps, mel_frames, linear_frames, text)
"""
try:
# Load the audio as numpy array
wav = audio.load_wav(wav_path, sr=hparams.sample_rate)
except FileNotFoundError: #catch missing wav exception
print('file {} present in csv metadata is not present in wav folder. skipping!'.format(
wav_path))
return None
#Trim lead/trail silences
if hparams.trim_silence:
wav = audio.trim_silence(wav, hparams)
#Pre-emphasize
preem_wav = audio.preemphasis(wav, hparams.preemphasis, hparams.preemphasize)
#rescale wav
if hparams.rescale:
wav = wav / np.abs(wav).max() * hparams.rescaling_max
preem_wav = preem_wav / np.abs(preem_wav).max() * hparams.rescaling_max
#Assert all audio is in [-1, 1]
if (wav > 1.).any() or (wav < -1.).any():
raise RuntimeError('wav has invalid value: {}'.format(wav_path))
if (preem_wav > 1.).any() or (preem_wav < -1.).any():
raise RuntimeError('wav has invalid value: {}'.format(wav_path))
#Mu-law quantize
if is_mulaw_quantize(hparams.input_type):
#[0, quantize_channels)
out = mulaw_quantize(wav, hparams.quantize_channels)
#Trim silences
start, end = audio.start_and_end_indices(out, hparams.silence_threshold)
wav = wav[start: end]
preem_wav = preem_wav[start: end]
out = out[start: end]
constant_values = mulaw_quantize(0, hparams.quantize_channels)
out_dtype = np.int16
elif is_mulaw(hparams.input_type):
#[-1, 1]
out = mulaw(wav, hparams.quantize_channels)
constant_values = mulaw(0., hparams.quantize_channels)
out_dtype = np.float32
else:
#[-1, 1]
out = wav
constant_values = 0.
out_dtype = np.float32
# Compute the mel scale spectrogram from the wav
mel_spectrogram = audio.melspectrogram(preem_wav, hparams).astype(np.float32)
mel_frames = mel_spectrogram.shape[1]
if mel_frames > hparams.max_mel_frames and hparams.clip_mels_length:
return None
#Compute the linear scale spectrogram from the wav
linear_spectrogram = audio.linearspectrogram(preem_wav, hparams).astype(np.float32)
linear_frames = linear_spectrogram.shape[1]
#sanity check
assert linear_frames == mel_frames
if hparams.use_lws:
#Ensure time resolution adjustement between audio and mel-spectrogram
fft_size = hparams.n_fft if hparams.win_size is None else hparams.win_size
l, r = audio.pad_lr(wav, fft_size, audio.get_hop_size(hparams))
#Zero pad audio signal
out = np.pad(out, (l, r), mode='constant', constant_values=constant_values)
else:
#Ensure time resolution adjustement between audio and mel-spectrogram
l_pad, r_pad = audio.librosa_pad_lr(wav, hparams.n_fft, audio.get_hop_size(hparams), hparams.wavenet_pad_sides)
#Reflect pad audio signal on the right (Just like it's done in Librosa to avoid frame inconsistency)
out = np.pad(out, (l_pad, r_pad), mode='constant', constant_values=constant_values)
assert len(out) >= mel_frames * audio.get_hop_size(hparams)
#time resolution adjustement
#ensure length of raw audio is multiple of hop size so that we can use
#transposed convolution to upsample
out = out[:mel_frames * audio.get_hop_size(hparams)]
assert len(out) % audio.get_hop_size(hparams) == 0
time_steps = len(out)
# Write the spectrogram and audio to disk
audio_filename = 'audio-{}.npy'.format(index)
mel_filename = 'mel-{}.npy'.format(index)
linear_filename = 'linear-{}.npy'.format(index)
np.save(os.path.join(wav_dir, audio_filename), out.astype(out_dtype), allow_pickle=False)
np.save(os.path.join(mel_dir, mel_filename), mel_spectrogram.T, allow_pickle=False)
np.save(os.path.join(linear_dir, linear_filename), linear_spectrogram.T, allow_pickle=False)
# Return a tuple describing this training example
return (audio_filename, mel_filename, linear_filename, time_steps, mel_frames, text)
def re_save_all(wav_path, audio_filename, mel_filename, linear_filename):
try:
# Load the audio as numpy array
aud = audio.load_audio(wav_path, sr=hparams.sample_rate)
except FileNotFoundError: #catch missing wav exception
print(
'file {} present in csv metadata is not present in wav folder. skipping!'
.format(wav_path))
return None
#Trim lead/trail silences
if hparams.trim_silence:
aud = audio.trim_silence(aud, hparams)
#Pre-emphasize
preem_aud = audio.preemphasis(aud, hparams.preemphasis,
hparams.preemphasize)
#rescale audio
if hparams.rescale:
aud = aud / np.abs(aud).max() * hparams.rescaling_max
preem_aud = preem_aud / np.abs(preem_aud).max() * hparams.rescaling_max
#Assert all audio is in [-1, 1]
if (aud > 1.).any() or (aud < -1.).any():
raise RuntimeError('audio has invalid value: {}'.format(wav_path))
if (preem_aud > 1.).any() or (preem_aud < -1.).any():
raise RuntimeError('audio has invalid value: {}'.format(wav_path))
#[-1, 1]
out = aud
constant_values = 0.
out_dtype = np.float32
# Compute the mel scale spectrogram from the audio
mel_spectrogram = audio.melspectrogram(preem_aud,
hparams).astype(np.float32)
mel_frames = mel_spectrogram.shape[1]
#Compute the linear scale spectrogram from the audui
linear_spectrogram = audio.linearspectrogram(preem_aud,
hparams).astype(np.float32)
linear_frames = linear_spectrogram.shape[1]
#sanity check
assert linear_frames == mel_frames
#Ensure time resolution adjustement between audio and mel-spectrogram
l_pad, r_pad = audio.librosa_pad_lr(aud, hparams.n_fft,
audio.get_hop_size(hparams),
hparams.wavenet_pad_sides)
#Reflect pad audio signal on the right (Just like it's done in Librosa to avoid frame inconsistency)
out = np.pad(out, (l_pad, r_pad),
mode='constant',
constant_values=constant_values)
assert len(out) >= mel_frames * audio.get_hop_size(hparams)
#time resolution adjustement
#ensure length of raw audio is multiple of hop size so that we can use
#transposed convolution to upsample
out = out[:mel_frames * audio.get_hop_size(hparams)]
assert len(out) % audio.get_hop_size(hparams) == 0
# Write the spectrogram and audio to disk
np.save(audio_filename, out.astype(out_dtype), allow_pickle=False)
np.save(mel_filename, mel_spectrogram.T, allow_pickle=False)
np.save(linear_filename, linear_spectrogram.T, allow_pickle=False)
def _process_utterance(out_dir, index, wav_path, text, hparams):
"""
Preprocesses a single utterance wav/text pair
this writes the mel scale spectogram to disk and return a tuple to write
to the train.txt file
Args:
- out_dir: the directory to write the msgpack into
- index: the numeric index to use in the spectogram filename
- wav_path: path to the audio file containing the speech input
- text: text spoken in the input audio file
- hparams: hyper parameters
Returns:
- A tuple: (audio_filename, mel_filename, linear_filename, time_steps, mel_frames, linear_frames, text)
"""
try:
# Load the audio as numpy array
wav = audio.load_wav(wav_path, sr=hparams.sample_rate)
except FileNotFoundError: # catch missing wav exception
print('file {} present in csv metadata is not present in wav folder. skipping!'.format(
wav_path))
return None
# Trim lead/trail silences
if hparams.trim_silence:
wav = audio.trim_silence(wav, hparams)
# Pre-emphasize
preem_wav = audio.preemphasis(wav, hparams.preemphasis, hparams.preemphasize)
# rescale wav
if hparams.rescale:
wav = wav / np.abs(wav).max() * hparams.rescaling_max
preem_wav = preem_wav / np.abs(preem_wav).max() * hparams.rescaling_max
# Assert all audio is in [-1, 1]
if (wav > 1.).any() or (wav < -1.).any():
raise RuntimeError('wav has invalid value: {}'.format(wav_path))
if (preem_wav > 1.).any() or (preem_wav < -1.).any():
raise RuntimeError('wav has invalid value: {}'.format(wav_path))
# [-1, 1]
out = wav
constant_values = 0.
out_dtype = np.float32
# Compute the mel scale spectrogram from the wav
mel_spectrogram = audio.melspectrogram(preem_wav, hparams).astype(np.float32)
mel_frames = mel_spectrogram.shape[1]
if mel_frames > hparams.max_mel_frames and hparams.clip_mels_length:
return None
# Compute the linear scale spectrogram from the wav
linear_spectrogram = audio.linearspectrogram(preem_wav, hparams).astype(np.float32)
linear_frames = linear_spectrogram.shape[1]
# sanity check
assert linear_frames == mel_frames
# Ensure time resolution adjustement between audio and mel-spectrogram
l_pad, r_pad = audio.librosa_pad_lr(wav, audio.get_hop_size(hparams), hparams.pad_sides)
# Reflect pad audio signal on the right (Just like it's done in Librosa to avoid frame inconsistency)
out = np.pad(out, (l_pad, r_pad), mode='constant', constant_values=constant_values)
assert len(out) >= mel_frames * audio.get_hop_size(hparams)
# time resolution adjustement
# ensure length of raw audio is multiple of hop size so that we can use
# transposed convolution to upsample
out = out[:mel_frames * audio.get_hop_size(hparams)]
assert len(out) % audio.get_hop_size(hparams) == 0
time_steps = len(out)
npz_filename = '{}.npz'.format(index)
r = hparams.outputs_per_step
if hparams.symmetric_mels:
_pad_value = -hparams.max_abs_value
else:
_pad_value = 0.
# +2r for head and tail silence
mel_spec = np.pad(mel_spectrogram.T, [[r, r], [0, 0]], 'constant', constant_values=_pad_value)
linear_spec = np.pad(linear_spectrogram.T, [[r, r], [0, 0]], 'constant', constant_values=_pad_value)
target_length = len(linear_spec)
target_frames = (target_length // r + 1) * r
num_pad = target_frames - target_length
if num_pad != 0:
linear_spec = np.pad(linear_spec, ((0, num_pad), (0, 0)), "constant", constant_values=_pad_value)
mel_spec = np.pad(mel_spec, ((0, num_pad), (0, 0)), "constant", constant_values=_pad_value)
stop_token = np.concatenate(
[np.zeros(target_frames - 1, dtype=np.float32), np.ones(1, dtype=np.float32)],
axis=0)
data = {
'mel': mel_spec,
'linear': linear_spec,
'audio': out.astype(out_dtype),
'input_data': np.asarray(text_to_sequence(text)),
'time_steps': time_steps,
'mel_frames': target_frames,
'text': text,
'stop_token': stop_token,
}
dumps_msgpack(data, os.path.join(out_dir, npz_filename))
# Return a tuple describing this training example
return npz_filename, time_steps, mel_frames, text
