def _get_next_example(self):
"""Gets a single example (input, mel_target, token_target, linear_target, mel_length) from_ disk
"""
if self._train_offset >= len(self._train_meta):
self._train_offset = 0
np.random.shuffle(self._train_meta)
meta = self._train_meta[self._train_offset]
self._train_offset += 1
text = meta[5]
input_data = np.asarray(text_to_sequence(text, self._cleaner_names),
dtype=np.int32)
# mel_target = np.load(meta[1])
# audio-P00001A-001-20170001P00001A0001_00.npy
# E:/data/stcmds/SV2TTS/synthesizer/audio/audio-P00001A-001-20170001P00001A0001_00.npy
# wav_fpath = Path(r'E:\data\stcmds\stcmds\wavs')
parts = Path(meta[0]).parts
name_parts = parts[-1].split('-')
name_parts[-1] = name_parts[-1].split('_')[0] + '.wav'
wav_fpath = Path(*parts[:3], parts[2], 'wavs', *name_parts[1:])
wav = load_wav(wav_fpath, hparams.sample_rate)
if hparams.rescale:
wav = wav / np.abs(wav).max() * hparams.rescaling_max
mel_target = melspectrogram(wav, hparams).T
#Create parallel sequences containing zeros to represent a non finished sequence
token_target = np.asarray([0.] * (len(mel_target) - 1))
embed_target = np.load(meta[2])
return input_data, mel_target, token_target, embed_target, len(
mel_target)
def _get_test_groups(self):
meta = self._test_meta[self._test_offset]
self._test_offset += 1
text = meta[5]
input_data = np.asarray(text_to_sequence(text, self._cleaner_names),
dtype=np.int32)
# mel_target = np.load(meta[1]) # os.path.join(self._mel_dir, meta[1])
parts = Path(meta[0]).parts
name_parts = parts[-1].split('-')
name_parts[-1] = name_parts[-1].split('_')[0] + '.wav'
wav_fpath = Path(*parts[:3], parts[2], 'wavs', *name_parts[1:])
wav = load_wav(wav_fpath, hparams.sample_rate)
if hparams.rescale:
wav = wav / np.abs(wav).max() * hparams.rescaling_max
mel_target = melspectrogram(wav, hparams).T
#Create parallel sequences containing zeros to represent a non finished sequence
token_target = np.asarray([0.] * (len(mel_target) - 1))
embed_target = np.load(
meta[2]) # os.path.join(self._embed_dir, meta[2])
return input_data, mel_target, token_target, embed_target, len(
mel_target)
def process_audio_file(vfile, args, gpu_id):
fulldir = vfile.replace('intervals', 'preprocessed') #windows下需要改正路径，这里win下是\\
fulldir = fulldir[:fulldir.rfind('.')] # ignore extension
os.makedirs(fulldir, exist_ok=True)
wavpath = path.join(fulldir, 'audio.wav')
specpath = path.join(fulldir, 'mels.npz')
wav = audio.load_wav(wavpath, hp.sample_rate)
spec = audio.melspectrogram(wav, hp)
lspec = audio.linearspectrogram(wav, hp)
np.savez_compressed(specpath, spec=spec, lspec=lspec)
def process_video_file(vfile, args, gpu_id):
video_stream = cv2.VideoCapture(vfile)
frames = []
while 1:
still_reading, frame = video_stream.read()
if not still_reading:
video_stream.release()
break
frames.append(frame)
fulldir = vfile.replace('/intervals/', '/preprocessed/')
fulldir = vfile[:vfile.rfind('.')] # ignore extension
os.makedirs(fulldir, exist_ok=True)
wavpath = path.join(fulldir, 'audio.wav')
specpath = path.join(fulldir, 'mels.npz')
command = template.format(vfile, hp.sample_rate, wavpath)
subprocess.call(command, shell=True)
wav = audio.load_wav(wavpath, hp.sample_rate)
spec = audio.melspectrogram(wav, hp)
lspec = audio.linearspectrogram(wav, hp)
np.savez_compressed(specpath, spec=spec, lspec=lspec)
batches = [
frames[i:i + args.batch_size]
for i in range(0, len(frames), args.batch_size)
]
i = -1
for fb in batches:
preds = fa[gpu_id].get_detections_for_batch(np.asarray(fb))
for j, f in enumerate(preds):
i += 1
if f is None:
continue
cv2.imwrite(path.join(fulldir, '{}.jpg'.format(i)), f[0])
def _process_utterance(mel_dir, linear_dir, wav_dir, index, wav_path, text,
hparams):
"""
Preprocesses a single utterance wav/text pair
this writes the mel scale spectogram to disk and return a tuple to write
to the train.txt file
Args:
- mel_dir: the directory to write the mel spectograms into
- linear_dir: the directory to write the linear spectrograms into
- wav_dir: the directory to write the preprocessed wav into
- index: the numeric index to use in the spectogram filename
- wav_path: path to the audio file containing the speech input
- text: text spoken in the input audio file
- hparams: hyper parameters
Returns:
- A tuple: (audio_filename, mel_filename, linear_filename, time_steps, mel_frames, linear_frames, text)
"""
try:
# Load the audio as numpy array
wav = audio.load_wav(wav_path, sr=hparams.sample_rate)
except FileNotFoundError: #catch missing wav exception
print(
'file {} present in csv metadata is not present in wav folder. skipping!'
.format(wav_path))
return None
#Pre-emphasize
wav = audio.preemphasis(wav, hparams.preemphasis, hparams.preemphasize)
#rescale wav
if hparams.rescale:
wav = wav / np.abs(wav).max() * hparams.rescaling_max
#Assert all audio is in [-1, 1]
if (wav > 1.).any() or (wav < -1.).any():
raise RuntimeError('wav has invalid value: {}'.format(wav))
#M-AILABS extra silence specific
if hparams.trim_silence:
wav = audio.trim_silence(wav, hparams)
out = wav
constant_values = 0.
out_dtype = np.float32
# Compute the mel scale spectrogram from the wav
mel_spectrogram = audio.melspectrogram(wav, hparams).astype(np.float32)
mel_frames = mel_spectrogram.shape[1]
if mel_frames > hparams.max_mel_frames and hparams.clip_mels_length:
return None
#Compute the linear scale spectrogram from the wav
linear_spectrogram = audio.linearspectrogram(wav,
hparams).astype(np.float32)
linear_frames = linear_spectrogram.shape[1]
#sanity check
assert linear_frames == mel_frames
if hparams.use_lws:
#Ensure time resolution adjustement between audio and mel-spectrogram
fft_size = hparams.n_fft if hparams.win_size is None else hparams.win_size
l, r = audio.pad_lr(wav, fft_size, audio.get_hop_size(hparams))
#Zero pad audio signal
out = np.pad(out, (l, r),
mode='constant',
constant_values=constant_values)
else:
#Ensure time resolution adjustement between audio and mel-spectrogram
l_pad, r_pad = audio.librosa_pad_lr(wav, hparams.n_fft,
audio.get_hop_size(hparams))
#Reflect pad audio signal on the right (Just like it's done in Librosa to avoid frame inconsistency)
out = np.pad(out, (l_pad, r_pad),
mode='constant',
constant_values=constant_values)
assert len(out) >= mel_frames * audio.get_hop_size(hparams)
#time resolution adjustement
#ensure length of raw audio is multiple of hop size so that we can use
#transposed convolution to upsample
out = out[:mel_frames * audio.get_hop_size(hparams)]
assert len(out) % audio.get_hop_size(hparams) == 0
time_steps = len(out)
# Write the spectrogram and audio to disk
audio_filename = 'audio-{}.npy'.format(index)
mel_filename = 'mel-{}.npy'.format(index)
linear_filename = 'linear-{}.npy'.format(index)
embed_filename = 'embed-{}.npy'.format(index)
np.save(os.path.join(wav_dir, audio_filename),
out.astype(out_dtype),
allow_pickle=False)
np.save(os.path.join(mel_dir, mel_filename),
mel_spectrogram.T,
allow_pickle=False)
np.save(os.path.join(linear_dir, linear_filename),
linear_spectrogram.T,
allow_pickle=False)
# Return a tuple describing this training example
return (audio_filename, mel_filename, linear_filename, embed_filename,
time_steps, mel_frames, text)