def process_audio_file(vfile, args, gpu_id):
fulldir = vfile.replace('intervals', 'preprocessed') #windows下需要改正路径，这里win下是\\
fulldir = fulldir[:fulldir.rfind('.')] # ignore extension
os.makedirs(fulldir, exist_ok=True)
wavpath = path.join(fulldir, 'audio.wav')
specpath = path.join(fulldir, 'mels.npz')
wav = audio.load_wav(wavpath, hp.sample_rate)
spec = audio.melspectrogram(wav, hp)
lspec = audio.linearspectrogram(wav, hp)
np.savez_compressed(specpath, spec=spec, lspec=lspec)
def process_video_file(vfile, args, gpu_id):
video_stream = cv2.VideoCapture(vfile)
frames = []
while 1:
still_reading, frame = video_stream.read()
if not still_reading:
video_stream.release()
break
frames.append(frame)
fulldir = vfile.replace('/intervals/', '/preprocessed/')
fulldir = vfile[:vfile.rfind('.')] # ignore extension
os.makedirs(fulldir, exist_ok=True)
wavpath = path.join(fulldir, 'audio.wav')
specpath = path.join(fulldir, 'mels.npz')
command = template.format(vfile, hp.sample_rate, wavpath)
subprocess.call(command, shell=True)
wav = audio.load_wav(wavpath, hp.sample_rate)
spec = audio.melspectrogram(wav, hp)
lspec = audio.linearspectrogram(wav, hp)
np.savez_compressed(specpath, spec=spec, lspec=lspec)
batches = [
frames[i:i + args.batch_size]
for i in range(0, len(frames), args.batch_size)
]
i = -1
for fb in batches:
preds = fa[gpu_id].get_detections_for_batch(np.asarray(fb))
for j, f in enumerate(preds):
i += 1
if f is None:
continue
cv2.imwrite(path.join(fulldir, '{}.jpg'.format(i)), f[0])
def _process_utterance(mel_dir, linear_dir, wav_dir, index, wav_path, text,
hparams):
"""
Preprocesses a single utterance wav/text pair
this writes the mel scale spectogram to disk and return a tuple to write
to the train.txt file
Args:
- mel_dir: the directory to write the mel spectograms into
- linear_dir: the directory to write the linear spectrograms into
- wav_dir: the directory to write the preprocessed wav into
- index: the numeric index to use in the spectogram filename
- wav_path: path to the audio file containing the speech input
- text: text spoken in the input audio file
- hparams: hyper parameters
Returns:
- A tuple: (audio_filename, mel_filename, linear_filename, time_steps, mel_frames, linear_frames, text)
"""
try:
# Load the audio as numpy array
wav = audio.load_wav(wav_path, sr=hparams.sample_rate)
except FileNotFoundError: #catch missing wav exception
print(
'file {} present in csv metadata is not present in wav folder. skipping!'
.format(wav_path))
return None
#Pre-emphasize
wav = audio.preemphasis(wav, hparams.preemphasis, hparams.preemphasize)
#rescale wav
if hparams.rescale:
wav = wav / np.abs(wav).max() * hparams.rescaling_max
#Assert all audio is in [-1, 1]
if (wav > 1.).any() or (wav < -1.).any():
raise RuntimeError('wav has invalid value: {}'.format(wav))
#M-AILABS extra silence specific
if hparams.trim_silence:
wav = audio.trim_silence(wav, hparams)
out = wav
constant_values = 0.
out_dtype = np.float32
# Compute the mel scale spectrogram from the wav
mel_spectrogram = audio.melspectrogram(wav, hparams).astype(np.float32)
mel_frames = mel_spectrogram.shape[1]
if mel_frames > hparams.max_mel_frames and hparams.clip_mels_length:
return None
#Compute the linear scale spectrogram from the wav
linear_spectrogram = audio.linearspectrogram(wav,
hparams).astype(np.float32)
linear_frames = linear_spectrogram.shape[1]
#sanity check
assert linear_frames == mel_frames
if hparams.use_lws:
#Ensure time resolution adjustement between audio and mel-spectrogram
fft_size = hparams.n_fft if hparams.win_size is None else hparams.win_size
l, r = audio.pad_lr(wav, fft_size, audio.get_hop_size(hparams))
#Zero pad audio signal
out = np.pad(out, (l, r),
mode='constant',
constant_values=constant_values)
else:
#Ensure time resolution adjustement between audio and mel-spectrogram
l_pad, r_pad = audio.librosa_pad_lr(wav, hparams.n_fft,
audio.get_hop_size(hparams))
#Reflect pad audio signal on the right (Just like it's done in Librosa to avoid frame inconsistency)
out = np.pad(out, (l_pad, r_pad),
mode='constant',
constant_values=constant_values)
assert len(out) >= mel_frames * audio.get_hop_size(hparams)
#time resolution adjustement
#ensure length of raw audio is multiple of hop size so that we can use
#transposed convolution to upsample
out = out[:mel_frames * audio.get_hop_size(hparams)]
assert len(out) % audio.get_hop_size(hparams) == 0
time_steps = len(out)
# Write the spectrogram and audio to disk
audio_filename = 'audio-{}.npy'.format(index)
mel_filename = 'mel-{}.npy'.format(index)
linear_filename = 'linear-{}.npy'.format(index)
embed_filename = 'embed-{}.npy'.format(index)
np.save(os.path.join(wav_dir, audio_filename),
out.astype(out_dtype),
allow_pickle=False)
np.save(os.path.join(mel_dir, mel_filename),
mel_spectrogram.T,
allow_pickle=False)
np.save(os.path.join(linear_dir, linear_filename),
linear_spectrogram.T,
allow_pickle=False)
# Return a tuple describing this training example
return (audio_filename, mel_filename, linear_filename, embed_filename,
time_steps, mel_frames, text)