def compute_mel(wav, f0_clean, duration=None, basename=None, snr=None):
if duration is not None:
# Compute mel-scale spectrogram from preprocessed wav with duration
mel_spectrogram, energy, clipt = get_mel_and_energy(wav,
duration,
norm=False)
# Compute rescaled energy (0 to 1)
min_, max_ = hp.energy_min, hp.energy_max
energy_rescaled = (energy - min_) / (max_ - min_)
energy_rescaled = np.clip(energy_rescaled, 0, 1)
# Compute normalized pitch
warnings.filterwarnings('error')
# Get f0 from noisy dataset.
f0 = get_f0_noisy(wav, duration)
if (f0 == 0.).all() or (energy_rescaled == 0.).all():
if (f0 == 0.).all():
print("all zero f0! basename:{} SNR: {}".format(
basename, snr))
else:
print("all zero energy! basename:{} SNR: {}".format(
basename, snr))
try:
f0_norm = utils.speaker_normalization(f0)
except Warning:
index_nonzero = (f0 > -1e10)
mean_f0, std_f0 = np.mean(f0[index_nonzero]), np.std(
f0[index_nonzero])
print(
'Warning was raised as an exception! basename: {}, mean_f0={} std_f0={}'
.format(basename, mean_f0,
std_f0)) #, f0, mean_f0, std_f0, f0_file_path)
f0_norm = utils.speaker_normalization(f0_clean)
warnings.simplefilter("ignore", DeprecationWarning)
mel_filename = '{}-mel-{}.npy'.format(hp.dataset, basename)
f0_filename = '{}-f0-{}.npy'.format(hp.dataset, basename)
energy_filename = '{}-energy-{}.npy'.format(hp.dataset, basename)
# Sanity check
mel_clean = np.load(
os.path.join(out_dir, 'mel_clean', mel_filename))
assert mel_clean.shape == mel_spectrogram.T.shape, "Computed mel should be the same size with pre-calculated one."
return (mel_spectrogram.T, f0, f0_norm, energy,
energy_rescaled), (mel_filename, f0_filename,
energy_filename), clipt
else:
# Get mel without any prior info
print("Oh finally i got here..")
exit(0)
mel_spectrogram, _, _ = Audio.tools.get_mel_from_wav(
torch.FloatTensor(wav), norm=False)
return mel_spectrogram.T
def _processing_data(hparams, full_path, spk_label, spk_emb, gender, npz_name,
pbar, i):
if gender == 'M':
lo, hi = 50, 250
elif gender == 'F':
lo, hi = 100, 600
else:
raise ValueError
prng = RandomState(int(random.random()))
x, fs = librosa.load(full_path, sr=hparams.sample_rate)
assert fs == hparams.sample_rate
if x.shape[0] % hparams.hop_size == 0:
x = np.concatenate((x, np.array([1e-06])), axis=0)
y = signal.filtfilt(b, a, x)
wav = y * 0.96 + (prng.rand(y.shape[0]) - 0.5) * 1e-06
# compute spectrogram
D = pySTFT(wav).T
D_mel = np.dot(D, mel_basis)
D_db = 20 * np.log10(np.maximum(min_level, D_mel)) - hparams.ref_level_db
S = (D_db + 100) / 100
# extract f0
f0_rapt = sptk.rapt(wav.astype(np.float32) * 32768,
fs,
hparams.hop_size,
min=lo,
max=hi,
otype=2)
index_nonzero = (f0_rapt != -1e10)
mean_f0, std_f0 = np.mean(f0_rapt[index_nonzero]), np.std(
f0_rapt[index_nonzero])
f0_norm = speaker_normalization(f0_rapt, index_nonzero, mean_f0, std_f0)
assert len(S) == len(f0_rapt)
data = {
'mel': S.astype(np.float32),
'f0': f0_norm.astype(np.float32),
'spk_label': spk_label
}
if spk_emb is not None:
data['spk_emb'] = spk_emb
np.savez(npz_name, **data)
pbar.update(i)
def extract_f0(wav, fs):
f0_rapt = sptk.rapt(wav.astype(np.float32) * 32768,
fs,
256,
min=lo,
max=hi,
otype=2)
index_nonzero = (f0_rapt != -1e10)
mean_f0, std_f0 = np.mean(f0_rapt[index_nonzero]), np.std(
f0_rapt[index_nonzero])
f0_norm = speaker_normalization(f0_rapt, index_nonzero, mean_f0, std_f0)
f0_quantized = quantize_f0_numpy(f0_norm)[0]
f0_onehot = f0_quantized[np.newaxis, :, :]
print(f0_onehot.shape)
if f0_onehot.shape[1] <= 192:
f0_onehot, _ = pad_seq_to_2(f0_onehot, 192)
return torch.from_numpy(f0_onehot).to(device)
y = signal.filtfilt(b, a, x)
wav = y * 0.96 + (prng.rand(y.shape[0]) - 0.5) * 1e-06
# compute spectrogram
D = pySTFT(wav).T
D_mel = np.dot(D, mel_basis)
D_db = 20 * np.log10(np.maximum(min_level, D_mel)) - 16
S = (D_db + 100) / 100
# extract f0
f0_rapt = sptk.rapt(wav.astype(np.float32) * 32768,
fs,
256,
min=lo,
max=hi,
otype=2)
index_nonzero = (f0_rapt != -1e10)
mean_f0, std_f0 = np.mean(f0_rapt[index_nonzero]), np.std(
f0_rapt[index_nonzero])
f0_norm = speaker_normalization(f0_rapt, index_nonzero, mean_f0,
std_f0)
assert len(S) == len(f0_rapt)
np.save(os.path.join(targetDir, basename),
S.astype(np.float32),
allow_pickle=False)
np.save(os.path.join(targetDir_f0, basename),
f0_norm.astype(np.float32),
allow_pickle=False)
print("All done in {:.3f}s".format(time.perf_counter() - start_time_total))
def synthesize_with_reference(idx_info, name, noisy_input, audio_path, tg_path,
speaker_id, inspection):
global model, vocoder, step
start_time = time.perf_counter()
# Prepare Reference Data
if speaker_id is not None:
spker_embed_path = os.path.join(
hp.preprocessed_path, "spker_embed",
"{}-spker_embed-{}.npy".format(hp.dataset, speaker_id))
speaker_embed = torch.from_numpy(np.load(spker_embed_path)).to(device)
else:
try:
# VCTK fileformat
speaker_id = name.split("_")[0]
spker_embed_path = os.path.join(
hp.preprocessed_path, "spker_embed",
"{}-spker_embed-{}.npy".format(hp.dataset, speaker_id))
speaker_embed = torch.from_numpy(
np.load(spker_embed_path)).to(device)
except:
# General cases
speaker_id = None
speaker_embed = torch.from_numpy(
embedding.predict_embedding(speaker_embedder, audio_path))
# Outdir
outdir = os.path.join(hp.test_path(),
"{}_by_{}_{}".format(name, speaker_id, step))
if not os.path.exists(outdir):
os.makedirs(outdir)
text = utils.get_transcript(
os.path.join(audio_path.replace(".wav", ".txt")))
if not os.path.isfile(tg_path):
tg_path = "NO TextGrid"
_, wav = read(audio_path)
if noisy_input:
f0 = sptk.rapt(wav.astype(np.float32) * hp.max_wav_value,
hp.sampling_rate,
hp.encoder_hidden,
min=hp.f0_min,
max=hp.f0_max,
otype=2) # log f0
f0 = np.exp(f0)
else:
f0, _ = pw.dio(wav.astype(np.float64),
hp.sampling_rate,
frame_period=hp.hop_length / hp.sampling_rate *
1000)
mel, energy, _ = Audio.tools.get_mel_from_wav(
torch.FloatTensor(np.array(wav)))
mel = mel.T.numpy().astype(np.float32)
energy = energy.numpy().astype(np.float32)
utils.plot_data([(mel.T, f0, energy)], ['Reference Spectrogram'],
filename=os.path.join(
outdir,
'{}_{}_{}.png'.format("Reference", name,
text[:100])))
else:
f0, energy, mel = get_processed_data_from_wav(audio_path, tg_path,
noisy_input)
utils.plot_data([(mel.T, f0, energy)], ['Reference Spectrogram'],
filename=os.path.join(
outdir,
'{}_{}_{}.png'.format("Reference", name,
text[:100])))
# Prepare Audio Inputs
energy = (energy - hp.energy_min) / (hp.energy_max - hp.energy_min)
f0_norm = utils.speaker_normalization(f0)
mel, mel_len, energy, f0, f0_norm = preprocess_audio(
mel, energy, f0, f0_norm)
print("\n\n---------------- [{}/{}]: {} ----------------".format(
idx_info[0] + 1, idx_info[1],
audio_path.split('/')[-1]))
print('Audio Path:', audio_path)
print('TextGrid Path:', tg_path)
print('Speaker ID:', speaker_id)
# Synthesize
success = 0
for sentence in sentences:
text = preprocess_text(sentence)
synthesize(outdir, model, vocoder, text, sentence, speaker_embed,
speaker_id, inspection, mel, mel_len, f0, f0_norm, energy,
args.duration_control, args.pitch_control,
args.energy_control)
success += 1
print("Synthesized {} out of {} in {:.3f}s".format(
success, len(sentences),
time.perf_counter() - start_time))