def test_file_wav(self, file_name): """ Function to extract vocals from wav file. """ sess = tf.Session() self.load_model(sess, log_dir = config.log_dir) mix_stft = utils.file_to_stft(file_name) out_feats = self.process_file(mix_stft, sess)
def test_model_yam(self):
"""
Function to extract vocals from wav file.
"""
sess = tf.Session()
self.load_model(sess, log_dir=config.log_dir)
voc_stft = utils.file_to_stft('./Bria_000_VoU67.wav')
back_stft = utils.file_to_stft('./Bria_000_Back.wav')
mix_stft = np.clip(
(voc_stft[:len(back_stft)] + back_stft[:len(voc_stft)]) / 2, 0.0,
1.0)
feats = utils.input_to_feats('./Bria_000_VoU67.wav')
out_feats = self.process_file(mix_stft, sess)
self.plot_features(feats, out_feats)
out_featss = np.concatenate(
(out_feats[:feats.shape[0], :-2], feats[:out_feats.shape[0], -2:]),
axis=-1)
utils.feats_to_audio(out_featss[:3000], 'Bree_output')
def synth_file(file_name,
file_path=config.wav_dir,
show_plots=True,
save_file=True):
if file_name.startswith('ikala'):
file_name = file_name[6:]
file_path = config.wav_dir
utils.write_ori_ikala(os.path.join(file_path, file_name), file_name)
mode = 0
elif file_name.startswith('mir'):
file_name = file_name[4:]
file_path = config.wav_dir_mir
utils.write_ori_ikala(os.path.join(file_path, file_name), file_name)
mode = 0
elif file_name.startswith('med'):
file_name = file_name[4:]
file_path = config.wav_dir_med
utils.write_ori_med(os.path.join(file_path, file_name), file_name)
mode = 2
else:
mode = 1
file_path = './'
stat_file = h5py.File(config.stat_dir + 'stats.hdf5', mode='r')
max_feat = np.array(stat_file["feats_maximus"])
min_feat = np.array(stat_file["feats_minimus"])
max_voc = np.array(stat_file["voc_stft_maximus"])
min_voc = np.array(stat_file["voc_stft_minimus"])
max_back = np.array(stat_file["back_stft_maximus"])
min_back = np.array(stat_file["back_stft_minimus"])
max_mix = np.array(max_voc) + np.array(max_back)
with tf.Graph().as_default():
input_placeholder = tf.placeholder(tf.float32,
shape=(config.batch_size,
config.max_phr_len,
config.input_features),
name='input_placeholder')
with tf.variable_scope('First_Model') as scope:
harm, ap, f0, vuv = modules.nr_wavenet(input_placeholder)
# harmy = harm_1+harm
if config.use_gan:
with tf.variable_scope('Generator') as scope:
gen_op = modules.GAN_generator(harm)
# with tf.variable_scope('Discriminator') as scope:
# D_real = modules.GAN_discriminator(target_placeholder[:,:,:60],input_placeholder)
# scope.reuse_variables()
# D_fake = modules.GAN_discriminator(gen_op,input_placeholder)
saver = tf.train.Saver(max_to_keep=config.max_models_to_keep)
init_op = tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer())
sess = tf.Session()
sess.run(init_op)
ckpt = tf.train.get_checkpoint_state(config.log_dir_m1)
if ckpt and ckpt.model_checkpoint_path:
print("Using the model in %s" % ckpt.model_checkpoint_path)
saver.restore(sess, ckpt.model_checkpoint_path)
mix_stft = utils.file_to_stft(os.path.join(file_path, file_name),
mode=mode)
targs = utils.input_to_feats(os.path.join(file_path, file_name),
mode=mode)
import pdb
pdb.set_trace()
# f0_sac = utils.file_to_sac(os.path.join(file_path,file_name))
# f0_sac = (f0_sac-min_feat[-2])/(max_feat[-2]-min_feat[-2])
in_batches, nchunks_in = utils.generate_overlapadd(mix_stft)
in_batches = in_batches / max_mix
# in_batches = utils.normalize(in_batches, 'mix_stft', mode=config.norm_mode_in)
val_outer = []
first_pred = []
cleaner = []
gan_op = []
for in_batch in in_batches:
val_harm, val_ap, val_f0, val_vuv = sess.run(
[harm, ap, f0, vuv], feed_dict={input_placeholder: in_batch})
if config.use_gan:
val_op = sess.run(gen_op,
feed_dict={input_placeholder: in_batch})
gan_op.append(val_op)
# first_pred.append(harm1)
# cleaner.append(val_harm)
val_harm = val_harm
val_outs = np.concatenate((val_harm, val_ap, val_f0, val_vuv),
axis=-1)
val_outer.append(val_outs)
val_outer = np.array(val_outer)
val_outer = utils.overlapadd(val_outer, nchunks_in)
val_outer[:, -1] = np.round(val_outer[:, -1])
val_outer = val_outer[:targs.shape[0], :]
val_outer = np.clip(val_outer, 0.0, 1.0)
import pdb
pdb.set_trace()
#Test purposes only
# first_pred = np.array(first_pred)
# first_pred = utils.overlapadd(first_pred, nchunks_in)
# cleaner = np.array(cleaner)
# cleaner = utils.overlapadd(cleaner, nchunks_in)
if config.use_gan:
gan_op = np.array(gan_op)
gan_op = utils.overlapadd(gan_op, nchunks_in)
targs = (targs - min_feat) / (max_feat - min_feat)
# first_pred = (first_pred-min_feat[:60])/(max_feat[:60]-min_feat[:60])
# cleaner = (cleaner-min_feat[:60])/(max_feat[:60]-min_feat[:60])
# ax1 = plt.subplot(311)
# plt.imshow(targs[:,:60].T, origin='lower', aspect='auto')
# # ax1.set_title("Harmonic Spectral Envelope", fontsize = 10)
# ax2 = plt.subplot(312)
# plt.imshow(targs[:,60:64].T, origin='lower', aspect='auto')
# # ax2.set_title("Aperiodicity Envelope", fontsize = 10)
# ax3 = plt.subplot(313)
# plt.plot(targs[:,-2])
# ax3.set_title("Fundamental Frequency Contour", fontsize = 10)
if show_plots:
# import pdb;pdb.set_trace()
ins = val_outer[:, :60]
outs = targs[:, :60]
plt.figure(1)
ax1 = plt.subplot(211)
plt.imshow(ins.T, origin='lower', aspect='auto')
ax1.set_title("Predicted Harm ", fontsize=10)
ax2 = plt.subplot(212)
plt.imshow(outs.T, origin='lower', aspect='auto')
ax2.set_title("Ground Truth Harm ", fontsize=10)
# ax1 = plt.subplot(413)
# plt.imshow(first_pred.T, origin='lower', aspect='auto')
# ax1.set_title("Initial Prediction ", fontsize = 10)
# ax2 = plt.subplot(412)
# plt.imshow(cleaner.T, origin='lower', aspect='auto')
# ax2.set_title("Residual Added ", fontsize = 10)
if config.use_gan:
plt.figure(5)
ax1 = plt.subplot(411)
plt.imshow(ins.T, origin='lower', aspect='auto')
ax1.set_title("Predicted Harm ", fontsize=10)
ax2 = plt.subplot(414)
plt.imshow(outs.T, origin='lower', aspect='auto')
ax2.set_title("Ground Truth Harm ", fontsize=10)
ax1 = plt.subplot(412)
plt.imshow(gan_op.T, origin='lower', aspect='auto')
ax1.set_title("GAN output ", fontsize=10)
ax1 = plt.subplot(413)
plt.imshow((gan_op[:ins.shape[0], :] + ins).T,
origin='lower',
aspect='auto')
ax1.set_title("GAN output ", fontsize=10)
plt.figure(2)
ax1 = plt.subplot(211)
plt.imshow(val_outer[:, 60:-2].T, origin='lower', aspect='auto')
ax1.set_title("Predicted Aperiodic Part", fontsize=10)
ax2 = plt.subplot(212)
plt.imshow(targs[:, 60:-2].T, origin='lower', aspect='auto')
ax2.set_title("Ground Truth Aperiodic Part", fontsize=10)
plt.figure(3)
f0_output = val_outer[:, -2] * (
(max_feat[-2] - min_feat[-2]) + min_feat[-2])
f0_output = f0_output * (1 - targs[:, -1])
f0_output[f0_output == 0] = np.nan
plt.plot(f0_output, label="Predicted Value")
f0_gt = targs[:, -2] * (
(max_feat[-2] - min_feat[-2]) + min_feat[-2])
f0_gt = f0_gt * (1 - targs[:, -1])
f0_gt[f0_gt == 0] = np.nan
plt.plot(f0_gt, label="Ground Truth")
f0_difference = np.nan_to_num(abs(f0_gt - f0_output))
f0_greater = np.where(f0_difference > config.f0_threshold)
diff_per = f0_greater[0].shape[0] / len(f0_output)
plt.suptitle("Percentage correct = " +
'{:.3%}'.format(1 - diff_per))
# import pdb;pdb.set_trace()
# import pdb;pdb.set_trace()
# uu = f0_sac[:,0]*(1-f0_sac[:,1])
# uu[uu == 0] = np.nan
# plt.plot(uu, label="Sac f0")
plt.legend()
plt.figure(4)
ax1 = plt.subplot(211)
plt.plot(val_outer[:, -1])
ax1.set_title("Predicted Voiced/Unvoiced", fontsize=10)
ax2 = plt.subplot(212)
plt.plot(targs[:, -1])
ax2.set_title("Ground Truth Voiced/Unvoiced", fontsize=10)
plt.show()
if save_file:
val_outer = np.ascontiguousarray(val_outer *
(max_feat - min_feat) + min_feat)
targs = np.ascontiguousarray(targs * (max_feat - min_feat) +
min_feat)
# import pdb;pdb.set_trace()
# val_outer = np.ascontiguousarray(utils.denormalize(val_outer,'feats', mode=config.norm_mode_out))
try:
utils.feats_to_audio(val_outer,
file_name[:-4] + '_synth_pred_f0')
print("File saved to %s" % config.val_dir + file_name[:-4] +
'_synth_pred_f0.wav')
except:
print("Couldn't synthesize with predicted f0")
try:
val_outer[:, -2:] = targs[:, -2:]
utils.feats_to_audio(val_outer,
file_name[:-4] + '_synth_ori_f0')
print("File saved to %s" % config.val_dir + file_name[:-4] +
'_synth_ori_f0.wav')
except:
print("Couldn't synthesize with original f0")
def eval_file():
file_path = config.wav_dir
# log_dir = './log_ikala_notrain/'
log_dir = config.log_dir
mode = 0
stat_file = h5py.File(config.stat_dir + 'stats.hdf5', mode='r')
max_feat = np.array(stat_file["feats_maximus"])
min_feat = np.array(stat_file["feats_minimus"])
max_voc = np.array(stat_file["voc_stft_maximus"])
min_voc = np.array(stat_file["voc_stft_minimus"])
max_back = np.array(stat_file["back_stft_maximus"])
min_back = np.array(stat_file["back_stft_minimus"])
max_mix = np.array(max_voc) + np.array(max_back)
with tf.Graph().as_default():
input_placeholder = tf.placeholder(tf.float32,
shape=(config.batch_size,
config.max_phr_len,
config.input_features),
name='input_placeholder')
with tf.variable_scope('First_Model') as scope:
harm, ap, f0, vuv = modules.nr_wavenet(input_placeholder)
saver = tf.train.Saver(max_to_keep=config.max_models_to_keep)
init_op = tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer())
sess = tf.Session()
sess.run(init_op)
ckpt = tf.train.get_checkpoint_state(log_dir)
if ckpt and ckpt.model_checkpoint_path:
print("Using the model in %s" % ckpt.model_checkpoint_path)
# saver.restore(sess, ckpt.model_checkpoint_path)
saver.restore(sess, './log/model.ckpt-59')
# import pdb;pdb.set_trace()
files = [
x for x in os.listdir(config.wav_dir)
if x.endswith('.wav') and not x.startswith('.')
]
diffs = []
count = 0
for file_name in files:
count += 1
mix_stft = utils.file_to_stft(os.path.join(file_path, file_name),
mode=mode)
targs = utils.input_to_feats(os.path.join(file_path, file_name),
mode=mode)
# f0_sac = utils.file_to_sac(os.path.join(file_path,file_name))
# f0_sac = (f0_sac-min_feat[-2])/(max_feat[-2]-min_feat[-2])
in_batches, nchunks_in = utils.generate_overlapadd(mix_stft)
in_batches = in_batches / max_mix
# in_batches = utils.normalize(in_batches, 'mix_stft', mode=config.norm_mode_in)
val_outer = []
first_pred = []
cleaner = []
gan_op = []
for in_batch in in_batches:
val_harm, val_ap, val_f0, val_vuv = sess.run(
[harm, ap, f0, vuv],
feed_dict={input_placeholder: in_batch})
if config.use_gan:
val_op = sess.run(gen_op,
feed_dict={input_placeholder: in_batch})
gan_op.append(val_op)
# first_pred.append(harm1)
# cleaner.append(val_harm)
val_harm = val_harm
val_outs = np.concatenate((val_harm, val_ap, val_f0, val_vuv),
axis=-1)
val_outer.append(val_outs)
val_outer = np.array(val_outer)
val_outer = utils.overlapadd(val_outer, nchunks_in)
val_outer[:, -1] = np.round(val_outer[:, -1])
val_outer = val_outer[:targs.shape[0], :]
val_outer = np.clip(val_outer, 0.0, 1.0)
#Test purposes only
# first_pred = np.array(first_pred)
# first_pred = utils.overlapadd(first_pred, nchunks_in)
# cleaner = np.array(cleaner)
# cleaner = utils.overlapadd(cleaner, nchunks_in)
f0_output = val_outer[:, -2] * (
(max_feat[-2] - min_feat[-2]) + min_feat[-2])
f0_output = f0_output * (1 - targs[:, -1])
f0_output = utils.new_base_to_hertz(f0_output)
f0_gt = targs[:, -2]
f0_gt = f0_gt * (1 - targs[:, -1])
f0_gt = utils.new_base_to_hertz(f0_gt)
f0_outputs = []
gt_outputs = []
for i, f0_o in enumerate(f0_output):
f0_outputs.append(
str(i * 0.00580498866 * 10000000) + ' ' + str(f0_o))
for i, f0_o in enumerate(f0_gt):
gt_outputs.append(
str(i * 0.00580498866 * 10000000) + ' ' + str(f0_o))
utils.list_to_file(
f0_outputs, './ikala_eval/net_out/' + file_name[:-4] + '.pv')
utils.list_to_file(gt_outputs,
'./ikala_eval/sac_gt/' + file_name[:-4] + '.pv')
# f0_difference = np.nan_to_num(abs(f0_gt-f0_output))
# f0_greater = np.where(f0_difference>config.f0_threshold)
# diff_per = f0_greater[0].shape[0]/len(f0_output)
# diffs.append(str(1-diff_per))
utils.progress(count, len(files))
def synth_file(file_name,
file_path=config.wav_dir,
show_plots=True,
save_file=True):
debug = False
if file_name.startswith('ikala'):
file_name = file_name[6:]
file_path = config.wav_dir
utils.write_ori_ikala(os.path.join(file_path, file_name), file_name)
elif file_name.startswith('mir'):
file_name = file_name[4:]
file_path = config.wav_dir_mir
utils.write_ori_ikala(os.path.join(file_path, file_name), file_name)
stat_file = h5py.File(config.stat_dir + 'stats.hdf5', mode='r')
max_feat = np.array(stat_file["feats_maximus"])
min_feat = np.array(stat_file["feats_minimus"])
max_voc = np.array(stat_file["voc_stft_maximus"])
min_voc = np.array(stat_file["voc_stft_minimus"])
max_back = np.array(stat_file["back_stft_maximus"])
min_back = np.array(stat_file["back_stft_minimus"])
max_mix = np.array(max_voc) + np.array(max_back)
with tf.Graph().as_default():
input_placeholder = tf.placeholder(tf.float32,
shape=(1, config.max_phr_len,
config.input_features),
name='input_placeholder')
tf.summary.histogram('conditioning', input_placeholder)
input_placeholder_2 = tf.placeholder(tf.float32,
shape=(1, config.max_phr_len,
config.output_features),
name='input_placeholder')
tf.summary.histogram('inputs', input_placeholder)
output, vuv = modules.wavenet(input_placeholder_2, input_placeholder)
saver = tf.train.Saver()
init_op = tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer())
sess = tf.Session()
sess.run(init_op)
ckpt = tf.train.get_checkpoint_state(config.log_dir)
if ckpt and ckpt.model_checkpoint_path:
print("Using the model in %s" % ckpt.model_checkpoint_path)
saver.restore(sess, ckpt.model_checkpoint_path)
mix_stft = utils.file_to_stft(os.path.join(file_path, file_name))
mix_stft = mix_stft / max_mix
lent = len(mix_stft)
mix_stft_in = np.pad(mix_stft, [(0, config.max_phr_len), (0, 0)],
'constant')
# import pdb;pdb.set_trace()
targs = utils.input_to_feats(os.path.join(file_path, file_name))
outputs = np.zeros((1, config.max_phr_len, config.output_features))
opus = []
i = 0
for index in range(lent):
inputs = mix_stft_in[index:index + config.max_phr_len, :]
if outputs.shape[1] > config.max_phr_len:
inpy = inputs.reshape(1, config.max_phr_len, -1)
outpy = outputs[:, -config.max_phr_len:, :]
else:
inpy = inputs.reshape(1, config.max_phr_len, -1)
outpy = outputs
# import pdb;pdb.set_trace()
op, vu = sess.run([output, vuv],
feed_dict={
input_placeholder: inpy,
input_placeholder_2: outpy
})
op = np.concatenate((op, vu), axis=-1)
if debug:
plt.figure(1)
plt.subplot(311)
plt.imshow(np.log(inputs.T), aspect='auto', origin='lower')
plt.subplot(312)
plt.imshow(targs[index:index + config.max_phr_len, :].T,
aspect='auto',
origin='lower')
plt.subplot(313)
plt.imshow(outpy.reshape(config.max_phr_len, -1).T,
aspect='auto',
origin='lower')
plt.show()
import pdb
pdb.set_trace()
if index > config.max_phr_len:
outputs = np.append(outputs, op[:, -1:, :], axis=1)
else:
outputs[:, :index, :] = op[:, :index, :]
val_outer = outputs[0]
val_outer[:, -1] = np.round(val_outer[:, -1])
val_outer = val_outer[:targs.shape[0], :]
val_outer = np.clip(val_outer, 0.0, 1.0)
#Test purposes only
# targs = utils.normalize(targs, 'feats', mode=config.norm_mode_out)
targs = (targs - min_feat) / (max_feat - min_feat)
if show_plots:
# import pdb;pdb.set_trace()
ins = val_outer[:, :60]
outs = targs[:, :60]
plt.figure(1)
ax1 = plt.subplot(211)
plt.imshow(ins.T, origin='lower', aspect='auto')
ax1.set_title("Predicted Harm ", fontsize=10)
ax2 = plt.subplot(212)
plt.imshow(outs.T, origin='lower', aspect='auto')
ax2.set_title("Ground Truth Harm ", fontsize=10)
plt.figure(2)
ax1 = plt.subplot(211)
plt.imshow(val_outer[:, 60:-2].T, origin='lower', aspect='auto')
ax1.set_title("Predicted Aperiodic Part", fontsize=10)
ax2 = plt.subplot(212)
plt.imshow(targs[:, 60:-2].T, origin='lower', aspect='auto')
ax2.set_title("Ground Truth Aperiodic Part", fontsize=10)
plt.figure(3)
uu = val_outer[:, -2] * (1 - targs[:, -1])
uu[uu == 0] = np.nan
plt.plot(uu, label="Predicted Value")
uu = targs[:, -2] * (1 - targs[:, -1])
uu[uu == 0] = np.nan
plt.plot(uu, label="Ground Truth")
# uu = f0_sac[:,0]*(1-f0_sac[:,1])
# uu[uu == 0] = np.nan
# plt.plot(uu, label="Sac f0")
plt.legend()
plt.figure(4)
ax1 = plt.subplot(211)
plt.plot(val_outer[:, -1])
ax1.set_title("Predicted Voiced/Unvoiced", fontsize=10)
ax2 = plt.subplot(212)
plt.plot(targs[:, -1])
ax2.set_title("Ground Truth Voiced/Unvoiced", fontsize=10)
plt.show()
if save_file:
val_outer = np.ascontiguousarray(val_outer *
(max_feat - min_feat) + min_feat)
targs = np.ascontiguousarray(targs * (max_feat - min_feat) +
min_feat)
# val_outer = np.ascontiguousarray(utils.denormalize(val_outer,'feats', mode=config.norm_mode_out))
try:
utils.feats_to_audio(val_outer,
file_name[:-4] + '_synth_pred_f0')
print("File saved to %s" % config.val_dir + file_name[:-4] +
'_synth_pred_f0.wav')
except:
print("Couldn't synthesize with predicted f0")
try:
val_outer[:, -2:] = targs[:, -2:]
utils.feats_to_audio(val_outer,
file_name[:-4] + '_synth_ori_f0')
print("File saved to %s" % config.val_dir + file_name[:-4] +
'_synth_ori_f0.wav')
except:
print("Couldn't synthesize with original f0")