def test_model_yam(self): """ Function to extract vocals from wav file. """ sess = tf.Session() self.load_model(sess, log_dir=config.log_dir) voc_stft = utils.file_to_stft('./Bria_000_VoU67.wav') back_stft = utils.file_to_stft('./Bria_000_Back.wav') mix_stft = np.clip( (voc_stft[:len(back_stft)] + back_stft[:len(voc_stft)]) / 2, 0.0, 1.0) feats = utils.input_to_feats('./Bria_000_VoU67.wav') out_feats = self.process_file(mix_stft, sess) self.plot_features(feats, out_feats) out_featss = np.concatenate( (out_feats[:feats.shape[0], :-2], feats[:out_feats.shape[0], -2:]), axis=-1) utils.feats_to_audio(out_featss[:3000], 'Bree_output')
def main(): # maximus=np.zeros(66) # minimus=np.ones(66)*1000 wav_files = [x for x in os.listdir(config.wav_dir) if x.endswith('.wav')] count = 0 for lf in wav_files: # print(lf) audio, fs = sf.read(os.path.join(config.wav_dir, lf)) vocals = np.array(audio[:, 1]) mixture = np.clip(audio[:, 0] + audio[:, 1], 0.0, 1.0) backing = np.array(audio[:, 0]) voc_stft = abs(utils.stft(vocals)) mix_stft = abs(utils.stft(mixture)) back_stft = abs(utils.stft(backing)) assert voc_stft.shape == mix_stft.shape out_feats = utils.input_to_feats(os.path.join(config.wav_dir, lf)) out_feats = np.concatenate( ((out_feats, np.zeros((1, out_feats.shape[1]))))) assert out_feats.shape[0] == voc_stft.shape[0] np.save(config.dir_npy + lf[:-4] + '_voc_stft', voc_stft) np.save(config.dir_npy + lf[:-4] + '_mix_stft', mix_stft) np.save(config.dir_npy + lf[:-4] + '_back_stft', back_stft) np.save(config.dir_npy + lf[:-4] + '_synth_feats', out_feats) count += 1 utils.progress(count, len(wav_files)) import pdb pdb.set_trace()
def synth_file(file_name, file_path=config.wav_dir, show_plots=True, save_file=True): if file_name.startswith('ikala'): file_name = file_name[6:] file_path = config.wav_dir utils.write_ori_ikala(os.path.join(file_path, file_name), file_name) mode = 0 elif file_name.startswith('mir'): file_name = file_name[4:] file_path = config.wav_dir_mir utils.write_ori_ikala(os.path.join(file_path, file_name), file_name) mode = 0 elif file_name.startswith('med'): file_name = file_name[4:] file_path = config.wav_dir_med utils.write_ori_med(os.path.join(file_path, file_name), file_name) mode = 2 else: mode = 1 file_path = './' stat_file = h5py.File(config.stat_dir + 'stats.hdf5', mode='r') max_feat = np.array(stat_file["feats_maximus"]) min_feat = np.array(stat_file["feats_minimus"]) max_voc = np.array(stat_file["voc_stft_maximus"]) min_voc = np.array(stat_file["voc_stft_minimus"]) max_back = np.array(stat_file["back_stft_maximus"]) min_back = np.array(stat_file["back_stft_minimus"]) max_mix = np.array(max_voc) + np.array(max_back) with tf.Graph().as_default(): input_placeholder = tf.placeholder(tf.float32, shape=(config.batch_size, config.max_phr_len, config.input_features), name='input_placeholder') with tf.variable_scope('First_Model') as scope: harm, ap, f0, vuv = modules.nr_wavenet(input_placeholder) # harmy = harm_1+harm if config.use_gan: with tf.variable_scope('Generator') as scope: gen_op = modules.GAN_generator(harm) # with tf.variable_scope('Discriminator') as scope: # D_real = modules.GAN_discriminator(target_placeholder[:,:,:60],input_placeholder) # scope.reuse_variables() # D_fake = modules.GAN_discriminator(gen_op,input_placeholder) saver = tf.train.Saver(max_to_keep=config.max_models_to_keep) init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer()) sess = tf.Session() sess.run(init_op) ckpt = tf.train.get_checkpoint_state(config.log_dir_m1) if ckpt and ckpt.model_checkpoint_path: print("Using the model in %s" % ckpt.model_checkpoint_path) saver.restore(sess, ckpt.model_checkpoint_path) mix_stft = utils.file_to_stft(os.path.join(file_path, file_name), mode=mode) targs = utils.input_to_feats(os.path.join(file_path, file_name), mode=mode) import pdb pdb.set_trace() # f0_sac = utils.file_to_sac(os.path.join(file_path,file_name)) # f0_sac = (f0_sac-min_feat[-2])/(max_feat[-2]-min_feat[-2]) in_batches, nchunks_in = utils.generate_overlapadd(mix_stft) in_batches = in_batches / max_mix # in_batches = utils.normalize(in_batches, 'mix_stft', mode=config.norm_mode_in) val_outer = [] first_pred = [] cleaner = [] gan_op = [] for in_batch in in_batches: val_harm, val_ap, val_f0, val_vuv = sess.run( [harm, ap, f0, vuv], feed_dict={input_placeholder: in_batch}) if config.use_gan: val_op = sess.run(gen_op, feed_dict={input_placeholder: in_batch}) gan_op.append(val_op) # first_pred.append(harm1) # cleaner.append(val_harm) val_harm = val_harm val_outs = np.concatenate((val_harm, val_ap, val_f0, val_vuv), axis=-1) val_outer.append(val_outs) val_outer = np.array(val_outer) val_outer = utils.overlapadd(val_outer, nchunks_in) val_outer[:, -1] = np.round(val_outer[:, -1]) val_outer = val_outer[:targs.shape[0], :] val_outer = np.clip(val_outer, 0.0, 1.0) import pdb pdb.set_trace() #Test purposes only # first_pred = np.array(first_pred) # first_pred = utils.overlapadd(first_pred, nchunks_in) # cleaner = np.array(cleaner) # cleaner = utils.overlapadd(cleaner, nchunks_in) if config.use_gan: gan_op = np.array(gan_op) gan_op = utils.overlapadd(gan_op, nchunks_in) targs = (targs - min_feat) / (max_feat - min_feat) # first_pred = (first_pred-min_feat[:60])/(max_feat[:60]-min_feat[:60]) # cleaner = (cleaner-min_feat[:60])/(max_feat[:60]-min_feat[:60]) # ax1 = plt.subplot(311) # plt.imshow(targs[:,:60].T, origin='lower', aspect='auto') # # ax1.set_title("Harmonic Spectral Envelope", fontsize = 10) # ax2 = plt.subplot(312) # plt.imshow(targs[:,60:64].T, origin='lower', aspect='auto') # # ax2.set_title("Aperiodicity Envelope", fontsize = 10) # ax3 = plt.subplot(313) # plt.plot(targs[:,-2]) # ax3.set_title("Fundamental Frequency Contour", fontsize = 10) if show_plots: # import pdb;pdb.set_trace() ins = val_outer[:, :60] outs = targs[:, :60] plt.figure(1) ax1 = plt.subplot(211) plt.imshow(ins.T, origin='lower', aspect='auto') ax1.set_title("Predicted Harm ", fontsize=10) ax2 = plt.subplot(212) plt.imshow(outs.T, origin='lower', aspect='auto') ax2.set_title("Ground Truth Harm ", fontsize=10) # ax1 = plt.subplot(413) # plt.imshow(first_pred.T, origin='lower', aspect='auto') # ax1.set_title("Initial Prediction ", fontsize = 10) # ax2 = plt.subplot(412) # plt.imshow(cleaner.T, origin='lower', aspect='auto') # ax2.set_title("Residual Added ", fontsize = 10) if config.use_gan: plt.figure(5) ax1 = plt.subplot(411) plt.imshow(ins.T, origin='lower', aspect='auto') ax1.set_title("Predicted Harm ", fontsize=10) ax2 = plt.subplot(414) plt.imshow(outs.T, origin='lower', aspect='auto') ax2.set_title("Ground Truth Harm ", fontsize=10) ax1 = plt.subplot(412) plt.imshow(gan_op.T, origin='lower', aspect='auto') ax1.set_title("GAN output ", fontsize=10) ax1 = plt.subplot(413) plt.imshow((gan_op[:ins.shape[0], :] + ins).T, origin='lower', aspect='auto') ax1.set_title("GAN output ", fontsize=10) plt.figure(2) ax1 = plt.subplot(211) plt.imshow(val_outer[:, 60:-2].T, origin='lower', aspect='auto') ax1.set_title("Predicted Aperiodic Part", fontsize=10) ax2 = plt.subplot(212) plt.imshow(targs[:, 60:-2].T, origin='lower', aspect='auto') ax2.set_title("Ground Truth Aperiodic Part", fontsize=10) plt.figure(3) f0_output = val_outer[:, -2] * ( (max_feat[-2] - min_feat[-2]) + min_feat[-2]) f0_output = f0_output * (1 - targs[:, -1]) f0_output[f0_output == 0] = np.nan plt.plot(f0_output, label="Predicted Value") f0_gt = targs[:, -2] * ( (max_feat[-2] - min_feat[-2]) + min_feat[-2]) f0_gt = f0_gt * (1 - targs[:, -1]) f0_gt[f0_gt == 0] = np.nan plt.plot(f0_gt, label="Ground Truth") f0_difference = np.nan_to_num(abs(f0_gt - f0_output)) f0_greater = np.where(f0_difference > config.f0_threshold) diff_per = f0_greater[0].shape[0] / len(f0_output) plt.suptitle("Percentage correct = " + '{:.3%}'.format(1 - diff_per)) # import pdb;pdb.set_trace() # import pdb;pdb.set_trace() # uu = f0_sac[:,0]*(1-f0_sac[:,1]) # uu[uu == 0] = np.nan # plt.plot(uu, label="Sac f0") plt.legend() plt.figure(4) ax1 = plt.subplot(211) plt.plot(val_outer[:, -1]) ax1.set_title("Predicted Voiced/Unvoiced", fontsize=10) ax2 = plt.subplot(212) plt.plot(targs[:, -1]) ax2.set_title("Ground Truth Voiced/Unvoiced", fontsize=10) plt.show() if save_file: val_outer = np.ascontiguousarray(val_outer * (max_feat - min_feat) + min_feat) targs = np.ascontiguousarray(targs * (max_feat - min_feat) + min_feat) # import pdb;pdb.set_trace() # val_outer = np.ascontiguousarray(utils.denormalize(val_outer,'feats', mode=config.norm_mode_out)) try: utils.feats_to_audio(val_outer, file_name[:-4] + '_synth_pred_f0') print("File saved to %s" % config.val_dir + file_name[:-4] + '_synth_pred_f0.wav') except: print("Couldn't synthesize with predicted f0") try: val_outer[:, -2:] = targs[:, -2:] utils.feats_to_audio(val_outer, file_name[:-4] + '_synth_ori_f0') print("File saved to %s" % config.val_dir + file_name[:-4] + '_synth_ori_f0.wav') except: print("Couldn't synthesize with original f0")
def eval_file(): file_path = config.wav_dir # log_dir = './log_ikala_notrain/' log_dir = config.log_dir mode = 0 stat_file = h5py.File(config.stat_dir + 'stats.hdf5', mode='r') max_feat = np.array(stat_file["feats_maximus"]) min_feat = np.array(stat_file["feats_minimus"]) max_voc = np.array(stat_file["voc_stft_maximus"]) min_voc = np.array(stat_file["voc_stft_minimus"]) max_back = np.array(stat_file["back_stft_maximus"]) min_back = np.array(stat_file["back_stft_minimus"]) max_mix = np.array(max_voc) + np.array(max_back) with tf.Graph().as_default(): input_placeholder = tf.placeholder(tf.float32, shape=(config.batch_size, config.max_phr_len, config.input_features), name='input_placeholder') with tf.variable_scope('First_Model') as scope: harm, ap, f0, vuv = modules.nr_wavenet(input_placeholder) saver = tf.train.Saver(max_to_keep=config.max_models_to_keep) init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer()) sess = tf.Session() sess.run(init_op) ckpt = tf.train.get_checkpoint_state(log_dir) if ckpt and ckpt.model_checkpoint_path: print("Using the model in %s" % ckpt.model_checkpoint_path) # saver.restore(sess, ckpt.model_checkpoint_path) saver.restore(sess, './log/model.ckpt-59') # import pdb;pdb.set_trace() files = [ x for x in os.listdir(config.wav_dir) if x.endswith('.wav') and not x.startswith('.') ] diffs = [] count = 0 for file_name in files: count += 1 mix_stft = utils.file_to_stft(os.path.join(file_path, file_name), mode=mode) targs = utils.input_to_feats(os.path.join(file_path, file_name), mode=mode) # f0_sac = utils.file_to_sac(os.path.join(file_path,file_name)) # f0_sac = (f0_sac-min_feat[-2])/(max_feat[-2]-min_feat[-2]) in_batches, nchunks_in = utils.generate_overlapadd(mix_stft) in_batches = in_batches / max_mix # in_batches = utils.normalize(in_batches, 'mix_stft', mode=config.norm_mode_in) val_outer = [] first_pred = [] cleaner = [] gan_op = [] for in_batch in in_batches: val_harm, val_ap, val_f0, val_vuv = sess.run( [harm, ap, f0, vuv], feed_dict={input_placeholder: in_batch}) if config.use_gan: val_op = sess.run(gen_op, feed_dict={input_placeholder: in_batch}) gan_op.append(val_op) # first_pred.append(harm1) # cleaner.append(val_harm) val_harm = val_harm val_outs = np.concatenate((val_harm, val_ap, val_f0, val_vuv), axis=-1) val_outer.append(val_outs) val_outer = np.array(val_outer) val_outer = utils.overlapadd(val_outer, nchunks_in) val_outer[:, -1] = np.round(val_outer[:, -1]) val_outer = val_outer[:targs.shape[0], :] val_outer = np.clip(val_outer, 0.0, 1.0) #Test purposes only # first_pred = np.array(first_pred) # first_pred = utils.overlapadd(first_pred, nchunks_in) # cleaner = np.array(cleaner) # cleaner = utils.overlapadd(cleaner, nchunks_in) f0_output = val_outer[:, -2] * ( (max_feat[-2] - min_feat[-2]) + min_feat[-2]) f0_output = f0_output * (1 - targs[:, -1]) f0_output = utils.new_base_to_hertz(f0_output) f0_gt = targs[:, -2] f0_gt = f0_gt * (1 - targs[:, -1]) f0_gt = utils.new_base_to_hertz(f0_gt) f0_outputs = [] gt_outputs = [] for i, f0_o in enumerate(f0_output): f0_outputs.append( str(i * 0.00580498866 * 10000000) + ' ' + str(f0_o)) for i, f0_o in enumerate(f0_gt): gt_outputs.append( str(i * 0.00580498866 * 10000000) + ' ' + str(f0_o)) utils.list_to_file( f0_outputs, './ikala_eval/net_out/' + file_name[:-4] + '.pv') utils.list_to_file(gt_outputs, './ikala_eval/sac_gt/' + file_name[:-4] + '.pv') # f0_difference = np.nan_to_num(abs(f0_gt-f0_output)) # f0_greater = np.where(f0_difference>config.f0_threshold) # diff_per = f0_greater[0].shape[0]/len(f0_output) # diffs.append(str(1-diff_per)) utils.progress(count, len(files))
def synth_file(file_name, file_path=config.wav_dir, show_plots=True, save_file=True): debug = False if file_name.startswith('ikala'): file_name = file_name[6:] file_path = config.wav_dir utils.write_ori_ikala(os.path.join(file_path, file_name), file_name) elif file_name.startswith('mir'): file_name = file_name[4:] file_path = config.wav_dir_mir utils.write_ori_ikala(os.path.join(file_path, file_name), file_name) stat_file = h5py.File(config.stat_dir + 'stats.hdf5', mode='r') max_feat = np.array(stat_file["feats_maximus"]) min_feat = np.array(stat_file["feats_minimus"]) max_voc = np.array(stat_file["voc_stft_maximus"]) min_voc = np.array(stat_file["voc_stft_minimus"]) max_back = np.array(stat_file["back_stft_maximus"]) min_back = np.array(stat_file["back_stft_minimus"]) max_mix = np.array(max_voc) + np.array(max_back) with tf.Graph().as_default(): input_placeholder = tf.placeholder(tf.float32, shape=(1, config.max_phr_len, config.input_features), name='input_placeholder') tf.summary.histogram('conditioning', input_placeholder) input_placeholder_2 = tf.placeholder(tf.float32, shape=(1, config.max_phr_len, config.output_features), name='input_placeholder') tf.summary.histogram('inputs', input_placeholder) output, vuv = modules.wavenet(input_placeholder_2, input_placeholder) saver = tf.train.Saver() init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer()) sess = tf.Session() sess.run(init_op) ckpt = tf.train.get_checkpoint_state(config.log_dir) if ckpt and ckpt.model_checkpoint_path: print("Using the model in %s" % ckpt.model_checkpoint_path) saver.restore(sess, ckpt.model_checkpoint_path) mix_stft = utils.file_to_stft(os.path.join(file_path, file_name)) mix_stft = mix_stft / max_mix lent = len(mix_stft) mix_stft_in = np.pad(mix_stft, [(0, config.max_phr_len), (0, 0)], 'constant') # import pdb;pdb.set_trace() targs = utils.input_to_feats(os.path.join(file_path, file_name)) outputs = np.zeros((1, config.max_phr_len, config.output_features)) opus = [] i = 0 for index in range(lent): inputs = mix_stft_in[index:index + config.max_phr_len, :] if outputs.shape[1] > config.max_phr_len: inpy = inputs.reshape(1, config.max_phr_len, -1) outpy = outputs[:, -config.max_phr_len:, :] else: inpy = inputs.reshape(1, config.max_phr_len, -1) outpy = outputs # import pdb;pdb.set_trace() op, vu = sess.run([output, vuv], feed_dict={ input_placeholder: inpy, input_placeholder_2: outpy }) op = np.concatenate((op, vu), axis=-1) if debug: plt.figure(1) plt.subplot(311) plt.imshow(np.log(inputs.T), aspect='auto', origin='lower') plt.subplot(312) plt.imshow(targs[index:index + config.max_phr_len, :].T, aspect='auto', origin='lower') plt.subplot(313) plt.imshow(outpy.reshape(config.max_phr_len, -1).T, aspect='auto', origin='lower') plt.show() import pdb pdb.set_trace() if index > config.max_phr_len: outputs = np.append(outputs, op[:, -1:, :], axis=1) else: outputs[:, :index, :] = op[:, :index, :] val_outer = outputs[0] val_outer[:, -1] = np.round(val_outer[:, -1]) val_outer = val_outer[:targs.shape[0], :] val_outer = np.clip(val_outer, 0.0, 1.0) #Test purposes only # targs = utils.normalize(targs, 'feats', mode=config.norm_mode_out) targs = (targs - min_feat) / (max_feat - min_feat) if show_plots: # import pdb;pdb.set_trace() ins = val_outer[:, :60] outs = targs[:, :60] plt.figure(1) ax1 = plt.subplot(211) plt.imshow(ins.T, origin='lower', aspect='auto') ax1.set_title("Predicted Harm ", fontsize=10) ax2 = plt.subplot(212) plt.imshow(outs.T, origin='lower', aspect='auto') ax2.set_title("Ground Truth Harm ", fontsize=10) plt.figure(2) ax1 = plt.subplot(211) plt.imshow(val_outer[:, 60:-2].T, origin='lower', aspect='auto') ax1.set_title("Predicted Aperiodic Part", fontsize=10) ax2 = plt.subplot(212) plt.imshow(targs[:, 60:-2].T, origin='lower', aspect='auto') ax2.set_title("Ground Truth Aperiodic Part", fontsize=10) plt.figure(3) uu = val_outer[:, -2] * (1 - targs[:, -1]) uu[uu == 0] = np.nan plt.plot(uu, label="Predicted Value") uu = targs[:, -2] * (1 - targs[:, -1]) uu[uu == 0] = np.nan plt.plot(uu, label="Ground Truth") # uu = f0_sac[:,0]*(1-f0_sac[:,1]) # uu[uu == 0] = np.nan # plt.plot(uu, label="Sac f0") plt.legend() plt.figure(4) ax1 = plt.subplot(211) plt.plot(val_outer[:, -1]) ax1.set_title("Predicted Voiced/Unvoiced", fontsize=10) ax2 = plt.subplot(212) plt.plot(targs[:, -1]) ax2.set_title("Ground Truth Voiced/Unvoiced", fontsize=10) plt.show() if save_file: val_outer = np.ascontiguousarray(val_outer * (max_feat - min_feat) + min_feat) targs = np.ascontiguousarray(targs * (max_feat - min_feat) + min_feat) # val_outer = np.ascontiguousarray(utils.denormalize(val_outer,'feats', mode=config.norm_mode_out)) try: utils.feats_to_audio(val_outer, file_name[:-4] + '_synth_pred_f0') print("File saved to %s" % config.val_dir + file_name[:-4] + '_synth_pred_f0.wav') except: print("Couldn't synthesize with predicted f0") try: val_outer[:, -2:] = targs[:, -2:] utils.feats_to_audio(val_outer, file_name[:-4] + '_synth_ori_f0') print("File saved to %s" % config.val_dir + file_name[:-4] + '_synth_ori_f0.wav') except: print("Couldn't synthesize with original f0")