def lpc_to_mag(v_lpc, fft_len=4096): ''' Computed the magnitude spectrum from LPC coefficients using approximation by FFT method. ''' v_imp = np.r_[1, np.zeros(fft_len-1)] v_imp_filt = lfilter(np.array([1.0]), v_lpc, v_imp) v_mag = np.absolute(np.fft.fft(v_imp_filt)) v_mag = la.remove_hermitian_half(v_mag[None,:])[0] return v_mag
def get_formant_locations_from_raw_long_frame(v_sig, v_pm, nx, fft_len): ''' nx: frame index ''' #v_sig, fs = la.read_audio_file(wavfile) # Epoch detection: #v_pm_sec, v_voi = la.reaper_epoch_detection(wavfile) #v_pm = lu.round_to_int(v_pm_sec * fs) # Raw-long Frame extraction: v_frm_long = v_sig[v_pm[nx-2]:v_pm[nx+2]+1] # Win: left_len = v_pm[nx] - v_pm[nx-2] right_len = v_pm[nx+2] - v_pm[nx] v_win = la.gen_non_symmetric_win(left_len, right_len, np.hanning, b_norm=False) v_frm_long_win = v_frm_long * v_win # Spectrum: v_mag = np.absolute(np.fft.fft(v_frm_long_win, n=fft_len)) v_mag_db = la.db(la.remove_hermitian_half(v_mag[None,:])[0]) # Formant extraction -LPC method:-------------------------------------------------- v_lpc, v_e, v_refl = lpc(v_frm_long_win, 120) b_use_lpc_roots = False if b_use_lpc_roots: v_lpc_roots = np.roots(v_lpc) v_lpc_angles = np.angle(v_lpc_roots) v_lpc_angles = v_lpc_angles[v_lpc_angles>=0] v_lpc_angles = np.sort(v_lpc_angles) fft_len_half = 1 + fft_len / 2 v_lpc_roots_bins = v_lpc_angles * fft_len_half / np.pi v_lpc_mag = lpc_to_mag(v_lpc, fft_len=fft_len) v_lpc_mag_db = la.db(v_lpc_mag) v_lpc_mag_db = v_lpc_mag_db - np.mean(v_lpc_mag_db) + np.mean(v_mag_db) v_frmnts_bins, v_frmnts_gains_db = get_formant_locations_from_spec_env(v_lpc_mag_db) # Getting bandwidth: fft_len_half = 1 + fft_len / 2 v_vall_bins = get_formant_locations_from_spec_env(-v_lpc_mag_db)[0] v_vall_bins = np.r_[0, v_vall_bins, fft_len_half-1] nfrmnts = v_frmnts_bins.size v_frmnts_bw = np.zeros(nfrmnts) - 1.0 for nx_f in xrange(nfrmnts): #Left slope: curr_frmnt_bin = v_frmnts_bins[nx_f] curr_vall_l_bin = v_vall_bins[nx_f] curr_vall_r_bin = v_vall_bins[nx_f+1] curr_midp_l = int((curr_frmnt_bin + curr_vall_l_bin) / 2.0) curr_midp_r = int((curr_frmnt_bin + curr_vall_r_bin) / 2.0) # Protection: if curr_midp_l==curr_frmnt_bin: curr_midp_l = curr_vall_l_bin if curr_midp_r==curr_frmnt_bin: curr_midp_r = curr_vall_r_bin #print(nx_f) # 27 y 32 #if ((nx==73) and (nx_f==27)): import ipdb; ipdb.set_trace(context=8) # breakpoint c4f78f1e // slope_l = (v_frmnts_gains_db[nx_f] - v_lpc_mag_db[curr_midp_l]) / (curr_frmnt_bin - curr_midp_l).astype(float) slope_r = (v_frmnts_gains_db[nx_f] - v_lpc_mag_db[curr_midp_r]) / (curr_frmnt_bin - curr_midp_r).astype(float) slope_ave = (slope_l - slope_r) / 2.0 v_frmnts_bw[nx_f] = 1.0 / slope_ave # Filtering by bandwidth: bw_thress = 7.0 v_frmnts_bins = v_frmnts_bins[v_frmnts_bw<bw_thress] v_frmnts_gains_db = v_frmnts_gains_db[v_frmnts_bw<bw_thress] v_frmnts_bw = v_frmnts_bw[v_frmnts_bw<bw_thress] # Computing frame short:-------------------------------- # Win: left_len_short = v_pm[nx] - v_pm[nx-1] right_len_short = v_pm[nx+1] - v_pm[nx] v_win_short = la.gen_non_symmetric_win(left_len_short, right_len_short, np.hanning, b_norm=False) v_frm_short = v_sig[v_pm[nx-1]:v_pm[nx+1]+1] v_frm_short_win = v_frm_short * v_win_short shift = v_pm[nx] - v_pm[nx-1] # Formant extraction - True envelope method:---------------------------------------- # Not finished. #v_true_env_db = la.true_envelope(v_mag_db[None,:], in_type='db', ncoeffs=400, thres_db=0.1)[0] if False: plt.figure(); plt.plot(v_mag_db); plt.plot(v_lpc_mag_db); plt.grid(); plt.show() return v_mag_db, v_lpc_mag_db, v_frmnts_bins, v_frmnts_gains_db, v_frmnts_bw, v_frm_short_win, shift
def speech_interp_with_anchors(wavfile_a, wavfile_b, nx_strt_a, nx_strt_b, nframes, fft_len): # MagPhase analysis: m_mag_a, m_real_a, m_imag_a, v_f0_a, fs, v_shift_a = mp.analysis_lossless(wavfile_a) m_mag_b, m_real_b, m_imag_b, v_f0_b, fs, v_shift_b = mp.analysis_lossless(wavfile_b) v_pm_a = la.shift_to_pm(v_shift_a) v_pm_b = la.shift_to_pm(v_shift_b) v_sig_a, fs = la.read_audio_file(wavfile_a) v_sig_b, fs = la.read_audio_file(wavfile_b) # Get formants: v_mag_db_a_dummy, v_lpc_mag_db_a, v_frmnts_bins_a, v_frmnts_gains_db_a, v_frmnts_bw_a, v_frm_short_a, shift_a = get_formant_locations_from_raw_long_frame(v_sig_a, v_pm_a, nx_strt_a, fft_len) v_mag_db_b_dummy, v_lpc_mag_db_b, v_frmnts_bins_b, v_frmnts_gains_db_b, v_frmnts_bw_b, v_frm_short_b, shift_b = get_formant_locations_from_raw_long_frame(v_sig_b, v_pm_b, nx_strt_a+nframes, fft_len) # Formant mapping:---------------------------------------------------------------- v_frmnts_bins_a_filt, v_frmnts_bins_b_filt = formant_mapping(v_frmnts_bins_a, v_frmnts_gains_db_a, v_frmnts_bins_b, v_frmnts_gains_db_b, fft_len) v_shifts_syn = np.zeros(nframes, dtype='int') m_frms_syn = np.zeros((nframes, fft_len)) for nx_frm in xrange(nframes): sp_weight = nx_frm / (nframes-1.0) nx_a = nx_strt_a + nx_frm nx_b = nx_strt_b + nx_frm # Computing mag spectrum: v_mag_db_a = get_formant_locations_from_raw_long_frame(v_sig_a, v_pm_a, nx_a, fft_len)[0] v_mag_db_b = get_formant_locations_from_raw_long_frame(v_sig_b, v_pm_b, nx_b, fft_len)[0] # NOT FINISHED !! # Warping:--------------------------------------------------------------------- # True envelope: v_true_env_db_a = la.true_envelope(v_mag_db_a[None,:], in_type='db', ncoeffs=400, thres_db=0.1)[0] v_true_env_db_b = la.true_envelope(v_mag_db_b[None,:], in_type='db', ncoeffs=400, thres_db=0.1)[0] v_sp_env_db_a_warp = warp_mag_spec(v_true_env_db_a, v_frmnts_bins_a_filt, v_frmnts_bins_b_filt, fft_len, sp_weight) v_sp_env_db_b_warp = warp_mag_spec(v_true_env_db_b, v_frmnts_bins_b_filt, v_frmnts_bins_a_filt, fft_len, (1-sp_weight)) #v_sp_env_db_a_warp = warp_mag_spec(v_lpc_mag_db_a, v_frmnts_bins_a_filt, v_frmnts_bins_b_filt, fft_len, sp_weight) #v_sp_env_db_b_warp = warp_mag_spec(v_lpc_mag_db_b, v_frmnts_bins_b_filt, v_frmnts_bins_a_filt, fft_len, (1-sp_weight)) # Spectral envelope mix:------------------------------------------------------- v_sp_env_db_targ = v_sp_env_db_a_warp * (1.0-sp_weight) + v_sp_env_db_b_warp * sp_weight # Source mix:------------------------------------------------------------------ v_spec_diff_db_a = v_sp_env_db_targ - v_true_env_db_a v_spec_diff_db_b = v_sp_env_db_targ - v_true_env_db_b # Filtering (FFT filter): v_frm_short_a_ext_filt = fft_filter(v_frm_short_a, shift_a, v_spec_diff_db_a, fft_len) v_frm_short_b_ext_filt = fft_filter(v_frm_short_b, shift_b, v_spec_diff_db_b, fft_len) # Mix signal: v_frm_short_ext_filt = v_frm_short_a_ext_filt * (1.0-sp_weight) + v_frm_short_b_ext_filt * sp_weight #v_frm_short_ext_filt = v_frm_short_a_ext_filt # Mix shifts: shift_mix = lu.round_to_int(shift_a * (1.0-sp_weight) + shift_b * sp_weight) # Save: v_shifts_syn[nx_frm] = shift_mix m_frms_syn[nx_frm, :] = v_frm_short_ext_filt if False: plt.figure(); plt.plot(v_frm_short_a_ext_filt); plt.plot(v_frm_short_b_ext_filt); plt.grid(); plt.show() plt.figure(); plt.plot(v_frm_short_a_ext_filt); plt.plot(v_frm_short_b_ext_filt); plt.plot(v_frm_short_ext_filt); plt.grid(); plt.show() # Merge: m_frms_syn_dc = np.fft.fftshift(m_frms_syn, axes=1) m_fft_syn = la.remove_hermitian_half(np.fft.fft(m_frms_syn_dc)) m_mag_syn, m_real_syn, m_imag_syn = compute_lossless_spec_feats(m_fft_syn) m_mag_merged = np.vstack((m_mag_a[:nx_strt_a,:] , m_mag_syn , m_mag_b[(nx_strt_b+nframes):,:])) m_real_merged = np.vstack((m_real_a[:nx_strt_a,:] , m_real_syn , m_real_b[(nx_strt_b+nframes):,:])) m_imag_merged = np.vstack((m_imag_a[:nx_strt_a,:] , m_imag_syn , m_imag_b[(nx_strt_b+nframes):,:])) v_shift_merged = np.r_[ v_shift_a[:nx_strt_a] , v_shifts_syn , v_shift_b[(nx_strt_b+nframes):] ] v_sig_merged = synthesis_from_lossless(m_mag_merged, m_real_merged, m_imag_merged, v_shift_merged) return v_sig_merged, fs
def get_formant_locations_from_raw_long_frame(v_sig, v_pm, nx, fft_len): ''' nx: frame index ''' #v_sig, fs = la.read_audio_file(wavfile) # Epoch detection: #v_pm_sec, v_voi = la.reaper_epoch_detection(wavfile) #v_pm = lu.round_to_int(v_pm_sec * fs) # Raw-long Frame extraction: v_frm_long = v_sig[v_pm[nx-2]:v_pm[nx+2]+1] # Win: left_len = v_pm[nx] - v_pm[nx-2] right_len = v_pm[nx+2] - v_pm[nx] v_win = la.gen_non_symmetric_win(left_len, right_len, np.hanning, b_norm=False) v_frm_long_win = v_frm_long * v_win # Spectrum: v_mag = la.remove_hermitian_half(np.absolute(np.fft.fft(v_frm_long_win, n=fft_len)[None,:]))[0] v_mag_db = la.db(v_mag) # Mel warping: alpha = 0.50 # 0.55 - 0.60 #ncoeffs = 2048 # must be even v_mag_mel = la.sp_mel_warp(v_mag[None,:], fft_len/2, alpha=alpha, in_type=3)[0] v_sp_cmplx = la.build_min_phase_from_mag_spec(v_mag_mel[None,:])[0] v_sp_cmplx_ext = la.add_hermitian_half(v_sp_cmplx[None,:], data_type='complex')[0] v_frm_long_win_mel = np.fft.ifft(v_sp_cmplx_ext).real if False: plt.close('all') pl(la.db(v_mag)) pl(la.db(np.absolute(v_sp_cmplx))) pl(v_frm_long_win_mel) # Formant extraction -LPC method:-------------------------------------------------- n_lpc_coeffs = 30 # 40 v_lpc_mel, v_e, v_refl = lpc(v_frm_long_win_mel, n_lpc_coeffs) v_lpc_mag_mel = lpc_to_mag(v_lpc_mel, fft_len=fft_len) v_lpc_mag_mel_db = la.db(v_lpc_mag_mel) v_lpc_mag_mel_db = v_lpc_mag_mel_db - np.mean(v_lpc_mag_mel_db) + np.mean(la.db(v_mag_mel)) v_frmnts_bins_mel, v_frmnts_gains_db = get_formant_locations_from_spec_env(v_lpc_mag_mel_db) # Getting bandwidth: fft_len_half = 1 + fft_len / 2 v_vall_bins = get_formant_locations_from_spec_env(-v_lpc_mag_mel_db)[0] v_vall_bins = np.r_[0, v_vall_bins, fft_len_half-1] nfrmnts = v_frmnts_bins_mel.size v_frmnts_bw_mel = np.zeros(nfrmnts) - 1.0 for nx_f in xrange(nfrmnts): #Left slope: curr_frmnt_bin = v_frmnts_bins_mel[nx_f] curr_vall_l_bin = v_vall_bins[nx_f] curr_vall_r_bin = v_vall_bins[nx_f+1] curr_midp_l = int((curr_frmnt_bin + curr_vall_l_bin) / 2.0) curr_midp_r = int((curr_frmnt_bin + curr_vall_r_bin) / 2.0) # Protection: if curr_midp_l==curr_frmnt_bin: curr_midp_l = curr_vall_l_bin if curr_midp_r==curr_frmnt_bin: curr_midp_r = curr_vall_r_bin #print(nx_f) # 27 y 32 slope_l = (v_frmnts_gains_db[nx_f] - v_lpc_mag_mel_db[curr_midp_l]) / (curr_frmnt_bin - curr_midp_l).astype(float) slope_r = (v_frmnts_gains_db[nx_f] - v_lpc_mag_mel_db[curr_midp_r]) / (curr_frmnt_bin - curr_midp_r).astype(float) slope_ave = (slope_l - slope_r) / 2.0 v_frmnts_bw_mel[nx_f] = 1.0 / slope_ave # Filtering by bandwidth: # bw_thress = 7.0 # v_frmnts_bins_mel = v_frmnts_bins_mel[v_frmnts_bw_mel<bw_thress] # v_frmnts_gains_db = v_frmnts_gains_db[v_frmnts_bw_mel<bw_thress] # v_frmnts_bw_mel = v_frmnts_bw_mel[v_frmnts_bw_mel<bw_thress] # Computing frame short:-------------------------------- # Win: left_len_short = v_pm[nx] - v_pm[nx-1] right_len_short = v_pm[nx+1] - v_pm[nx] v_win_short = la.gen_non_symmetric_win(left_len_short, right_len_short, np.hanning, b_norm=False) v_frm_short = v_sig[v_pm[nx-1]:v_pm[nx+1]+1] v_frm_short_win = v_frm_short * v_win_short shift = v_pm[nx] - v_pm[nx-1] # Formant extraction - True envelope method:---------------------------------------- # Not finished. #v_true_env_db = la.true_envelope(v_mag_db[None,:], in_type='db', ncoeffs=400, thres_db=0.1)[0] if True: plt.figure(); plt.plot(la.db(v_mag_mel)); plt.plot(v_lpc_mag_mel_db); plt.grid(); plt.show() #pl(v_mag_db) if True: import ipdb; ipdb.set_trace(context=8) # breakpoint 906d26d6 // return v_mag_db, v_lpc_mag_mel_db, v_frmnts_bins_mel, v_frmnts_gains_db, v_frmnts_bw_mel, v_frm_short_win, shift