def dtw_multiprocess(multiprocess_datum): pair = multiprocess_datum[0] dtw_dir = multiprocess_datum[1] normal_speaker = multiprocess_datum[2] dys_speaker = multiprocess_datum[3] """Need to find the path with mcep features, then apply the path to mel log features (features)""" """Also need to make this multiprocessing""" mcep_normal_path = pair['normal'].replace(config.directories.features, config.directories.mcep) mcep_dys_path = pair['dys'].replace(config.directories.features, config.directories.mcep) normal_spec = joblib.load(pair['normal']) dys_spec = joblib.load(pair['dys']) normal_mcep = (joblib.load(mcep_normal_path))['mcep'] dys_mcep = (joblib.load(mcep_dys_path))['mcep'] try: res = dtw(normal_mcep, dys_mcep, step_pattern="symmetricP2", open_begin=False, open_end=False) normal_path = res.path[:, 0] dys_path = res.path[:, 1] warped_normal_spect = normal_spec[normal_path] warped_dys_spect = dys_spec[dys_path] """Name the file""" utterance_name = ((pair['normal'].split('/')[-1]).replace(normal_speaker, ''))[1:] dump_path = os.path.join(dtw_dir, normal_speaker + '_to_' + dys_speaker + '_' + utterance_name) data_to_dump = {'normal': warped_normal_spect, 'dys': warped_dys_spect} joblib.dump(data_to_dump, dump_path) except: print("No alignment found...")
def _step(self, action, render=True): state_out, reward, done, info = super(VTLRefMaskedActionDTWEnv, self)._step(action, render) if self.current_step >= int( self.max_episode_duration / self.timestep) - 1: ref_obs = np.zeros(self.state_dim - len(state_out)) else: ref_obs = self.get_current_ref_obs() ref_full_vtl_state = np.concatenate( (self.cur_reference['tract_params'][:, :], self.cur_reference['glottis_params'][:, :]), axis=-1) ep_ar = np.array(self.episode_states)[:, self.selected_ref_param_idx] ref_ar = ref_full_vtl_state[:, self.selected_ref_param_idx] dtw_res_ar = dtwalign.dtw(ep_ar, ref_ar, open_end=True, step_pattern="symmetricP2") # if max(abs(ref_ar[ep_ar.shape[0], :] - ep_ar[-1, :])) > 0.8: # done = True if dtw_res_ar.distance > 5 and self.current_step > 2: done = True state_out = np.concatenate((state_out, ref_obs)) return state_out, reward, done, info
def calc_alignment(self, s, reference): return dtwalign.dtw(s, reference, dist=self.dist_params['dist'], step_pattern=self.dist_params['step_pattern'], open_end=self.dist_params['open_end'], dist_only=False)
def show_dtw_path(series_1, series_2, window_size=-1, mode=0): if mode == 0: path, sim = _dtw_path(series_1, series_2, window_size) size = max(len(series_1), len(series_2)) matrix_path = np.zeros((size, size), dtype=np.float) for i, j in path: matrix_path[i, j] = 1 for i in range(size): matrix_path[i, i] = 0.3 plt.figure(figsize=(10, 10)) plt.imshow(matrix_path, cmap="gray_r") elif mode == 1: if window_size > -1: res = dtwalign.dtw(series_1, series_2, dist="euclidean", step_pattern='symmetric2', window_type='sakoechiba', window_size=window_size) else: res = dtwalign.dtw(series_1, series_2, dist="euclidean", step_pattern='symmetric2') res.plot_path() elif mode == 2: d, paths = dtaidistance.dtw.warping_paths(series_1, series_2, window=20) best_path = dtaidistance.dtw.best_path(paths) dtaidistance.dtw_visualisation.plot_warpingpaths( series_1, series_2, paths, best_path) else: if window_size > -1: d, paths = dtaidistance.dtw.warping_paths(series_1, series_2, window=window_size) else: d, paths = dtaidistance.dtw.warping_paths(series_1, series_2) best_path = dtaidistance.dtw.best_path(paths) return series_1, series_2, paths, best_path
def calc_distance(self, s, reference): if self.dist_params['name'] == 'soft-DTW': dist_func = SoftDTW(open_end=self.dist_params['open_end'], dist=self.dist_params['dist']) return dist_func( torch.from_numpy(s).to(self.device), torch.from_numpy(reference).to( self.device)).detach().cpu().item() elif self.dist_params['name'] == 'dtwalign': return dtwalign.dtw(s, reference, dist=self.dist_params['dist'], step_pattern=self.dist_params['step_pattern'], open_end=self.dist_params['open_end'], dist_only=True).normalized_distance else: raise KeyError( f"unknown name of the dist: {self.dist_params['name']}")
def _assert_dist(self, pattern, win_name, win_size, open_begin, open_end): if open_begin and win_name != "none": """ Results with open-begin and using window differ between R and Python because R implementation doesn't consider zero-padded row separately (there should be no problem in practical use...) """ pass elif (open_begin or open_end) and win_name == "itakura": """ itakura window requires closed end """ pass elif (open_begin or open_end) and not _get_pattern(pattern).is_normalizable: """ partial matching requires normalizable step pattern """ pass elif open_begin and _get_pattern(pattern).normalize_guide != "N": """ open-begin matching requires "N"-normalizable step pattern """ pass else: # get R result rdtw = DtwR(pattern, win_name, win_size, False, open_end, open_begin) rdtw.fit(self.X[int(open_begin)][int(open_end)]) # get Python result pydtw = dtw(self.x[int(open_begin)][int(open_end)], self.y[int(open_begin)][int(open_end)], "euclidean", win_name, win_size, pattern, False, open_begin, open_end) # assert assert_almost_equal(rdtw.distance, pydtw.distance) if _get_pattern(pattern).is_normalizable: """ only normalizable pattern can be asserted for normalization """ assert_almost_equal(rdtw.normalized_distance, pydtw.normalized_distance)
def _assert_dist(self,pattern,win_name,win_size,open_begin,open_end): if open_begin and win_name != "none": """ Results with open-begin and using window differ between R and Python because R implementation doesn't consider zero-padded row separately (there should be no problem in practical use...) """ pass elif (open_begin or open_end) and win_name == "itakura": """ itakura window requires closed end """ pass elif (open_begin or open_end) and not _get_pattern(pattern).is_normalizable: """ partial matching requires normalizable step pattern """ pass elif open_begin and _get_pattern(pattern).normalize_guide != "N": """ open-begin matching requires "N"-normalizable step pattern """ pass else: # get R result rdtw = DtwR(pattern,win_name,win_size,False,open_end,open_begin) rdtw.fit(self.X[int(open_begin)][int(open_end)]) # get Python result pydtw = dtw( self.x[int(open_begin)][int(open_end)], self.y[int(open_begin)][int(open_end)], "euclidean",win_name,win_size,pattern, False,open_begin,open_end ) # assert assert_almost_equal(rdtw.distance,pydtw.distance) if _get_pattern(pattern).is_normalizable: """ only normalizable pattern can be asserted for normalization """ assert_almost_equal(rdtw.normalized_distance,pydtw.normalized_distance)
# plt.ylabel('MaxGy') # plt.title('Passing The Door - Holding Style') # plt.legend() # plt.show() import numpy as np import matplotlib.pyplot as plt from dtwalign import dtw np.random.seed(1234) # test data x = np.sin(2 * np.pi * 3.1 * np.linspace(0, 1, 101)) x += np.random.rand(x.size) y = np.sin(2 * np.pi * 3 * np.linspace(0, 1, 120)) y += np.random.rand(y.size) res = dtw(x, y) print(len(x)) print(len(y)) print("dtw distance: {}".format(res.distance)) print("dtw normalized distance: {}".format(res.normalized_distance)) plt.plot(x, label="query") plt.plot(y, label="reference") plt.legend() #plt.ylim(-1,3) plt.show() # dtw distance """
IDaa, paa, rmsaccraw, Type = processdata(P, MagnitudeAcc, Type, 0) fs = 25 cutoff = 1.5 alpha = calalpha(cutoff, fs) plpf = LPF(alpha, paa) plpf = LPF(alpha, plpf) rmsacclpf = LPF(alpha, rmsaccraw) rmsacclpf = LPF(alpha, rmsacclpf) IDaa, pa, acc, typearr = processdata(plpf, rmsacclpf, Type, 300) return acc if __name__ == '__main__': totaldung = pd.read_csv('Downstair2.csv') #dung totalsai = pd.read_csv('Downstair4.csv') #sai magdung = np.array(process(totaldung)) magsai = np.array(process(totalsai)) res = dtw(magdung, magsai) print("dtw distance: {}".format(res.distance)) print("dtw normalized distance: {}".format(res.normalized_distance)) plt.plot(magdung, label="query") plt.plot(magsai, label="reference") plt.legend() plt.show()
def validate_and_summarize(self, env, writer, **kwargs): state = env.reset(offset=1) env.render() episode_done = [False] * kwargs['num_workers'] dtw_dists = [[] for _ in range(kwargs['num_workers'])] last_path_points = [[] for _ in range(kwargs['num_workers'])] vt_predictions = [[] for _ in range(kwargs['num_workers'])] embeds_predictions = [[] for _ in range(kwargs['num_workers'])] while not all(episode_done): action = self.policy_net.get_action(state) action = action.detach().cpu().numpy() action = np.clip(action, -1, 1) next_state, reward, done, info = env.step(action) agent_state = state[:, :self.agent_state_dim] predicted_next_agent_state = self.model_dynamics_target( torch.from_numpy(agent_state).float().to(self.device), torch.from_numpy(action).float().to( self.device)).detach().cpu().numpy() audio_dim = env.get_attr('audio_dim')[0] vt_pred = predicted_next_agent_state[:, :-audio_dim] embeds_pred = predicted_next_agent_state[:, -audio_dim:] # print(reward) # dtw_dist = info['dtw_dist'] env.render() state = next_state for worker_idx in range(kwargs['num_workers']): dtw_dists[worker_idx].append(info[worker_idx]['dtw_dist']) last_path_points[worker_idx].append( info[worker_idx]['last_path_point']) vt_predictions[worker_idx].append(vt_pred[worker_idx]) embeds_predictions[worker_idx].append( (embeds_pred[worker_idx])) if done[worker_idx]: episode_done[worker_idx] = True name = f'{self._env_name}_BackpropIntoPolicy_' + f'{self.step}' save_dir = f'../../../runs/{self._env_name}_backprop_{self._dt}/' fname = os.path.join(save_dir, name + ".mp4") fnames = env.dump_episode( fname=os.path.abspath(os.path.join(save_dir, name))) episode_history = env.get_episode_history( remotes=[worker_idx])[0] video_data = torchvision.io.read_video(fnames[0] + ".mp4", start_pts=0, end_pts=None, pts_unit='sec') dtw_res = dtwalign.dtw( episode_history['embeds'], episode_history['ref']['acoustics'], dist=kwargs['distance']['dist'], step_pattern=kwargs['distance']['step_pattern'], open_end=False) #prepare predictions vt_preds = np.array(vt_predictions[worker_idx]) embeds_preds = np.array( embeds_predictions[worker_idx]).reshape( -1, episode_history['embeds'].shape[-1]) if worker_idx == 0: self.summarize(writer, episode_history, video_data, dtw_res, dtw_dists[worker_idx], last_path_points[worker_idx], vt_preds, embeds_preds) state[worker_idx] = env.reset(remotes=[worker_idx])
def get_current_ref_obs(self): # print(self.current_step) if self.current_step <= 2: ref_full_vtl_state = np.concatenate( (self.cur_reference['tract_params'][self.current_step + 1, :], self.cur_reference['glottis_params'][self.current_step + 1, :])) ref_obs = ref_full_vtl_state[self.selected_ref_param_idx] if "ACOUSTICS" in self.selected_ref_params: cols_per_step = int(self.timestep / 1000 / self.preproc_params['winstep']) ref_ac_obs = self.cur_reference['acoustics'][ self.current_step * cols_per_step:(self.current_step + 1) * cols_per_step, :].flatten().squeeze() ref_obs = np.concatenate((ref_obs, ref_ac_obs)) else: # dtw # ep_states = np.array(self.episode_states)[:, -self.audio_dim:] # acoustic dtw ref_ac = self.cur_reference['acoustics'] ep_ac = np.array(self.episode_states)[:, -self.audio_dim:].reshape( -1, ref_ac.shape[-1]) dtw_res_ac = dtwalign.dtw(ep_ac, ref_ac, open_end=True, step_pattern="symmetricP2") last_ref_matched_elem = dtw_res_ac.path[-1, 1] #artic dtw # TODO: do smth with art dtw as well ref_full_vtl_state = np.concatenate( (self.cur_reference['tract_params'][:, :], self.cur_reference['glottis_params'][:, :]), axis=-1) ep_ar = np.array(self.episode_states)[:, self.selected_ref_param_idx] ref_ar = ref_full_vtl_state[:, self.selected_ref_param_idx] dtw_res_ar = dtwalign.dtw(ep_ar, ref_ar, open_end=True, step_pattern="symmetricP2") print(dtw_res_ar.path) last_ref_matched_elem_ar = dtw_res_ar.path[-1, 1] if "ACOUSTICS" in self.selected_ref_params: cols_per_step = int(self.timestep / 1000 / self.preproc_params['winstep']) last_matched_step = (last_ref_matched_elem + 1) // cols_per_step last_matched_step = last_ref_matched_elem_ar last_matched_step = min( self.cur_reference['tract_params'].shape[0] - 2, last_matched_step) print(last_matched_step, self.current_step) ref_ac_obs = self.cur_reference['acoustics'][ (last_matched_step) * cols_per_step:(last_matched_step + 1) * cols_per_step, :].flatten().squeeze() ref_full_vtl_state = np.concatenate( (self.cur_reference['tract_params'][last_matched_step + 1, :], self.cur_reference['glottis_params'][last_matched_step + 1, :])) ref_obs = ref_full_vtl_state[self.selected_ref_param_idx] ref_obs = np.concatenate((ref_obs, ref_ac_obs)) return ref_obs
actualtimemovingarr = changepo(actualtimemovingarr) if 56 < palpha < 58: if desnopredict == 1: IDaa, pa, acc, typearr = xulytim(profileup) startprofile, endprofile, timeaccarrprofile, startendtimeparrprofile, desprofile, desnoprofile, palpha = plotcheck( pa, acc, IDaa) #print(f'Up {startprofile} - {endprofile}') else: IDaa, pa, acc, typearr = xulytim(profiledown) startprofile, endprofile, timeaccarrprofile, startendtimeparrprofile, desprofile, desnoprofile, palpha = plotcheck( pa, acc, IDaa) #print(f'Down {startprofile} - {endprofile}') startpredict = np.array(startpredict) startprofile = np.array(startprofile) startres = dtw(startpredict, startprofile) endpredict = np.array(endpredict) endprofile = np.array(endprofile) endres = dtw(endpredict, endprofile) predicttimemovingarr = [] th = 25 if startres.distance < th and endres.distance < th: movingtimepredict = abs(timeaccarrpredict[-1] - timeaccarrpredict[0]) predicttimemovingarr.append(timeaccarrpredict[0]) predicttimemovingarr.append(timeaccarrpredict[-1]) levelpredict = dnf(movingtimepredict) elif startres.distance > th and endres.distance < th: movingtimepredict = abs( timeaccarrpredict[-1] - startendtimeparrpredict[0]) - 1.5