def dtw_multiprocess(multiprocess_datum):
pair = multiprocess_datum[0]
dtw_dir = multiprocess_datum[1]
normal_speaker = multiprocess_datum[2]
dys_speaker = multiprocess_datum[3]
"""Need to find the path with mcep features, then apply the path to mel log features (features)"""
"""Also need to make this multiprocessing"""
mcep_normal_path = pair['normal'].replace(config.directories.features, config.directories.mcep)
mcep_dys_path = pair['dys'].replace(config.directories.features, config.directories.mcep)
normal_spec = joblib.load(pair['normal'])
dys_spec = joblib.load(pair['dys'])
normal_mcep = (joblib.load(mcep_normal_path))['mcep']
dys_mcep = (joblib.load(mcep_dys_path))['mcep']
try:
res = dtw(normal_mcep, dys_mcep, step_pattern="symmetricP2", open_begin=False, open_end=False)
normal_path = res.path[:, 0]
dys_path = res.path[:, 1]
warped_normal_spect = normal_spec[normal_path]
warped_dys_spect = dys_spec[dys_path]
"""Name the file"""
utterance_name = ((pair['normal'].split('/')[-1]).replace(normal_speaker, ''))[1:]
dump_path = os.path.join(dtw_dir, normal_speaker + '_to_' + dys_speaker + '_' + utterance_name)
data_to_dump = {'normal': warped_normal_spect, 'dys': warped_dys_spect}
joblib.dump(data_to_dump, dump_path)
except:
print("No alignment found...")
def _step(self, action, render=True):
state_out, reward, done, info = super(VTLRefMaskedActionDTWEnv,
self)._step(action, render)
if self.current_step >= int(
self.max_episode_duration / self.timestep) - 1:
ref_obs = np.zeros(self.state_dim - len(state_out))
else:
ref_obs = self.get_current_ref_obs()
ref_full_vtl_state = np.concatenate(
(self.cur_reference['tract_params'][:, :],
self.cur_reference['glottis_params'][:, :]),
axis=-1)
ep_ar = np.array(self.episode_states)[:, self.selected_ref_param_idx]
ref_ar = ref_full_vtl_state[:, self.selected_ref_param_idx]
dtw_res_ar = dtwalign.dtw(ep_ar,
ref_ar,
open_end=True,
step_pattern="symmetricP2")
# if max(abs(ref_ar[ep_ar.shape[0], :] - ep_ar[-1, :])) > 0.8:
# done = True
if dtw_res_ar.distance > 5 and self.current_step > 2:
done = True
state_out = np.concatenate((state_out, ref_obs))
return state_out, reward, done, info
def calc_alignment(self, s, reference):
return dtwalign.dtw(s,
reference,
dist=self.dist_params['dist'],
step_pattern=self.dist_params['step_pattern'],
open_end=self.dist_params['open_end'],
dist_only=False)
def show_dtw_path(series_1, series_2, window_size=-1, mode=0):
if mode == 0:
path, sim = _dtw_path(series_1, series_2, window_size)
size = max(len(series_1), len(series_2))
matrix_path = np.zeros((size, size), dtype=np.float)
for i, j in path:
matrix_path[i, j] = 1
for i in range(size):
matrix_path[i, i] = 0.3
plt.figure(figsize=(10, 10))
plt.imshow(matrix_path, cmap="gray_r")
elif mode == 1:
if window_size > -1:
res = dtwalign.dtw(series_1,
series_2,
dist="euclidean",
step_pattern='symmetric2',
window_type='sakoechiba',
window_size=window_size)
else:
res = dtwalign.dtw(series_1,
series_2,
dist="euclidean",
step_pattern='symmetric2')
res.plot_path()
elif mode == 2:
d, paths = dtaidistance.dtw.warping_paths(series_1,
series_2,
window=20)
best_path = dtaidistance.dtw.best_path(paths)
dtaidistance.dtw_visualisation.plot_warpingpaths(
series_1, series_2, paths, best_path)
else:
if window_size > -1:
d, paths = dtaidistance.dtw.warping_paths(series_1,
series_2,
window=window_size)
else:
d, paths = dtaidistance.dtw.warping_paths(series_1, series_2)
best_path = dtaidistance.dtw.best_path(paths)
return series_1, series_2, paths, best_path
def calc_distance(self, s, reference):
if self.dist_params['name'] == 'soft-DTW':
dist_func = SoftDTW(open_end=self.dist_params['open_end'],
dist=self.dist_params['dist'])
return dist_func(
torch.from_numpy(s).to(self.device),
torch.from_numpy(reference).to(
self.device)).detach().cpu().item()
elif self.dist_params['name'] == 'dtwalign':
return dtwalign.dtw(s,
reference,
dist=self.dist_params['dist'],
step_pattern=self.dist_params['step_pattern'],
open_end=self.dist_params['open_end'],
dist_only=True).normalized_distance
else:
raise KeyError(
f"unknown name of the dist: {self.dist_params['name']}")
def _assert_dist(self, pattern, win_name, win_size, open_begin, open_end):
if open_begin and win_name != "none":
"""
Results with open-begin and using window differ between R and Python because
R implementation doesn't consider zero-padded row separately
(there should be no problem in practical use...)
"""
pass
elif (open_begin or open_end) and win_name == "itakura":
"""
itakura window requires closed end
"""
pass
elif (open_begin
or open_end) and not _get_pattern(pattern).is_normalizable:
"""
partial matching requires normalizable step pattern
"""
pass
elif open_begin and _get_pattern(pattern).normalize_guide != "N":
"""
open-begin matching requires "N"-normalizable step pattern
"""
pass
else:
# get R result
rdtw = DtwR(pattern, win_name, win_size, False, open_end,
open_begin)
rdtw.fit(self.X[int(open_begin)][int(open_end)])
# get Python result
pydtw = dtw(self.x[int(open_begin)][int(open_end)],
self.y[int(open_begin)][int(open_end)], "euclidean",
win_name, win_size, pattern, False, open_begin,
open_end)
# assert
assert_almost_equal(rdtw.distance, pydtw.distance)
if _get_pattern(pattern).is_normalizable:
"""
only normalizable pattern can be asserted for normalization
"""
assert_almost_equal(rdtw.normalized_distance,
pydtw.normalized_distance)
def _assert_dist(self,pattern,win_name,win_size,open_begin,open_end):
if open_begin and win_name != "none":
"""
Results with open-begin and using window differ between R and Python because
R implementation doesn't consider zero-padded row separately
(there should be no problem in practical use...)
"""
pass
elif (open_begin or open_end) and win_name == "itakura":
"""
itakura window requires closed end
"""
pass
elif (open_begin or open_end) and not _get_pattern(pattern).is_normalizable:
"""
partial matching requires normalizable step pattern
"""
pass
elif open_begin and _get_pattern(pattern).normalize_guide != "N":
"""
open-begin matching requires "N"-normalizable step pattern
"""
pass
else:
# get R result
rdtw = DtwR(pattern,win_name,win_size,False,open_end,open_begin)
rdtw.fit(self.X[int(open_begin)][int(open_end)])
# get Python result
pydtw = dtw(
self.x[int(open_begin)][int(open_end)],
self.y[int(open_begin)][int(open_end)],
"euclidean",win_name,win_size,pattern,
False,open_begin,open_end
)
# assert
assert_almost_equal(rdtw.distance,pydtw.distance)
if _get_pattern(pattern).is_normalizable:
"""
only normalizable pattern can be asserted for normalization
"""
assert_almost_equal(rdtw.normalized_distance,pydtw.normalized_distance)
# plt.ylabel('MaxGy')
# plt.title('Passing The Door - Holding Style')
# plt.legend()
# plt.show()
import numpy as np
import matplotlib.pyplot as plt
from dtwalign import dtw
np.random.seed(1234)
# test data
x = np.sin(2 * np.pi * 3.1 * np.linspace(0, 1, 101))
x += np.random.rand(x.size)
y = np.sin(2 * np.pi * 3 * np.linspace(0, 1, 120))
y += np.random.rand(y.size)
res = dtw(x, y)
print(len(x))
print(len(y))
print("dtw distance: {}".format(res.distance))
print("dtw normalized distance: {}".format(res.normalized_distance))
plt.plot(x, label="query")
plt.plot(y, label="reference")
plt.legend()
#plt.ylim(-1,3)
plt.show()
# dtw distance
"""
IDaa, paa, rmsaccraw, Type = processdata(P, MagnitudeAcc, Type, 0)
fs = 25
cutoff = 1.5
alpha = calalpha(cutoff, fs)
plpf = LPF(alpha, paa)
plpf = LPF(alpha, plpf)
rmsacclpf = LPF(alpha, rmsaccraw)
rmsacclpf = LPF(alpha, rmsacclpf)
IDaa, pa, acc, typearr = processdata(plpf, rmsacclpf, Type, 300)
return acc
if __name__ == '__main__':
totaldung = pd.read_csv('Downstair2.csv') #dung
totalsai = pd.read_csv('Downstair4.csv') #sai
magdung = np.array(process(totaldung))
magsai = np.array(process(totalsai))
res = dtw(magdung, magsai)
print("dtw distance: {}".format(res.distance))
print("dtw normalized distance: {}".format(res.normalized_distance))
plt.plot(magdung, label="query")
plt.plot(magsai, label="reference")
plt.legend()
plt.show()
def validate_and_summarize(self, env, writer, **kwargs):
state = env.reset(offset=1)
env.render()
episode_done = [False] * kwargs['num_workers']
dtw_dists = [[] for _ in range(kwargs['num_workers'])]
last_path_points = [[] for _ in range(kwargs['num_workers'])]
vt_predictions = [[] for _ in range(kwargs['num_workers'])]
embeds_predictions = [[] for _ in range(kwargs['num_workers'])]
while not all(episode_done):
action = self.policy_net.get_action(state)
action = action.detach().cpu().numpy()
action = np.clip(action, -1, 1)
next_state, reward, done, info = env.step(action)
agent_state = state[:, :self.agent_state_dim]
predicted_next_agent_state = self.model_dynamics_target(
torch.from_numpy(agent_state).float().to(self.device),
torch.from_numpy(action).float().to(
self.device)).detach().cpu().numpy()
audio_dim = env.get_attr('audio_dim')[0]
vt_pred = predicted_next_agent_state[:, :-audio_dim]
embeds_pred = predicted_next_agent_state[:, -audio_dim:]
# print(reward)
# dtw_dist = info['dtw_dist']
env.render()
state = next_state
for worker_idx in range(kwargs['num_workers']):
dtw_dists[worker_idx].append(info[worker_idx]['dtw_dist'])
last_path_points[worker_idx].append(
info[worker_idx]['last_path_point'])
vt_predictions[worker_idx].append(vt_pred[worker_idx])
embeds_predictions[worker_idx].append(
(embeds_pred[worker_idx]))
if done[worker_idx]:
episode_done[worker_idx] = True
name = f'{self._env_name}_BackpropIntoPolicy_' + f'{self.step}'
save_dir = f'../../../runs/{self._env_name}_backprop_{self._dt}/'
fname = os.path.join(save_dir, name + ".mp4")
fnames = env.dump_episode(
fname=os.path.abspath(os.path.join(save_dir, name)))
episode_history = env.get_episode_history(
remotes=[worker_idx])[0]
video_data = torchvision.io.read_video(fnames[0] + ".mp4",
start_pts=0,
end_pts=None,
pts_unit='sec')
dtw_res = dtwalign.dtw(
episode_history['embeds'],
episode_history['ref']['acoustics'],
dist=kwargs['distance']['dist'],
step_pattern=kwargs['distance']['step_pattern'],
open_end=False)
#prepare predictions
vt_preds = np.array(vt_predictions[worker_idx])
embeds_preds = np.array(
embeds_predictions[worker_idx]).reshape(
-1, episode_history['embeds'].shape[-1])
if worker_idx == 0:
self.summarize(writer, episode_history, video_data,
dtw_res, dtw_dists[worker_idx],
last_path_points[worker_idx], vt_preds,
embeds_preds)
state[worker_idx] = env.reset(remotes=[worker_idx])
def get_current_ref_obs(self):
# print(self.current_step)
if self.current_step <= 2:
ref_full_vtl_state = np.concatenate(
(self.cur_reference['tract_params'][self.current_step + 1, :],
self.cur_reference['glottis_params'][self.current_step +
1, :]))
ref_obs = ref_full_vtl_state[self.selected_ref_param_idx]
if "ACOUSTICS" in self.selected_ref_params:
cols_per_step = int(self.timestep / 1000 /
self.preproc_params['winstep'])
ref_ac_obs = self.cur_reference['acoustics'][
self.current_step * cols_per_step:(self.current_step + 1) *
cols_per_step, :].flatten().squeeze()
ref_obs = np.concatenate((ref_obs, ref_ac_obs))
else:
# dtw
# ep_states = np.array(self.episode_states)[:, -self.audio_dim:]
# acoustic dtw
ref_ac = self.cur_reference['acoustics']
ep_ac = np.array(self.episode_states)[:, -self.audio_dim:].reshape(
-1, ref_ac.shape[-1])
dtw_res_ac = dtwalign.dtw(ep_ac,
ref_ac,
open_end=True,
step_pattern="symmetricP2")
last_ref_matched_elem = dtw_res_ac.path[-1, 1]
#artic dtw
# TODO: do smth with art dtw as well
ref_full_vtl_state = np.concatenate(
(self.cur_reference['tract_params'][:, :],
self.cur_reference['glottis_params'][:, :]),
axis=-1)
ep_ar = np.array(self.episode_states)[:,
self.selected_ref_param_idx]
ref_ar = ref_full_vtl_state[:, self.selected_ref_param_idx]
dtw_res_ar = dtwalign.dtw(ep_ar,
ref_ar,
open_end=True,
step_pattern="symmetricP2")
print(dtw_res_ar.path)
last_ref_matched_elem_ar = dtw_res_ar.path[-1, 1]
if "ACOUSTICS" in self.selected_ref_params:
cols_per_step = int(self.timestep / 1000 /
self.preproc_params['winstep'])
last_matched_step = (last_ref_matched_elem +
1) // cols_per_step
last_matched_step = last_ref_matched_elem_ar
last_matched_step = min(
self.cur_reference['tract_params'].shape[0] - 2,
last_matched_step)
print(last_matched_step, self.current_step)
ref_ac_obs = self.cur_reference['acoustics'][
(last_matched_step) *
cols_per_step:(last_matched_step + 1) *
cols_per_step, :].flatten().squeeze()
ref_full_vtl_state = np.concatenate(
(self.cur_reference['tract_params'][last_matched_step +
1, :],
self.cur_reference['glottis_params'][last_matched_step +
1, :]))
ref_obs = ref_full_vtl_state[self.selected_ref_param_idx]
ref_obs = np.concatenate((ref_obs, ref_ac_obs))
return ref_obs
actualtimemovingarr = changepo(actualtimemovingarr)
if 56 < palpha < 58:
if desnopredict == 1:
IDaa, pa, acc, typearr = xulytim(profileup)
startprofile, endprofile, timeaccarrprofile, startendtimeparrprofile, desprofile, desnoprofile, palpha = plotcheck(
pa, acc, IDaa)
#print(f'Up {startprofile} - {endprofile}')
else:
IDaa, pa, acc, typearr = xulytim(profiledown)
startprofile, endprofile, timeaccarrprofile, startendtimeparrprofile, desprofile, desnoprofile, palpha = plotcheck(
pa, acc, IDaa)
#print(f'Down {startprofile} - {endprofile}')
startpredict = np.array(startpredict)
startprofile = np.array(startprofile)
startres = dtw(startpredict, startprofile)
endpredict = np.array(endpredict)
endprofile = np.array(endprofile)
endres = dtw(endpredict, endprofile)
predicttimemovingarr = []
th = 25
if startres.distance < th and endres.distance < th:
movingtimepredict = abs(timeaccarrpredict[-1] -
timeaccarrpredict[0])
predicttimemovingarr.append(timeaccarrpredict[0])
predicttimemovingarr.append(timeaccarrpredict[-1])
levelpredict = dnf(movingtimepredict)
elif startres.distance > th and endres.distance < th:
movingtimepredict = abs(
timeaccarrpredict[-1] -
startendtimeparrpredict[0]) - 1.5
