def _load_embedding_dicts(self, folder):
try:
out = []
for json_filename in sorted(
Tools.list_files(folder, end='.json', substr='embed_')):
pos = Tools.fname(json_filename).replace('embed_', '').replace(
'.json', '')
root_folder = os.path.join(folder, pos)
with open(json_filename, 'r') as f:
D = json.load(f)
X = np.zeros((len(D), len(D[list(D)[0]])), dtype=float)
frames = sorted(list(D))
for ii, frame in enumerate(frames):
X[ii, :] = np.array(D[frame])
out.append((root_folder, frames, X, self._reduce(X)))
if self.averaged:
X_avg = np.stack([pos_data[2] for pos_data in out], axis=0)
X_avg = np.mean(X_avg, axis=0)
for ii, (fldr, frms, _, rdcd) in enumerate(out):
out[ii] = (fldr, frms, X_avg, rdcd)
return out
except Exception as e:
Tools.debug(e)
Tools.debug(folder, ex=0)
def __init__(self, root=None):
self.POS = tuple([
Tools.fname(f) for f in Tools.list_dirs(Tools.list_dirs(root)[0])
])
Tools.debug(self.POS)
if 'steady' in self.POS:
self.POS = 'steady'
elif 'middle' in self.POS:
self.POS = 'middle'
else:
self.POS = self.POS[0]
am = AlignMatrix(root)
self.alignments = am.load()
def visualize(self, in_folder, ou_folder):
Tools.makedirs(ou_folder)
nn = self._nearest_neighbor(in_folder)
fourcc = cv2.VideoWriter_fourcc(*'mp4v')
writer = cv2.VideoWriter(
os.path.join(ou_folder,
in_folder.split('/')[-1] + '.mp4'), fourcc, 16,
(480, 480))
frames = [
Tools.fname(f)
for f in Tools.list_files(os.path.join(in_folder, self.POS),
end='.jpg')
]
cv_plot = CVLine((240, 480), minx=0, maxx=len(frames), miny=0, maxy=1)
main_frm = np.zeros((480, 480, 3), dtype=np.uint8)
for fii, frame in Tools.tqdm_pbar(enumerate(frames),
Tools.fname(in_folder),
total=len(frames)):
frame = cv2.imread(os.path.join(in_folder, self.POS, frame))
try:
ancho = cv2.imread(
os.path.join(nn, self.POS,
self.alignments[in_folder][nn][fii][0]))
except Exception as e:
Tools.debug(self.alignments[in_folder][nn])
Tools.debug(e, ex=0)
frame = cv2.resize(frame, (240, 240))
ancho = cv2.resize(ancho, (240, 240))
main_frm[:240, :240, :] = frame[:, :, :]
main_frm[:240, 240:, :] = ancho[:, :, :]
R = self._reward_score(in_folder, fii)
main_frm[240:, :, :] = cv_plot.plot((fii, R))
writer.write(main_frm)
writer.release()
def _compute_cost_matrix(self, distances):
accumulated_cost = np.zeros(distances.shape)
Y = distances.shape[0]
dy = 10
for yii in range(distances.shape[0]):
accumulated_cost[yii, 0] = distances[yii, 0]
for xii in range(1, distances.shape[1]):
for yii in range(distances.shape[0]):
try:
accumulated_cost[yii, xii] = \
distances[yii, xii] + \
np.min(accumulated_cost[max(
0, yii - dy):min(Y, yii + dy), xii - 1])
except Exception as e:
Tools.debug(distances.shape, max(0, yii - 3),
min(yii + 3, distances.shape[0]), xii - 1)
raise e
# Tools.render(self._heatmap(accumulated_cost), waitkey=10)
return accumulated_cost
def _reward_score(self, trial, index):
cum_gain = 0.
cum_weight = 0.
for match_trial in self.alignments[trial]:
match_frm, weight = self.alignments[trial][match_trial][index]
match_nmr = int(''.join(filter(str.isdigit, match_frm)))
try:
match_min = min(
int(''.join(filter(str.isdigit, Tools.fname(f))))
for f in Tools.list_files(
os.path.join(match_trial, self.POS), end='.jpg'))
match_max = max(
int(''.join(filter(str.isdigit, Tools.fname(f))))
for f in Tools.list_files(
os.path.join(match_trial, self.POS), end='.jpg'))
except Exception as e:
Tools.debug(os.path.join(Tools.pathstr(match_trial), self.POS))
Tools.debug(e, ex=0)
cum_gain += (match_nmr - match_min) / (match_max - match_min)
cum_weight += 1
return cum_gain / cum_weight
def _align(self, trial_root, anchor_root, lock):
"""
project trial sequence onto anchor sequence with modified dtw
"""
try:
frm_a, emb_a = self._init_embeddings(anchor_root, lock)
frm_t, emb_t = self._init_embeddings(trial_root, lock)
dist_matrix = self._compute_dist_matrix(emb_t, emb_a)
cost_matrix = self._compute_cost_matrix(dist_matrix)
idx_path = self._compute_path(cost_matrix)
path = []
for point in idx_path:
path.append(
(frm_a[point[1]], 1e-3 / dist_matrix[point[1], point[0]]))
# self._plot(path)
return path
except Exception as e:
Tools.debug(e, ex=1)
sys.exit(1)
