def _load_embedding_dicts(self, folder): try: out = [] for json_filename in sorted( Tools.list_files(folder, end='.json', substr='embed_')): pos = Tools.fname(json_filename).replace('embed_', '').replace( '.json', '') root_folder = os.path.join(folder, pos) with open(json_filename, 'r') as f: D = json.load(f) X = np.zeros((len(D), len(D[list(D)[0]])), dtype=float) frames = sorted(list(D)) for ii, frame in enumerate(frames): X[ii, :] = np.array(D[frame]) out.append((root_folder, frames, X, self._reduce(X))) if self.averaged: X_avg = np.stack([pos_data[2] for pos_data in out], axis=0) X_avg = np.mean(X_avg, axis=0) for ii, (fldr, frms, _, rdcd) in enumerate(out): out[ii] = (fldr, frms, X_avg, rdcd) return out except Exception as e: Tools.debug(e) Tools.debug(folder, ex=0)
def __init__(self, root=None): self.POS = tuple([ Tools.fname(f) for f in Tools.list_dirs(Tools.list_dirs(root)[0]) ]) Tools.debug(self.POS) if 'steady' in self.POS: self.POS = 'steady' elif 'middle' in self.POS: self.POS = 'middle' else: self.POS = self.POS[0] am = AlignMatrix(root) self.alignments = am.load()
def visualize(self, in_folder, ou_folder): Tools.makedirs(ou_folder) nn = self._nearest_neighbor(in_folder) fourcc = cv2.VideoWriter_fourcc(*'mp4v') writer = cv2.VideoWriter( os.path.join(ou_folder, in_folder.split('/')[-1] + '.mp4'), fourcc, 16, (480, 480)) frames = [ Tools.fname(f) for f in Tools.list_files(os.path.join(in_folder, self.POS), end='.jpg') ] cv_plot = CVLine((240, 480), minx=0, maxx=len(frames), miny=0, maxy=1) main_frm = np.zeros((480, 480, 3), dtype=np.uint8) for fii, frame in Tools.tqdm_pbar(enumerate(frames), Tools.fname(in_folder), total=len(frames)): frame = cv2.imread(os.path.join(in_folder, self.POS, frame)) try: ancho = cv2.imread( os.path.join(nn, self.POS, self.alignments[in_folder][nn][fii][0])) except Exception as e: Tools.debug(self.alignments[in_folder][nn]) Tools.debug(e, ex=0) frame = cv2.resize(frame, (240, 240)) ancho = cv2.resize(ancho, (240, 240)) main_frm[:240, :240, :] = frame[:, :, :] main_frm[:240, 240:, :] = ancho[:, :, :] R = self._reward_score(in_folder, fii) main_frm[240:, :, :] = cv_plot.plot((fii, R)) writer.write(main_frm) writer.release()
def _compute_cost_matrix(self, distances): accumulated_cost = np.zeros(distances.shape) Y = distances.shape[0] dy = 10 for yii in range(distances.shape[0]): accumulated_cost[yii, 0] = distances[yii, 0] for xii in range(1, distances.shape[1]): for yii in range(distances.shape[0]): try: accumulated_cost[yii, xii] = \ distances[yii, xii] + \ np.min(accumulated_cost[max( 0, yii - dy):min(Y, yii + dy), xii - 1]) except Exception as e: Tools.debug(distances.shape, max(0, yii - 3), min(yii + 3, distances.shape[0]), xii - 1) raise e # Tools.render(self._heatmap(accumulated_cost), waitkey=10) return accumulated_cost
def _reward_score(self, trial, index): cum_gain = 0. cum_weight = 0. for match_trial in self.alignments[trial]: match_frm, weight = self.alignments[trial][match_trial][index] match_nmr = int(''.join(filter(str.isdigit, match_frm))) try: match_min = min( int(''.join(filter(str.isdigit, Tools.fname(f)))) for f in Tools.list_files( os.path.join(match_trial, self.POS), end='.jpg')) match_max = max( int(''.join(filter(str.isdigit, Tools.fname(f)))) for f in Tools.list_files( os.path.join(match_trial, self.POS), end='.jpg')) except Exception as e: Tools.debug(os.path.join(Tools.pathstr(match_trial), self.POS)) Tools.debug(e, ex=0) cum_gain += (match_nmr - match_min) / (match_max - match_min) cum_weight += 1 return cum_gain / cum_weight
def _align(self, trial_root, anchor_root, lock): """ project trial sequence onto anchor sequence with modified dtw """ try: frm_a, emb_a = self._init_embeddings(anchor_root, lock) frm_t, emb_t = self._init_embeddings(trial_root, lock) dist_matrix = self._compute_dist_matrix(emb_t, emb_a) cost_matrix = self._compute_cost_matrix(dist_matrix) idx_path = self._compute_path(cost_matrix) path = [] for point in idx_path: path.append( (frm_a[point[1]], 1e-3 / dist_matrix[point[1], point[0]])) # self._plot(path) return path except Exception as e: Tools.debug(e, ex=1) sys.exit(1)