def _construct_matrix(self, root):
def align(trial_root, anchor_root):
A = Aligner()
path = A.align(trial_root, anchor_root)
return (trial_root, anchor_root, path)
pool = multiprocessing.Pool(Config.N_WORKERS)
lock = multiprocessing.Manager().Lock()
tasks = []
for trial_root in Tools.list_dirs(root):
tasks.append((root, trial_root, lock))
# try:
for _ in Tools.tqdm_pbar(pool.imap_unordered(self._align, tasks),
"ALIGNING (multiprocessing pool)",
total=len(tasks)):
pass
def _init_model(self, data_root, n_components, n_clusters):
# Load embeddings
trials_embeddings = []
for trial_folder in Tools.tqdm_pbar(Tools.list_dirs(data_root),
'LOAD TCN EMBEDDINGS'):
trials_embeddings.append(self._load_embeddings(trial_folder))
# compute distance matrix & perform dimensionality reduction
dist_matrix = self._compute_dist_matrix(trials_embeddings)
reduce_embd = self._reduce(dist_matrix, n_components=n_components)
# do clustering
labels = self._cluster(dist_matrix, n_clusters=n_clusters)
# write results to csv
self._write_csv(data_root, trials_embeddings, reduce_embd, labels)
def _load_videos(self, path):
"""
read frames of left color perspective to working memory
Args:
path: string - path to video folder
Returns:
frame_buffers: list of list containing frames of videos as cv2
imgs
"""
vidnames = ('color-recording-left.avi', )
frame_buffers = []
for vidname in Tools.tqdm_pbar(vidnames, "LOADING " + path):
frame_buffers.append(self._load_video(os.path.join(path, vidname)))
return frame_buffers
def copy(self, root, in_pos, ou_pos):
"""
extract frames from video files
Args:
root: string - path to store video files
in_pos: string - camera perspective (e.g. left, middle, right)
ou_pos: string - name for camera perspective directory in
dataset
"""
for trial in Tools.tqdm_pbar(Tools.list_dirs(root), 'COPYING'):
Tools.pyout(trial)
fname = '_'.join(Tools.fname(trial).split('_')[:-1])
Tools.makedirs(os.path.join(trial, ou_pos))
vid_folder = self._find(fname)
frames = []
for frame in Tools.list_files(os.path.join(trial, in_pos)):
frames.append(int(Tools.fname(frame).split('.')[0]))
path = os.path.join(vid_folder,
'color-recording-' + ou_pos + '.avi')
if not os.path.isfile(path):
path = path.replace('.avi', '-x265.mp4')
cap = cv2.VideoCapture(path)
if not cap.isOpened():
Tools.pyout("ERROR OPENING VideoCapture")
raise FileNotFoundError('in copy(): "' + path + '"')
ii = 0
with tqdm(total=len(frames)) as pbar:
while cap.isOpened():
ret, frame = cap.read()
if ret is True:
if ii in frames:
img = np.copy(frame)
cv2.imwrite(
os.path.join(trial, ou_pos,
str(ii).zfill(5) + '.jpg'), img)
pbar.update(1)
else:
break
ii += 1
def visualize(self, in_folder, ou_folder):
Tools.makedirs(ou_folder)
nn = self._nearest_neighbor(in_folder)
fourcc = cv2.VideoWriter_fourcc(*'mp4v')
writer = cv2.VideoWriter(
os.path.join(ou_folder,
in_folder.split('/')[-1] + '.mp4'), fourcc, 16,
(480, 480))
frames = [
Tools.fname(f)
for f in Tools.list_files(os.path.join(in_folder, self.POS),
end='.jpg')
]
cv_plot = CVLine((240, 480), minx=0, maxx=len(frames), miny=0, maxy=1)
main_frm = np.zeros((480, 480, 3), dtype=np.uint8)
for fii, frame in Tools.tqdm_pbar(enumerate(frames),
Tools.fname(in_folder),
total=len(frames)):
frame = cv2.imread(os.path.join(in_folder, self.POS, frame))
try:
ancho = cv2.imread(
os.path.join(nn, self.POS,
self.alignments[in_folder][nn][fii][0]))
except Exception as e:
Tools.debug(self.alignments[in_folder][nn])
Tools.debug(e, ex=0)
frame = cv2.resize(frame, (240, 240))
ancho = cv2.resize(ancho, (240, 240))
main_frm[:240, :240, :] = frame[:, :, :]
main_frm[:240, 240:, :] = ancho[:, :, :]
R = self._reward_score(in_folder, fii)
main_frm[240:, :, :] = cv_plot.plot((fii, R))
writer.write(main_frm)
writer.release()
def _compute_dist_matrix(self, embeddings):
N = len(embeddings)
dist_matrix = np.zeros((N, ) * 2, dtype=float)
tasks = []
for ii in range(len(embeddings)):
for jj in range(ii + 1, len(embeddings)):
tasks.append((embeddings[ii][1], embeddings[jj][1], ii, jj))
pool = multiprocessing.Pool(15)
try:
for retval in Tools.tqdm_pbar(pool.imap_unordered(
self._dtw, tasks),
description='DISTANCE MATRIX',
total=len(tasks)):
ii, jj = retval[0]
dist_matrix[ii, jj], dist_matrix[jj, ii] = (retval[1], ) * 2
finally:
pool.close()
pool.join()
return dist_matrix
def walk_dirs(self, ou_path):
"""
find trials and store
Args:
ou_path: string - path to output file
"""
recording_paths = []
for f0 in Tools.tqdm_pbar(Tools.list_dirs(self.root), "GATHERING"):
fname = Tools.fname(f0)
if fname == '2017-11-09-to-encode':
for f1 in Tools.list_dirs(f0):
for f2 in Tools.list_dirs(f1):
if self._valid_folder(f2):
if self._length(f2, encoded=False) >= 60:
recording_paths.append(f2)
elif fname == '2017-11-06-still some parts to compress':
for f1 in Tools.list_dirs(f0):
if self._valid_folder(f1):
if self._length(f1, encoded=False) >= 60:
recording_paths.append(f1)
elif fname == 'toencode':
for f1 in Tools.list_dirs(f0):
fname1 = Tools.fname(f1)
if "copy" not in fname1 and not fname1 == "encoded":
for f2 in Tools.list_dirs(f1):
if self._valid_folder(f2):
if self._length(f2, encoded=False) >= 60:
recording_paths.append(f2)
else:
for f1 in Tools.list_dirs(f0):
if self._valid_folder(f1):
if self._length(f1) >= 60:
recording_paths.append(f1)
with open(ou_path, 'w+') as f:
f.write('\n'.join(recording_paths))
import cv2
import os
import shutil
from src.utils.tools import Tools
from src.segmenter_grasping import SegmenterGrasping
IN_ROOT = ''
OU_ROOT = ''
RGB = False # whether videos are in RGB (true) format or BGR (false) format
# MANUALLY SEGMENT RECORDINGS
s = SegmenterGrasping()
for trial in Tools.tqdm_pbar(Tools.list_dirs(IN_ROOT),
description="SEGMENTING"):
s.segment(trial, os.path.join(OU_ROOT, Tools.fname(trial)))
# COPY ACTUAL FRAMES TO OU_ROOT
for trial in Tools.tqdm_pbar(Tools.list_dirs(OU_ROOT),
description="MOVING FRAMES"):
for pos in Tools.list_dirs(trial):
with open(os.path.join(pos, 'frames.txt'), 'r') as f:
for path in f:
shutil.copy(
path.replace('\n', ''),
os.path.join(pos, Tools.fname(path.replace('\n', ''))))
os.remove(os.path.join(pos, 'frames.txt'))
# BGR->RGB, RESIZE, AND CROP IMAGES
for trial in Tools.tqdm_pbar(Tools.list_dirs(IN_ROOT), description="CROPPING"):
for pos in Tools.list_dirs(trial):
def _load_matrix(self, root):
for trial_root in Tools.tqdm_pbar(Tools.list_dirs(root),
"LOADING ALIGNMENT DATA"):
pass
IN_ROOT = ''
OU_ROOT = ''
RGB = False # whether videos are in RGB (true) format or BGR (false) format
# GARHER TRIALS
gatherer = GatherTrials(IN_ROOT)
gatherer.walk_dirs('fileslist.txt')
# MANUAL SEGMENTATION
segmenter = Segmenter()
tasks = []
with open('fileslist.txt', 'r') as f:
for line in f:
tasks.append(line.replace('\n', ''))
for task in Tools.tqdm_pbar(tasks, description="SEGMENTING", total=len(tasks)):
segmenter.segment_video(task, OU_ROOT)
os.remove('fileslist.txt')
# EXTRACT SELECTED FRAMES FROM VIDEOS
copyer = Copyer()
copyer.copy(OU_ROOT, 'left', 'middle')
copyer.copy(OU_ROOT, 'left', 'right')
copyer.copy(OU_ROOT, 'middle', 'left')
# BGR->RGB, RESIZE, AND CROP IMAGES
for trial in tqdm(Tools.list_dirs(IN_ROOT)):
for pos in Tools.list_dirs(trial):
for frame_pth in Tools.list_files(pos, end='.jpg'):
if RGB:
img = cv2.imread(frame_pth)
