def __init__(self,
x_train_path,
y_train_path,
batch_size,
dim,
color_dict,
shuffle=True,
validation_split=0.1):
x_train = pims.Video(x_train_path)
y_train = pims.Video(y_train_path)
indices = np.arange(len(y_train))
np.random.shuffle(indices)
val_length = int(len(y_train) * validation_split)
validation = indices[:val_length]
indices = indices[val_length:]
# pims Videos cannot exist in two generators at once. To get around this,
# construct two versions of x_train and y_train.
x_train_copy = pims.Video(x_train_path)
y_train_copy = pims.Video(y_train_path)
self.datagen = DataGenerator(x_train,
y_train,
batch_size,
dim,
color_dict,
shuffle=shuffle,
indices=indices)
self.validation_datagen = DataGenerator(x_train_copy,
y_train_copy,
batch_size,
dim,
color_dict,
shuffle=shuffle,
indices=validation,
distort_images=False)
def getbatch(self,idx):
X = []
Y = []
#Read a batch of clips from files
for i in idx:
idfs = list(range(25*5))
np.random.shuffle(idfs)
idfs = idfs[:self.no_frames] # keep only 2 random frames per clip on train mode
ok = True
try:
#print(self.basepath + '/' + self.part + '/X/'+self.filelistX[i])
Xj = pims.Video(self.basepath + '/' + self.part + '/X/'+self.filelistX[i])[idfs]
Xj = np.array(Xj, dtype='float32') / 255.
#print(self.basepath + '/' + self.part + '/Y/'+self.filelistY[i])
Yj = pims.Video(self.basepath + '/' + self.part + '/Y/'+self.filelistY[i])[idfs]
Yj = np.array(Yj, dtype='float32') / 255.
except:
print('Error clip number '+ str(i) + ' at '+ self.filelistX[i] + ' OR '+ self.filelistY[i])
ok = False
if ok:
X.append(Xj)
Y.append(Yj)
# make numpy and reshape
X = np.asarray(X)
X = X.reshape((X.shape[0]*X.shape[1], X.shape[2], X.shape[3], X.shape[4]))
Y = np.asarray(Y)
Y = Y.reshape((Y.shape[0]*Y.shape[1], Y.shape[2], Y.shape[3], Y.shape[4]))
return X*2 - 1, Y*2 - 1
def getAllFrames(clipname):
print(clipname)
# open one video clip sample
try:
data = pims.Video(root_dataset + '/' + clipname)
except:
data = pims.Video(clipname)
data = np.array(data, dtype='float32')
length = data.shape[0]
return data[:125] / 255.
def generate_data(max_samples, batchsize, part): #part = train|dev|test
while 1:
samples = list(range(0, max_samples, batchsize))
#np.random.shuffle(samples)
for i in samples:
X = []
Y = []
#Read a batch of clips from files
j = 0
while len(X) < batchsize:
if part == 'train':
idxs = list(range(25 * 5))
np.random.shuffle(idxs)
idxs = idxs[:
2] # keep only 2 random frames per clip on train mode
else:
idxs = [50, 100
] # only evaluate frames 50 and 100 on eval mode
ok = True
path = root_dataset + '/' + part + '/X/X' + str(i + j) + '.mp4'
# pdb.set_trace()
try:
Xj = pims.Video(root_dataset + '/' + part + '/X/X' +
str(i + j) + '.mp4')[idxs]
Xj = np.array(Xj, dtype='float32') / 255.
Yj = pims.Video(root_dataset + '/' + part + '/Y/Y' +
str(i + j) + '.mp4')[idxs]
Yj = np.array(Yj, dtype='float32') / 255.
except:
print('Error clip number ' + str(i + j) + ' at ' +
root_dataset + '/train/X/X' + str(i + j) + '.mp4' +
' OR ' + root_dataset + '/train/Y/Y' + str(i + j) +
'.mp4')
ok = False
if i + j >= max_samples: j = 0
if ok:
X.append(Xj)
Y.append(Yj)
j = j + 1
# make numpy and reshape
X = np.asarray(X)
X = X.reshape(
(X.shape[0] * X.shape[1], X.shape[2], X.shape[3], X.shape[4]))
Y = np.asarray(Y)
Y = Y.reshape(
(Y.shape[0] * Y.shape[1], Y.shape[2], Y.shape[3], Y.shape[4]))
yield (X, Y)
def draw_ROIs(folder):
df = pd.read_csv(os.path.join(folder, 'analysis_rois.csv'), index_col=0)
for col in df.columns:
for i in range(5):
print('new column!')
for vid in df.index:
roinr = col[:2]
roitype = col[2:]
if pd.isnull(df.loc[vid, col]):
video = pims.Video(os.path.join(folder, vid))
if roitype == 'rectangle' or roitype == 'rect':
roi = str(cv2.selectROI('Select a rectangular ROI for ROI nr. {}'.format(roinr), video[0]))
cv2.destroyWindow('Select a rectangular ROI for ROI nr. {}'.format(roinr))
elif roitype == 'polygon':
pgd = PolygonDrawer('Draw a POLYGON', video[0])
roi = str(pgd.run())
elif roitype == 'circle':
cd = CircleDrawer('Draw a CIRCLE ROI for ROI nr. {}'.format(roinr), video[0])
roi = str(cd.run(fill=False))
else:
print(roinr, roitype)
raise BaseException
df.loc[vid, col] = roi
df.to_csv(os.path.join(folder, 'analysis_rois.csv'))
print(os.path.join(folder, 'analysis_rois.csv'), 'updated!')
def get_split_ROIs(folder):
"""Utility function to get ROIs from multiple videos. Grabs a frame from each video, asks for the ROI,
saves the video paths and ROIs as a csv file for the downstream processes"""
videos = []
rois = []
region = []
for video in os.listdir(folder):
if video.endswith('.mp4'):
vid = pims.Video(os.path.join(folder, video))
for i in range(2):
if i == 0:
roi = cv2.selectROI("Select LEFT crop", img=vid[0])
cv2.destroyWindow("Select LEFT crop")
videos.append(video)
rois.append(roi)
region.append('L')
elif i == 1:
roi = cv2.selectROI("Select RIGHT crop", img=vid[0])
cv2.destroyWindow("Select RIGHT crop")
videos.append(video)
rois.append(roi)
region.append('R')
df = pd.DataFrame(data=rois, index=[videos, region])
df.to_csv(os.path.join(folder, "splitROIs.csv"))
print("splitROIs.csv saved to:", os.path.join(folder, "splitROIs.csv"))
def configureRecord(self):
if self.videoPath != "":
self.w = self.roiBound[2] - self.roiBound[0]
self.h = self.roiBound[3] - self.roiBound[1]
logging.debug('%s, %s, %s', "configurerecord", self.w, self.h)
self.codecChoices = {'DIVX': cv2.VideoWriter_fourcc(*'DIVX'),
'MJPG': cv2.VideoWriter_fourcc('M','J','P','G'),
'FFV1': cv2.VideoWriter_fourcc('F','F','V','1')}
fourcc = self.codecChoices.get(self.configRecWindow.codec.currentText())
self.recordingPath = self.configRecWindow.textbox.toPlainText()
if self.configRecWindow.fourRec.isChecked() == True:
i = 2
else:
i = 1
w = 2 * 512 #TODO: include this in gui (check above)
h = i * 512
size = (w, h)
## fps = self.frameRate
fps = self.configRecWindow.fps.value() #fixed playback fps
self.out = cv2.VideoWriter(self.recordingPath, fourcc, fps, size)
self.merged_frame = np.empty([h, w, 3], dtype = np.uint8)
logging.debug('%s, %s', self.recordingPath, self.merged_frame.shape)
self.recordBtn.setEnabled(True)
videofile2 = self.configRecWindow.videoTextbox.toPlainText() #second video
logging.debug(videofile2)
if videofile2 != "":
# self.cap2 = cv2.VideoCapture(videofile2)
self.cap2 = pims.Video(videofile2)
self.configRecWindow.close()
self.seekSlider.setValue(1) #reset to beginning
# self.showContours.setChecked(False) #uncheck show contours
self.clear_data() #clear data
def _parse_video(self, video_path, label):
"""Extract image sequence from video.
return:
images_per_video: 4D ndarray
in shape "num_frames x height x weight x channel".
Note:
Now we assume num_frames is the same for all the videos, and image sequence
is uniformly distributed along the time dimesnion.
"""
# print(type(video_path)) <class 'bytes'>
# print(video_path) b'../../data/UCF-101/ApplyEyeMakeup/v_ApplyEyeMakeup_g08_c04.avi'
v = pims.Video(video_path.decode())
length = len(v)
step = np.floor(length / self.num_frames)
self.sample_index = [
np.random.randint(i * step, (i + 1) * step)
for i in range(self.num_frames)
]
samples = [v[i] for i in self.sample_index]
images_per_video = np.stack(samples)
return images_per_video, label
def collect_lengths(folder):
videos = [mp4 for mp4 in os.listdir(folder) if (mp4.endswith(".mp4"))]
if not os.path.exists(os.path.join(folder, 'lengths.csv')):
df = pd.DataFrame(index=videos,
columns=[
'line', 'frame_shape', 'known_length_px',
'known_length_cm'
])
df.to_csv(os.path.join(folder, 'lengths.csv'))
df = pd.read_csv(os.path.join(folder, 'lengths.csv'), index_col=0)
for vid in df.index:
if pd.isnull(df.loc[vid, 'line']):
video = pims.Video(os.path.join(folder, vid))
frame_shape = video.frame_shape
df.loc[vid, 'frame_shape'] = str(frame_shape)
ld = LineDrawer("Draw a LINE, RIGHT click when you're ready!",
video[0])
line = ld.run(thickness=1)
df.loc[vid, 'line'] = str(line)
known_length_px = sqrt(
(line[1][1] - line[0][1])**2 +
(line[1][0] - line[0][0])**2) # Euclidean distance
df.loc[vid, 'known_length_px'] = known_length_px
df.to_csv(os.path.join(folder, 'lengths.csv'))
df.to_csv(os.path.join(folder, 'lengths.csv'))
def set_video(self, file, hands = False, size = (120,160), factor = 2):
video = pims.Video(file)
frames = video[::factor]
self.frames = list(map(lambda f,s: m.imresize(f, tuple(s)).astype(np.float32), frames, [list(size)]*len(frames)))
video.close()
if hands:
self.hands = np.uint8(np.load(file.replace(".mp4",".npy").replace("_color",""))[::factor])
def extract_scene_frames(trailer_id):
video_file_path = get_trailer_video_path(trailer_id)
efl = find_exemplar_frames(video_file_path)
vid = pims.Video(video_file_path)
if efl == []:
raise Exception('Empty list of scene exemplar frame numbers!')
exemplar_frame_path = get_trailer_frame_path(trailer_id)
if os.path.exists(exemplar_frame_path):
print('{} already exists!'.format(trailer_id))
return
else:
os.mkdir(exemplar_frame_path)
# extract the exemplar numbered frames from the trailer video
exemplar_frames = vid[efl]
for i, exemplar_frame in enumerate(exemplar_frames):
frame_file_name = '%03d.png'%(i)
frame_file_path = os.path.join(exemplar_frame_path, frame_file_name)
im = Image.fromarray(exemplar_frame)
im.save(frame_file_path)
# print(frame_file_path)
efl_file_path = get_frame_list_path(trailer_id)
with open(efl_file_path, 'w+') as f:
for i,fm in enumerate(efl):
f.write('{},{}\n'.format(i,fm))
def getMaskROIs(folder, shape, frame_no=0, save=True):
"""Utility function to get ROIs from multiple videos. Grabs a frame from each video, asks for the ROI,
saves the video paths and ROIs as a csv file for the downstream processes
:shape: 'rectangular' or 'polygon'
"""
videos = []
rois = []
for video in os.listdir(folder):
if video.endswith('.mp4'):
vid = pims.Video(os.path.join(folder, video))
if shape == "rectangular":
roi = cv2.selectROI("Select ROI", vid[frame_no])
cv2.destroyWindow("Select ROI")
videos.append(video)
rois.append(roi)
if shape == "polygon":
pgd = PolygonDrawer("PolygonDrawer", img=vid[frame_no])
# cv2.destroyWindow("PolygonDrawer")
roi = pgd.run()
videos.append(video)
rois.append(roi)
if save:
df = pd.DataFrame(data=rois, index=videos)
df.to_csv(os.path.join(folder, "maskROIs.csv"))
print("maskROIs.csv saved to:", os.path.join(folder, "maskROIs.csv"))
else:
return videos, rois
def get_video_info(filename, verbose = False):
"""
Args:
filename: path to video file
Returns: a dictionary with the video metadata
"""
v = pims.Video(filename)
if verbose:
print(v)
#JG: v.framerate worked on the Alice computer but not on Eve where I got
# AttributeError: 'ImageIOReader' object has no attribute 'duration'
# thus in case v.framerate doesn't work, we try to get the framerate from ffpeg
try:
frame_rate = v.frame_rate
except:
frame_rate = get_frame_rate(filename)
# JG: similar for v.duration
try:
duration = v.duration
except:
duration = len(v)
video_info = {
'frame_rate': frame_rate,
'duration': duration
}
return video_info
def maskVideo_rect(videoPath, outputFolder, reader='ImageIO'):
"""Function containing a pipeline which masks a desired ROI in a given video (provided by the maskROIs.csv file)
and saves the result"""
# get the maskCoords from the csv file
df = pd.read_csv(os.path.join(os.path.split(videoPath)[0], "maskROIs.csv"),
index_col=0)
maskCoords = tuple(df.loc[os.path.split(videoPath)[-1]])
# reader
if reader == 'ImageIO':
video = pims.ImageIOReader(videoPath)
elif reader == 'PyAV':
video = pims.Video(videoPath)
fps = video.frame_rate
# pipeline
r = maskCoords
masking_pipeline = pims.pipeline(maskFrame_rect)
kwargs = dict(maskCoords=r)
processed_video = masking_pipeline(video, **kwargs)
# writing to disk
try:
os.mkdir(outputFolder)
except OSError:
print(
outputFolder,
"already exists! Continuing with the process without creating it.")
outputFilename = os.path.join(
outputFolder,
os.path.split(videoPath)[-1].strip(".mp4") + "_masked.mp4")
imageio.mimwrite(outputFilename, processed_video, fps=fps)
def __getitem__(self, vidInd):
data = torch.zeros(3, self.imgSize, self.imgSize)
vidFold = os.path.splitext(self.videoPaths[vidInd])[0]
vidName = os.path.basename(vidFold)
if not self.videoPaths[vidInd] in self.FPSDict.keys():
self.FPSDict[self.videoPaths[vidInd]] = utils.getVideoFPS(
self.videoPaths[vidInd])
fps = self.FPSDict[self.videoPaths[vidInd]]
shotBounds = np.genfromtxt("{}/result.csv".format(vidFold))
shotInd = np.random.randint(len(shotBounds))
#Selecting frame indexes
shotBounds = torch.tensor(shotBounds[shotInd]).float()
frameInd = torch.distributions.uniform.Uniform(
shotBounds[0], shotBounds[1] + 1).sample().long()
video = pims.Video(self.videoPaths[vidInd])
gt = [1]
return self.preproc(video[frameInd.item()]).unsqueeze(0), torch.tensor(
gt).float().unsqueeze(0), [vidName]
def _parse_video(self, video_path):
"""Extract image sequence from video.
return:
images_per_video: 4D ndarray
in shape "num_frames x height x weight x channel",
or 4D tensor in shape "num_frames x channel x height x weight ".
Note:
Now we assume num_frames is the same for all the videos, and image sequence
is uniformly distributed along the time dimesnion.
"""
v = pims.Video(video_path)
length = len(v)
step = np.floor(length / self.num_frames)
self.sample_index = [np.random.randint(i*step, (i+1)*step) for i in range(self.num_frames)]
if self.transform:
samples = [self.transform(v[i]) for i in self.sample_index]
else:
samples = [v[i] for i in self.sample_index]
images_per_video = torch.stack(samples)
return images_per_video
def getVideoFPS(videoPath):
''' Get the number of frame per sencond of a video.'''
pimsVid = pims.Video(videoPath)
if hasattr(pimsVid, "_frame_rate"):
return float(pims.Video(videoPath)._frame_rate)
else:
res = subprocess.check_output(
"ffprobe -v 0 -of csv=p=0 -select_streams v:0 -show_entries stream=r_frame_rate {}"
.format(videoPath),
shell=True)
res = str(res)[:str(res).find("\\n")].replace("'", "").replace(
"b", "").split("/")
fps = int(res[0]) / int(res[1])
return fps
def show_videos():
files = glob.glob("labels/*.mp4")
labels_dict = pickle.load(open("label_complete.pikle", 'rb'),
encoding='latin1')
draw_hand = True
draw_body = True
draw_ball = True
empty = False
empty_image = np.zeros((480, 640, 3))
cv2.imshow("Action Dataset", empty_image)
for file in files:
name = file.split("/")[-1]
video_labels = labels_dict[name]
frames = pims.Video(file)
for i, frame_label in enumerate(video_labels):
frame = empty_image if empty else frames[i]
frame = draw_bodypose(
frame, frame_label["body"].astype(int)) if draw_body else frame
frame = draw_handpose(
frame, frame_label["hand"].astype(int)) if draw_hand else frame
frame = draw_ballpose(
frame, frame_label["ball"].astype(int)) if draw_ball else frame
cv2.imshow("Action Dataset", frame[:, :, [2, 1, 0]])
k = cv2.waitKey(10)
if k == 27: return
if k == 98: draw_ball = not draw_ball #B
if k == 101: empty = not empty #E
if k == 104: draw_hand = not draw_hand #H
if k == 106: draw_body = not draw_body #J
if k == 32:
while cv2.waitKey(30) != 32:
continue
def mkv2tiff(file_path, ds_factor=4):
'''
Input:
- file_path: path to mkv file
- ds_factor: factor to downsample by, default = 4, choose 1 for no
downsampling
Output:
- NA, saves new downsampled and converted file to same path as original
with '_ds' appended to end of filename.
'''
print('Accessing Video')
vid = pims.Video(file_path)
save_name = file_path.replace('.mkv', '_ds')
dims = vid[0].shape
vid_ds = np.zeros((int(len(vid)/ds_factor), dims[0], dims[1]))
frame_ds = 0
for frame in range(0, len(vid), ds_factor):
if frame + ds_factor < len(vid):
slice = np.array(vid[frame:frame+ds_factor])[:,:,:,0]
vid_ds[frame_ds, :, :] = np.round(np.mean(slice, axis=0))
frame_ds += 1
if frame % 500 == 0:
print('frame {} of {}'.format(frame, len(vid)))
else:
continue
skimage.io.imsave(save_name + '.tiff', skimage.img_as_uint(vid_ds/vid_ds.max()))
def get_frames(file, size = (120,160)):
try:
frames = pims.Video(file)
frames = list(map(lambda f,s: m.imresize(f, tuple(s)).astype(np.float32), frames, [list(size)]*len(frames)))
return frames
except:
print("Error in file = {}".format(file))
return None
def get_frames(file_path, frames, gaussian_filter_width=None, roi = None):
"""
opens the video and returns the requested frames as numpy array
Args:
file_path: path to video file
frames: integer or list of integers of the frames to be returned
gaussian_filter_width: if not None apply Gaussian filter
roi: region of interest, this allows to limit the search to a region of interest with the frame, the structure is
roi = [roi_center, roi_dimension], where
roi_center = [ro, co], roi_dimension = [h, w], where
ro, co is the center of the roi (row, columns) and
w, h is the width and the height of the roi
Note that roi dimensions w, h should be odd numbers!
"""
images = []
if not hasattr(frames, '__len__'):
frames = [frames]
if not roi is None:
[roi_center, roi_dimension] = roi
v = pims.Video(file_path)
video = rgb2gray_pipeline(v)
if not gaussian_filter_width is None:
gaussian_filter_pipeline = pipeline(gaussian_filter)
video = gaussian_filter_pipeline(video, gaussian_filter_width)
frame_shape = np.shape(video[frames[0]])
for frame in frames:
image = video[frame]
# reduce image to roi
if not roi is None:
[roi_center, roi_dimension] = roi
image_roi = image[
int(roi_center[0] - (roi_dimension[0] - 1) / 2): int(
roi_center[0] + (roi_dimension[0] + 1) / 2),
int(roi_center[1] - (roi_dimension[1] - 1) / 2): int(roi_center[1] + (roi_dimension[1] + 1) / 2)
]
else:
image_roi = image
images.append(image_roi)
return images
def background(path, method):
images = pims.Video(str(path))
for m in method:
try:
handle = getattr(background_methods, m)
except:
raise RuntimeError(f'{m} is not a valid background method.')
result = handle(images)
io.imsave(f'{m}.png', result)
def __init__(self):
self.micron_per_pixel = 0.15192872980868
self.feature_diameter = 2.8 # um
self.radius = int(
np.round(self.feature_diameter / 2.0 / self.micron_per_pixel))
if self.radius % 2 == 0:
self.radius + 1
self.frames = pims.as_gray(pims.Video('core/sample.mp4'))
def __next__(self):
if self.videoInd == len(self.videoPaths):
raise StopIteration
L = self.evalL
self.sumL += L
videoPath = self.videoPaths[self.videoInd]
video = pims.Video(videoPath)
vidName = os.path.basename(os.path.splitext(videoPath)[0])
fps = utils.getVideoFPS(videoPath)
shotBounds = np.genfromtxt("../data/{}/{}/result.csv".format(
self.dataset, vidName))
shotInds = np.arange(self.shotInd,
min(self.shotInd + L, len(shotBounds)))
if not self.randomFrame:
frameInds = self.regularlySpacedFrames(
shotBounds[shotInds]).reshape(-1)
else:
shotBoundsToUse = torch.tensor(
shotBounds[shotInds.astype(int)]).float()
frameInds = torch.distributions.uniform.Uniform(
shotBoundsToUse[:, 0], shotBoundsToUse[:, 1] + 1).sample(
(1, )).long()
frameInds = frameInds.transpose(dim0=0, dim1=1)
frameInds = np.array(frameInds.contiguous().view(-1))
try:
frameSeq = torch.cat(list(
map(lambda x: self.preproc(video[x]).unsqueeze(0),
np.array(frameInds))),
dim=0)
except IndexError:
print(
vidName, "max frame", frameInds.max(),
np.genfromtxt("../data/{}/{}/result.csv".format(
self.dataset, vidName)).max())
sys.exit(0)
gt = getGT(self.dataset,
vidName)[self.shotInd:min(self.shotInd +
L, len(shotBounds))]
if shotInds[-1] + 1 == len(shotBounds):
self.shotInd = 0
self.videoInd += 1
else:
self.shotInd += L
return frameSeq.unsqueeze(0), torch.tensor(gt).float().unsqueeze(
0), vidName, torch.tensor(frameInds).int()
def loadVideo(self):
if os.path.isfile(self.localMasterDirectory + self.videofile):
print(self.videofile + ' present in local path.', file = sys.stderr)
else:
print(self.videofile + ' not present in local path. Trying to find it remotely', file = sys.stderr)
# Try to download it from the cloud
subprocess.call(['rclone', 'copy', self.cloudMasterDirectory + self.videofile, self.localMasterDirectory + self.videofile.split(self.videofile.split('/')[-1])[0]], stderr = self.fnull)
if not os.path.isfile(self.localMasterDirectory + self.videofile):
print(self.videofile + ' not present in remote path. Trying to find h264 file and convert it to mp4', file = sys.stderr)
if not os.path.isfile(self.localMasterDirectory + self.videofile.replace('.mp4', '.h264')):
subprocess.call(['rclone', 'copy', self.cloudMasterDirectory + self.videofile.replace('.mp4', '.h264'), self.localMasterDirectory + self.videofile.split(self.videofile.split('/')[-1])[0]], stderr = self.fnull)
if not os.path.isfile(self.localMasterDirectory + self.videofile.replace('.mp4', '.h264')):
print('Unable to find ' + self.videofile.replace('.mp4', '.h264'), file = sys.stderr)
raise Exception
# Convert it to mpeg
subprocess.call(['ffmpeg', '-i', self.localMasterDirectory + self.videofile.replace('.mp4', '.h264'), '-c:v', 'copy', self.localMasterDirectory + self.videofile])
if os.stat(self.localMasterDirectory + self.videofile).st_size >= os.stat(self.localMasterDirectory + self.videofile.replace('.mp4', '.h264')).st_size:
try:
vid = pims.Video(self.localMasterDirectory + self.videofile)
vid.close()
os.remove(self.localMasterDirectory + self.videofile.replace('.mp4', '.h264'))
except Exception as e:
self._print(e)
self._print('Unable to convert ' + self.videofile)
raise Exception
print('Uploading ' + self.videofile + ' to cloud', file = sys.stderr)
subprocess.call(['rclone', 'copy', self.localMasterDirectory + self.videofile, self.cloudMasterDirectory + self.videofile.split(self.videofile.split('/')[-1])[0]], stderr = self.fnull)
self._print(self.videofile + ' converted and uploaded to cloud')
#Grab info on video
cap = pims.Video(self.localMasterDirectory + self.videofile)
self.height = int(cap.frame_shape[0])
self.width = int(cap.frame_shape[1])
self.frame_rate = int(cap.frame_rate)
try:
self.frames = min(int(cap.get_metadata()['duration']*cap.frame_rate), 12*60*60*self.frame_rate)
except AttributeError:
self.frames = min(int(cap.duration*cap.frame_rate), 12*60*60*self.frame_rate)
cap.close()
def generate_body_labels():
body_estimation = Body('pytorch-openpose/model/body_pose_model.pth')
hand_estimation = Hand('pytorch-openpose/model/hand_pose_model.pth')
def get_body(img):
candidate, subset = body_estimation(img)
candidate = get_joints(candidate, subset)
subset = np.array(
[[i if candidate[i, 0] > -1 else -1 for i in range(18)]])
return candidate
def get_hand(img, body):
subset = np.array([[i if body[i, 0] > -1 else -1 for i in range(18)]])
hands_list = util.handDetect(body, subset, img)
hand_joints = np.ones((2, 23, 2)) * -1
for x, y, w, is_left in hands_list:
hand = 0 if is_left else 1
peaks = hand_estimation(img[y:y + w, x:x + w, :])
peaks[:, 0] = np.where(peaks[:, 0] == 0, peaks[:, 0],
peaks[:, 0] + x)
peaks[:, 1] = np.where(peaks[:, 1] == 0, peaks[:, 1],
peaks[:, 1] + y)
hand_joints[hand, :-2] = peaks
hand_joints[hand, -2:] = np.array([[x, y], [w, w]])
return hand_joints
files = glob.glob("out.mp4")
#label = pd.read_csv("labels2.csv")
hand_size = 2 * 23 * 2
skl = 18 * 2
print("total:", len(files))
for file in files:
print(file)
video = []
i = 0
start = timer()
images = pims.Video(file)
total = len(images)
videos = np.ones((total, 1 + hand_size + skl))
np.save("./labels_cut.npy", videos)
for frame, img in enumerate(images):
# ball = label[(label.video==file.split("/")[-1]) & (label.frame == frame)].values
# ball = ball[:,-2:].reshape(-1)
#img = img[:,:,[2,0,1]]
body = get_body(img)
hand = get_hand(img, body)
videos[frame, 0] = frame
videos[frame, 1:skl + 1] = body.flatten()
videos[frame, skl + 1:] = hand.flatten()
if frame % 500 == 0:
print("{}/{} time/frame = {}".format(
frame, total, (timer() - start) / (frame + 1)))
np.save("./labels_cut.npy", videos)
def load_video_file(self, videofile=None):
if videofile:
# For testing purposes
frame_array = pims.Video(videofile)
return (videofile, frame_array)
file_name, _ = QFileDialog.getOpenFileName(
self, 'QFileDialog.getOpenFileName()', '',
'All Files(*);;PythonFiles(*.py)')
# Convert the file to an array of frames
if file_name:
frame_array = None
try:
# Return the file path and frame array
frame_array = pims.Video(file_name)
return (file_name, frame_array)
except AVError as err:
msg = QMessageBox()
msg.setText(f'Try loading a different video file. \n{err}')
msg.exec_()
return err
def _readBlock(self, block):
min_t = block*self.blocksize
max_t = min((block+1)*self.blocksize, int(self.frames/self.frame_rate))
ad = np.empty(shape = (self.height, self.width, max_t - min_t), dtype = 'uint8')
cap = pims.Video(self.localMasterDirectory + self.videofile)
counter = 0
for i in range(min_t, max_t):
current_frame = i*self.frame_rate
frame = cap[current_frame]
ad[:,:,counter] = 0.2125 * frame[:,:,0] + 0.7154 * frame[:,:,1] + 0.0721 * frame[:,:,2]
counter += 1
return ad
def downloadVideo(self):
videoName = self.videofile.split('/')[-1]
localDirectory = self.videofile.replace(videoName,'')
if os.path.isfile(self.videofile):
return
self._print(self.videofile + ' not present in local path. Trying to find it remotely')
subprocess.call(['rclone', 'copy', self.remVideoDirectory + videoName, localDirectory], stderr = self.fnull)
if not os.path.isfile(self.videofile):
self._print(self.videofile + ' not present in remote path. Trying to find h264 file and convert it to mp4')
if not os.path.isfile(self.videofile.replace('.mp4', '.h264')):
subprocess.call(['rclone', 'copy', self.remVideoDirectory + videoName.replace('.mp4', '.h264'), localDirectory], stderr = self.fnull)
if not os.path.isfile(self.videofile.replace('.mp4', '.h264')):
self._print('Unable to find ' + self.remVideoDirectory + videoName.replace('.mp4', '.h264'))
raise Exception
subprocess.call(['ffmpeg', '-i', self.videofile.replace('.mp4', '.h264'), '-c:v', 'copy', self.videofile])
if os.stat(self.videofile).st_size >= os.stat(self.videofile.replace('.mp4', '.h264')).st_size:
try:
vid = pims.Video(self.videofile)
vid.close()
os.remove(self.videofile.replace('.mp4', '.h264'))
except Exception as e:
self._print(e)
self._print('Unable to convert ' + self.videofile)
raise Exception
subprocess.call(['rclone', 'copy', self.videofile, self.remVideoDirectory], stderr = self.fnull)
self._print(self.videofile + ' converted and uploaded to ' + self.remVideoDirectory)
#Grab info on video
cap = pims.Video(self.videofile)
self.height = int(cap.frame_shape[0])
self.width = int(cap.frame_shape[1])
self.frame_rate = int(cap.frame_rate)
try:
self.frames = min(int(cap.get_metadata()['duration']*cap.frame_rate), 12*60*60*self.frame_rate)
except AttributeError:
self.frames = min(int(cap.duration*cap.frame_rate), 12*60*60*self.frame_rate)
cap.close()
def recognition_video():
class_names = ["no gesture","vattene","vieniqui","perfetto","furbo","cheduepalle","chevuoi","daccordo","seipazzo","combinato","freganiente","ok","cosatifarei","basta","prendere","noncenepiu","fame","tantotempo","buonissimo","messidaccordo","sonostufo","not anticipated"]
best_samples = ["Sample0913","Sample0742","Sample0707","Sample0777"]
with open("results/results_anticipation_RTM_8691.pkl","rb") as file:
predictions = pickle.load(file)
org = (20, 30)
org_box = (30, 30)
fontScale = 1
color_bue = (200, 0, 0)
color_green = (0,200, 0)
color_red = ( 0, 0, 200)
thickness = 2
corrects = 0
total = 0
JI_i,JI_u = 0,0
font = cv2.FONT_HERSHEY_SIMPLEX
fourcc = cv2.VideoWriter_fourcc(*'XVID')
out = cv2.VideoWriter('results/recognition_montalbano.avi', fourcc, 40.0, (640,480))
for s in best_samples:
for prediction in predictions:
pred = np.array(prediction["pred"])
label = np.array(prediction["label"])
JI_i += sum(pred==label)
JI_u += len(label)
probs = prediction["probs"]
file = prediction["file"]
acc = sum(pred==label)/len(pred)
sample = file.split("/")[-1]
corrects += sum(pred==label)
acc = sum(pred==label)/len(label)
total += len(pred)
if sample == s:
print("{} -> acc {:.2f}%".format(sample,acc*100))
video = pims.Video("samples/{}_color.mp4".format(sample))
for frame, p, l, prob in zip(video,pred,label,probs):
frame = cv2.cvtColor(frame, cv2.COLOR_RGB2BGR)
frame = cv2.rectangle(frame, (0,0), (640,70), (200,200,200), cv2.FILLED)
frame = cv2.putText(frame, "Label:", (10,30), font, fontScale, (0,0,0), thickness)
frame = cv2.putText(frame, "video 2x", (530,20), font, 0.8, (0,0,0))
frame = cv2.putText(frame, "Recog.:", (10,60), font, fontScale, (0,0,0), thickness)
frame = cv2.putText(frame, "{}".format(class_names[l]), (130,30), font, fontScale, (0,0,0), thickness)
color = color_green if p==l else color_red
frame = cv2.putText(frame, "{}".format(class_names[p]), (130,60), font, fontScale, color, thickness)
out.write(frame)
break
out.release()
print(JI_i/JI_u)
