def learn_msr_video(name, offsets_1, offsets_2): ''' ''' depth_file = name + "depth.bin" color_file = name + "rgb.avi" skeleton_file = name + "skeleton.txt" ''' Read data from each video/sequence ''' try: depthIms, maskIms = read_MSR_depth_ims(depth_file) depthIms *= maskIms depthIms /= 10 colorIms = read_MSR_color_ims(color_file) skels_world, _ = read_MSR_skeletons(skeleton_file) skels_world[:,2]/=10 except: print "Error getting frame features" return -1,-1 all_features = [] all_labels = [] for i in xrange(0, len(depthIms), 25): im_depth = depthIms[i] skel_pos = skel2depth(skels_world[i], rez=[240,320]) # ''' --- NOTE THIS 10 PX OFFSET IN THE MSR DATASET !!! --- ''' # skel_pos[:,0] -= 10 features, pixel_labels = learn_frame(im_depth, skel_pos, offsets_1, offsets_2) ''' Stack features ''' all_features += [features] all_labels += [pixel_labels] return np.concatenate(all_features), np.concatenate(all_labels)
def learn_msr_video(name, offsets_1, offsets_2):
'''
'''
depth_file = name + "depth.bin"
color_file = name + "rgb.avi"
skeleton_file = name + "skeleton.txt"
''' Read data from each video/sequence '''
try:
depthIms, maskIms = read_MSR_depth_ims(depth_file)
depthIms *= maskIms
depthIms /= 10
colorIms = read_MSR_color_ims(color_file)
skels_world, _ = read_MSR_skeletons(skeleton_file)
skels_world[:, 2] /= 10
except:
print "Error getting frame features"
return -1, -1
all_features = []
all_labels = []
for i in xrange(0, len(depthIms), 25):
im_depth = depthIms[i]
skel_pos = skel2depth(skels_world[i], rez=[240, 320])
# ''' --- NOTE THIS 10 PX OFFSET IN THE MSR DATASET !!! --- '''
# skel_pos[:,0] -= 10
features, pixel_labels = learn_frame(im_depth, skel_pos, offsets_1,
offsets_2)
''' Stack features '''
all_features += [features]
all_labels += [pixel_labels]
return np.concatenate(all_features), np.concatenate(all_labels)
def visualize_data(name):
'''
'''
depth_file = name + "depth.bin"
color_file = name + "rgb.avi"
skeleton_file = name + "skeleton.txt"
''' Read data from each video/sequence '''
try:
depthIms, maskIms = read_MSR_depth_ims(depth_file)
colorIms = read_MSR_color_ims(color_file)
skels_world, skels_im = read_MSR_skeletons(skeleton_file)
except:
print "Error reading data"
return -1
framecount = np.minimum(depthIms.shape[0], colorIms.shape[0])
''' View all data'''
for frame in xrange(framecount):
depth = depthIms[frame]
mask = maskIms[frame] > 0
color = colorIms[frame]
# color = np.dstack([color[:,:,2],color[:,:,1],color[:,:,0]])
''' Skeleton in world (w) and image (i) coordinates '''
skel_w = skels_world[frame]
skel_i = world2depth(skel_w, rez=[240,320])
''' Calculate hogs '''
grayIm = (rgb2gray(color) * 255).astype(np.uint8)
person_mask, bounding_boxes, labels = extract_people(grayIm, mask>0)
rez = grayIm[bounding_boxes[0]].shape
''' Plot skeletons on color image'''
# color = color*(mask[:,:,None]==0)
# from pylab import *
# embed()
# blurIm = (grayIm*mask).astype(np.float)
# blurIm = color*mask[:,:,None]
# blurIm = cv2.GaussianBlur(color, (15,15), 10)
# blurIm = cv2.GaussianBlur(mask[:,:,None]*color, (9,9), 5)
# blurIm = cv2.blur(mask[:,:,None]*color, (5,5))
# cv2.imshow("b", blurIm/float(blurIm.max()))
''' Visualization '''
# color = display_skeletons(color, skel_i, color=(0,200,0))
# blurIm = display_skeletons(blurIm, skel_i, color=(0,200,0))
cv2.imshow("Depth", depth/float(depth.max()))
cv2.imshow("RGB", color)
# cv2.imshow("RGB_blur", color*(mask[:,:,None]==0) + blurIm*(mask[:,:,None]>0))
# cv2.imshow("RGB masked", color*(mask[:,:,None]>0))
ret = cv2.waitKey(10)
if ret >= 0:
break
def visualize_data(name):
'''
'''
depth_file = name + "depth.bin"
color_file = name + "rgb.avi"
skeleton_file = name + "skeleton.txt"
''' Read data from each video/sequence '''
try:
depthIms, maskIms = read_MSR_depth_ims(depth_file)
colorIms = read_MSR_color_ims(color_file)
skels_world, skels_im = read_MSR_skeletons(skeleton_file)
except:
print "Error reading data"
return -1
framecount = np.minimum(depthIms.shape[0], colorIms.shape[0])
''' View all data'''
for frame in xrange(framecount):
depth = depthIms[frame]
mask = maskIms[frame] > 0
color = colorIms[frame]
# color = np.dstack([color[:,:,2],color[:,:,1],color[:,:,0]])
''' Skeleton in world (w) and image (i) coordinates '''
skel_w = skels_world[frame]
skel_i = world2depth(skel_w, rez=[240, 320])
''' Calculate hogs '''
grayIm = (rgb2gray(color) * 255).astype(np.uint8)
person_mask, bounding_boxes, labels = extract_people(grayIm, mask > 0)
rez = grayIm[bounding_boxes[0]].shape
''' Plot skeletons on color image'''
# color = color*(mask[:,:,None]==0)
# from pylab import *
# embed()
# blurIm = (grayIm*mask).astype(np.float)
# blurIm = color*mask[:,:,None]
# blurIm = cv2.GaussianBlur(color, (15,15), 10)
# blurIm = cv2.GaussianBlur(mask[:,:,None]*color, (9,9), 5)
# blurIm = cv2.blur(mask[:,:,None]*color, (5,5))
# cv2.imshow("b", blurIm/float(blurIm.max()))
''' Visualization '''
# color = display_skeletons(color, skel_i, color=(0,200,0))
# blurIm = display_skeletons(blurIm, skel_i, color=(0,200,0))
cv2.imshow("Depth", depth / float(depth.max()))
cv2.imshow("RGB", color)
# cv2.imshow("RGB_blur", color*(mask[:,:,None]==0) + blurIm*(mask[:,:,None]>0))
# cv2.imshow("RGB masked", color*(mask[:,:,None]>0))
ret = cv2.waitKey(10)
if ret >= 0:
break
def compute_features(name, vis=False, person_rez=[144, 72]):
'''
---Parameters---
filename : base filename for depth/color/skel
vis: turn visualize on?
person_rez : all hog/hof features should be the same dimensions, so we resize the people to this resolution
---Return---
features: features in a dictionary
'''
''' Get filenames '''
depth_file = name + "depth.bin"
color_file = name + "rgb.avi"
skeleton_file = name + "skeleton.txt"
''' Read data from each video/sequence '''
try:
depthIms, maskIms = read_MSR_depth_ims(depth_file)
colorIms = read_MSR_color_ims(color_file)
skels_world, skels_im = read_MSR_skeletons(skeleton_file)
except:
print "Error reading data"
return -1
#print depthIms.shape, colorIms.shape
framecount = np.minimum(depthIms.shape[0], colorIms.shape[0])
dataset_features = {
'framecount': framecount,
'hog': [],
'hof': [],
'skel_image': [],
'skel_world': []
}
grayIm_prev = None
''' View all data'''
for frame in xrange(framecount):
depth = depthIms[frame]
mask = maskIms[frame]
color = colorIms[frame]
# Skeleton in world (w) and image (i) coordinates
skel_w = skels_world[frame]
skel_i = world2depth(skel_w, rez=[240, 320])
''' Calculate hogs '''
grayIm = (rgb2gray(color) * 255).astype(np.uint8)
hogIm = np.zeros_like(depth)
person_mask, bounding_boxes, labels = extract_people(grayIm, mask > 0)
rez = grayIm[bounding_boxes[0]].shape
#hog_input_im = sm.imresize(grayIm[bounding_boxes[0]], person_rez)
hog_input_im = cv2.resize(grayIm[bounding_boxes[0]],
(person_rez[1], person_rez[0]))
hogData, hogImBox = hog(hog_input_im, orientations=4, visualise=True)
#hogIm[bounding_boxes[0]] = sm.imresize(hogImBox, [rez[0],rez[1]])
hogIm[bounding_boxes[0]] = cv2.resize(hogImBox, (rez[1], rez[0]))
# hogIm[bounding_boxes[0]] = hogImBox
hogIm *= person_mask
''' Calculate HOF '''
hofIm = np.zeros_like(depth)
if grayIm_prev is None:
grayIm_prev = np.copy(grayIm)
continue
else:
flow = getFlow(grayIm_prev[bounding_boxes[0]],
grayIm[bounding_boxes[0]])
rez = flow.shape
bounding_boxes = (bounding_boxes[0][0], bounding_boxes[0][1],
slice(0, 2))
#hof_input_im = np.dstack([sm.imresize(flow[0], [person_rez[0],person_rez[1]]),
# sm.imresize(flow[1], [person_rez[0],person_rez[1]])])i
hof_input_im = np.dstack([
cv2.resize(flow[0], (person_rez[1], person_rez[0])),
cv2.resize(flow[1], (person_rez[1], person_rez[0]))
])
hofData, hofImBox = hof(hof_input_im,
orientations=5,
visualise=True)
#hofIm[bounding_boxes[:2]] = sm.imresize(hofImBox, [rez[0],rez[1]])
hofIm[bounding_boxes[:2]] = cv2.resize(hogImBox, (rez[1], rez[0]))
hofIm *= person_mask
grayIm_prev = np.copy(grayIm)
''' Add features '''
dataset_features['hog'] += [hogData]
dataset_features['hof'] += [hofData]
dataset_features['skel_image'] += [skel_i]
dataset_features['skel_world'] += [skel_w]
''' Plot skeletons on color image'''
if vis:
color = display_skeletons(color, skel_i)
''' Visualization '''
cv2.imshow("Depth", depth / float(depth.max()))
cv2.imshow("HOG", hogIm / float(hogIm.max()))
cv2.imshow("RGB", color)
cv2.imshow("RGB masked", color * (mask[:, :, None] > 0))
cv2.imshow("HOF", hofIm / float(hofIm.max()))
ret = cv2.waitKey(10)
if ret >= 0:
break
print "Done calculating ", name
return dataset_features
def main_infer(rf_name=None):
'''
'''
if rf_name is None:
import os
files = os.listdir('Saved_Params/')
rf_name = 'Saved_Params/' + files[-1]
print "Classifier file:",rf_name
# Load classifier data
data = pickle.load(open(rf_name))
rf = data['rf']
offsets_1, offsets_2 = data['offsets']
''' Read data from each video/sequence '''
depthIms = []
skels_world = []
if 1:
# VP = KinectPlayer(base_dir='/Users/colin/Data/Office_25Feb2013/', device=2, get_depth=True, get_color=False, get_skeleton=True, get_mask=False)
VP = KinectPlayer(base_dir='/Users/colin/Data/Room_close_26Feb13/', device=1, get_depth=True, get_color=False, get_skeleton=True, get_mask=False)
_,_ = VP.get_n_skeletons(50)
depthIms, skels_world = VP.get_n_skeletons(100)
# depthIms = np.array(depthIms)[:,:,::-1]
else:
# name = 'a01_s01_e02_'
name = 'a01_s10_e02_'
# name = 'a02_s06_e02_'
# name = 'a05_s02_e02_'
depth_file = name + "depth.bin"
color_file = name + "rgb.avi"
skeleton_file = name + "skeleton.txt"
try:
depthIms, maskIms = read_MSR_depth_ims(depth_file)
depthIms *= maskIms
depthIms /= 10
# colorIms = read_MSR_color_ims(color_file)
skels_world, skels_im = read_MSR_skeletons(skeleton_file)
skels_world[:,2]/=10
except:
print "Error reading data"
''' Process data '''
all_pred_ims = []
for i in xrange(1, len(depthIms), 1):
# try:
if 1:
print i
''' Get frame data '''
im_depth = depthIms[i]
# im_depth[160:, :] = 0
skel_pos = skel2depth(skels_world[i], rez=[240,320])
# ''' --- NOTE THIS 10 PX OFFSET IN THE MSR DATASET !!! --- '''
# skel_pos[:,0] -= 10
skel_pos_pred, im_predict = infer_pose(im_depth, rf, offsets_1, offsets_2)
# Overlay skeletons
if 1:
# colorIm = colorIms[i]
# im_predict = colorIm
cv2.imshow("forest_prediction", im_predict/float(im_predict.max()))
im_predict = np.repeat(im_depth[:,:,None].astype(np.float), 3, -1)
# embed()
im_predict[im_depth>0] -= im_depth[im_depth>0].min()
im_predict /= float(im_predict.max()/255.)
im_predict = im_predict.astype(np.uint8)
# im_predict = np.repeat(im_predict[:,:,None], 3, -1)
# im_predict /= float(im_predict.max())*255
# im_predict = im_predict.astype(np.uint8)
im_predict = display_skeletons(im_predict, skel_pos, (255,0,0), SKEL_DISPLAY_MODE)
im_predict = display_skeletons(im_predict, skel_pos_pred, (0,255,0), SKEL_DISPLAY_MODE)
im_predict = display_skeletons(im_predict, skel_pos, (255,0,0), SKEL_DISPLAY_MODE)
im_predict = display_skeletons(im_predict, skel_pos_pred, (0,255,0), SKEL_DISPLAY_MODE)
# embed()
# max_ = (im_predict * (im_predict < 255)).max()
all_pred_ims += [im_predict]
''' Visualize '''
if 1:
cv2.putText(im_predict, "Blue=Truth", (10, 210), cv2.FONT_HERSHEY_DUPLEX, .5, (int(im_predict.max()/2), 0, 0))
cv2.putText(im_predict, "Green=Predict", (10, 230), cv2.FONT_HERSHEY_DUPLEX, .5, (0, int(im_predict.max()/2), 0))
cv2.imshow("prediction", im_predict)
ret = cv2.waitKey(10)
if ret > 0: break
time.sleep(.5)
# except:
# print "Frame failed:", i
# break
embed()
def compute_features(name, vis=False, person_rez=[144,72]):
'''
---Parameters---
filename : base filename for depth/color/skel
vis: turn visualize on?
person_rez : all hog/hof features should be the same dimensions, so we resize the people to this resolution
---Return---
features: features in a dictionary
'''
''' Get filenames '''
depth_file = name + "depth.bin"
color_file = name + "rgb.avi"
skeleton_file = name + "skeleton.txt"
''' Read data from each video/sequence '''
try:
depthIms, maskIms = read_MSR_depth_ims(depth_file)
colorIms = read_MSR_color_ims(color_file)
skels_world, skels_im = read_MSR_skeletons(skeleton_file)
except:
print "Error reading data"
return -1
#print depthIms.shape, colorIms.shape
framecount = np.minimum(depthIms.shape[0], colorIms.shape[0])
dataset_features = {'framecount':framecount, 'hog':[], 'hof':[], 'skel_image':[], 'skel_world':[]}
grayIm_prev = None
''' View all data'''
for frame in xrange(framecount):
depth = depthIms[frame]
mask = maskIms[frame]
color = colorIms[frame]
# Skeleton in world (w) and image (i) coordinates
skel_w = skels_world[frame]
skel_i = world2depth(skel_w, rez=[240,320])
''' Calculate hogs '''
grayIm = (rgb2gray(color) * 255).astype(np.uint8)
hogIm = np.zeros_like(depth)
person_mask, bounding_boxes, labels = extract_people(grayIm, mask>0)
rez = grayIm[bounding_boxes[0]].shape
#hog_input_im = sm.imresize(grayIm[bounding_boxes[0]], person_rez)
hog_input_im = cv2.resize(grayIm[bounding_boxes[0]], (person_rez[1], person_rez[0]))
hogData, hogImBox = hog(hog_input_im, orientations=4, visualise=True)
#hogIm[bounding_boxes[0]] = sm.imresize(hogImBox, [rez[0],rez[1]])
hogIm[bounding_boxes[0]] = cv2.resize(hogImBox, (rez[1],rez[0]))
# hogIm[bounding_boxes[0]] = hogImBox
hogIm *= person_mask
''' Calculate HOF '''
hofIm = np.zeros_like(depth)
if grayIm_prev is None:
grayIm_prev = np.copy(grayIm)
continue
else:
flow = getFlow(grayIm_prev[bounding_boxes[0]], grayIm[bounding_boxes[0]])
rez = flow.shape
bounding_boxes = (bounding_boxes[0][0], bounding_boxes[0][1], slice(0,2))
#hof_input_im = np.dstack([sm.imresize(flow[0], [person_rez[0],person_rez[1]]),
# sm.imresize(flow[1], [person_rez[0],person_rez[1]])])i
hof_input_im = np.dstack([cv2.resize(flow[0], (person_rez[1],person_rez[0])), cv2.resize(flow[1], (person_rez[1], person_rez[0]))])
hofData, hofImBox = hof(hof_input_im, orientations=5, visualise=True)
#hofIm[bounding_boxes[:2]] = sm.imresize(hofImBox, [rez[0],rez[1]])
hofIm[bounding_boxes[:2]] = cv2.resize(hogImBox, (rez[1],rez[0]))
hofIm *= person_mask
grayIm_prev = np.copy(grayIm)
''' Add features '''
dataset_features['hog'] += [hogData]
dataset_features['hof'] += [hofData]
dataset_features['skel_image'] += [skel_i]
dataset_features['skel_world'] += [skel_w]
''' Plot skeletons on color image'''
if vis:
color = display_skeletons(color, skel_i)
''' Visualization '''
cv2.imshow("Depth", depth/float(depth.max()))
cv2.imshow("HOG", hogIm/float(hogIm.max()))
cv2.imshow("RGB", color)
cv2.imshow("RGB masked", color*(mask[:,:,None]>0))
cv2.imshow("HOF", hofIm/float(hofIm.max()))
ret = cv2.waitKey(10)
if ret >= 0:
break
print "Done calculating ", name
return dataset_features
def main_infer(rf_name=None):
'''
'''
if rf_name is None:
import os
files = os.listdir('Saved_Params/')
rf_name = 'Saved_Params/' + files[-1]
print "Classifier file:", rf_name
# Load classifier data
data = pickle.load(open(rf_name))
rf = data['rf']
offsets_1, offsets_2 = data['offsets']
''' Read data from each video/sequence '''
depthIms = []
skels_world = []
if 1:
# VP = KinectPlayer(base_dir='/Users/colin/Data/Office_25Feb2013/', device=2, get_depth=True, get_color=False, get_skeleton=True, get_mask=False)
VP = KinectPlayer(base_dir='/Users/colin/Data/Room_close_26Feb13/',
device=1,
get_depth=True,
get_color=False,
get_skeleton=True,
get_mask=False)
_, _ = VP.get_n_skeletons(50)
depthIms, skels_world = VP.get_n_skeletons(100)
# depthIms = np.array(depthIms)[:,:,::-1]
else:
# name = 'a01_s01_e02_'
name = 'a01_s10_e02_'
# name = 'a02_s06_e02_'
# name = 'a05_s02_e02_'
depth_file = name + "depth.bin"
color_file = name + "rgb.avi"
skeleton_file = name + "skeleton.txt"
try:
depthIms, maskIms = read_MSR_depth_ims(depth_file)
depthIms *= maskIms
depthIms /= 10
# colorIms = read_MSR_color_ims(color_file)
skels_world, skels_im = read_MSR_skeletons(skeleton_file)
skels_world[:, 2] /= 10
except:
print "Error reading data"
''' Process data '''
all_pred_ims = []
for i in xrange(1, len(depthIms), 1):
# try:
if 1:
print i
''' Get frame data '''
im_depth = depthIms[i]
# im_depth[160:, :] = 0
skel_pos = skel2depth(skels_world[i], rez=[240, 320])
# ''' --- NOTE THIS 10 PX OFFSET IN THE MSR DATASET !!! --- '''
# skel_pos[:,0] -= 10
skel_pos_pred, im_predict = infer_pose(im_depth, rf, offsets_1,
offsets_2)
# Overlay skeletons
if 1:
# colorIm = colorIms[i]
# im_predict = colorIm
cv2.imshow("forest_prediction",
im_predict / float(im_predict.max()))
im_predict = np.repeat(im_depth[:, :, None].astype(np.float),
3, -1)
# embed()
im_predict[im_depth > 0] -= im_depth[im_depth > 0].min()
im_predict /= float(im_predict.max() / 255.)
im_predict = im_predict.astype(np.uint8)
# im_predict = np.repeat(im_predict[:,:,None], 3, -1)
# im_predict /= float(im_predict.max())*255
# im_predict = im_predict.astype(np.uint8)
im_predict = display_skeletons(im_predict, skel_pos,
(255, 0, 0), SKEL_DISPLAY_MODE)
im_predict = display_skeletons(im_predict, skel_pos_pred,
(0, 255, 0), SKEL_DISPLAY_MODE)
im_predict = display_skeletons(im_predict, skel_pos,
(255, 0, 0), SKEL_DISPLAY_MODE)
im_predict = display_skeletons(im_predict, skel_pos_pred,
(0, 255, 0), SKEL_DISPLAY_MODE)
# embed()
# max_ = (im_predict * (im_predict < 255)).max()
all_pred_ims += [im_predict]
''' Visualize '''
if 1:
cv2.putText(im_predict, "Blue=Truth", (10, 210),
cv2.FONT_HERSHEY_DUPLEX, .5,
(int(im_predict.max() / 2), 0, 0))
cv2.putText(im_predict, "Green=Predict", (10, 230),
cv2.FONT_HERSHEY_DUPLEX, .5,
(0, int(im_predict.max() / 2), 0))
cv2.imshow("prediction", im_predict)
ret = cv2.waitKey(10)
if ret > 0: break
time.sleep(.5)
# except:
# print "Frame failed:", i
# break
embed()
