def learn_msr_video(name, offsets_1, offsets_2): ''' ''' depth_file = name + "depth.bin" color_file = name + "rgb.avi" skeleton_file = name + "skeleton.txt" ''' Read data from each video/sequence ''' try: depthIms, maskIms = read_MSR_depth_ims(depth_file) depthIms *= maskIms depthIms /= 10 colorIms = read_MSR_color_ims(color_file) skels_world, _ = read_MSR_skeletons(skeleton_file) skels_world[:,2]/=10 except: print "Error getting frame features" return -1,-1 all_features = [] all_labels = [] for i in xrange(0, len(depthIms), 25): im_depth = depthIms[i] skel_pos = skel2depth(skels_world[i], rez=[240,320]) # ''' --- NOTE THIS 10 PX OFFSET IN THE MSR DATASET !!! --- ''' # skel_pos[:,0] -= 10 features, pixel_labels = learn_frame(im_depth, skel_pos, offsets_1, offsets_2) ''' Stack features ''' all_features += [features] all_labels += [pixel_labels] return np.concatenate(all_features), np.concatenate(all_labels)
def learn_msr_video(name, offsets_1, offsets_2): ''' ''' depth_file = name + "depth.bin" color_file = name + "rgb.avi" skeleton_file = name + "skeleton.txt" ''' Read data from each video/sequence ''' try: depthIms, maskIms = read_MSR_depth_ims(depth_file) depthIms *= maskIms depthIms /= 10 colorIms = read_MSR_color_ims(color_file) skels_world, _ = read_MSR_skeletons(skeleton_file) skels_world[:, 2] /= 10 except: print "Error getting frame features" return -1, -1 all_features = [] all_labels = [] for i in xrange(0, len(depthIms), 25): im_depth = depthIms[i] skel_pos = skel2depth(skels_world[i], rez=[240, 320]) # ''' --- NOTE THIS 10 PX OFFSET IN THE MSR DATASET !!! --- ''' # skel_pos[:,0] -= 10 features, pixel_labels = learn_frame(im_depth, skel_pos, offsets_1, offsets_2) ''' Stack features ''' all_features += [features] all_labels += [pixel_labels] return np.concatenate(all_features), np.concatenate(all_labels)
def visualize_data(name): ''' ''' depth_file = name + "depth.bin" color_file = name + "rgb.avi" skeleton_file = name + "skeleton.txt" ''' Read data from each video/sequence ''' try: depthIms, maskIms = read_MSR_depth_ims(depth_file) colorIms = read_MSR_color_ims(color_file) skels_world, skels_im = read_MSR_skeletons(skeleton_file) except: print "Error reading data" return -1 framecount = np.minimum(depthIms.shape[0], colorIms.shape[0]) ''' View all data''' for frame in xrange(framecount): depth = depthIms[frame] mask = maskIms[frame] > 0 color = colorIms[frame] # color = np.dstack([color[:,:,2],color[:,:,1],color[:,:,0]]) ''' Skeleton in world (w) and image (i) coordinates ''' skel_w = skels_world[frame] skel_i = world2depth(skel_w, rez=[240,320]) ''' Calculate hogs ''' grayIm = (rgb2gray(color) * 255).astype(np.uint8) person_mask, bounding_boxes, labels = extract_people(grayIm, mask>0) rez = grayIm[bounding_boxes[0]].shape ''' Plot skeletons on color image''' # color = color*(mask[:,:,None]==0) # from pylab import * # embed() # blurIm = (grayIm*mask).astype(np.float) # blurIm = color*mask[:,:,None] # blurIm = cv2.GaussianBlur(color, (15,15), 10) # blurIm = cv2.GaussianBlur(mask[:,:,None]*color, (9,9), 5) # blurIm = cv2.blur(mask[:,:,None]*color, (5,5)) # cv2.imshow("b", blurIm/float(blurIm.max())) ''' Visualization ''' # color = display_skeletons(color, skel_i, color=(0,200,0)) # blurIm = display_skeletons(blurIm, skel_i, color=(0,200,0)) cv2.imshow("Depth", depth/float(depth.max())) cv2.imshow("RGB", color) # cv2.imshow("RGB_blur", color*(mask[:,:,None]==0) + blurIm*(mask[:,:,None]>0)) # cv2.imshow("RGB masked", color*(mask[:,:,None]>0)) ret = cv2.waitKey(10) if ret >= 0: break
def visualize_data(name): ''' ''' depth_file = name + "depth.bin" color_file = name + "rgb.avi" skeleton_file = name + "skeleton.txt" ''' Read data from each video/sequence ''' try: depthIms, maskIms = read_MSR_depth_ims(depth_file) colorIms = read_MSR_color_ims(color_file) skels_world, skels_im = read_MSR_skeletons(skeleton_file) except: print "Error reading data" return -1 framecount = np.minimum(depthIms.shape[0], colorIms.shape[0]) ''' View all data''' for frame in xrange(framecount): depth = depthIms[frame] mask = maskIms[frame] > 0 color = colorIms[frame] # color = np.dstack([color[:,:,2],color[:,:,1],color[:,:,0]]) ''' Skeleton in world (w) and image (i) coordinates ''' skel_w = skels_world[frame] skel_i = world2depth(skel_w, rez=[240, 320]) ''' Calculate hogs ''' grayIm = (rgb2gray(color) * 255).astype(np.uint8) person_mask, bounding_boxes, labels = extract_people(grayIm, mask > 0) rez = grayIm[bounding_boxes[0]].shape ''' Plot skeletons on color image''' # color = color*(mask[:,:,None]==0) # from pylab import * # embed() # blurIm = (grayIm*mask).astype(np.float) # blurIm = color*mask[:,:,None] # blurIm = cv2.GaussianBlur(color, (15,15), 10) # blurIm = cv2.GaussianBlur(mask[:,:,None]*color, (9,9), 5) # blurIm = cv2.blur(mask[:,:,None]*color, (5,5)) # cv2.imshow("b", blurIm/float(blurIm.max())) ''' Visualization ''' # color = display_skeletons(color, skel_i, color=(0,200,0)) # blurIm = display_skeletons(blurIm, skel_i, color=(0,200,0)) cv2.imshow("Depth", depth / float(depth.max())) cv2.imshow("RGB", color) # cv2.imshow("RGB_blur", color*(mask[:,:,None]==0) + blurIm*(mask[:,:,None]>0)) # cv2.imshow("RGB masked", color*(mask[:,:,None]>0)) ret = cv2.waitKey(10) if ret >= 0: break
def compute_features(name, vis=False, person_rez=[144, 72]): ''' ---Parameters--- filename : base filename for depth/color/skel vis: turn visualize on? person_rez : all hog/hof features should be the same dimensions, so we resize the people to this resolution ---Return--- features: features in a dictionary ''' ''' Get filenames ''' depth_file = name + "depth.bin" color_file = name + "rgb.avi" skeleton_file = name + "skeleton.txt" ''' Read data from each video/sequence ''' try: depthIms, maskIms = read_MSR_depth_ims(depth_file) colorIms = read_MSR_color_ims(color_file) skels_world, skels_im = read_MSR_skeletons(skeleton_file) except: print "Error reading data" return -1 #print depthIms.shape, colorIms.shape framecount = np.minimum(depthIms.shape[0], colorIms.shape[0]) dataset_features = { 'framecount': framecount, 'hog': [], 'hof': [], 'skel_image': [], 'skel_world': [] } grayIm_prev = None ''' View all data''' for frame in xrange(framecount): depth = depthIms[frame] mask = maskIms[frame] color = colorIms[frame] # Skeleton in world (w) and image (i) coordinates skel_w = skels_world[frame] skel_i = world2depth(skel_w, rez=[240, 320]) ''' Calculate hogs ''' grayIm = (rgb2gray(color) * 255).astype(np.uint8) hogIm = np.zeros_like(depth) person_mask, bounding_boxes, labels = extract_people(grayIm, mask > 0) rez = grayIm[bounding_boxes[0]].shape #hog_input_im = sm.imresize(grayIm[bounding_boxes[0]], person_rez) hog_input_im = cv2.resize(grayIm[bounding_boxes[0]], (person_rez[1], person_rez[0])) hogData, hogImBox = hog(hog_input_im, orientations=4, visualise=True) #hogIm[bounding_boxes[0]] = sm.imresize(hogImBox, [rez[0],rez[1]]) hogIm[bounding_boxes[0]] = cv2.resize(hogImBox, (rez[1], rez[0])) # hogIm[bounding_boxes[0]] = hogImBox hogIm *= person_mask ''' Calculate HOF ''' hofIm = np.zeros_like(depth) if grayIm_prev is None: grayIm_prev = np.copy(grayIm) continue else: flow = getFlow(grayIm_prev[bounding_boxes[0]], grayIm[bounding_boxes[0]]) rez = flow.shape bounding_boxes = (bounding_boxes[0][0], bounding_boxes[0][1], slice(0, 2)) #hof_input_im = np.dstack([sm.imresize(flow[0], [person_rez[0],person_rez[1]]), # sm.imresize(flow[1], [person_rez[0],person_rez[1]])])i hof_input_im = np.dstack([ cv2.resize(flow[0], (person_rez[1], person_rez[0])), cv2.resize(flow[1], (person_rez[1], person_rez[0])) ]) hofData, hofImBox = hof(hof_input_im, orientations=5, visualise=True) #hofIm[bounding_boxes[:2]] = sm.imresize(hofImBox, [rez[0],rez[1]]) hofIm[bounding_boxes[:2]] = cv2.resize(hogImBox, (rez[1], rez[0])) hofIm *= person_mask grayIm_prev = np.copy(grayIm) ''' Add features ''' dataset_features['hog'] += [hogData] dataset_features['hof'] += [hofData] dataset_features['skel_image'] += [skel_i] dataset_features['skel_world'] += [skel_w] ''' Plot skeletons on color image''' if vis: color = display_skeletons(color, skel_i) ''' Visualization ''' cv2.imshow("Depth", depth / float(depth.max())) cv2.imshow("HOG", hogIm / float(hogIm.max())) cv2.imshow("RGB", color) cv2.imshow("RGB masked", color * (mask[:, :, None] > 0)) cv2.imshow("HOF", hofIm / float(hofIm.max())) ret = cv2.waitKey(10) if ret >= 0: break print "Done calculating ", name return dataset_features
def main_infer(rf_name=None): ''' ''' if rf_name is None: import os files = os.listdir('Saved_Params/') rf_name = 'Saved_Params/' + files[-1] print "Classifier file:",rf_name # Load classifier data data = pickle.load(open(rf_name)) rf = data['rf'] offsets_1, offsets_2 = data['offsets'] ''' Read data from each video/sequence ''' depthIms = [] skels_world = [] if 1: # VP = KinectPlayer(base_dir='/Users/colin/Data/Office_25Feb2013/', device=2, get_depth=True, get_color=False, get_skeleton=True, get_mask=False) VP = KinectPlayer(base_dir='/Users/colin/Data/Room_close_26Feb13/', device=1, get_depth=True, get_color=False, get_skeleton=True, get_mask=False) _,_ = VP.get_n_skeletons(50) depthIms, skels_world = VP.get_n_skeletons(100) # depthIms = np.array(depthIms)[:,:,::-1] else: # name = 'a01_s01_e02_' name = 'a01_s10_e02_' # name = 'a02_s06_e02_' # name = 'a05_s02_e02_' depth_file = name + "depth.bin" color_file = name + "rgb.avi" skeleton_file = name + "skeleton.txt" try: depthIms, maskIms = read_MSR_depth_ims(depth_file) depthIms *= maskIms depthIms /= 10 # colorIms = read_MSR_color_ims(color_file) skels_world, skels_im = read_MSR_skeletons(skeleton_file) skels_world[:,2]/=10 except: print "Error reading data" ''' Process data ''' all_pred_ims = [] for i in xrange(1, len(depthIms), 1): # try: if 1: print i ''' Get frame data ''' im_depth = depthIms[i] # im_depth[160:, :] = 0 skel_pos = skel2depth(skels_world[i], rez=[240,320]) # ''' --- NOTE THIS 10 PX OFFSET IN THE MSR DATASET !!! --- ''' # skel_pos[:,0] -= 10 skel_pos_pred, im_predict = infer_pose(im_depth, rf, offsets_1, offsets_2) # Overlay skeletons if 1: # colorIm = colorIms[i] # im_predict = colorIm cv2.imshow("forest_prediction", im_predict/float(im_predict.max())) im_predict = np.repeat(im_depth[:,:,None].astype(np.float), 3, -1) # embed() im_predict[im_depth>0] -= im_depth[im_depth>0].min() im_predict /= float(im_predict.max()/255.) im_predict = im_predict.astype(np.uint8) # im_predict = np.repeat(im_predict[:,:,None], 3, -1) # im_predict /= float(im_predict.max())*255 # im_predict = im_predict.astype(np.uint8) im_predict = display_skeletons(im_predict, skel_pos, (255,0,0), SKEL_DISPLAY_MODE) im_predict = display_skeletons(im_predict, skel_pos_pred, (0,255,0), SKEL_DISPLAY_MODE) im_predict = display_skeletons(im_predict, skel_pos, (255,0,0), SKEL_DISPLAY_MODE) im_predict = display_skeletons(im_predict, skel_pos_pred, (0,255,0), SKEL_DISPLAY_MODE) # embed() # max_ = (im_predict * (im_predict < 255)).max() all_pred_ims += [im_predict] ''' Visualize ''' if 1: cv2.putText(im_predict, "Blue=Truth", (10, 210), cv2.FONT_HERSHEY_DUPLEX, .5, (int(im_predict.max()/2), 0, 0)) cv2.putText(im_predict, "Green=Predict", (10, 230), cv2.FONT_HERSHEY_DUPLEX, .5, (0, int(im_predict.max()/2), 0)) cv2.imshow("prediction", im_predict) ret = cv2.waitKey(10) if ret > 0: break time.sleep(.5) # except: # print "Frame failed:", i # break embed()
def compute_features(name, vis=False, person_rez=[144,72]): ''' ---Parameters--- filename : base filename for depth/color/skel vis: turn visualize on? person_rez : all hog/hof features should be the same dimensions, so we resize the people to this resolution ---Return--- features: features in a dictionary ''' ''' Get filenames ''' depth_file = name + "depth.bin" color_file = name + "rgb.avi" skeleton_file = name + "skeleton.txt" ''' Read data from each video/sequence ''' try: depthIms, maskIms = read_MSR_depth_ims(depth_file) colorIms = read_MSR_color_ims(color_file) skels_world, skels_im = read_MSR_skeletons(skeleton_file) except: print "Error reading data" return -1 #print depthIms.shape, colorIms.shape framecount = np.minimum(depthIms.shape[0], colorIms.shape[0]) dataset_features = {'framecount':framecount, 'hog':[], 'hof':[], 'skel_image':[], 'skel_world':[]} grayIm_prev = None ''' View all data''' for frame in xrange(framecount): depth = depthIms[frame] mask = maskIms[frame] color = colorIms[frame] # Skeleton in world (w) and image (i) coordinates skel_w = skels_world[frame] skel_i = world2depth(skel_w, rez=[240,320]) ''' Calculate hogs ''' grayIm = (rgb2gray(color) * 255).astype(np.uint8) hogIm = np.zeros_like(depth) person_mask, bounding_boxes, labels = extract_people(grayIm, mask>0) rez = grayIm[bounding_boxes[0]].shape #hog_input_im = sm.imresize(grayIm[bounding_boxes[0]], person_rez) hog_input_im = cv2.resize(grayIm[bounding_boxes[0]], (person_rez[1], person_rez[0])) hogData, hogImBox = hog(hog_input_im, orientations=4, visualise=True) #hogIm[bounding_boxes[0]] = sm.imresize(hogImBox, [rez[0],rez[1]]) hogIm[bounding_boxes[0]] = cv2.resize(hogImBox, (rez[1],rez[0])) # hogIm[bounding_boxes[0]] = hogImBox hogIm *= person_mask ''' Calculate HOF ''' hofIm = np.zeros_like(depth) if grayIm_prev is None: grayIm_prev = np.copy(grayIm) continue else: flow = getFlow(grayIm_prev[bounding_boxes[0]], grayIm[bounding_boxes[0]]) rez = flow.shape bounding_boxes = (bounding_boxes[0][0], bounding_boxes[0][1], slice(0,2)) #hof_input_im = np.dstack([sm.imresize(flow[0], [person_rez[0],person_rez[1]]), # sm.imresize(flow[1], [person_rez[0],person_rez[1]])])i hof_input_im = np.dstack([cv2.resize(flow[0], (person_rez[1],person_rez[0])), cv2.resize(flow[1], (person_rez[1], person_rez[0]))]) hofData, hofImBox = hof(hof_input_im, orientations=5, visualise=True) #hofIm[bounding_boxes[:2]] = sm.imresize(hofImBox, [rez[0],rez[1]]) hofIm[bounding_boxes[:2]] = cv2.resize(hogImBox, (rez[1],rez[0])) hofIm *= person_mask grayIm_prev = np.copy(grayIm) ''' Add features ''' dataset_features['hog'] += [hogData] dataset_features['hof'] += [hofData] dataset_features['skel_image'] += [skel_i] dataset_features['skel_world'] += [skel_w] ''' Plot skeletons on color image''' if vis: color = display_skeletons(color, skel_i) ''' Visualization ''' cv2.imshow("Depth", depth/float(depth.max())) cv2.imshow("HOG", hogIm/float(hogIm.max())) cv2.imshow("RGB", color) cv2.imshow("RGB masked", color*(mask[:,:,None]>0)) cv2.imshow("HOF", hofIm/float(hofIm.max())) ret = cv2.waitKey(10) if ret >= 0: break print "Done calculating ", name return dataset_features
def main_infer(rf_name=None): ''' ''' if rf_name is None: import os files = os.listdir('Saved_Params/') rf_name = 'Saved_Params/' + files[-1] print "Classifier file:", rf_name # Load classifier data data = pickle.load(open(rf_name)) rf = data['rf'] offsets_1, offsets_2 = data['offsets'] ''' Read data from each video/sequence ''' depthIms = [] skels_world = [] if 1: # VP = KinectPlayer(base_dir='/Users/colin/Data/Office_25Feb2013/', device=2, get_depth=True, get_color=False, get_skeleton=True, get_mask=False) VP = KinectPlayer(base_dir='/Users/colin/Data/Room_close_26Feb13/', device=1, get_depth=True, get_color=False, get_skeleton=True, get_mask=False) _, _ = VP.get_n_skeletons(50) depthIms, skels_world = VP.get_n_skeletons(100) # depthIms = np.array(depthIms)[:,:,::-1] else: # name = 'a01_s01_e02_' name = 'a01_s10_e02_' # name = 'a02_s06_e02_' # name = 'a05_s02_e02_' depth_file = name + "depth.bin" color_file = name + "rgb.avi" skeleton_file = name + "skeleton.txt" try: depthIms, maskIms = read_MSR_depth_ims(depth_file) depthIms *= maskIms depthIms /= 10 # colorIms = read_MSR_color_ims(color_file) skels_world, skels_im = read_MSR_skeletons(skeleton_file) skels_world[:, 2] /= 10 except: print "Error reading data" ''' Process data ''' all_pred_ims = [] for i in xrange(1, len(depthIms), 1): # try: if 1: print i ''' Get frame data ''' im_depth = depthIms[i] # im_depth[160:, :] = 0 skel_pos = skel2depth(skels_world[i], rez=[240, 320]) # ''' --- NOTE THIS 10 PX OFFSET IN THE MSR DATASET !!! --- ''' # skel_pos[:,0] -= 10 skel_pos_pred, im_predict = infer_pose(im_depth, rf, offsets_1, offsets_2) # Overlay skeletons if 1: # colorIm = colorIms[i] # im_predict = colorIm cv2.imshow("forest_prediction", im_predict / float(im_predict.max())) im_predict = np.repeat(im_depth[:, :, None].astype(np.float), 3, -1) # embed() im_predict[im_depth > 0] -= im_depth[im_depth > 0].min() im_predict /= float(im_predict.max() / 255.) im_predict = im_predict.astype(np.uint8) # im_predict = np.repeat(im_predict[:,:,None], 3, -1) # im_predict /= float(im_predict.max())*255 # im_predict = im_predict.astype(np.uint8) im_predict = display_skeletons(im_predict, skel_pos, (255, 0, 0), SKEL_DISPLAY_MODE) im_predict = display_skeletons(im_predict, skel_pos_pred, (0, 255, 0), SKEL_DISPLAY_MODE) im_predict = display_skeletons(im_predict, skel_pos, (255, 0, 0), SKEL_DISPLAY_MODE) im_predict = display_skeletons(im_predict, skel_pos_pred, (0, 255, 0), SKEL_DISPLAY_MODE) # embed() # max_ = (im_predict * (im_predict < 255)).max() all_pred_ims += [im_predict] ''' Visualize ''' if 1: cv2.putText(im_predict, "Blue=Truth", (10, 210), cv2.FONT_HERSHEY_DUPLEX, .5, (int(im_predict.max() / 2), 0, 0)) cv2.putText(im_predict, "Green=Predict", (10, 230), cv2.FONT_HERSHEY_DUPLEX, .5, (0, int(im_predict.max() / 2), 0)) cv2.imshow("prediction", im_predict) ret = cv2.waitKey(10) if ret > 0: break time.sleep(.5) # except: # print "Frame failed:", i # break embed()