def transform_skels(skels, transformation, output='image'): ''' ---Parameters--- skels : list of skeletons in frame 1 transformation : 4x4 transform from frame 1 to 2 output : 'image' or 'world' for either coordinate system ---Result--- skels_out : skeletons in frame 2 ''' skels_out = [] for skel_c1 in skels: if np.all(skel_c1 != -1): skels_mask = skel_c1 == 0 # Convert to depth coordinate system skel_c1 = depth2world(skel2depth(skel_c1, [240, 320]), [240, 320]) # Transform from cam1 -> cam2 skel_c2 = np.dot(transformation[:3, :3], skel_c1.T).T + transformation[:3, 3] if len(skel_c2) != N_MSR_JOINTS: skel_c2 = kinect_to_msr_skel(skel_c2) skel_c2[skels_mask] = 0 if output == 'world': skels_out += [skel_c2] elif output == 'image': # Get skel in image (cam2) coordinates skel_im2 = world2depth(skel_c2, [240, 320]) skels_out += [skel_im2] return skels_out
def transform_skels(skels, transformation, output='image'): ''' ---Parameters--- skels : list of skeletons in frame 1 transformation : 4x4 transform from frame 1 to 2 output : 'image' or 'world' for either coordinate system ---Result--- skels_out : skeletons in frame 2 ''' skels_out = [] for skel_c1 in skels: if np.all(skel_c1 != -1): skels_mask = skel_c1 == 0 # Convert to depth coordinate system skel_c1 = depth2world(skel2depth(skel_c1, [240,320]), [240,320]) # Transform from cam1 -> cam2 skel_c2 = np.dot(transformation[:3,:3], skel_c1.T).T + transformation[:3,3] if len(skel_c2) != N_MSR_JOINTS: skel_c2 = kinect_to_msr_skel(skel_c2) skel_c2[skels_mask] = 0 if output=='world': skels_out += [skel_c2] elif output=='image': # Get skel in image (cam2) coordinates skel_im2 = world2depth(skel_c2, [240,320]) skels_out += [skel_im2] return skels_out
def visualize_data(name): ''' ''' depth_file = name + "depth.bin" color_file = name + "rgb.avi" skeleton_file = name + "skeleton.txt" ''' Read data from each video/sequence ''' try: depthIms, maskIms = read_MSR_depth_ims(depth_file) colorIms = read_MSR_color_ims(color_file) skels_world, skels_im = read_MSR_skeletons(skeleton_file) except: print "Error reading data" return -1 framecount = np.minimum(depthIms.shape[0], colorIms.shape[0]) ''' View all data''' for frame in xrange(framecount): depth = depthIms[frame] mask = maskIms[frame] > 0 color = colorIms[frame] # color = np.dstack([color[:,:,2],color[:,:,1],color[:,:,0]]) ''' Skeleton in world (w) and image (i) coordinates ''' skel_w = skels_world[frame] skel_i = world2depth(skel_w, rez=[240,320]) ''' Calculate hogs ''' grayIm = (rgb2gray(color) * 255).astype(np.uint8) person_mask, bounding_boxes, labels = extract_people(grayIm, mask>0) rez = grayIm[bounding_boxes[0]].shape ''' Plot skeletons on color image''' # color = color*(mask[:,:,None]==0) # from pylab import * # embed() # blurIm = (grayIm*mask).astype(np.float) # blurIm = color*mask[:,:,None] # blurIm = cv2.GaussianBlur(color, (15,15), 10) # blurIm = cv2.GaussianBlur(mask[:,:,None]*color, (9,9), 5) # blurIm = cv2.blur(mask[:,:,None]*color, (5,5)) # cv2.imshow("b", blurIm/float(blurIm.max())) ''' Visualization ''' # color = display_skeletons(color, skel_i, color=(0,200,0)) # blurIm = display_skeletons(blurIm, skel_i, color=(0,200,0)) cv2.imshow("Depth", depth/float(depth.max())) cv2.imshow("RGB", color) # cv2.imshow("RGB_blur", color*(mask[:,:,None]==0) + blurIm*(mask[:,:,None]>0)) # cv2.imshow("RGB masked", color*(mask[:,:,None]>0)) ret = cv2.waitKey(10) if ret >= 0: break
def visualize_data(name): ''' ''' depth_file = name + "depth.bin" color_file = name + "rgb.avi" skeleton_file = name + "skeleton.txt" ''' Read data from each video/sequence ''' try: depthIms, maskIms = read_MSR_depth_ims(depth_file) colorIms = read_MSR_color_ims(color_file) skels_world, skels_im = read_MSR_skeletons(skeleton_file) except: print "Error reading data" return -1 framecount = np.minimum(depthIms.shape[0], colorIms.shape[0]) ''' View all data''' for frame in xrange(framecount): depth = depthIms[frame] mask = maskIms[frame] > 0 color = colorIms[frame] # color = np.dstack([color[:,:,2],color[:,:,1],color[:,:,0]]) ''' Skeleton in world (w) and image (i) coordinates ''' skel_w = skels_world[frame] skel_i = world2depth(skel_w, rez=[240, 320]) ''' Calculate hogs ''' grayIm = (rgb2gray(color) * 255).astype(np.uint8) person_mask, bounding_boxes, labels = extract_people(grayIm, mask > 0) rez = grayIm[bounding_boxes[0]].shape ''' Plot skeletons on color image''' # color = color*(mask[:,:,None]==0) # from pylab import * # embed() # blurIm = (grayIm*mask).astype(np.float) # blurIm = color*mask[:,:,None] # blurIm = cv2.GaussianBlur(color, (15,15), 10) # blurIm = cv2.GaussianBlur(mask[:,:,None]*color, (9,9), 5) # blurIm = cv2.blur(mask[:,:,None]*color, (5,5)) # cv2.imshow("b", blurIm/float(blurIm.max())) ''' Visualization ''' # color = display_skeletons(color, skel_i, color=(0,200,0)) # blurIm = display_skeletons(blurIm, skel_i, color=(0,200,0)) cv2.imshow("Depth", depth / float(depth.max())) cv2.imshow("RGB", color) # cv2.imshow("RGB_blur", color*(mask[:,:,None]==0) + blurIm*(mask[:,:,None]>0)) # cv2.imshow("RGB masked", color*(mask[:,:,None]>0)) ret = cv2.waitKey(10) if ret >= 0: break
def compute_features(name, vis=False, person_rez=[144, 72]): ''' ---Parameters--- filename : base filename for depth/color/skel vis: turn visualize on? person_rez : all hog/hof features should be the same dimensions, so we resize the people to this resolution ---Return--- features: features in a dictionary ''' ''' Get filenames ''' depth_file = name + "depth.bin" color_file = name + "rgb.avi" skeleton_file = name + "skeleton.txt" ''' Read data from each video/sequence ''' try: depthIms, maskIms = read_MSR_depth_ims(depth_file) colorIms = read_MSR_color_ims(color_file) skels_world, skels_im = read_MSR_skeletons(skeleton_file) except: print "Error reading data" return -1 #print depthIms.shape, colorIms.shape framecount = np.minimum(depthIms.shape[0], colorIms.shape[0]) dataset_features = { 'framecount': framecount, 'hog': [], 'hof': [], 'skel_image': [], 'skel_world': [] } grayIm_prev = None ''' View all data''' for frame in xrange(framecount): depth = depthIms[frame] mask = maskIms[frame] color = colorIms[frame] # Skeleton in world (w) and image (i) coordinates skel_w = skels_world[frame] skel_i = world2depth(skel_w, rez=[240, 320]) ''' Calculate hogs ''' grayIm = (rgb2gray(color) * 255).astype(np.uint8) hogIm = np.zeros_like(depth) person_mask, bounding_boxes, labels = extract_people(grayIm, mask > 0) rez = grayIm[bounding_boxes[0]].shape #hog_input_im = sm.imresize(grayIm[bounding_boxes[0]], person_rez) hog_input_im = cv2.resize(grayIm[bounding_boxes[0]], (person_rez[1], person_rez[0])) hogData, hogImBox = hog(hog_input_im, orientations=4, visualise=True) #hogIm[bounding_boxes[0]] = sm.imresize(hogImBox, [rez[0],rez[1]]) hogIm[bounding_boxes[0]] = cv2.resize(hogImBox, (rez[1], rez[0])) # hogIm[bounding_boxes[0]] = hogImBox hogIm *= person_mask ''' Calculate HOF ''' hofIm = np.zeros_like(depth) if grayIm_prev is None: grayIm_prev = np.copy(grayIm) continue else: flow = getFlow(grayIm_prev[bounding_boxes[0]], grayIm[bounding_boxes[0]]) rez = flow.shape bounding_boxes = (bounding_boxes[0][0], bounding_boxes[0][1], slice(0, 2)) #hof_input_im = np.dstack([sm.imresize(flow[0], [person_rez[0],person_rez[1]]), # sm.imresize(flow[1], [person_rez[0],person_rez[1]])])i hof_input_im = np.dstack([ cv2.resize(flow[0], (person_rez[1], person_rez[0])), cv2.resize(flow[1], (person_rez[1], person_rez[0])) ]) hofData, hofImBox = hof(hof_input_im, orientations=5, visualise=True) #hofIm[bounding_boxes[:2]] = sm.imresize(hofImBox, [rez[0],rez[1]]) hofIm[bounding_boxes[:2]] = cv2.resize(hogImBox, (rez[1], rez[0])) hofIm *= person_mask grayIm_prev = np.copy(grayIm) ''' Add features ''' dataset_features['hog'] += [hogData] dataset_features['hof'] += [hofData] dataset_features['skel_image'] += [skel_i] dataset_features['skel_world'] += [skel_w] ''' Plot skeletons on color image''' if vis: color = display_skeletons(color, skel_i) ''' Visualization ''' cv2.imshow("Depth", depth / float(depth.max())) cv2.imshow("HOG", hogIm / float(hogIm.max())) cv2.imshow("RGB", color) cv2.imshow("RGB masked", color * (mask[:, :, None] > 0)) cv2.imshow("HOF", hofIm / float(hofIm.max())) ret = cv2.waitKey(10) if ret >= 0: break print "Done calculating ", name return dataset_features
for cam_index in range(cameraCount): #Skip if no entries if len(newComs[cam_index]) == 0: continue dists = [np.linalg.norm(u['coms'][-1] - x['com']) for x in newComs[cam_index]] closestPerson = np.argmin(dists) if dists[closestPerson] < 500: u['coms'].append(newComs[cam_index][closestPerson]['com']) u['times'].append(newComs[cam_index][closestPerson]['time']) u['cameras'].append(newComs[cam_index][closestPerson]['camera']) u['data'].append(newComs[cam_index][closestPerson]['data']) # centroid = world2depth(np.array([[-u['data'][-1]['com'][1], -u['data'][-1]['com'][0], u['data'][-1]['com'][2]]])).T[0]/2 #rez=[320,240] centroid = world2depth(np.array([u['data'][-1]['com']])).T[0]/2 #rez=[320,240] print centroid try: print u['data'][-1]['time'] ims.append(viewImage(u['cameras'][-1], u['data'][-1]['time'], coms=centroid)) except: print "Error viewing index:" + str(cam_index) + " time:" + " ".join(newComs[cam_index][closestPerson]['data']['time']) newComs[cam_index].pop(closestPerson) ''' If there isn't a relevant user then add a new user to the list ''' for cam_index in range(cameraCount): for u in newComs[cam_index]: userCount += 1
def main(deviceID, record, baseDir, frameDifferencePercent, getSkel, anonomize, viz, imgStoreCount=10): '''------------ Setup Kinect ------------''' ''' Physical Kinect ''' depthDevice = RealTimeDevice(device=deviceID, getDepth=True, getColor=True, getSkel=getSkel) depthDevice.start() maxFramerate = 60 minFramerate = 1.0/3.0 motionLagTime = 3 recentMotionTime = time.clock() ''' ------------- Main -------------- ''' ''' Setup time-based stuff ''' prevTime = 0 prevFrame = 0 prevFrameTime = 0 currentFrame = 0; ms = time.clock() diff = 0 prevSec = 0; secondCount = 0 prevSecondCountMax = 0 ''' Ensure base folder is there ''' if not os.path.isdir(baseDir): os.mkdir(baseDir) depthOld = [] colorOld = [] backgroundModel = None if viz: import cv2 from pyKinectTools.utils.DepthUtils import world2depth cv2.namedWindow("image") while 1: # try: if 1: depthDevice.update() colorRaw = depthDevice.colorIm depthRaw8 = depthDevice.depthIm users = depthDevice.users skel = None if len(depthOld) == imgStoreCount: depthOld.pop(0) ''' If framerate is too fast then skip ''' ''' Keep this after update to ensure fast enough kinect refresh ''' if (time.clock() - float(ms))*1000 < 1000.0/maxFramerate: continue if viz and 0: for i in depthDevice.user.users: tmpPx = depthDevice.user.get_user_pixels(i) if depthDevice.skel_cap.is_tracking(i): brightness = 50 else: brightness = 150 depthRaw8 = depthRaw8*(1-np.array(tmpPx).reshape([480,640])) depthRaw8 += brightness*(np.array(tmpPx).reshape([480,640])) d = None d = np.array(depthRaw8) d /= (np.nanmin([d.max(), 2**16])/256.0) d = d.astype(np.uint8) ''' Get new time info ''' currentFrame += 1 time_ = time.localtime() day = str(time_.tm_yday) hour = str(time_.tm_hour) minute = str(time_.tm_min) second = str(time_.tm_sec) ms = str(time.clock()) ms_str = str(ms)[str(ms).find(".")+1:] ''' Look at how much of the image has changed ''' if depthOld != []: diff = np.sum(np.logical_and((depthRaw8 - depthOld[0]) > 200, (depthRaw8 - depthOld[0]) < 20000)) / 307200.0 * 100 ''' We want to watch all video for at least 5 seconds after we seen motion ''' ''' This prevents problems where there is small motion that doesn't trigger the motion detector ''' if diff > frameDifferencePercent: recentMotionTime = time.clock() depthOld.append(depthRaw8) if anonomize: '''Background model''' if backgroundModel is None: bgSubtraction = AdaptiveMixtureOfGaussians(depthRaw8, maxGaussians=3, learningRate=0.2, decayRate=0.9, variance=100**2) backgroundModel = bgSubtraction.getModel() continue else: bgSubtraction.update(depthRaw8) backgroundModel = bgSubtraction.getModel() cv2.imshow("BG Model", backgroundModel/backgroundModel.max()) foregroundMask = bgSubtraction.getForeground(thresh=100) ''' Find people ''' foregroundMask, _, _ = extractPeople(depthRaw8, foregroundMask, minPersonPixThresh=5000, gradientFilter=True, gradThresh=15) else: foregroundMask = None ''' Write to file if there has been substantial change. ''' # if 1: if diff > frameDifferencePercent or time.clock()-prevFrameTime > 1/minFramerate or time.clock()-recentMotionTime < motionLagTime: if depthRaw8 != []: ''' Logical time ''' if second != prevSec: prevSecondCountMax = secondCount secondCount = 0 prevSec = second else: secondCount = int(secondCount) + 1 secondCount = str(secondCount) if len(ms_str) == 1: ms_str = '0' + ms_str if len(secondCount) == 1: secondCount = '0' + secondCount ''' Keep track of framerate ''' if prevTime != second: prevTime = second # print currentFrame - prevFrame, " fps. Diff = ", str(diff)[:4] + "%" #" #threads = ", len(processList), print "FPS: "+ str(prevSecondCountMax) + " Diff: " + str(diff)[:4] + "%" #" #threads = ", len(processList), prevFrame = currentFrame ''' Create folder/file names ''' depthDir = baseDir+'depth/'+day+"/"+hour+"/"+minute+"/device_"+str(deviceID) depthName = depthDir + "/depth_"+day+"_"+hour+"_"+minute+"_"+second+"_"+secondCount+"_"+ms_str+".png" colorDir = baseDir+'color/'+day+"/"+hour+"/"+minute+"/device_"+str(deviceID) colorName = colorDir + "/color_"+day+"_"+hour+"_"+minute+"_"+second+"_"+secondCount+"_"+ms_str+".jpg" if getSkel: skelDir = baseDir+'skel/'+day+"/"+hour+"/"+minute+"/device_"+str(deviceID) usersName = skelDir + "/skel_"+day+"_"+hour+"_"+minute+"_"+second+"_"+secondCount+"_"+ms_str+"_.dat" else: skelDir = None usersName = None if anonomize: maskDir = baseDir+'mask/'+day+"/"+hour+"/"+minute+"/device_"+str(deviceID) maskName = maskDir + "/mask_"+day+"_"+hour+"_"+minute+"_"+second+"_"+secondCount+"_"+ms_str+".jpg" else: maskDir = None maskName = None ''' Create folders if they doesn't exist ''' createDirectory(depthDir) createDirectory(colorDir) if getSkel: createDirectory(skelDir) if anonomize: createDirectory(maskDir) # if not os.path.isdir(depthDir): # for p in xrange(4, len(depthDir.split("/"))+1): # try: # os.mkdir("/".join(depthDir.split('/')[0:p])) # os.mkdir("/".join(colorDir.split('/')[0:p])) # if getSkel: # os.mkdir("/".join(skelDir.split('/')[0:p])) # if anonomize: # os.mkdir("/".join(maskDir.split('/')[0:p])) # except: # # print "error making dir" # pass ''' Save data ''' if record: save_frame(depthName, depthRaw8, colorName, colorRaw, usersName, users, maskName=maskName, mask=foregroundMask) prevFrameTime = time.clock() ''' Display skeletons ''' if viz and getSkel: # print "Users: ", len(users) for u_key in users.keys(): u = users[u_key] pt = world2depth(u.com) w = 10 d[pt[0]-w:pt[0]+w, pt[1]-w:pt[1]+w] = 255 w = 3 if u.tracked: print "Joints: ", len(u.jointPositions) for j in u.jointPositions.keys(): pt = world2depth(u.jointPositions[j]) d[pt[0]-w:pt[0]+w, pt[1]-w:pt[1]+w] = 200 if viz: if 1: cv2.imshow("imageD", d) if 0: # cv2.imshow("imageM", mask/float(mask.max())) cv2.imshow("imageM", colorRaw + (255-colorRaw)*(foregroundMask>0)[:,:,np.newaxis] + 50*(((foregroundMask)[:,:,np.newaxis]))) if 1: cv2.imshow("imageC", colorRaw) # cv2.imshow("image", colorRaw + (255-colorRaw)*(foregroundMask>0)[:,:,np.newaxis] + 50*(((foregroundMask)[:,:,np.newaxis]))) ret = cv2.waitKey(10) if ret >= 0: break
def compute_features(name, vis=False, person_rez=[144,72]): ''' ---Parameters--- filename : base filename for depth/color/skel vis: turn visualize on? person_rez : all hog/hof features should be the same dimensions, so we resize the people to this resolution ---Return--- features: features in a dictionary ''' ''' Get filenames ''' depth_file = name + "depth.bin" color_file = name + "rgb.avi" skeleton_file = name + "skeleton.txt" ''' Read data from each video/sequence ''' try: depthIms, maskIms = read_MSR_depth_ims(depth_file) colorIms = read_MSR_color_ims(color_file) skels_world, skels_im = read_MSR_skeletons(skeleton_file) except: print "Error reading data" return -1 #print depthIms.shape, colorIms.shape framecount = np.minimum(depthIms.shape[0], colorIms.shape[0]) dataset_features = {'framecount':framecount, 'hog':[], 'hof':[], 'skel_image':[], 'skel_world':[]} grayIm_prev = None ''' View all data''' for frame in xrange(framecount): depth = depthIms[frame] mask = maskIms[frame] color = colorIms[frame] # Skeleton in world (w) and image (i) coordinates skel_w = skels_world[frame] skel_i = world2depth(skel_w, rez=[240,320]) ''' Calculate hogs ''' grayIm = (rgb2gray(color) * 255).astype(np.uint8) hogIm = np.zeros_like(depth) person_mask, bounding_boxes, labels = extract_people(grayIm, mask>0) rez = grayIm[bounding_boxes[0]].shape #hog_input_im = sm.imresize(grayIm[bounding_boxes[0]], person_rez) hog_input_im = cv2.resize(grayIm[bounding_boxes[0]], (person_rez[1], person_rez[0])) hogData, hogImBox = hog(hog_input_im, orientations=4, visualise=True) #hogIm[bounding_boxes[0]] = sm.imresize(hogImBox, [rez[0],rez[1]]) hogIm[bounding_boxes[0]] = cv2.resize(hogImBox, (rez[1],rez[0])) # hogIm[bounding_boxes[0]] = hogImBox hogIm *= person_mask ''' Calculate HOF ''' hofIm = np.zeros_like(depth) if grayIm_prev is None: grayIm_prev = np.copy(grayIm) continue else: flow = getFlow(grayIm_prev[bounding_boxes[0]], grayIm[bounding_boxes[0]]) rez = flow.shape bounding_boxes = (bounding_boxes[0][0], bounding_boxes[0][1], slice(0,2)) #hof_input_im = np.dstack([sm.imresize(flow[0], [person_rez[0],person_rez[1]]), # sm.imresize(flow[1], [person_rez[0],person_rez[1]])])i hof_input_im = np.dstack([cv2.resize(flow[0], (person_rez[1],person_rez[0])), cv2.resize(flow[1], (person_rez[1], person_rez[0]))]) hofData, hofImBox = hof(hof_input_im, orientations=5, visualise=True) #hofIm[bounding_boxes[:2]] = sm.imresize(hofImBox, [rez[0],rez[1]]) hofIm[bounding_boxes[:2]] = cv2.resize(hogImBox, (rez[1],rez[0])) hofIm *= person_mask grayIm_prev = np.copy(grayIm) ''' Add features ''' dataset_features['hog'] += [hogData] dataset_features['hof'] += [hofData] dataset_features['skel_image'] += [skel_i] dataset_features['skel_world'] += [skel_w] ''' Plot skeletons on color image''' if vis: color = display_skeletons(color, skel_i) ''' Visualization ''' cv2.imshow("Depth", depth/float(depth.max())) cv2.imshow("HOG", hogIm/float(hogIm.max())) cv2.imshow("RGB", color) cv2.imshow("RGB masked", color*(mask[:,:,None]>0)) cv2.imshow("HOF", hofIm/float(hofIm.max())) ret = cv2.waitKey(10) if ret >= 0: break print "Done calculating ", name return dataset_features