def transform_skels(skels, transformation, output='image'):
'''
---Parameters---
skels : list of skeletons in frame 1
transformation : 4x4 transform from frame 1 to 2
output : 'image' or 'world' for either coordinate system
---Result---
skels_out : skeletons in frame 2
'''
skels_out = []
for skel_c1 in skels:
if np.all(skel_c1 != -1):
skels_mask = skel_c1 == 0
# Convert to depth coordinate system
skel_c1 = depth2world(skel2depth(skel_c1, [240, 320]), [240, 320])
# Transform from cam1 -> cam2
skel_c2 = np.dot(transformation[:3, :3],
skel_c1.T).T + transformation[:3, 3]
if len(skel_c2) != N_MSR_JOINTS:
skel_c2 = kinect_to_msr_skel(skel_c2)
skel_c2[skels_mask] = 0
if output == 'world':
skels_out += [skel_c2]
elif output == 'image':
# Get skel in image (cam2) coordinates
skel_im2 = world2depth(skel_c2, [240, 320])
skels_out += [skel_im2]
return skels_out
def transform_skels(skels, transformation, output='image'):
'''
---Parameters---
skels : list of skeletons in frame 1
transformation : 4x4 transform from frame 1 to 2
output : 'image' or 'world' for either coordinate system
---Result---
skels_out : skeletons in frame 2
'''
skels_out = []
for skel_c1 in skels:
if np.all(skel_c1 != -1):
skels_mask = skel_c1 == 0
# Convert to depth coordinate system
skel_c1 = depth2world(skel2depth(skel_c1, [240,320]), [240,320])
# Transform from cam1 -> cam2
skel_c2 = np.dot(transformation[:3,:3], skel_c1.T).T + transformation[:3,3]
if len(skel_c2) != N_MSR_JOINTS:
skel_c2 = kinect_to_msr_skel(skel_c2)
skel_c2[skels_mask] = 0
if output=='world':
skels_out += [skel_c2]
elif output=='image':
# Get skel in image (cam2) coordinates
skel_im2 = world2depth(skel_c2, [240,320])
skels_out += [skel_im2]
return skels_out
def visualize_data(name):
'''
'''
depth_file = name + "depth.bin"
color_file = name + "rgb.avi"
skeleton_file = name + "skeleton.txt"
''' Read data from each video/sequence '''
try:
depthIms, maskIms = read_MSR_depth_ims(depth_file)
colorIms = read_MSR_color_ims(color_file)
skels_world, skels_im = read_MSR_skeletons(skeleton_file)
except:
print "Error reading data"
return -1
framecount = np.minimum(depthIms.shape[0], colorIms.shape[0])
''' View all data'''
for frame in xrange(framecount):
depth = depthIms[frame]
mask = maskIms[frame] > 0
color = colorIms[frame]
# color = np.dstack([color[:,:,2],color[:,:,1],color[:,:,0]])
''' Skeleton in world (w) and image (i) coordinates '''
skel_w = skels_world[frame]
skel_i = world2depth(skel_w, rez=[240,320])
''' Calculate hogs '''
grayIm = (rgb2gray(color) * 255).astype(np.uint8)
person_mask, bounding_boxes, labels = extract_people(grayIm, mask>0)
rez = grayIm[bounding_boxes[0]].shape
''' Plot skeletons on color image'''
# color = color*(mask[:,:,None]==0)
# from pylab import *
# embed()
# blurIm = (grayIm*mask).astype(np.float)
# blurIm = color*mask[:,:,None]
# blurIm = cv2.GaussianBlur(color, (15,15), 10)
# blurIm = cv2.GaussianBlur(mask[:,:,None]*color, (9,9), 5)
# blurIm = cv2.blur(mask[:,:,None]*color, (5,5))
# cv2.imshow("b", blurIm/float(blurIm.max()))
''' Visualization '''
# color = display_skeletons(color, skel_i, color=(0,200,0))
# blurIm = display_skeletons(blurIm, skel_i, color=(0,200,0))
cv2.imshow("Depth", depth/float(depth.max()))
cv2.imshow("RGB", color)
# cv2.imshow("RGB_blur", color*(mask[:,:,None]==0) + blurIm*(mask[:,:,None]>0))
# cv2.imshow("RGB masked", color*(mask[:,:,None]>0))
ret = cv2.waitKey(10)
if ret >= 0:
break
def visualize_data(name):
'''
'''
depth_file = name + "depth.bin"
color_file = name + "rgb.avi"
skeleton_file = name + "skeleton.txt"
''' Read data from each video/sequence '''
try:
depthIms, maskIms = read_MSR_depth_ims(depth_file)
colorIms = read_MSR_color_ims(color_file)
skels_world, skels_im = read_MSR_skeletons(skeleton_file)
except:
print "Error reading data"
return -1
framecount = np.minimum(depthIms.shape[0], colorIms.shape[0])
''' View all data'''
for frame in xrange(framecount):
depth = depthIms[frame]
mask = maskIms[frame] > 0
color = colorIms[frame]
# color = np.dstack([color[:,:,2],color[:,:,1],color[:,:,0]])
''' Skeleton in world (w) and image (i) coordinates '''
skel_w = skels_world[frame]
skel_i = world2depth(skel_w, rez=[240, 320])
''' Calculate hogs '''
grayIm = (rgb2gray(color) * 255).astype(np.uint8)
person_mask, bounding_boxes, labels = extract_people(grayIm, mask > 0)
rez = grayIm[bounding_boxes[0]].shape
''' Plot skeletons on color image'''
# color = color*(mask[:,:,None]==0)
# from pylab import *
# embed()
# blurIm = (grayIm*mask).astype(np.float)
# blurIm = color*mask[:,:,None]
# blurIm = cv2.GaussianBlur(color, (15,15), 10)
# blurIm = cv2.GaussianBlur(mask[:,:,None]*color, (9,9), 5)
# blurIm = cv2.blur(mask[:,:,None]*color, (5,5))
# cv2.imshow("b", blurIm/float(blurIm.max()))
''' Visualization '''
# color = display_skeletons(color, skel_i, color=(0,200,0))
# blurIm = display_skeletons(blurIm, skel_i, color=(0,200,0))
cv2.imshow("Depth", depth / float(depth.max()))
cv2.imshow("RGB", color)
# cv2.imshow("RGB_blur", color*(mask[:,:,None]==0) + blurIm*(mask[:,:,None]>0))
# cv2.imshow("RGB masked", color*(mask[:,:,None]>0))
ret = cv2.waitKey(10)
if ret >= 0:
break
def compute_features(name, vis=False, person_rez=[144, 72]):
'''
---Parameters---
filename : base filename for depth/color/skel
vis: turn visualize on?
person_rez : all hog/hof features should be the same dimensions, so we resize the people to this resolution
---Return---
features: features in a dictionary
'''
''' Get filenames '''
depth_file = name + "depth.bin"
color_file = name + "rgb.avi"
skeleton_file = name + "skeleton.txt"
''' Read data from each video/sequence '''
try:
depthIms, maskIms = read_MSR_depth_ims(depth_file)
colorIms = read_MSR_color_ims(color_file)
skels_world, skels_im = read_MSR_skeletons(skeleton_file)
except:
print "Error reading data"
return -1
#print depthIms.shape, colorIms.shape
framecount = np.minimum(depthIms.shape[0], colorIms.shape[0])
dataset_features = {
'framecount': framecount,
'hog': [],
'hof': [],
'skel_image': [],
'skel_world': []
}
grayIm_prev = None
''' View all data'''
for frame in xrange(framecount):
depth = depthIms[frame]
mask = maskIms[frame]
color = colorIms[frame]
# Skeleton in world (w) and image (i) coordinates
skel_w = skels_world[frame]
skel_i = world2depth(skel_w, rez=[240, 320])
''' Calculate hogs '''
grayIm = (rgb2gray(color) * 255).astype(np.uint8)
hogIm = np.zeros_like(depth)
person_mask, bounding_boxes, labels = extract_people(grayIm, mask > 0)
rez = grayIm[bounding_boxes[0]].shape
#hog_input_im = sm.imresize(grayIm[bounding_boxes[0]], person_rez)
hog_input_im = cv2.resize(grayIm[bounding_boxes[0]],
(person_rez[1], person_rez[0]))
hogData, hogImBox = hog(hog_input_im, orientations=4, visualise=True)
#hogIm[bounding_boxes[0]] = sm.imresize(hogImBox, [rez[0],rez[1]])
hogIm[bounding_boxes[0]] = cv2.resize(hogImBox, (rez[1], rez[0]))
# hogIm[bounding_boxes[0]] = hogImBox
hogIm *= person_mask
''' Calculate HOF '''
hofIm = np.zeros_like(depth)
if grayIm_prev is None:
grayIm_prev = np.copy(grayIm)
continue
else:
flow = getFlow(grayIm_prev[bounding_boxes[0]],
grayIm[bounding_boxes[0]])
rez = flow.shape
bounding_boxes = (bounding_boxes[0][0], bounding_boxes[0][1],
slice(0, 2))
#hof_input_im = np.dstack([sm.imresize(flow[0], [person_rez[0],person_rez[1]]),
# sm.imresize(flow[1], [person_rez[0],person_rez[1]])])i
hof_input_im = np.dstack([
cv2.resize(flow[0], (person_rez[1], person_rez[0])),
cv2.resize(flow[1], (person_rez[1], person_rez[0]))
])
hofData, hofImBox = hof(hof_input_im,
orientations=5,
visualise=True)
#hofIm[bounding_boxes[:2]] = sm.imresize(hofImBox, [rez[0],rez[1]])
hofIm[bounding_boxes[:2]] = cv2.resize(hogImBox, (rez[1], rez[0]))
hofIm *= person_mask
grayIm_prev = np.copy(grayIm)
''' Add features '''
dataset_features['hog'] += [hogData]
dataset_features['hof'] += [hofData]
dataset_features['skel_image'] += [skel_i]
dataset_features['skel_world'] += [skel_w]
''' Plot skeletons on color image'''
if vis:
color = display_skeletons(color, skel_i)
''' Visualization '''
cv2.imshow("Depth", depth / float(depth.max()))
cv2.imshow("HOG", hogIm / float(hogIm.max()))
cv2.imshow("RGB", color)
cv2.imshow("RGB masked", color * (mask[:, :, None] > 0))
cv2.imshow("HOF", hofIm / float(hofIm.max()))
ret = cv2.waitKey(10)
if ret >= 0:
break
print "Done calculating ", name
return dataset_features
for cam_index in range(cameraCount): #Skip if no entries if len(newComs[cam_index]) == 0: continue dists = [np.linalg.norm(u['coms'][-1] - x['com']) for x in newComs[cam_index]] closestPerson = np.argmin(dists) if dists[closestPerson] < 500: u['coms'].append(newComs[cam_index][closestPerson]['com']) u['times'].append(newComs[cam_index][closestPerson]['time']) u['cameras'].append(newComs[cam_index][closestPerson]['camera']) u['data'].append(newComs[cam_index][closestPerson]['data']) # centroid = world2depth(np.array([[-u['data'][-1]['com'][1], -u['data'][-1]['com'][0], u['data'][-1]['com'][2]]])).T[0]/2 #rez=[320,240] centroid = world2depth(np.array([u['data'][-1]['com']])).T[0]/2 #rez=[320,240] print centroid try: print u['data'][-1]['time'] ims.append(viewImage(u['cameras'][-1], u['data'][-1]['time'], coms=centroid)) except: print "Error viewing index:" + str(cam_index) + " time:" + " ".join(newComs[cam_index][closestPerson]['data']['time']) newComs[cam_index].pop(closestPerson) ''' If there isn't a relevant user then add a new user to the list ''' for cam_index in range(cameraCount): for u in newComs[cam_index]: userCount += 1
def main(deviceID, record, baseDir, frameDifferencePercent, getSkel, anonomize, viz, imgStoreCount=10):
'''------------ Setup Kinect ------------'''
''' Physical Kinect '''
depthDevice = RealTimeDevice(device=deviceID, getDepth=True, getColor=True, getSkel=getSkel)
depthDevice.start()
maxFramerate = 60
minFramerate = 1.0/3.0
motionLagTime = 3
recentMotionTime = time.clock()
''' ------------- Main -------------- '''
''' Setup time-based stuff '''
prevTime = 0
prevFrame = 0
prevFrameTime = 0
currentFrame = 0;
ms = time.clock()
diff = 0
prevSec = 0;
secondCount = 0
prevSecondCountMax = 0
''' Ensure base folder is there '''
if not os.path.isdir(baseDir):
os.mkdir(baseDir)
depthOld = []
colorOld = []
backgroundModel = None
if viz:
import cv2
from pyKinectTools.utils.DepthUtils import world2depth
cv2.namedWindow("image")
while 1:
# try:
if 1:
depthDevice.update()
colorRaw = depthDevice.colorIm
depthRaw8 = depthDevice.depthIm
users = depthDevice.users
skel = None
if len(depthOld) == imgStoreCount:
depthOld.pop(0)
''' If framerate is too fast then skip '''
''' Keep this after update to ensure fast enough kinect refresh '''
if (time.clock() - float(ms))*1000 < 1000.0/maxFramerate:
continue
if viz and 0:
for i in depthDevice.user.users:
tmpPx = depthDevice.user.get_user_pixels(i)
if depthDevice.skel_cap.is_tracking(i):
brightness = 50
else:
brightness = 150
depthRaw8 = depthRaw8*(1-np.array(tmpPx).reshape([480,640]))
depthRaw8 += brightness*(np.array(tmpPx).reshape([480,640]))
d = None
d = np.array(depthRaw8)
d /= (np.nanmin([d.max(), 2**16])/256.0)
d = d.astype(np.uint8)
''' Get new time info '''
currentFrame += 1
time_ = time.localtime()
day = str(time_.tm_yday)
hour = str(time_.tm_hour)
minute = str(time_.tm_min)
second = str(time_.tm_sec)
ms = str(time.clock())
ms_str = str(ms)[str(ms).find(".")+1:]
''' Look at how much of the image has changed '''
if depthOld != []:
diff = np.sum(np.logical_and((depthRaw8 - depthOld[0]) > 200, (depthRaw8 - depthOld[0]) < 20000)) / 307200.0 * 100
''' We want to watch all video for at least 5 seconds after we seen motion '''
''' This prevents problems where there is small motion that doesn't trigger the motion detector '''
if diff > frameDifferencePercent:
recentMotionTime = time.clock()
depthOld.append(depthRaw8)
if anonomize:
'''Background model'''
if backgroundModel is None:
bgSubtraction = AdaptiveMixtureOfGaussians(depthRaw8, maxGaussians=3, learningRate=0.2, decayRate=0.9, variance=100**2)
backgroundModel = bgSubtraction.getModel()
continue
else:
bgSubtraction.update(depthRaw8)
backgroundModel = bgSubtraction.getModel()
cv2.imshow("BG Model", backgroundModel/backgroundModel.max())
foregroundMask = bgSubtraction.getForeground(thresh=100)
''' Find people '''
foregroundMask, _, _ = extractPeople(depthRaw8, foregroundMask, minPersonPixThresh=5000, gradientFilter=True, gradThresh=15)
else:
foregroundMask = None
''' Write to file if there has been substantial change. '''
# if 1:
if diff > frameDifferencePercent or time.clock()-prevFrameTime > 1/minFramerate or time.clock()-recentMotionTime < motionLagTime:
if depthRaw8 != []:
''' Logical time '''
if second != prevSec:
prevSecondCountMax = secondCount
secondCount = 0
prevSec = second
else:
secondCount = int(secondCount) + 1
secondCount = str(secondCount)
if len(ms_str) == 1:
ms_str = '0' + ms_str
if len(secondCount) == 1:
secondCount = '0' + secondCount
''' Keep track of framerate '''
if prevTime != second:
prevTime = second
# print currentFrame - prevFrame, " fps. Diff = ", str(diff)[:4] + "%" #" #threads = ", len(processList),
print "FPS: "+ str(prevSecondCountMax) + " Diff: " + str(diff)[:4] + "%" #" #threads = ", len(processList),
prevFrame = currentFrame
''' Create folder/file names '''
depthDir = baseDir+'depth/'+day+"/"+hour+"/"+minute+"/device_"+str(deviceID)
depthName = depthDir + "/depth_"+day+"_"+hour+"_"+minute+"_"+second+"_"+secondCount+"_"+ms_str+".png"
colorDir = baseDir+'color/'+day+"/"+hour+"/"+minute+"/device_"+str(deviceID)
colorName = colorDir + "/color_"+day+"_"+hour+"_"+minute+"_"+second+"_"+secondCount+"_"+ms_str+".jpg"
if getSkel:
skelDir = baseDir+'skel/'+day+"/"+hour+"/"+minute+"/device_"+str(deviceID)
usersName = skelDir + "/skel_"+day+"_"+hour+"_"+minute+"_"+second+"_"+secondCount+"_"+ms_str+"_.dat"
else:
skelDir = None
usersName = None
if anonomize:
maskDir = baseDir+'mask/'+day+"/"+hour+"/"+minute+"/device_"+str(deviceID)
maskName = maskDir + "/mask_"+day+"_"+hour+"_"+minute+"_"+second+"_"+secondCount+"_"+ms_str+".jpg"
else:
maskDir = None
maskName = None
''' Create folders if they doesn't exist '''
createDirectory(depthDir)
createDirectory(colorDir)
if getSkel:
createDirectory(skelDir)
if anonomize:
createDirectory(maskDir)
# if not os.path.isdir(depthDir):
# for p in xrange(4, len(depthDir.split("/"))+1):
# try:
# os.mkdir("/".join(depthDir.split('/')[0:p]))
# os.mkdir("/".join(colorDir.split('/')[0:p]))
# if getSkel:
# os.mkdir("/".join(skelDir.split('/')[0:p]))
# if anonomize:
# os.mkdir("/".join(maskDir.split('/')[0:p]))
# except:
# # print "error making dir"
# pass
''' Save data '''
if record:
save_frame(depthName, depthRaw8, colorName, colorRaw, usersName, users, maskName=maskName, mask=foregroundMask)
prevFrameTime = time.clock()
''' Display skeletons '''
if viz and getSkel:
# print "Users: ", len(users)
for u_key in users.keys():
u = users[u_key]
pt = world2depth(u.com)
w = 10
d[pt[0]-w:pt[0]+w, pt[1]-w:pt[1]+w] = 255
w = 3
if u.tracked:
print "Joints: ", len(u.jointPositions)
for j in u.jointPositions.keys():
pt = world2depth(u.jointPositions[j])
d[pt[0]-w:pt[0]+w, pt[1]-w:pt[1]+w] = 200
if viz:
if 1:
cv2.imshow("imageD", d)
if 0:
# cv2.imshow("imageM", mask/float(mask.max()))
cv2.imshow("imageM", colorRaw + (255-colorRaw)*(foregroundMask>0)[:,:,np.newaxis] + 50*(((foregroundMask)[:,:,np.newaxis])))
if 1:
cv2.imshow("imageC", colorRaw)
# cv2.imshow("image", colorRaw + (255-colorRaw)*(foregroundMask>0)[:,:,np.newaxis] + 50*(((foregroundMask)[:,:,np.newaxis])))
ret = cv2.waitKey(10)
if ret >= 0:
break
def compute_features(name, vis=False, person_rez=[144,72]):
'''
---Parameters---
filename : base filename for depth/color/skel
vis: turn visualize on?
person_rez : all hog/hof features should be the same dimensions, so we resize the people to this resolution
---Return---
features: features in a dictionary
'''
''' Get filenames '''
depth_file = name + "depth.bin"
color_file = name + "rgb.avi"
skeleton_file = name + "skeleton.txt"
''' Read data from each video/sequence '''
try:
depthIms, maskIms = read_MSR_depth_ims(depth_file)
colorIms = read_MSR_color_ims(color_file)
skels_world, skels_im = read_MSR_skeletons(skeleton_file)
except:
print "Error reading data"
return -1
#print depthIms.shape, colorIms.shape
framecount = np.minimum(depthIms.shape[0], colorIms.shape[0])
dataset_features = {'framecount':framecount, 'hog':[], 'hof':[], 'skel_image':[], 'skel_world':[]}
grayIm_prev = None
''' View all data'''
for frame in xrange(framecount):
depth = depthIms[frame]
mask = maskIms[frame]
color = colorIms[frame]
# Skeleton in world (w) and image (i) coordinates
skel_w = skels_world[frame]
skel_i = world2depth(skel_w, rez=[240,320])
''' Calculate hogs '''
grayIm = (rgb2gray(color) * 255).astype(np.uint8)
hogIm = np.zeros_like(depth)
person_mask, bounding_boxes, labels = extract_people(grayIm, mask>0)
rez = grayIm[bounding_boxes[0]].shape
#hog_input_im = sm.imresize(grayIm[bounding_boxes[0]], person_rez)
hog_input_im = cv2.resize(grayIm[bounding_boxes[0]], (person_rez[1], person_rez[0]))
hogData, hogImBox = hog(hog_input_im, orientations=4, visualise=True)
#hogIm[bounding_boxes[0]] = sm.imresize(hogImBox, [rez[0],rez[1]])
hogIm[bounding_boxes[0]] = cv2.resize(hogImBox, (rez[1],rez[0]))
# hogIm[bounding_boxes[0]] = hogImBox
hogIm *= person_mask
''' Calculate HOF '''
hofIm = np.zeros_like(depth)
if grayIm_prev is None:
grayIm_prev = np.copy(grayIm)
continue
else:
flow = getFlow(grayIm_prev[bounding_boxes[0]], grayIm[bounding_boxes[0]])
rez = flow.shape
bounding_boxes = (bounding_boxes[0][0], bounding_boxes[0][1], slice(0,2))
#hof_input_im = np.dstack([sm.imresize(flow[0], [person_rez[0],person_rez[1]]),
# sm.imresize(flow[1], [person_rez[0],person_rez[1]])])i
hof_input_im = np.dstack([cv2.resize(flow[0], (person_rez[1],person_rez[0])), cv2.resize(flow[1], (person_rez[1], person_rez[0]))])
hofData, hofImBox = hof(hof_input_im, orientations=5, visualise=True)
#hofIm[bounding_boxes[:2]] = sm.imresize(hofImBox, [rez[0],rez[1]])
hofIm[bounding_boxes[:2]] = cv2.resize(hogImBox, (rez[1],rez[0]))
hofIm *= person_mask
grayIm_prev = np.copy(grayIm)
''' Add features '''
dataset_features['hog'] += [hogData]
dataset_features['hof'] += [hofData]
dataset_features['skel_image'] += [skel_i]
dataset_features['skel_world'] += [skel_w]
''' Plot skeletons on color image'''
if vis:
color = display_skeletons(color, skel_i)
''' Visualization '''
cv2.imshow("Depth", depth/float(depth.max()))
cv2.imshow("HOG", hogIm/float(hogIm.max()))
cv2.imshow("RGB", color)
cv2.imshow("RGB masked", color*(mask[:,:,None]>0))
cv2.imshow("HOF", hofIm/float(hofIm.max()))
ret = cv2.waitKey(10)
if ret >= 0:
break
print "Done calculating ", name
return dataset_features