def main(get_depth, get_color, get_skeleton, get_mask, calculate_features, visualize, save_anonomized, device):
dev = device
ret = 0
backgroundTemplates = np.empty([1,1,1])
backgroundModel = None
backgroundCount = 20
bgPercentage = .05
prevDepthIm = None
day_dirs = os.listdir('depth/')
day_dirs = [x for x in day_dirs if x[0]!='.']
day_dirs.sort(key=lambda x: int(x))
hour_index = 0
minute_index=0
allFeatures = []
coms = []
orns = []
play_speed = 1
new_date_entered = False
framerate = 0
frame_prev = 0
frame_prev_time = time()
day_index = 0
while day_index < len(day_dirs):
if new_date_entered:
try:
day_index = day_dirs.index(day_new)
except:
print "Day not found"
day_index = 0
dayDir = day_dirs[day_index]
hour_dirs = os.listdir('depth/'+dayDir)
hour_dirs = [x for x in hour_dirs if x[0]!='.']
hour_dirs.sort(key=lambda x: int(x))
'''Hours'''
''' Check for new Hours index '''
if not new_date_entered:
if play_speed >= 0 and ret != keys_frame_left:
hour_index = 0
else:
hour_index = len(hour_dirs)-1
else:
try:
hour_index = hour_dirs.index(hour_new)
except:
print "Hour was not found"
hour_index = 0
while hour_index < len(hour_dirs):
hourDir = hour_dirs[hour_index]
minute_dirs = os.listdir('depth/'+dayDir+'/'+hourDir)
minute_dirs = [x for x in minute_dirs if x[0]!='.']
minute_dirs.sort(key=lambda x: int(x))
'''Minutes'''
''' Check for new minute index '''
if not new_date_entered:
if play_speed >= 0 and ret != keys_frame_left:
minute_index = 0
else:
minute_index = len(minute_dirs)-1
else:
try:
minute_index = minute_dirs.index(minute_new)
except:
print "Minute was not found"
minute_index = 0
''' Loop through this minute '''
while minute_index < len(minute_dirs):
minute_dir = minute_dirs[minute_index]
# Prevent from reading hidden files
if minute_dir[0] == '.':
continue
depth_files = []
skelFiles = []
# For each available device:
devices = os.listdir('depth/'+dayDir+'/'+hourDir+'/'+minute_dir)
devices = [x for x in devices if x[0]!='.' and x.find('tmp')<0]
devices.sort()
deviceID = "device_{0:d}".format(dev+1)
if not os.path.isdir('depth/'+dayDir+'/'+hourDir+'/'+minute_dir+'/'+deviceID):
continue
''' Sort files '''
if get_depth:
depthTmp = os.listdir('depth/'+dayDir+'/'+hourDir+'/'+minute_dir+'/'+deviceID)
tmpSort = [int(x.split('_')[-3])*100 + int(formatFileString(x.split('_')[-2])) for x in depthTmp]
depthTmp = np.array(depthTmp)[np.argsort(tmpSort)].tolist()
depth_files.append([x for x in depthTmp if x.find('.png')>=0])
if get_skeleton:
skelTmp = os.listdir('skel/'+dayDir+'/'+hourDir+'/'+minute_dir+'/'+deviceID)
tmpSort = [int(x.split('_')[-4])*100 + int(formatFileString(x.split('_')[-3])) for x in skelTmp]
skelTmp = np.array(skelTmp)[np.argsort(tmpSort)].tolist()
skelFiles.append([x for x in skelTmp if x.find('.dat')>=0])
if len(depth_files) == 0:
continue
if play_speed >= 0 and ret != keys_frame_left:
frame_id = 0
else:
frame_id = len(depth_files[dev])-1
while frame_id < len(depth_files[0]):
# while frame_id < len(depth_files[dev]):
depthFile = depth_files[0][frame_id]
# try:
if 1:
''' Load Depth '''
if get_depth:
depthIm = sm.imread('depth/'+dayDir+'/'+hourDir+'/'+minute_dir+'/'+deviceID+'/'+depthFile)
depthIm = np.array(depthIm, dtype=np.uint16)
''' Load Color '''
if get_color:
colorFile = 'color_'+depthFile[6:-4]+'.jpg'
colorIm = sm.imread('color/'+dayDir+'/'+hourDir+'/'+minute_dir+'/'+deviceID+'/'+colorFile)
# colorIm_g = colorIm.mean(-1, dtype=np.uint8)
colorIm_g = skimage.img_as_ubyte(skimage.color.rgb2gray(colorIm))
# colorIm_lab = skimage.color.rgb2lab(colorIm).astype(np.uint8)
''' Load Skeleton Data '''
if get_skeleton:
skelFile = 'skel_'+depthFile[6:-4]+'_.dat'
if os.path.isfile('skel/'+dayDir+'/'+hourDir+'/'+minute_dir+'/'+deviceID+'/'+skelFile):
with open('skel/'+dayDir+'/'+hourDir+'/'+minute_dir+'/'+deviceID+'/'+skelFile, 'rb') as inFile:
users = pickle.load(inFile)
else:
print "No user file:", skelFile
coms = [users[x]['com'] for x in users.keys() if users[x]['com'][2] > 0.0]
jointCount = 0
for i in users.keys():
user = users[i]
timestamp = depthFile[:-4].split('_')[1:] # Day, hour, minute, second, millisecond, Frame number in this second
depthIm = np.minimum(depthIm.astype(np.float), 5000)
fill_image(depthIm)
'''Background model'''
if backgroundModel is None:
bgSubtraction = AdaptiveMixtureOfGaussians(depthIm, maxGaussians=3, learningRate=0.01, decayRate=0.02, variance=300**2)
backgroundModel = bgSubtraction.getModel()
if get_color:
prevColorIm = colorIm_g.copy()
continue
else:
bgSubtraction.update(depthIm)
backgroundModel = bgSubtraction.getModel()
foregroundMask = bgSubtraction.get_foreground(thresh=50)
''' Find people '''
if get_skeleton:
ret = plotUsers(depthIm, users, device=deviceID, vis=True)
if get_mask:
foregroundMask, userBoundingBoxes, userLabels = extract_people(depthIm, foregroundMask, minPersonPixThresh=1500, gradientFilter=True, gradThresh=100)
''' Calculate user features '''
if calculate_features and get_color:
''' Color Optical Flow '''
flow = getFlow(prevColorIm, colorIm_g)
prevColorIm = colorIm_g.copy()
userCount = len(userBoundingBoxes)
for i in xrange(userCount):
userBox = userBoundingBoxes[i]
userMask = foregroundMask==i+1
allFeatures.append(computeUserFeatures(colorIm, depthIm, flow, userBox, time=timestamp, mask=userMask, windowSize=[96,72], visualise=False))
''' Or get CoM + orientation '''
if get_mask and not calculate_features:
userCount = len(userBoundingBoxes)
for i in xrange(userCount):
userBox = userBoundingBoxes[i]
userMask = foregroundMask==i+1
com, ornBasis = calculateBasicPose(depthIm, userMask)
coms.append(com)
orns.append(ornBasis[1])
allFeatures.append({'com':com, "orn":ornBasis, 'time':timestamp})
''' Visualization '''
if visualize:
tmpSecond = depthFile.split("_")[-3]
if len(tmpSecond) == 0:
tmpSecond = '0'+tmpSecond
if get_depth:
vv.imshow("Depth", depthIm/6000.)
vv.putText("Depth", "Day "+dayDir+" Time "+hourDir+":"+minute_dir+":"+tmpSecond, (5,220), size=15)
vv.putText("Depth", "Play speed: "+str(play_speed)+"x", (5,15), size=15)
vv.putText("Depth", str(int(framerate))+" fps", (275,15), size=15)
if get_color:
vv.putText(colorIm, "Day "+dayDir+" Time "+hourDir+":"+minute_dir+" Dev#"+str(dev), (10,220))
vv.imshow("I_orig", colorIm)
if get_mask:
# vv.imshow("I", colorIm*foregroundMask[:,:,np.newaxis])
vv.imshow("I_masked", colorIm + (255-colorIm)*(((foregroundMask)[:,:,np.newaxis])))
if get_mask:
vv.imshow("Mask", foregroundMask.astype(np.float)/float(foregroundMask.max()))
# vv.imshow("BG Model", backgroundModel.astype(np.float)/float(backgroundModel.max()))
''' Multi-camera map '''
if 0 and len(coms) > 0:
mapRez = [200,200]
mapIm = np.zeros(mapRez)
coms_np = np.array(coms)
xs = np.minimum(np.maximum(mapRez[0]+((coms_np[:,2]+500)/3000.*mapRez[0]).astype(np.int), 0),mapRez[0]-1)
ys = np.minimum(np.maximum(((coms_np[:,0]+500)/1500.*mapRez[0]).astype(np.int), 0), mapRez[1]-1)
mapIm[xs, ys] = 255
vv.imshow("Map", mapIm)
# scatter(coms_np[:,0], -coms_np[:,2])
'''3D Vis'''
if 0:
# figure = mlab.figure(1, fgcolor=(1,1,1), bgcolor=(0,0,0))
# from pyKinectTools.utils.DepthUtils import *
pts = depthIm2XYZ(depthIm).astype(np.int)
interval = 25
figure.scene.disable_render = True
mlab.clf()
# ss = mlab.points3d(-pts[::interval,0], pts[::interval,1], pts[::interval,2], colormap='Blues', vmin=1000., vmax=5000., mode='2dvertex')
ss = mlab.points3d(pts[::interval,0], pts[::interval,1], pts[::interval,2], 5.-(np.minimum(pts[::interval,2], 5000)/float((-pts[:,2]).max()))/1000., scale_factor=25., colormap='Blues')#, mode='2dvertex')
# , scale_factor=25.
mlab.view(azimuth=0, elevation=0, distance=3000., focalpoint=(0,0,0), figure=figure)#, reset_roll=False)
# mlab.roll(90)
currentView = mlab.view()
figure.scene.disable_render = False
mlab.draw()
# mlab.show()
# ss = mlab.points3d(pts[::interval,0], pts[::interval,1], pts[::interval,2], color=col, scale_factor=5)
# ss = mlab.points3d(pts[:,0], pts[:,1], pts[:,2], color=(1,1,1), scale_factor=5)
# ss = mlab.points3d(pts[:,0], pts[:,1], pts[:,2])
''' Playback control: Look at keyboard input '''
ret = vv.waitKey()
if frame_id - frame_prev > 0:
framerate = (frame_id - frame_prev) / (time() - frame_prev_time)
frame_prev = frame_id
frame_prev_time = time()
new_date_entered = False
if ret > 0:
# player_controls(ret)
# print "Ret is",ret
if ret == keys_ESC:
break
elif ret == keys_space:
print "Enter the following into the command line"
tmp = raw_input("Enter date: ")
day_new = tmp
tmp = raw_input("Enter hour: ")
hour_new = tmp
tmp = raw_input("Enter minute: ")
minute_new = tmp
print "New date:", day_new, hour_new, minute_new
new_date_entered = True
break
elif ret == keys_down_arrow:
play_speed = 0
elif ret == keys_left_arrow:
play_speed -= 1
elif ret == keys_right_arrow:
play_speed += 1
elif ret == keys_i:
embed()
elif ret == keys_frame_left:
frame_id -= 1
elif ret == keys_frame_right:
frame_id += 1
elif ret == keys_help:
display_help()
frame_id += play_speed
if save_anonomized and get_mask:
save_dir = 'color_masked/'+dayDir+'/'+hourDir+'/'+minute_dir+'/'+devices[dev]+'/'
createDirectory(save_dir)
sm.imsave(save_dir+'colorM_'+depthFile[6:-4]+'.jpg', colorIm*(1-foregroundMask))
# except:
# print "Erroneous frame"
# if visualize:
# vv.imshow("D", depthIm.astype(np.float)/5000)
# ret = vv.waitKey(10)
# End seconds
if ret == keys_ESC or new_date_entered:
break
if frame_id >= len(depth_files[0]):
minute_index += 1
elif frame_id < 0:
minute_index -= 1
break
# End hours
if ret == keys_ESC or new_date_entered:
break
if minute_index >= len(minute_dirs):
hour_index += 1
elif minute_index < 0:
hour_index -= 1
break
# End days
if ret == keys_ESC:
break
if new_date_entered:
break
if hour_index >= len(hour_dirs):
day_index += 1
elif hour_index < 0:
day_index -= 1
if day_index < 0:
day_index = 0
if ret == keys_ESC or day_index > len(day_dirs):
break
np.save("/media/Data/r40_cX_", allFeatures)
embed()
def main(deviceID, record, baseDir, frameDifferencePercent, getSkel, anonomize, viz, imgStoreCount=10):
'''------------ Setup Kinect ------------'''
''' Physical Kinect '''
depthDevice = RealTimeDevice(device=deviceID, getDepth=True, getColor=True, getSkel=getSkel)
depthDevice.start()
maxFramerate = 60
minFramerate = 1.0/3.0
motionLagTime = 3
recentMotionTime = time.clock()
''' ------------- Main -------------- '''
''' Setup time-based stuff '''
prevTime = 0
prevFrame = 0
prevFrameTime = 0
currentFrame = 0;
ms = time.clock()
diff = 0
prevSec = 0;
secondCount = 0
prevSecondCountMax = 0
''' Ensure base folder is there '''
if not os.path.isdir(baseDir):
os.mkdir(baseDir)
depthOld = []
colorOld = []
backgroundModel = None
if viz:
import cv2
from pyKinectTools.utils.DepthUtils import world2depth
cv2.namedWindow("image")
while 1:
# try:
if 1:
depthDevice.update()
colorRaw = depthDevice.colorIm
depthRaw8 = depthDevice.depthIm
users = depthDevice.users
skel = None
if len(depthOld) == imgStoreCount:
depthOld.pop(0)
''' If framerate is too fast then skip '''
''' Keep this after update to ensure fast enough kinect refresh '''
if (time.clock() - float(ms))*1000 < 1000.0/maxFramerate:
continue
if viz and 0:
for i in depthDevice.user.users:
tmpPx = depthDevice.user.get_user_pixels(i)
if depthDevice.skel_cap.is_tracking(i):
brightness = 50
else:
brightness = 150
depthRaw8 = depthRaw8*(1-np.array(tmpPx).reshape([480,640]))
depthRaw8 += brightness*(np.array(tmpPx).reshape([480,640]))
d = None
d = np.array(depthRaw8)
d /= (np.nanmin([d.max(), 2**16])/256.0)
d = d.astype(np.uint8)
''' Get new time info '''
currentFrame += 1
time_ = time.localtime()
day = str(time_.tm_yday)
hour = str(time_.tm_hour)
minute = str(time_.tm_min)
second = str(time_.tm_sec)
ms = str(time.clock())
ms_str = str(ms)[str(ms).find(".")+1:]
''' Look at how much of the image has changed '''
if depthOld != []:
diff = np.sum(np.logical_and((depthRaw8 - depthOld[0]) > 200, (depthRaw8 - depthOld[0]) < 20000)) / 307200.0 * 100
''' We want to watch all video for at least 5 seconds after we seen motion '''
''' This prevents problems where there is small motion that doesn't trigger the motion detector '''
if diff > frameDifferencePercent:
recentMotionTime = time.clock()
depthOld.append(depthRaw8)
if anonomize:
'''Background model'''
if backgroundModel is None:
bgSubtraction = AdaptiveMixtureOfGaussians(depthRaw8, maxGaussians=3, learningRate=0.2, decayRate=0.9, variance=100**2)
backgroundModel = bgSubtraction.getModel()
continue
else:
bgSubtraction.update(depthRaw8)
backgroundModel = bgSubtraction.getModel()
cv2.imshow("BG Model", backgroundModel/backgroundModel.max())
foregroundMask = bgSubtraction.getForeground(thresh=100)
''' Find people '''
foregroundMask, _, _ = extractPeople(depthRaw8, foregroundMask, minPersonPixThresh=5000, gradientFilter=True, gradThresh=15)
else:
foregroundMask = None
''' Write to file if there has been substantial change. '''
# if 1:
if diff > frameDifferencePercent or time.clock()-prevFrameTime > 1/minFramerate or time.clock()-recentMotionTime < motionLagTime:
if depthRaw8 != []:
''' Logical time '''
if second != prevSec:
prevSecondCountMax = secondCount
secondCount = 0
prevSec = second
else:
secondCount = int(secondCount) + 1
secondCount = str(secondCount)
if len(ms_str) == 1:
ms_str = '0' + ms_str
if len(secondCount) == 1:
secondCount = '0' + secondCount
''' Keep track of framerate '''
if prevTime != second:
prevTime = second
# print currentFrame - prevFrame, " fps. Diff = ", str(diff)[:4] + "%" #" #threads = ", len(processList),
print "FPS: "+ str(prevSecondCountMax) + " Diff: " + str(diff)[:4] + "%" #" #threads = ", len(processList),
prevFrame = currentFrame
''' Create folder/file names '''
depthDir = baseDir+'depth/'+day+"/"+hour+"/"+minute+"/device_"+str(deviceID)
depthName = depthDir + "/depth_"+day+"_"+hour+"_"+minute+"_"+second+"_"+secondCount+"_"+ms_str+".png"
colorDir = baseDir+'color/'+day+"/"+hour+"/"+minute+"/device_"+str(deviceID)
colorName = colorDir + "/color_"+day+"_"+hour+"_"+minute+"_"+second+"_"+secondCount+"_"+ms_str+".jpg"
if getSkel:
skelDir = baseDir+'skel/'+day+"/"+hour+"/"+minute+"/device_"+str(deviceID)
usersName = skelDir + "/skel_"+day+"_"+hour+"_"+minute+"_"+second+"_"+secondCount+"_"+ms_str+"_.dat"
else:
skelDir = None
usersName = None
if anonomize:
maskDir = baseDir+'mask/'+day+"/"+hour+"/"+minute+"/device_"+str(deviceID)
maskName = maskDir + "/mask_"+day+"_"+hour+"_"+minute+"_"+second+"_"+secondCount+"_"+ms_str+".jpg"
else:
maskDir = None
maskName = None
''' Create folders if they doesn't exist '''
createDirectory(depthDir)
createDirectory(colorDir)
if getSkel:
createDirectory(skelDir)
if anonomize:
createDirectory(maskDir)
# if not os.path.isdir(depthDir):
# for p in xrange(4, len(depthDir.split("/"))+1):
# try:
# os.mkdir("/".join(depthDir.split('/')[0:p]))
# os.mkdir("/".join(colorDir.split('/')[0:p]))
# if getSkel:
# os.mkdir("/".join(skelDir.split('/')[0:p]))
# if anonomize:
# os.mkdir("/".join(maskDir.split('/')[0:p]))
# except:
# # print "error making dir"
# pass
''' Save data '''
if record:
save_frame(depthName, depthRaw8, colorName, colorRaw, usersName, users, maskName=maskName, mask=foregroundMask)
prevFrameTime = time.clock()
''' Display skeletons '''
if viz and getSkel:
# print "Users: ", len(users)
for u_key in users.keys():
u = users[u_key]
pt = world2depth(u.com)
w = 10
d[pt[0]-w:pt[0]+w, pt[1]-w:pt[1]+w] = 255
w = 3
if u.tracked:
print "Joints: ", len(u.jointPositions)
for j in u.jointPositions.keys():
pt = world2depth(u.jointPositions[j])
d[pt[0]-w:pt[0]+w, pt[1]-w:pt[1]+w] = 200
if viz:
if 1:
cv2.imshow("imageD", d)
if 0:
# cv2.imshow("imageM", mask/float(mask.max()))
cv2.imshow("imageM", colorRaw + (255-colorRaw)*(foregroundMask>0)[:,:,np.newaxis] + 50*(((foregroundMask)[:,:,np.newaxis])))
if 1:
cv2.imshow("imageC", colorRaw)
# cv2.imshow("image", colorRaw + (255-colorRaw)*(foregroundMask>0)[:,:,np.newaxis] + 50*(((foregroundMask)[:,:,np.newaxis])))
ret = cv2.waitKey(10)
if ret >= 0:
break
def main(get_depth, get_color, get_skeleton, get_mask, calculate_features,
visualize, save_anonomized, device):
dev = device
ret = 0
backgroundTemplates = np.empty([1, 1, 1])
backgroundModel = None
backgroundCount = 20
bgPercentage = .05
prevDepthIm = None
day_dirs = os.listdir('depth/')
day_dirs = [x for x in day_dirs if x[0] != '.']
day_dirs.sort(key=lambda x: int(x))
hour_index = 0
minute_index = 0
allFeatures = []
coms = []
orns = []
play_speed = 1
new_date_entered = False
framerate = 0
frame_prev = 0
frame_prev_time = time()
day_index = 0
while day_index < len(day_dirs):
if new_date_entered:
try:
day_index = day_dirs.index(day_new)
except:
print "Day not found"
day_index = 0
dayDir = day_dirs[day_index]
hour_dirs = os.listdir('depth/' + dayDir)
hour_dirs = [x for x in hour_dirs if x[0] != '.']
hour_dirs.sort(key=lambda x: int(x))
'''Hours'''
''' Check for new Hours index '''
if not new_date_entered:
if play_speed >= 0 and ret != keys_frame_left:
hour_index = 0
else:
hour_index = len(hour_dirs) - 1
else:
try:
hour_index = hour_dirs.index(hour_new)
except:
print "Hour was not found"
hour_index = 0
while hour_index < len(hour_dirs):
hourDir = hour_dirs[hour_index]
minute_dirs = os.listdir('depth/' + dayDir + '/' + hourDir)
minute_dirs = [x for x in minute_dirs if x[0] != '.']
minute_dirs.sort(key=lambda x: int(x))
'''Minutes'''
''' Check for new minute index '''
if not new_date_entered:
if play_speed >= 0 and ret != keys_frame_left:
minute_index = 0
else:
minute_index = len(minute_dirs) - 1
else:
try:
minute_index = minute_dirs.index(minute_new)
except:
print "Minute was not found"
minute_index = 0
''' Loop through this minute '''
while minute_index < len(minute_dirs):
minute_dir = minute_dirs[minute_index]
# Prevent from reading hidden files
if minute_dir[0] == '.':
continue
depth_files = []
skelFiles = []
# For each available device:
devices = os.listdir('depth/' + dayDir + '/' + hourDir + '/' +
minute_dir)
devices = [
x for x in devices if x[0] != '.' and x.find('tmp') < 0
]
devices.sort()
deviceID = "device_{0:d}".format(dev + 1)
if not os.path.isdir('depth/' + dayDir + '/' + hourDir + '/' +
minute_dir + '/' + deviceID):
continue
''' Sort files '''
if get_depth:
depthTmp = os.listdir('depth/' + dayDir + '/' + hourDir +
'/' + minute_dir + '/' + deviceID)
tmpSort = [
int(x.split('_')[-3]) * 100 +
int(formatFileString(x.split('_')[-2]))
for x in depthTmp
]
depthTmp = np.array(depthTmp)[np.argsort(tmpSort)].tolist()
depth_files.append(
[x for x in depthTmp if x.find('.png') >= 0])
if get_skeleton:
skelTmp = os.listdir('skel/' + dayDir + '/' + hourDir +
'/' + minute_dir + '/' + deviceID)
tmpSort = [
int(x.split('_')[-4]) * 100 +
int(formatFileString(x.split('_')[-3]))
for x in skelTmp
]
skelTmp = np.array(skelTmp)[np.argsort(tmpSort)].tolist()
skelFiles.append(
[x for x in skelTmp if x.find('.dat') >= 0])
if len(depth_files) == 0:
continue
if play_speed >= 0 and ret != keys_frame_left:
frame_id = 0
else:
frame_id = len(depth_files[dev]) - 1
while frame_id < len(depth_files[0]):
# while frame_id < len(depth_files[dev]):
depthFile = depth_files[0][frame_id]
# try:
if 1:
''' Load Depth '''
if get_depth:
depthIm = sm.imread('depth/' + dayDir + '/' +
hourDir + '/' + minute_dir +
'/' + deviceID + '/' +
depthFile)
depthIm = np.array(depthIm, dtype=np.uint16)
''' Load Color '''
if get_color:
colorFile = 'color_' + depthFile[6:-4] + '.jpg'
colorIm = sm.imread('color/' + dayDir + '/' +
hourDir + '/' + minute_dir +
'/' + deviceID + '/' +
colorFile)
# colorIm_g = colorIm.mean(-1, dtype=np.uint8)
colorIm_g = skimage.img_as_ubyte(
skimage.color.rgb2gray(colorIm))
# colorIm_lab = skimage.color.rgb2lab(colorIm).astype(np.uint8)
''' Load Skeleton Data '''
if get_skeleton:
skelFile = 'skel_' + depthFile[6:-4] + '_.dat'
if os.path.isfile('skel/' + dayDir + '/' +
hourDir + '/' + minute_dir +
'/' + deviceID + '/' + skelFile):
with open(
'skel/' + dayDir + '/' + hourDir +
'/' + minute_dir + '/' + deviceID +
'/' + skelFile, 'rb') as inFile:
users = pickle.load(inFile)
else:
print "No user file:", skelFile
coms = [
users[x]['com'] for x in users.keys()
if users[x]['com'][2] > 0.0
]
jointCount = 0
for i in users.keys():
user = users[i]
timestamp = depthFile[:-4].split(
'_'
)[1:] # Day, hour, minute, second, millisecond, Frame number in this second
depthIm = np.minimum(depthIm.astype(np.float), 5000)
fill_image(depthIm)
'''Background model'''
if backgroundModel is None:
bgSubtraction = AdaptiveMixtureOfGaussians(
depthIm,
maxGaussians=3,
learningRate=0.01,
decayRate=0.02,
variance=300**2)
backgroundModel = bgSubtraction.getModel()
if get_color:
prevColorIm = colorIm_g.copy()
continue
else:
bgSubtraction.update(depthIm)
backgroundModel = bgSubtraction.getModel()
foregroundMask = bgSubtraction.get_foreground(
thresh=50)
''' Find people '''
if get_skeleton:
ret = plotUsers(depthIm,
users,
device=deviceID,
vis=True)
if get_mask:
foregroundMask, userBoundingBoxes, userLabels = extract_people(
depthIm,
foregroundMask,
minPersonPixThresh=1500,
gradientFilter=True,
gradThresh=100)
''' Calculate user features '''
if calculate_features and get_color:
''' Color Optical Flow '''
flow = getFlow(prevColorIm, colorIm_g)
prevColorIm = colorIm_g.copy()
userCount = len(userBoundingBoxes)
for i in xrange(userCount):
userBox = userBoundingBoxes[i]
userMask = foregroundMask == i + 1
allFeatures.append(
computeUserFeatures(colorIm,
depthIm,
flow,
userBox,
time=timestamp,
mask=userMask,
windowSize=[96, 72],
visualise=False))
''' Or get CoM + orientation '''
if get_mask and not calculate_features:
userCount = len(userBoundingBoxes)
for i in xrange(userCount):
userBox = userBoundingBoxes[i]
userMask = foregroundMask == i + 1
com, ornBasis = calculateBasicPose(
depthIm, userMask)
coms.append(com)
orns.append(ornBasis[1])
allFeatures.append({
'com': com,
"orn": ornBasis,
'time': timestamp
})
''' Visualization '''
if visualize:
tmpSecond = depthFile.split("_")[-3]
if len(tmpSecond) == 0:
tmpSecond = '0' + tmpSecond
if get_depth:
vv.imshow("Depth", depthIm / 6000.)
vv.putText("Depth",
"Day " + dayDir + " Time " +
hourDir + ":" + minute_dir + ":" +
tmpSecond, (5, 220),
size=15)
vv.putText("Depth",
"Play speed: " + str(play_speed) +
"x", (5, 15),
size=15)
vv.putText("Depth",
str(int(framerate)) + " fps",
(275, 15),
size=15)
if get_color:
vv.putText(
colorIm,
"Day " + dayDir + " Time " + hourDir +
":" + minute_dir + " Dev#" + str(dev),
(10, 220))
vv.imshow("I_orig", colorIm)
if get_mask:
# vv.imshow("I", colorIm*foregroundMask[:,:,np.newaxis])
vv.imshow(
"I_masked", colorIm + (255 - colorIm) *
(((foregroundMask)[:, :, np.newaxis])))
if get_mask:
vv.imshow(
"Mask",
foregroundMask.astype(np.float) /
float(foregroundMask.max()))
# vv.imshow("BG Model", backgroundModel.astype(np.float)/float(backgroundModel.max()))
''' Multi-camera map '''
if 0 and len(coms) > 0:
mapRez = [200, 200]
mapIm = np.zeros(mapRez)
coms_np = np.array(coms)
xs = np.minimum(
np.maximum(
mapRez[0] +
((coms_np[:, 2] + 500) / 3000. *
mapRez[0]).astype(np.int), 0),
mapRez[0] - 1)
ys = np.minimum(
np.maximum(((coms_np[:, 0] + 500) / 1500. *
mapRez[0]).astype(np.int), 0),
mapRez[1] - 1)
mapIm[xs, ys] = 255
vv.imshow("Map", mapIm)
# scatter(coms_np[:,0], -coms_np[:,2])
'''3D Vis'''
if 0:
# figure = mlab.figure(1, fgcolor=(1,1,1), bgcolor=(0,0,0))
# from pyKinectTools.utils.DepthUtils import *
pts = depthIm2XYZ(depthIm).astype(np.int)
interval = 25
figure.scene.disable_render = True
mlab.clf()
# ss = mlab.points3d(-pts[::interval,0], pts[::interval,1], pts[::interval,2], colormap='Blues', vmin=1000., vmax=5000., mode='2dvertex')
ss = mlab.points3d(
pts[::interval, 0],
pts[::interval, 1],
pts[::interval, 2],
5. -
(np.minimum(pts[::interval, 2], 5000) /
float((-pts[:, 2]).max())) / 1000.,
scale_factor=25.,
colormap='Blues') #, mode='2dvertex')
# , scale_factor=25.
mlab.view(azimuth=0,
elevation=0,
distance=3000.,
focalpoint=(0, 0, 0),
figure=figure) #, reset_roll=False)
# mlab.roll(90)
currentView = mlab.view()
figure.scene.disable_render = False
mlab.draw()
# mlab.show()
# ss = mlab.points3d(pts[::interval,0], pts[::interval,1], pts[::interval,2], color=col, scale_factor=5)
# ss = mlab.points3d(pts[:,0], pts[:,1], pts[:,2], color=(1,1,1), scale_factor=5)
# ss = mlab.points3d(pts[:,0], pts[:,1], pts[:,2])
''' Playback control: Look at keyboard input '''
ret = vv.waitKey()
if frame_id - frame_prev > 0:
framerate = (frame_id - frame_prev) / (
time() - frame_prev_time)
frame_prev = frame_id
frame_prev_time = time()
new_date_entered = False
if ret > 0:
# player_controls(ret)
# print "Ret is",ret
if ret == keys_ESC:
break
elif ret == keys_space:
print "Enter the following into the command line"
tmp = raw_input("Enter date: ")
day_new = tmp
tmp = raw_input("Enter hour: ")
hour_new = tmp
tmp = raw_input("Enter minute: ")
minute_new = tmp
print "New date:", day_new, hour_new, minute_new
new_date_entered = True
break
elif ret == keys_down_arrow:
play_speed = 0
elif ret == keys_left_arrow:
play_speed -= 1
elif ret == keys_right_arrow:
play_speed += 1
elif ret == keys_i:
embed()
elif ret == keys_frame_left:
frame_id -= 1
elif ret == keys_frame_right:
frame_id += 1
elif ret == keys_help:
display_help()
frame_id += play_speed
if save_anonomized and get_mask:
save_dir = 'color_masked/' + dayDir + '/' + hourDir + '/' + minute_dir + '/' + devices[
dev] + '/'
createDirectory(save_dir)
sm.imsave(
save_dir + 'colorM_' + depthFile[6:-4] + '.jpg',
colorIm * (1 - foregroundMask))
# except:
# print "Erroneous frame"
# if visualize:
# vv.imshow("D", depthIm.astype(np.float)/5000)
# ret = vv.waitKey(10)
# End seconds
if ret == keys_ESC or new_date_entered:
break
if frame_id >= len(depth_files[0]):
minute_index += 1
elif frame_id < 0:
minute_index -= 1
break
# End hours
if ret == keys_ESC or new_date_entered:
break
if minute_index >= len(minute_dirs):
hour_index += 1
elif minute_index < 0:
hour_index -= 1
break
# End days
if ret == keys_ESC:
break
if new_date_entered:
break
if hour_index >= len(hour_dirs):
day_index += 1
elif hour_index < 0:
day_index -= 1
if day_index < 0:
day_index = 0
if ret == keys_ESC or day_index > len(day_dirs):
break
np.save("/media/Data/r40_cX_", allFeatures)
embed()
def main(device_id, record, base_dir, frame_difference_percent, get_skeleton, anonomize, viz, motion_lag_time=10, min_fps=0.5):
'''
---parameters---
device_id
record
base_dir
frame_difference_percent
get_skeleton
anonomize
viz
motion_lag_time
min_fps
'''
'''------------ Setup Kinect ------------'''
''' Physical Kinect '''
depthDevice = RealTimeDevice(device=device_id, get_depth=True, get_color=True, get_skeleton=get_skeleton)
depthDevice.start()
maxFramerate = 100
minFramerate = min_fps
recentMotionTime = time.time()
imgStoreCount = 100
''' ------------- Main -------------- '''
''' Setup time-based stuff '''
prevTime = 0
prevFrame = 0
prevFrameTime = 0
currentFrame = 0
ms = time.time()
prevFPSTime = time.time()
diff = 0
prevSec = 0;
secondCount = 0
prevSecondCountMax = 0
''' Ensure base folder is there '''
createDirectory(base_dir)
skel_dir = None
skel_filename = None
maskDir = None
maskName = None
depthOld = []
backgroundModel = None
while 1:
# try:
if 1:
depthDevice.update()
colorRaw = depthDevice.colorIm
depthRaw = depthDevice.depthIm
users = depthDevice.users
skel = None
if len(depthOld) == imgStoreCount:
depthOld.pop(0)
''' If framerate is too fast then skip '''
''' Keep this after update to ensure fast enough kinect refresh '''
if (time.time() - float(ms))*1000 < 1000.0/maxFramerate:
continue
''' Get new time info '''
time_ = time.localtime()
day = str(time_.tm_yday)
hour = str(time_.tm_hour)
minute = str(time_.tm_min)
second = str(time_.tm_sec)
ms = str(time.time())
ms_str = ms
''' Look at how much of the image has changed '''
if depthOld != []:
mask = (depthRaw != 0) * (depthOld[0] != 0)
diff = (((depthRaw - depthOld[0]).astype(np.int16) > 200)*mask).sum() / (float(mask.sum()) * 100.)
''' We want to watch all video for at least 5 seconds after we seen motion '''
''' This prevents problems where there is small motion that doesn't trigger the motion detector '''
if diff > frame_difference_percent:
recentMotionTime = time.time()
depthOld.append(depthRaw)
if anonomize:
'''Background model'''
if backgroundModel is None:
bgSubtraction = AdaptiveMixtureOfGaussians(depthRaw, maxGaussians=3, learningRate=0.2, decayRate=0.9, variance=100**2)
backgroundModel = bgSubtraction.getModel()
continue
else:
bgSubtraction.update(depthRaw)
backgroundModel = bgSubtraction.getModel()
cv2.imshow("BG Model", backgroundModel/backgroundModel.max())
foregroundMask = bgSubtraction.get_foreground(thresh=100)
''' Find people '''
foregroundMask, _, _ = extract_people(depthRaw, foregroundMask, minPersonPixThresh=5000, gradientFilter=True, gradThresh=15)
else:
foregroundMask = None
''' Write to file if there has been substantial change. '''
if diff > frame_difference_percent or time.time()-prevFrameTime > 1/minFramerate or time.time()-recentMotionTime < motion_lag_time:
currentFrame += 1
if depthRaw != []:
''' Logical time '''
if second != prevSec:
prevSecondCountMax = secondCount
secondCount = 0
prevSec = second
else:
secondCount = int(secondCount) + 1
secondCount = str(secondCount)
if len(ms_str) == 1:
ms_str = '0' + ms_str
if len(secondCount) == 1:
secondCount = '0' + secondCount
''' Keep track of framerate '''
if prevTime != second:
prevTime = second
print "FPS: {0:.1f} Diff: {1:.1f}%".format((currentFrame-prevFrame)/(time.time()-prevFPSTime), diff)
prevFrame = currentFrame
prevFPSTime = time.time()
''' Create folder/file names '''
if record:
if 1:
''' Version 1. 2012 '''
ms_str = str(ms)[str(ms).find(".")+1:]
depth_dir = base_dir+'depth/'+day+"/"+hour+"/"+minute+"/device_"+str(device_id)
depth_filename = depth_dir + "/depth_"+day+"_"+hour+"_"+minute+"_"+second+"_"+secondCount+"_"+ms_str+".png"
color_dir = base_dir+'color/'+day+"/"+hour+"/"+minute+"/device_"+str(device_id)
color_filename = color_dir + "/color_"+day+"_"+hour+"_"+minute+"_"+second+"_"+secondCount+"_"+ms_str+".jpg"
if get_skeleton:
skel_dir = base_dir+'skel/'+day+"/"+hour+"/"+minute+"/device_"+str(device_id)
skel_filename = skel_dir + "/skel_"+day+"_"+hour+"_"+minute+"_"+second+"_"+secondCount+"_"+ms_str+"_.dat"
if anonomize:
maskDir = base_dir+'mask/'+day+"/"+hour+"/"+minute+"/device_"+str(device_id)
maskName = maskDir + "/mask_"+day+"_"+hour+"_"+minute+"_"+second+"_"+secondCount+"_"+ms_str+".jpg"
else:
''' Version 2. April 2013 '''
base_sub_dir = "{:s}device_{:d}/{:s}/{:s}/{:s}".format(base_dir,device_id,day,hour,minute)
depth_dir = "{:s}/depth".format(base_sub_dir)
color_dir = "{:s}/color".format(base_sub_dir)
depth_filename = "{:s}/depth_{:s}_{:s}_{:s}_{:s}_{:s}_{:s}_.png".format(depth_dir,day,hour,minute,second,secondCount,ms_str)
color_filename = "{:s}/color_{:s}_{:s}_{:s}_{:s}_{:s}_{:s}_.jpg".format(color_dir,day,hour,minute,second,secondCount,ms_str)
if get_skeleton:
skel_dir = "{:s}/skel".format(base_sub_dir)
skel_filename = "{:s}/skel_{:s}_{:s}_{:s}_{:s}_{:s}_{:s}_.dat".format(skel_dir,day,hour,minute,second,secondCount,ms_str)
if anonomize:
maskDir = "{:s}/mask".format(base_sub_dir)
maskName = "{:s}/mask_{:s}_{:s}_{:s}_{:s}_{:s}_{:s}_.jpg".format(skel_dir,day,hour,minute,second,secondCount,ms_str)
''' Create folders if they doesn't exist '''
createDirectory(depth_dir)
createDirectory(color_dir)
if get_skeleton:
createDirectory(skel_dir)
if anonomize:
createDirectory(maskDir)
''' Save data '''
save_frame(depth_filename, depthRaw, color_filename, colorRaw, skel_filename, users, maskName=maskName, mask=foregroundMask)
prevFrameTime = time.time()
''' Display skeletons '''
if viz:
d = np.array(depthRaw)
d /= (np.nanmin([d.max(), 2**16])/256.0)
d = d.astype(np.uint8)
if get_skeleton:
for u_key in users.keys():
u = users[u_key]
pt = skel2depth(np.array([u.com]))[0]
w = 10
d[pt[1]-w:pt[1]+w, pt[0]-w:pt[0]+w] = 255
w = 3
if u.tracked:
pts = skel2depth(np.array(u.jointPositions.values()), d.shape)
d = display_skeletons(d, pts, (100,0,0), skel_type='Kinect')
if 1:
cv2.imshow("imageD", d)
if 0:
cv2.imshow("imageM", colorRaw + (255-colorRaw)*(foregroundMask>0)[:,:,np.newaxis] + 50*(((foregroundMask)[:,:,np.newaxis])))
if 1:
cv2.imshow("imageC", colorRaw)
ret = cv2.waitKey(10)
if ret >= 0:
break
