class FaceRecognition(ParallelPDU):
""" PDU that receives cropped images (head only)
and saves them to file """
QUEUE = 'face-recognition'
ORDERED_DELIVERY = True
POOL_SIZE = 8
UNFINISHED_TASKS_THRESHOLD = 2 * POOL_SIZE
BEST_PERSON_THRESHOLD = 0.75
UPGRADE_FACE_SAMPLES_THRESHOLD = 0.85
def __init__(self, **kwargs):
kwargs['heavy_preprocess'] = betaface_recognition
super(FaceRecognition, self).__init__(**kwargs)
self.session_tracker = SessionTracker()
def light_postprocess(self, matches, image_dict):
self.logger.info("Received matches from BetaFace: %r" % matches)
# No persons means we're out of here.
if len(matches) == 0:
self.logger.info("Bailing out because of empty matches from "
"BetaFace")
return
# No strong bias towards a given person also gets us out of here.
max_probability = max(matches.values())
if max_probability < self.BEST_PERSON_THRESHOLD:
self.logger.info("Out of the matches returned by BetaFace, the "
"maximum confidence is only %.2f "
"(should have been at least %.2f)"\
% (max_probability, self.BEST_PERSON_THRESHOLD))
return
# Get the person with the maximal probability from BetaFace
person_name = matches.keys()[matches.values().index(max_probability)]
self.logger.info("MOST_PROBABLE_PERSON = %s" % person_name)
# Send event to room. This will cause interactivity within the lab.
message_to_room = {'event_type': 'person_appeared',
'person_name': person_name}
self.send_to('room', message_to_room)
if 'session_id' in image_dict.keys():
self.session_tracker.track_event(image_dict['session_id'],
0, # set the person name globally on the session
{'person_name': person_name})
# Send cropped image to UpgradeFaceSamples only if detection confidence
# is really really high. Otherwise it's not worth it and we will
# pollute the data structures in BetaFace API.
if max_probability >= self.UPGRADE_FACE_SAMPLES_THRESHOLD:
upgrade_message = {'person_name': person_name}
upgrade_message.update(image_dict)
self.send_to('upgrade-face-samples', upgrade_message)
else:
self.logger.info("It isn't worth it to send a face sample because "
"maximal confidence was only %.2f (should have "
"been at least %.2f" %\
(max_probability,
self.UPGRADE_FACE_SAMPLES_THRESHOLD))
class Dashboard(PDU):
QUEUE = 'dashboard'
def __init__(self, **kwargs):
super(Dashboard, self).__init__(**kwargs)
self.dashboard_cache = DashboardCache()
self.session_tracker = SessionTracker()
def process_message(self, message):
push_to_redis = self.get_pushing_function(message)
push_to_redis(sensor_id=message['sensor_id'],
sensor_type=message['sensor_type'],
measurement_type=message['type'],
measurement=json.dumps(message))
# Send the RGB image to SessionsStore only if the message has a
# session_id. This means that, at kinect-level, there is an active
# tracking session.
if message['type'] == 'image_rgb' and message.get('session_id', None):
sid = message['session_id']
time = message['created_at']
mappings = {'image_rgb': message['image_rgb']}
self.session_tracker.track_event(sid, time, mappings)
def get_pushing_function(self, message):
if(message['sensor_type'] == "arduino"):
return self.dashboard_cache.lpush
else:
return self.dashboard_cache.put
class HeadCrop(ParallelPDU):
""" PDU that receives images and skeletons from Router
and crops images (head only) """
QUEUE = 'head-crop'
POOL_SIZE = 20
UNFINISHED_TASKS_THRESHOLD = 2 * POOL_SIZE
def __init__(self, **kwargs):
kwargs['heavy_preprocess'] = crop_head
super(HeadCrop, self).__init__(**kwargs)
self.last_image = None
self.last_image_at = None
self.last_skeleton = None
self.last_skeleton_at = None
self.session_tracker = SessionTracker()
def process_message(self, message):
# Step 1 - always update last_image/last_skeleton
if (message['type'] == 'image_rgb' and
message['sensor_type'] == 'kinect'):
self.last_image = message['image_rgb']
if not 'encoder_name' in self.last_image:
self.last_image['encoder_name'] = 'raw'
self.last_image_at = message['created_at']
elif message['type'] == 'skeleton' and\
message['sensor_type'] == 'kinect':
self.last_skeleton = message['skeleton_2D']
self.last_skeleton_at = message['created_at']
message['hack'] = {}
message['hack']['last_image'] = copy.copy(self.last_image)
message['hack']['last_image_at'] = self.last_image_at
message['hack']['last_skeleton'] = copy.copy(self.last_skeleton)
message['hack']['last_skeleton_at'] = self.last_skeleton_at
super(HeadCrop, self).process_message(message)
def light_postprocess(self, cropped_head, image_dict):
# Route cropped images to face-recognition
if cropped_head is not None:
self.log("Sending an image to face recognition")
self._send_to_recognition(cropped_head)
self.session_tracker.track_event(image_dict['session_id'],
image_dict['created_at'],
{"head": cropped_head})
def _send_to_recognition(self, image):
""" Send a given image to face recognition. """
self.send_to('face-recognition', {'head_image': image})
def __init__(self, **kwargs):
kwargs['heavy_preprocess'] = crop_head
super(HeadCrop, self).__init__(**kwargs)
self.last_image = None
self.last_image_at = None
self.last_skeleton = None
self.last_skeleton_at = None
self.session_tracker = SessionTracker()
def __init__(self, **kwargs):
super(RoomPosition, self).__init__(**kwargs)
self.session_tracker = SessionTracker()
self.dashboard_cache = DashboardCache()
class RoomPosition(PDU):
''' PDU that reads skeleton messages from kinect and translates the position
of the torso (from kinect coordinates) to room coordinates using the
sensor position value that is sent by kinect.
INPUT:
Message from kinect of skeleton type with sensor_position field
The sensor_position needs the following fields:
- X,Y,Z: the position of the sensor wrt the room
- alpha: the rotation around y axis
- beta: the rotation around x axis
- gamma: the rotation around z axis
The message needs a skeleton_3D key with the positions of the torso
OUTPUT:
Sends a message with a subject_position key set to a dictionary with
X,Y,Z in room coordinates. The message is sent to the subject-position
queue
'''
QUEUE = 'room-position'
#torso is always first in list
JOINTS = ['torso', 'head', 'neck', 'left_shoulder', 'right_shoulder',
'left_elbow', 'right_elbow', 'left_hand', 'right_hand',
'left_hip', 'right_hip', 'left_knee', 'right_knee',
'left_foot', 'right_foot']
def __init__(self, **kwargs):
super(RoomPosition, self).__init__(**kwargs)
self.session_tracker = SessionTracker()
self.dashboard_cache = DashboardCache()
def process_message(self, message):
if message['type'] != 'skeleton':
return
sensor_position = message['sensor_position']
""" We are doing rotation using the euler angles
see http://en.wikipedia.org/wiki/Rotation_matrix
"""
alpha = sensor_position['alpha']
beta = sensor_position['beta']
gamma = sensor_position['gamma']
rx = rot_x(beta)
ry = rot_y(alpha)
rz = rot_z(gamma)
trans = tanslation(sensor_position['X'], sensor_position['Y'], sensor_position['Z'])
# some temp variables manipulation follows as the cv library uses
# output parameters on multilications
temp_mat = cv.CreateMat(4,4, cv.CV_64F)
rot_mat = cv.CreateMat(4,4, cv.CV_64F)
cv.MatMul(ry,rx,temp_mat)
cv.MatMul(temp_mat, rz, rot_mat)
cv.MatMul(trans, rot_mat, temp_mat)
N = len(self.JOINTS)
pos = cv.CreateMat(4, N, cv.CV_64F)
temp_pos = cv.CreateMat(4, N, cv.CV_64F)
skeleton_3D = message['skeleton_3D']
for i in range(N):
joint = self.JOINTS[i]
if joint in skeleton_3D:
joint_pos = skeleton_3D[joint]
temp_pos[0,i] = joint_pos['X']
temp_pos[1,i] = joint_pos['Y']
temp_pos[2,i] = joint_pos['Z']
temp_pos[3,i] = 1
cv.MatMul(temp_mat, temp_pos, pos)
skeleton_in_room = {}
for i in range(N):
joint = self.JOINTS[i]
if joint in skeleton_3D:
skeleton_in_room[joint] = {
'X': pos[0, i],
'Y': pos[1, i],
'Z': pos[2, i],
}
position_message = {
'X': pos[0,0],
'Y': pos[1,0],
'Z': pos[2,0],
}
self.log('Found position %s' % position_message)
if message.get('session_id', None):
sid = message['session_id']
time = message['created_at']
self.session_tracker.track_event(sid, time, {'skeleton_in_room': skeleton_in_room})
# Keep it simple and put X, Y, Z as top-level keys for this
# measurement of the current tracking session.
self.session_tracker.track_event(sid, time, position_message)
dashboard_message = {
'created_at': message['created_at'],
'sensor_id': message['sensor_id'],
'sensor_position': sensor_position,
}
dashboard_message.update(position_message)
# Send subject position to Redis
self.dashboard_cache.lpush(sensor_id=message['sensor_id'],
sensor_type=message['sensor_type'],
measurement_type='subject_position',
measurement=json.dumps(dashboard_message))
return None
def __init__(self, **kwargs):
kwargs['heavy_preprocess'] = betaface_recognition
super(FaceRecognition, self).__init__(**kwargs)
self.session_tracker = SessionTracker()
