def _map_binocular(self, p0, p1):
if '3d'not in p0['method'] or '3d' not in p1['method']:
return None
#find the nearest intersection point of the two gaze lines
# a line is defined by two point
s0_center = self.eye0_to_World( np.array( p0['sphere']['center'] ) )
s1_center = self.eye1_to_World( np.array( p1['sphere']['center'] ) )
s0_normal = np.dot( self.rotation_matricies[0], np.array( p0['circle_3d']['normal'] ) )
s1_normal = np.dot( self.rotation_matricies[1], np.array( p1['circle_3d']['normal'] ) )
gaze_line0 = [ s0_center, s0_center + s0_normal ]
gaze_line1 = [ s1_center, s1_center + s1_normal ]
nearest_intersection_point , intersection_distance = math_helper.nearest_intersection( gaze_line0, gaze_line1 )
if nearest_intersection_point is not None :
self.last_gaze_distance = np.sqrt( nearest_intersection_point.dot( nearest_intersection_point ) )
image_point, _ = cv2.projectPoints( np.array([nearest_intersection_point]) , np.array([0.0,0.0,0.0]) , np.array([0.0,0.0,0.0]) , self.camera_matrix , self.dist_coefs )
image_point = image_point.reshape(-1,2)
image_point = normalize( image_point[0], self.world_frame_size , flip_y = True)
if self.visualizer.window:
gaze0_3d = s0_normal * self.last_gaze_distance + s0_center
gaze1_3d = s1_normal * self.last_gaze_distance + s1_center
self.gaze_pts_debug0.append( gaze0_3d)
self.gaze_pts_debug1.append( gaze1_3d)
if nearest_intersection_point is not None:
self.intersection_points_debug.append( nearest_intersection_point )
self.sphere0['center'] = s0_center #eye camera coordinates
self.sphere0['radius'] = p0['sphere']['radius']
self.sphere1['center'] = s1_center #eye camera coordinates
self.sphere1['radius'] = p1['sphere']['radius']
if nearest_intersection_point is None :
return None
confidence = (p0['confidence'] + p1['confidence'])/2.
ts = (p0['timestamp'] + p1['timestamp'])/2.
g = { 'norm_pos':image_point,
'eye_centers_3d':{0:s0_center.tolist(),1:s1_center.tolist()},
'gaze_normals_3d':{0:s0_normal.tolist(),1:s1_normal.tolist()},
'gaze_point_3d':nearest_intersection_point.tolist(),
'confidence':confidence,
'timestamp':ts,
'base':[p0,p1]}
return g
def _map_binocular(self, p0, p1):
if '3d' not in p0['method'] or '3d' not in p1['method']:
return None
#find the nearest intersection point of the two gaze lines
#eye ball centers in world coords
s0_center = self.eye0_to_World( np.array( p0['sphere']['center'] ) )
s1_center = self.eye1_to_World( np.array( p1['sphere']['center'] ) )
#eye line of sight in world coords
s0_normal = np.dot( self.rotation_matricies[0], np.array( p0['circle_3d']['normal'] ) )
s1_normal = np.dot( self.rotation_matricies[1], np.array( p1['circle_3d']['normal'] ) )
# See Lech Swirski: "Gaze estimation on glasses-based stereoscopic displays"
# Chapter: 7.4.2 Cyclopean gaze estimate
#the cyclop is the avg of both lines of sight
cyclop_normal = (s0_normal+s1_normal)/2.
cyclop_center = (s0_center+s1_center)/2.
# We use it to define a viewing plane.
gaze_plane = np.cross(cyclop_normal , s1_center-s0_center)
gaze_plane = gaze_plane/np.linalg.norm(gaze_plane)
#project lines of sight onto the gaze plane
s0_norm_on_plane = s0_normal - np.dot(gaze_plane,s0_normal)*gaze_plane
s1_norm_on_plane = s1_normal - np.dot(gaze_plane,s1_normal)*gaze_plane
#create gaze lines on this plane
gaze_line0 = [ s0_center, s0_center + s0_norm_on_plane ]
gaze_line1 = [ s1_center, s1_center + s1_norm_on_plane ]
#find the intersection of left and right line of sight.
nearest_intersection_point , intersection_distance = math_helper.nearest_intersection( gaze_line0, gaze_line1 )
if nearest_intersection_point is not None :
cyclop_gaze = nearest_intersection_point-cyclop_center
self.last_gaze_distance = np.sqrt( cyclop_gaze.dot( cyclop_gaze ) )
image_point, _ = cv2.projectPoints( np.array([nearest_intersection_point]) , np.array([0.0,0.0,0.0]) , np.array([0.0,0.0,0.0]) , self.camera_matrix , self.dist_coefs )
image_point = image_point.reshape(-1,2)
image_point = normalize( image_point[0], self.world_frame_size , flip_y = True)
if self.visualizer.window:
gaze0_3d = s0_normal * self.last_gaze_distance + s0_center
gaze1_3d = s1_normal * self.last_gaze_distance + s1_center
self.gaze_pts_debug0.append( gaze0_3d)
self.gaze_pts_debug1.append( gaze1_3d)
if nearest_intersection_point is not None:
self.intersection_points_debug.append( nearest_intersection_point )
self.sphere0['center'] = s0_center #eye camera coordinates
self.sphere0['radius'] = p0['sphere']['radius']
self.sphere1['center'] = s1_center #eye camera coordinates
self.sphere1['radius'] = p1['sphere']['radius']
if nearest_intersection_point is None :
return None
confidence = min(p0['confidence'],p1['confidence'])
ts = (p0['timestamp'] + p1['timestamp'])/2.
g = { 'norm_pos':image_point,
'eye_centers_3d':{0:s0_center.tolist(),1:s1_center.tolist()},
'gaze_normals_3d':{0:s0_normal.tolist(),1:s1_normal.tolist()},
'gaze_point_3d':nearest_intersection_point.tolist(),
'confidence':confidence,
'timestamp':ts,
'base_data':[p0,p1]}
return g
def _map_binocular(self, p0, p1):
if '3d' not in p0['method'] or '3d' not in p1['method']:
return None
#find the nearest intersection point of the two gaze lines
#eye ball centers in world coords
s0_center = self.eye0_to_World( np.array( p0['sphere']['center'] ) )
s1_center = self.eye1_to_World( np.array( p1['sphere']['center'] ) )
#eye line of sight in world coords
s0_normal = np.dot( self.rotation_matricies[0], np.array( p0['circle_3d']['normal'] ) )
s1_normal = np.dot( self.rotation_matricies[1], np.array( p1['circle_3d']['normal'] ) )
# See Lech Swirski: "Gaze estimation on glasses-based stereoscopic displays"
# Chapter: 7.4.2 Cyclopean gaze estimate
#the cyclop is the avg of both lines of sight
cyclop_normal = (s0_normal+s1_normal)/2.
cyclop_center = (s0_center+s1_center)/2.
# We use it to define a viewing plane.
gaze_plane = np.cross(cyclop_normal , s1_center-s0_center)
gaze_plane = gaze_plane/np.linalg.norm(gaze_plane)
#project lines of sight onto the gaze plane
s0_norm_on_plane = s0_normal - np.dot(gaze_plane,s0_normal)*gaze_plane
s1_norm_on_plane = s1_normal - np.dot(gaze_plane,s1_normal)*gaze_plane
#create gaze lines on this plane
gaze_line0 = [ s0_center, s0_center + s0_norm_on_plane ]
gaze_line1 = [ s1_center, s1_center + s1_norm_on_plane ]
#find the intersection of left and right line of sight.
nearest_intersection_point , intersection_distance = math_helper.nearest_intersection( gaze_line0, gaze_line1 )
if nearest_intersection_point is not None :
cyclop_gaze = nearest_intersection_point-cyclop_center
self.last_gaze_distance = np.sqrt( cyclop_gaze.dot( cyclop_gaze ) )
image_point, _ = cv2.projectPoints( np.array([nearest_intersection_point]) , np.array([0.0,0.0,0.0]) , np.array([0.0,0.0,0.0]) , self.camera_matrix , self.dist_coefs )
image_point = image_point.reshape(-1,2)
image_point = normalize( image_point[0], self.world_frame_size , flip_y = True)
if self.visualizer.window:
gaze0_3d = s0_normal * self.last_gaze_distance + s0_center
gaze1_3d = s1_normal * self.last_gaze_distance + s1_center
self.gaze_pts_debug0.append( gaze0_3d)
self.gaze_pts_debug1.append( gaze1_3d)
if nearest_intersection_point is not None:
self.intersection_points_debug.append( nearest_intersection_point )
self.sphere0['center'] = s0_center #eye camera coordinates
self.sphere0['radius'] = p0['sphere']['radius']
self.sphere1['center'] = s1_center #eye camera coordinates
self.sphere1['radius'] = p1['sphere']['radius']
if nearest_intersection_point is None :
return None
confidence = min(p0['confidence'],p1['confidence'])
ts = (p0['timestamp'] + p1['timestamp'])/2.
g = { 'norm_pos':image_point,
'eye_centers_3d':{0:s0_center.tolist(),1:s1_center.tolist()},
'gaze_normals_3d':{0:s0_normal.tolist(),1:s1_normal.tolist()},
'gaze_point_3d':nearest_intersection_point.tolist(),
'confidence':confidence,
'timestamp':ts,
'base_data':[p0,p1]}
return g
def _map_binocular(self, p0, p1):
if "3d" not in p0["method"] or "3d" not in p1["method"]:
return None
# find the nearest intersection point of the two gaze lines
# eye ball centers in world coords
s0_center = self.eye0_to_World(np.array(p0["sphere"]["center"]))
s1_center = self.eye1_to_World(np.array(p1["sphere"]["center"]))
# eye line of sight in world coords
s0_normal = np.dot(
self.rotation_matricies[0], np.array(p0["circle_3d"]["normal"])
)
s1_normal = np.dot(
self.rotation_matricies[1], np.array(p1["circle_3d"]["normal"])
)
# See Lech Swirski: "Gaze estimation on glasses-based stereoscopic displays"
# Chapter: 7.4.2 Cyclopean gaze estimate
# the cyclop is the avg of both lines of sight
cyclop_normal = (s0_normal + s1_normal) / 2.0
cyclop_center = (s0_center + s1_center) / 2.0
# We use it to define a viewing plane.
gaze_plane = np.cross(cyclop_normal, s1_center - s0_center)
gaze_plane = gaze_plane / np.linalg.norm(gaze_plane)
# project lines of sight onto the gaze plane
s0_norm_on_plane = s0_normal - np.dot(gaze_plane, s0_normal) * gaze_plane
s1_norm_on_plane = s1_normal - np.dot(gaze_plane, s1_normal) * gaze_plane
# create gaze lines on this plane
gaze_line0 = [s0_center, s0_center + s0_norm_on_plane]
gaze_line1 = [s1_center, s1_center + s1_norm_on_plane]
# find the intersection of left and right line of sight.
(
nearest_intersection_point,
intersection_distance,
) = math_helper.nearest_intersection(gaze_line0, gaze_line1)
if nearest_intersection_point is not None and self.backproject:
cyclop_gaze = nearest_intersection_point - cyclop_center
self.last_gaze_distance = np.sqrt(cyclop_gaze.dot(cyclop_gaze))
image_point = self.g_pool.capture.intrinsics.projectPoints(
np.array([nearest_intersection_point])
)
image_point = image_point.reshape(-1, 2)
image_point = normalize(
image_point[0], self.g_pool.capture.intrinsics.resolution, flip_y=True
)
image_point = _clamp_norm_point(image_point)
if hasattr(self, "visualizer") and self.visualizer.window:
gaze0_3d = s0_normal * self.last_gaze_distance + s0_center
gaze1_3d = s1_normal * self.last_gaze_distance + s1_center
self.gaze_pts_debug0.append(gaze0_3d)
self.gaze_pts_debug1.append(gaze1_3d)
if nearest_intersection_point is not None:
self.intersection_points_debug.append(nearest_intersection_point)
self.sphere0["center"] = s0_center # eye camera coordinates
self.sphere0["radius"] = p0["sphere"]["radius"]
self.sphere1["center"] = s1_center # eye camera coordinates
self.sphere1["radius"] = p1["sphere"]["radius"]
if nearest_intersection_point is None:
return None
confidence = min(p0["confidence"], p1["confidence"])
ts = (p0["timestamp"] + p1["timestamp"]) / 2.0
g = {
"topic": "gaze.3d.01.",
"eye_centers_3d": {0: s0_center.tolist(), 1: s1_center.tolist()},
"gaze_normals_3d": {0: s0_normal.tolist(), 1: s1_normal.tolist()},
"gaze_point_3d": nearest_intersection_point.tolist(),
"confidence": confidence,
"timestamp": ts,
"base_data": [p0, p1],
}
if self.backproject:
g["norm_pos"] = image_point
return g
def _predict_single(self, x):
assert x.ndim == 1, x
assert x.shape[0] == _BINOCULAR_FEATURE_COUNT, x
# find the nearest intersection point of the two gaze lines
# eye ball centers in world coords
s1_center = self._eye1_to_World(x[_MONOCULAR_SPHERE_CENTER])
s0_center = self._eye0_to_World(x[_BINOCULAR_SPHERE_CENTER])
# eye line of sight in world coords
s1_normal = np.dot(self.rotation_matricies[1], x[_MONOCULAR_PUPIL_NORMAL])
s0_normal = np.dot(self.rotation_matricies[0], x[_BINOCULAR_PUPIL_NORMAL])
# See Lech Swirski: "Gaze estimation on glasses-based stereoscopic displays"
# Chapter: 7.4.2 Cyclopean gaze estimate
# the cyclop is the avg of both lines of sight
cyclop_normal = (s0_normal + s1_normal) / 2.0
cyclop_center = (s0_center + s1_center) / 2.0
# We use it to define a viewing plane.
gaze_plane = np.cross(cyclop_normal, s1_center - s0_center)
gaze_plane = gaze_plane / np.linalg.norm(gaze_plane)
# project lines of sight onto the gaze plane
s0_norm_on_plane = s0_normal - np.dot(gaze_plane, s0_normal) * gaze_plane
s1_norm_on_plane = s1_normal - np.dot(gaze_plane, s1_normal) * gaze_plane
# create gaze lines on this plane
gaze_line0 = [s0_center, s0_center + s0_norm_on_plane]
gaze_line1 = [s1_center, s1_center + s1_norm_on_plane]
# find the intersection of left and right line of sight.
(
nearest_intersection_point,
intersection_distance,
) = math_helper.nearest_intersection(gaze_line0, gaze_line1)
if nearest_intersection_point is None:
return None
# Check if gaze is in front of camera. If it is not, flip direction.
if nearest_intersection_point[-1] < 0:
nearest_intersection_point *= -1.0
g = {
"eye_centers_3d": {"0": s0_center.tolist(), "1": s1_center.tolist()},
"gaze_normals_3d": {"0": s0_normal.tolist(), "1": s1_normal.tolist()},
"gaze_point_3d": nearest_intersection_point.tolist(),
}
if self.intrinsics is not None:
cyclop_gaze = nearest_intersection_point - cyclop_center
self.last_gaze_distance = np.sqrt(cyclop_gaze.dot(cyclop_gaze))
image_point = self.intrinsics.projectPoints(
np.array([nearest_intersection_point])
)
image_point = image_point.reshape(-1, 2)
image_point = normalize(
image_point[0], self.intrinsics.resolution, flip_y=True
)
image_point = _clamp_norm_point(image_point)
g["norm_pos"] = image_point
return g
def map_binocular_intersect(self, pupil_pts_0, pupil_pts_1 ,frame ):
# maps gaze with binocular mapping
# requires each list to contain at least one item!
# returns 1 gaze point at minimum
gaze_pts = []
p0 = pupil_pts_0.pop(0)
p1 = pupil_pts_1.pop(0)
while True:
#find the nearest intersection point of the two gaze lines
# a line is defined by two point
s0_center = self.eye0_to_World( np.array( p0['sphere']['center'] ) )
s1_center = self.eye1_to_World( np.array( p1['sphere']['center'] ) )
s0_normal = np.dot( self.rotation_matrix0, np.array( p0['circle_3d']['normal'] ) )
s1_normal = np.dot( self.rotation_matrix1, np.array( p1['circle_3d']['normal'] ) )
gaze_line0 = [ s0_center, s0_center + s0_normal ]
gaze_line1 = [ s1_center, s1_center + s1_normal ]
nearest_intersection_point , intersection_distance = math_helper.nearest_intersection( gaze_line0, gaze_line1 )
if nearest_intersection_point is not None :
self.last_gaze_distance = np.sqrt( nearest_intersection_point.dot( nearest_intersection_point ) )
image_point, _ = cv2.projectPoints( np.array([nearest_intersection_point]) , np.array([0.0,0.0,0.0]) , np.array([0.0,0.0,0.0]) , self.camera_matrix , self.dist_coefs )
image_point = image_point.reshape(-1,2)
image_point = normalize( image_point[0], (frame.width, frame.height) , flip_y = True)
confidence = (p0['confidence'] + p1['confidence'])/2.
ts = (p0['timestamp'] + p1['timestamp'])/2.
gaze_pts.append({ 'norm_pos':image_point,
'eye_centers_3d':{0:s0_center.tolist(),1:s1_center.tolist()},
'gaze_normals_3d':{0:s0_normal.tolist(),1:s1_normal.tolist()},
'gaze_point_3d':nearest_intersection_point.tolist(),
'confidence':confidence,
'timestamp':ts,
'base':[p0,p1]})
else:
logger.debug('No gaze line intersection point found')
if self.visualizer.window:
gaze0_3d = s0_normal * self.last_gaze_distance + s0_center
gaze1_3d = s1_normal * self.last_gaze_distance + s1_center
self.gaze_pts_debug0.append( gaze0_3d)
self.gaze_pts_debug1.append( gaze1_3d)
if nearest_intersection_point is not None:
self.intersection_points_debug.append( nearest_intersection_point )
self.sphere0['center'] = s0_center #eye camera coordinates
self.sphere0['radius'] = p0['sphere']['radius']
self.sphere1['center'] = s1_center #eye camera coordinates
self.sphere1['radius'] = p1['sphere']['radius']
# keep sample with higher timestamp and increase the one with lower timestamp
if p0['timestamp'] <= p1['timestamp'] and pupil_pts_0:
p0 = pupil_pts_0.pop(0)
continue
elif p1['timestamp'] <= p0['timestamp'] and pupil_pts_1:
p1 = pupil_pts_1.pop(0)
continue
elif pupil_pts_0 and not pupil_pts_1:
p0 = pupil_pts_0.pop(0)
elif pupil_pts_1 and not pupil_pts_0:
p1 = pupil_pts_1.pop(0)
else:
break
return gaze_pts
def _map_binocular(self, p0, p1):
if '3d' not in p0['method'] or '3d' not in p1['method']:
return None
#find the nearest intersection point of the two gaze lines
# a line is defined by two point
s0_center = self.eye0_to_World(np.array(p0['sphere']['center']))
s1_center = self.eye1_to_World(np.array(p1['sphere']['center']))
s0_normal = np.dot(self.rotation_matricies[0],
np.array(p0['circle_3d']['normal']))
s1_normal = np.dot(self.rotation_matricies[1],
np.array(p1['circle_3d']['normal']))
gaze_line0 = [s0_center, s0_center + s0_normal]
gaze_line1 = [s1_center, s1_center + s1_normal]
nearest_intersection_point, intersection_distance = math_helper.nearest_intersection(
gaze_line0, gaze_line1)
if nearest_intersection_point is not None:
self.last_gaze_distance = np.sqrt(
nearest_intersection_point.dot(nearest_intersection_point))
image_point, _ = cv2.projectPoints(
np.array([nearest_intersection_point]),
np.array([0.0, 0.0, 0.0]), np.array([0.0, 0.0, 0.0]),
self.camera_matrix, self.dist_coefs)
image_point = image_point.reshape(-1, 2)
image_point = normalize(image_point[0],
self.world_frame_size,
flip_y=True)
if self.visualizer.window:
gaze0_3d = s0_normal * self.last_gaze_distance + s0_center
gaze1_3d = s1_normal * self.last_gaze_distance + s1_center
self.gaze_pts_debug0.append(gaze0_3d)
self.gaze_pts_debug1.append(gaze1_3d)
if nearest_intersection_point is not None:
self.intersection_points_debug.append(
nearest_intersection_point)
self.sphere0['center'] = s0_center #eye camera coordinates
self.sphere0['radius'] = p0['sphere']['radius']
self.sphere1['center'] = s1_center #eye camera coordinates
self.sphere1['radius'] = p1['sphere']['radius']
if nearest_intersection_point is None:
return None
confidence = (p0['confidence'] + p1['confidence']) / 2.
ts = (p0['timestamp'] + p1['timestamp']) / 2.
g = {
'norm_pos': image_point,
'eye_centers_3d': {
0: s0_center.tolist(),
1: s1_center.tolist()
},
'gaze_normals_3d': {
0: s0_normal.tolist(),
1: s1_normal.tolist()
},
'gaze_point_3d': nearest_intersection_point.tolist(),
'confidence': confidence,
'timestamp': ts,
'base': [p0, p1]
}
return g