def on_click(self,img_pos,button,action):
"""
gets called when the user clicks in the window screen
"""
hdpi_factor = float(glfwGetFramebufferSize(glfwGetCurrentContext())[0]/glfwGetWindowSize(glfwGetCurrentContext())[0])
pos = glfwGetCursorPos(glfwGetCurrentContext())
pos = pos[0]*hdpi_factor,pos[1]*hdpi_factor
#drag the seek point
if action == GLFW_PRESS:
screen_seek_pos = self.seek_bar_to_screen((self.current_frame_index,0))
dist = abs(pos[0]-screen_seek_pos[0])+abs(pos[1]-screen_seek_pos[1])
if dist < 20:
self.drag_mode=True
self.was_playing = self.g_pool.play
self.g_pool.play = False
elif action == GLFW_RELEASE:
if self.drag_mode:
x, _ = self.screen_to_seek_bar(pos)
x = int(min(self.frame_count-5,max(0,x)))
try:
self.cap.seek_to_frame(x)
except:
pass
self.g_pool.new_seek = True
self.drag_mode=False
self.g_pool.play = self.was_playing
def on_click(self,img_pos,button,action):
"""
gets called when the user clicks in the window screen
"""
hdpi_factor = float(glfwGetFramebufferSize(glfwGetCurrentContext())[0]/glfwGetWindowSize(glfwGetCurrentContext())[0])
pos = glfwGetCursorPos(glfwGetCurrentContext())
pos = pos[0]*hdpi_factor,pos[1]*hdpi_factor
#drag the seek point
if action == GLFW_PRESS:
screen_in_mark_pos = self.bar_space_to_screen((self.in_mark,0))
screen_out_mark_pos = self.bar_space_to_screen((self.out_mark,0))
#in mark
dist = abs(pos[0]-screen_in_mark_pos[0])+abs(pos[1]-screen_in_mark_pos[1])
if dist < 10:
if self.distance_in_pix(self.in_mark,self.capture.get_frame_index()) > 20:
self.drag_in=True
return
#out mark
dist = abs(pos[0]-screen_out_mark_pos[0])+abs(pos[1]-screen_out_mark_pos[1])
if dist < 10:
if self.distance_in_pix(self.out_mark,self.capture.get_frame_index()) > 20:
self.drag_out=True
elif action == GLFW_RELEASE:
if self.drag_out or self.drag_in:
logger.info("Section: "+self.get_string())
self.drag_out = False
self.drag_in = False
def recent_events(self, events):
frame = events.get("frame")
if not frame:
return
if self.drag_offset is not None:
pos = glfwGetCursorPos(glfwGetCurrentContext())
pos = normalize(pos, glfwGetWindowSize(glfwGetCurrentContext()))
pos = denormalize(
pos, (frame.img.shape[1], frame.img.shape[0])
) # Position in img pixels
self.pos[0] = pos[0] + self.drag_offset[0]
self.pos[1] = pos[1] + self.drag_offset[1]
if self.watermark is not None:
# keep in image bounds, do this even when not dragging because the image sizes could change.
self.pos[1] = max(
0,
min(frame.img.shape[0] - self.watermark.shape[0], max(self.pos[1], 0)),
)
self.pos[0] = max(
0,
min(frame.img.shape[1] - self.watermark.shape[1], max(self.pos[0], 0)),
)
pos = int(self.pos[0]), int(self.pos[1])
img = frame.img
roi = (
slice(pos[1], pos[1] + self.watermark.shape[0]),
slice(pos[0], pos[0] + self.watermark.shape[1]),
)
w_roi = slice(0, img.shape[0] - pos[1]), slice(0, img.shape[1] - pos[0])
img[roi] = self.watermark[w_roi] * self.alpha_mask[w_roi] + img[roi] * (
1 - self.alpha_mask[w_roi]
)
def on_click(self, img_pos, button, action):
"""
gets called when the user clicks in the window screen
"""
hdpi_factor = float(
glfwGetFramebufferSize(glfwGetCurrentContext())[0] /
glfwGetWindowSize(glfwGetCurrentContext())[0])
pos = glfwGetCursorPos(glfwGetCurrentContext())
pos = pos[0] * hdpi_factor, pos[1] * hdpi_factor
#drag the seek point
if action == GLFW_PRESS:
screen_seek_pos = self.seek_bar_to_screen(
(self.current_frame_index, 0))
dist = abs(pos[0] - screen_seek_pos[0]) + abs(pos[1] -
screen_seek_pos[1])
if dist < 20:
self.drag_mode = True
self.was_playing = self.g_pool.play
self.g_pool.play = False
elif action == GLFW_RELEASE:
if self.drag_mode:
x, _ = self.screen_to_seek_bar(pos)
x = int(min(self.frame_count - 5, max(0, x)))
try:
self.cap.seek_to_frame(x)
except:
pass
self.g_pool.new_seek = True
self.drag_mode = False
self.g_pool.play = self.was_playing
def init_gui(self):
#lets make a menu entry in the sidebar
self.menu = ui.Scrolling_Menu('view add edit annotations')
self.g_pool.gui.append(self.menu)
#add a button to close the plugin
self.menu.append(ui.Button('close', self.close))
self.menu.append(
ui.Info_Text(
"Annotations recorded with capture are displayed when this plugin is loaded. New annotations can be added with the interface below."
))
self.menu.append(
ui.Info_Text(
"If you want to revert annotations to the recorded state, stop player, delete the annotations file in the recording and reopen player."
))
self.menu.append(ui.Text_Input('new_annotation_name', self))
self.menu.append(ui.Text_Input('new_annotation_hotkey', self))
self.menu.append(ui.Button('add annotation type', self.add_annotation))
self.sub_menu = ui.Growing_Menu('Events - click to remove')
self.menu.append(self.sub_menu)
self.update_buttons()
self.on_window_resize(glfwGetCurrentContext(),
*glfwGetWindowSize(glfwGetCurrentContext()))
self.glfont = fontstash.Context()
self.glfont.add_font('opensans', get_opensans_font_path())
self.glfont.set_size(24)
#self.glfont.set_color_float((0.2,0.5,0.9,1.0))
self.glfont.set_align_string(v_align='center', h_align='middle')
def on_click(self,img_pos,button,action):
"""
gets called when the user clicks in the window screen
"""
hdpi_factor = float(glfwGetFramebufferSize(glfwGetCurrentContext())[0]/glfwGetWindowSize(glfwGetCurrentContext())[0])
pos = glfwGetCursorPos(glfwGetCurrentContext())
pos = pos[0]*hdpi_factor,pos[1]*hdpi_factor
#drag the seek point
if action == GLFW_PRESS:
screen_in_mark_pos = self.bar_space_to_screen((self.in_mark,0))
screen_out_mark_pos = self.bar_space_to_screen((self.out_mark,0))
#in mark
dist = abs(pos[0]-screen_in_mark_pos[0])+abs(pos[1]-screen_in_mark_pos[1])
if dist < 10:
if self.distance_in_pix(self.in_mark,self.capture.get_frame_index()) > 20:
self.drag_in=True
return
#out mark
dist = abs(pos[0]-screen_out_mark_pos[0])+abs(pos[1]-screen_out_mark_pos[1])
if dist < 10:
if self.distance_in_pix(self.out_mark,self.capture.get_frame_index()) > 20:
self.drag_out=True
elif action == GLFW_RELEASE:
if self.drag_out or self.drag_in:
logger.info("Section: "+self.get_string())
self.drag_out = False
self.drag_in = False
def recent_events(self, events):
frame = events.get("frame")
if not frame:
return
if self.drag_offset is not None:
pos = glfwGetCursorPos(glfwGetCurrentContext())
pos = normalize(pos, glfwGetWindowSize(glfwGetCurrentContext()))
pos = denormalize(
pos, (frame.img.shape[1], frame.img.shape[0])
) # Position in img pixels
self.pos[0] = pos[0] + self.drag_offset[0]
self.pos[1] = pos[1] + self.drag_offset[1]
if self.watermark is not None:
# keep in image bounds, do this even when not dragging because the image sizes could change.
self.pos[1] = max(
0,
min(frame.img.shape[0] - self.watermark.shape[0], max(self.pos[1], 0)),
)
self.pos[0] = max(
0,
min(frame.img.shape[1] - self.watermark.shape[1], max(self.pos[0], 0)),
)
pos = int(self.pos[0]), int(self.pos[1])
img = frame.img
roi = (
slice(pos[1], pos[1] + self.watermark.shape[0]),
slice(pos[0], pos[0] + self.watermark.shape[1]),
)
w_roi = slice(0, img.shape[0] - pos[1]), slice(0, img.shape[1] - pos[0])
img[roi] = self.watermark[w_roi] * self.alpha_mask[w_roi] + img[roi] * (
1 - self.alpha_mask[w_roi]
)
def init_ui(self):
super().init_ui()
self.menu.label = "Offline Calibration"
self.glfont = fontstash.Context()
self.glfont.add_font('opensans', ui.get_opensans_font_path())
self.glfont.set_color_float((1., 1., 1., 1.))
self.glfont.set_align_string(v_align='right', h_align='top')
def use_as_natural_features():
self.manual_ref_positions.extend(self.circle_marker_positions)
self.manual_ref_positions.sort(key=lambda mr: mr['index'])
def jump_next_natural_feature():
self.manual_ref_positions.sort(key=lambda mr: mr['index'])
current = self.g_pool.capture.get_frame_index()
for nf in self.manual_ref_positions:
if nf['index'] > current:
self.notify_all({'subject': 'seek_control.should_seek',
'index': nf['index']})
return
logger.error('No further natural feature available')
def clear_natural_features():
self.manual_ref_positions = []
self.menu.append(ui.Info_Text('"Detection" searches for circle markers in the world video.'))
# self.menu.append(ui.Button('Redetect', self.start_marker_detection))
slider = ui.Slider('detection_progress', self, label='Detection Progress', setter=lambda _: _)
slider.display_format = '%3.0f%%'
self.menu.append(slider)
toggle_label = 'Cancel circle marker detection' if self.process_pipe else 'Start circle marker detection'
self.toggle_detection_button = ui.Button(toggle_label, self.toggle_marker_detection)
self.menu.append(self.toggle_detection_button)
self.menu.append(ui.Separator())
self.menu.append(ui.Button('Use calibration markers as natural features', use_as_natural_features))
self.menu.append(ui.Button('Jump to next natural feature', jump_next_natural_feature))
self.menu.append(ui.Switch('manual_ref_edit_mode', self, label="Natural feature edit mode"))
self.menu.append(ui.Button('Clear natural features', clear_natural_features))
self.menu.append(ui.Info_Text('Calibration only considers pupil data that has an equal or higher confidence than the minimum calibration confidence.'))
self.menu.append(ui.Slider('min_calibration_confidence', self.g_pool,
step=.01, min=0.0, max=1.0,
label='Minimum calibration confidence'))
self.menu.append(ui.Button('Add section', self.append_section))
# set to minimum height
self.timeline = ui.Timeline('Calibration Sections', self.draw_sections, self.draw_labels, 1)
self.g_pool.user_timelines.append(self.timeline)
for sec in self.sections:
self.append_section_menu(sec)
self.on_window_resize(glfw.glfwGetCurrentContext(), *glfw.glfwGetWindowSize(glfw.glfwGetCurrentContext()))
def init_gui(self):
self.on_window_resize(glfwGetCurrentContext(),*glfwGetWindowSize(glfwGetCurrentContext()))
# initialize the menu
self.menu = ui.Scrolling_Menu('Trial Events on Seek Bar')
# load the configuration of last session
self.menu.configuration = self.menu_conf
# add menu to the window
self.g_pool.gui.append(self.menu)
# add ui elements to the menu
self.menu.append(ui.Button('Close', self.unset_alive))
def update(self,frame,recent_pupil_positions,events):
if frame.index == self.out_mark or frame.index == self.in_mark:
self.g_pool.play=False
if self.drag_in:
x,y = glfwGetCursorPos(glfwGetCurrentContext())
x,_ = self.screen_to_bar_space((x,y))
self.in_mark = x
elif self.drag_out:
x,y = glfwGetCursorPos(glfwGetCurrentContext())
x,_ = self.screen_to_bar_space((x,y))
self.out_mark = x
def update(self, frame, events):
if frame.index == self.out_mark or frame.index == self.in_mark:
self.g_pool.play = False
if self.drag_in:
x, y = glfwGetCursorPos(glfwGetCurrentContext())
x, _ = self.screen_to_bar_space((x, y))
self.in_mark = x
elif self.drag_out:
x, y = glfwGetCursorPos(glfwGetCurrentContext())
x, _ = self.screen_to_bar_space((x, y))
self.out_mark = x
def open_window(self):
if not self._window:
monitor = None
# open with same aspect ratio as surface
surface_aspect_ratio = (self.surface.real_world_size["x"] /
self.surface.real_world_size["y"])
win_h = 640
win_w = int(win_h / surface_aspect_ratio)
self._window = glfw.glfwCreateWindow(
win_h,
win_w,
"Reference Surface: " + self.surface.name,
monitor=monitor,
share=glfw.glfwGetCurrentContext(),
)
glfw.glfwSetWindowPos(
self._window,
self.window_position_default[0],
self.window_position_default[1],
)
self.trackball = gl_utils.trackball.Trackball()
self.input = {"down": False, "mouse": (0, 0)}
# Register callbacks
glfw.glfwSetFramebufferSizeCallback(self._window, self.on_resize)
glfw.glfwSetKeyCallback(self._window, self.on_window_key)
glfw.glfwSetWindowCloseCallback(self._window, self.on_close)
glfw.glfwSetMouseButtonCallback(self._window,
self.on_window_mouse_button)
glfw.glfwSetCursorPosCallback(self._window, self.on_pos)
glfw.glfwSetScrollCallback(self._window, self.on_scroll)
self.on_resize(self._window,
*glfw.glfwGetFramebufferSize(self._window))
# gl_state settings
active_window = glfw.glfwGetCurrentContext()
glfw.glfwMakeContextCurrent(self._window)
gl_utils.basic_gl_setup()
gl_utils.make_coord_system_norm_based()
# refresh speed settings
glfw.glfwSwapInterval(0)
glfw.glfwMakeContextCurrent(active_window)
def open_window(self):
if not self._window:
monitor = None
# open with same aspect ratio as surface
surface_aspect_ratio = (
self.surface.real_world_size["x"] / self.surface.real_world_size["y"]
)
win_h = 640
win_w = int(win_h / surface_aspect_ratio)
self._window = glfw.glfwCreateWindow(
win_h,
win_w,
"Reference Surface: " + self.surface.name,
monitor=monitor,
share=glfw.glfwGetCurrentContext(),
)
glfw.glfwSetWindowPos(
self._window,
self.window_position_default[0],
self.window_position_default[1],
)
self.trackball = gl_utils.trackball.Trackball()
self.input = {"down": False, "mouse": (0, 0)}
# Register callbacks
glfw.glfwSetFramebufferSizeCallback(self._window, self.on_resize)
glfw.glfwSetKeyCallback(self._window, self.on_window_key)
glfw.glfwSetWindowCloseCallback(self._window, self.on_close)
glfw.glfwSetMouseButtonCallback(self._window, self.on_window_mouse_button)
glfw.glfwSetCursorPosCallback(self._window, self.on_pos)
glfw.glfwSetScrollCallback(self._window, self.on_scroll)
self.on_resize(self._window, *glfw.glfwGetFramebufferSize(self._window))
# gl_state settings
active_window = glfw.glfwGetCurrentContext()
glfw.glfwMakeContextCurrent(self._window)
gl_utils.basic_gl_setup()
gl_utils.make_coord_system_norm_based()
# refresh speed settings
glfw.glfwSwapInterval(0)
glfw.glfwMakeContextCurrent(active_window)
def seek_bar_to_screen(self, pos):
width, height = glfwGetWindowSize(glfwGetCurrentContext())
x, y = pos
y = 1 - y
x = x * (width - 2 * self.padding) + self.padding
y = y * (height - 2 * self.padding) + self.padding
return x, y
def gl_display(self):
glMatrixMode(GL_PROJECTION)
glPushMatrix()
glLoadIdentity()
width, height = glfwGetWindowSize(glfwGetCurrentContext())
h_pad = self.padding / width
v_pad = self.padding / height
gluOrtho2D(-h_pad, 1 + h_pad, -v_pad, 1 + v_pad) # gl coord convention
glMatrixMode(GL_MODELVIEW)
glPushMatrix()
glLoadIdentity()
if self.drag_mode:
color1 = (0., .8, .5, .5)
color2 = (0., .8, .5, 1.)
else:
color1 = (.25, .8, .8, .5)
color2 = (.25, .8, .8, 1.)
draw_gl_polyline([(0, 0), (self.norm_seek_pos, 0)], color=color1)
draw_gl_polyline([(self.norm_seek_pos, 0), (1, 0)],
color=(.5, .5, .5, .5))
draw_gl_point((self.norm_seek_pos, 0), color=color1, size=40)
draw_gl_point((self.norm_seek_pos, 0), color=color2, size=10)
glMatrixMode(GL_PROJECTION)
glPopMatrix()
glMatrixMode(GL_MODELVIEW)
glPopMatrix()
def on_click(self, img_pos, button, action):
"""
gets called when the user clicks in the window screen
"""
pos = glfwGetCursorPos(glfwGetCurrentContext())
#drag the seek point
if action == GLFW_PRESS:
screen_in_mark_pos = self.bar_space_to_screen((self.in_mark, 0))
screen_out_mark_pos = self.bar_space_to_screen((self.out_mark, 0))
#in mark
dist = abs(pos[0] -
screen_in_mark_pos[0]) + abs(pos[1] -
screen_in_mark_pos[1])
if dist < 10:
if self.distance_in_pix(self.in_mark,
self.capture.get_frame_index()) > 20:
self.drag_in = True
return
#out mark
dist = abs(pos[0] -
screen_out_mark_pos[0]) + abs(pos[1] -
screen_out_mark_pos[1])
if dist < 10:
if self.distance_in_pix(self.out_mark,
self.capture.get_frame_index()) > 20:
self.drag_out = True
elif action == GLFW_RELEASE:
self.drag_out = False
self.drag_in = False
def _gl_state_settings(window):
active_window = glfw.glfwGetCurrentContext()
glfw.glfwMakeContextCurrent(window)
gl_utils.basic_gl_setup()
gl_utils.make_coord_system_norm_based()
glfw.glfwSwapInterval(0)
glfw.glfwMakeContextCurrent(active_window)
def visualize(self, frame, alpha, scale, show_ellipses, pupil_positions):
if not self.initialized:
return
requested_eye_frame_idx = self.eye_world_frame_map[frame.index]
# 1. do we need a new frame?
if requested_eye_frame_idx != self.current_eye_frame.index:
if requested_eye_frame_idx == self.source.get_frame_index() + 2:
# if we just need to seek by one frame, its faster to just read one and and throw it away.
self.source.get_frame()
if requested_eye_frame_idx != self.source.get_frame_index() + 1:
self.source.seek_to_frame(requested_eye_frame_idx)
try:
self.current_eye_frame = self.source.get_frame()
except EndofVideoError:
logger.info("Reached the end of the eye video for eye video {}.".format(self.eyeid))
# 2. dragging image
if self.drag_offset is not None:
x, y = glfwGetCursorPos(glfwGetCurrentContext())
pos = x * self.hdpi_fac, y * self.hdpi_fac
pos = normalize(pos, self.g_pool.camera_render_size)
# Position in img pixels
pos = denormalize(pos, (frame.img.shape[1], frame.img.shape[0]))
self.pos = int(pos[0] + self.drag_offset[0]), int(pos[1] + self.drag_offset[1])
# 3. keep in image bounds, do this even when not dragging because the image video_sizes could change.
video_size = round(self.current_eye_frame.width * scale), round(self.current_eye_frame.height * scale)
# frame.img.shape[0] is height, frame.img.shape[1] is width of screen
self.pos = (min(frame.img.shape[1] - video_size[0], max(self.pos[0], 0)),
min(frame.img.shape[0] - video_size[1], max(self.pos[1], 0)))
# 4. vflipping images, converting to greyscale
eyeimage = self.current_eye_frame.gray
eyeimage = cv2.cvtColor(eyeimage, cv2.COLOR_GRAY2BGR)
if show_ellipses:
try:
pp = next((pp for pp in pupil_positions if pp['id'] == self.eyeid and pp['timestamp'] == self.current_eye_frame.timestamp))
except StopIteration:
pass
else:
el = pp['ellipse']
conf = int(pp.get('model_confidence', pp.get('confidence', 0.1)) * 255)
el_points = getEllipsePts((el['center'], el["axes"], el['angle']))
cv2.polylines(eyeimage, [np.asarray(el_points,dtype='i')], True, (0, 0, 255, conf), thickness=1)
cv2.circle(eyeimage,(int(el['center'][0]),int(el['center'][1])), 5, (0, 0, 255, conf), thickness=-1)
#flip and scale
eyeimage = cv2.resize(eyeimage, (0, 0), fx=scale, fy=scale)
if self.hflip:
eyeimage = np.fliplr(eyeimage)
if self.vflip:
eyeimage = np.flipud(eyeimage)
transparent_image_overlay(self.pos, eyeimage, frame.img, alpha)
def _switch_to_current_context(self):
previous_context = glfw.glfwGetCurrentContext()
glfw.glfwMakeContextCurrent(self.__gl_handle)
try:
yield
finally:
glfw.glfwMakeContextCurrent(previous_context)
def update(self,frame,events):
if self.drag_offset is not None:
pos = glfwGetCursorPos(glfwGetCurrentContext())
pos = normalize(pos,glfwGetWindowSize(glfwGetCurrentContext()))
pos = denormalize(pos,(frame.img.shape[1],frame.img.shape[0]) ) # Position in img pixels
self.pos[0] = pos[0]+self.drag_offset[0]
self.pos[1] = pos[1]+self.drag_offset[1]
if self.watermark is not None:
#keep in image bounds, do this even when not dragging because the image sizes could change.
self.pos[1] = min(frame.img.shape[0]-self.watermark.shape[0],max(self.pos[1],0))
self.pos[0] = min(frame.img.shape[1]-self.watermark.shape[1],max(self.pos[0],0))
pos = int(self.pos[0]),int(self.pos[1])
img = frame.img
roi = slice(pos[1],pos[1]+self.watermark.shape[0]),slice(pos[0],pos[0]+self.watermark.shape[1])
img[roi] = self.watermark*self.alpha_mask + img[roi]*(1-self.alpha_mask)
def on_click(self,img_pos,button,action):
"""
gets called when the user clicks in the window screen
"""
pos = glfwGetCursorPos(glfwGetCurrentContext())
#drag the seek point
if action == GLFW_PRESS:
screen_in_mark_pos = self.bar_space_to_screen((self.in_mark,0))
screen_out_mark_pos = self.bar_space_to_screen((self.out_mark,0))
#in mark
dist = abs(pos[0]-screen_in_mark_pos[0])+abs(pos[1]-screen_in_mark_pos[1])
if dist < 10:
if self.distance_in_pix(self.in_mark,self.capture.get_frame_index()) > 20:
self.drag_in=True
return
#out mark
dist = abs(pos[0]-screen_out_mark_pos[0])+abs(pos[1]-screen_out_mark_pos[1])
if dist < 10:
if self.distance_in_pix(self.out_mark,self.capture.get_frame_index()) > 20:
self.drag_out=True
elif action == GLFW_RELEASE:
self.drag_out = False
self.drag_in = False
def toggle_depth_display():
def on_depth_mouse_button(window, button, action, mods):
if button == glfw.GLFW_MOUSE_BUTTON_LEFT and action == glfw.GLFW_PRESS:
self.mouse_drag = True
if (button == glfw.GLFW_MOUSE_BUTTON_LEFT
and action == glfw.GLFW_RELEASE):
self.mouse_drag = False
if self.depth_window is None:
self.pitch = 0
self.yaw = 0
win_size = glfw.glfwGetWindowSize(self.g_pool.main_window)
self.depth_window = glfw.glfwCreateWindow(
win_size[0], win_size[1], "3D Point Cloud")
glfw.glfwSetMouseButtonCallback(self.depth_window,
on_depth_mouse_button)
active_window = glfw.glfwGetCurrentContext()
glfw.glfwMakeContextCurrent(self.depth_window)
gl_utils.basic_gl_setup()
gl_utils.make_coord_system_norm_based()
# refresh speed settings
glfw.glfwSwapInterval(0)
glfw.glfwMakeContextCurrent(active_window)
def toggle_depth_display():
def on_depth_mouse_button(window, button, action, mods):
if button == glfw.GLFW_MOUSE_BUTTON_LEFT and action == glfw.GLFW_PRESS:
self.mouse_drag = True
if (
button == glfw.GLFW_MOUSE_BUTTON_LEFT
and action == glfw.GLFW_RELEASE
):
self.mouse_drag = False
if self.depth_window is None:
self.pitch = 0
self.yaw = 0
win_size = glfw.glfwGetWindowSize(self.g_pool.main_window)
self.depth_window = glfw.glfwCreateWindow(
win_size[0], win_size[1], "3D Point Cloud"
)
glfw.glfwSetMouseButtonCallback(
self.depth_window, on_depth_mouse_button
)
active_window = glfw.glfwGetCurrentContext()
glfw.glfwMakeContextCurrent(self.depth_window)
gl_utils.basic_gl_setup()
gl_utils.make_coord_system_norm_based()
# refresh speed settings
glfw.glfwSwapInterval(0)
glfw.glfwMakeContextCurrent(active_window)
def _gl_state_settings(window):
active_window = glfw.glfwGetCurrentContext()
glfw.glfwMakeContextCurrent(window)
gl_utils.basic_gl_setup()
gl_utils.make_coord_system_norm_based()
glfw.glfwSwapInterval(0)
glfw.glfwMakeContextCurrent(active_window)
def seek_bar_to_screen(self,pos):
width,height = glfwGetWindowSize(glfwGetCurrentContext())
x,y=pos
y = 1-y
x = x*(width-2*self.padding)+self.padding
y = y*(height-2*self.padding)+self.padding
return x,y
def gl_display(self):
glMatrixMode(GL_PROJECTION)
glPushMatrix()
glLoadIdentity()
width,height = glfwGetWindowSize(glfwGetCurrentContext())
h_pad = self.padding/width
v_pad = self.padding/height
gluOrtho2D(-h_pad, 1+h_pad, -v_pad, 1+v_pad) # gl coord convention
glMatrixMode(GL_MODELVIEW)
glPushMatrix()
glLoadIdentity()
if self.drag_mode:
color1 = (0.,.8,.5,.5)
color2 = (0.,.8,.5,1.)
else:
color1 = (.25,.8,.8,.5)
color2 = (.25,.8,.8,1.)
draw_gl_polyline( [(0,0),(self.norm_seek_pos,0)],color=color1)
draw_gl_polyline( [(self.norm_seek_pos,0),(1,0)],color=(.5,.5,.5,.5))
draw_gl_point((self.norm_seek_pos,0),color=color1,size=40)
draw_gl_point((self.norm_seek_pos,0),color=color2,size=10)
glMatrixMode(GL_PROJECTION)
glPopMatrix()
glMatrixMode(GL_MODELVIEW)
glPopMatrix()
def update(self,frame,events):
for eye_index in self.showeyes:
requested_eye_frame_idx = self.eye_world_frame_map[eye_index][frame.index]
#1. do we need a new frame?
if requested_eye_frame_idx != self.eye_frames[eye_index].index:
# do we need to seek?
if requested_eye_frame_idx == self.eye_cap[eye_index].get_frame_index()+1:
# if we just need to seek by one frame, its faster to just read one and and throw it away.
_ = self.eye_cap[eye_index].get_frame()
if requested_eye_frame_idx != self.eye_cap[eye_index].get_frame_index():
# only now do I need to seek
self.eye_cap[eye_index].seek_to_frame(requested_eye_frame_idx)
# reading the new eye frame frame
try:
self.eye_frames[eye_index] = self.eye_cap[eye_index].get_frame()
except EndofVideoFileError:
logger.warning("Reached the end of the eye video for eye video %s."%eye_index)
else:
#our old frame is still valid because we are doing upsampling
pass
#2. dragging image
if self.drag_offset[eye_index] is not None:
pos = glfwGetCursorPos(glfwGetCurrentContext())
pos = normalize(pos,glfwGetWindowSize(glfwGetCurrentContext()))
pos = denormalize(pos,(frame.img.shape[1],frame.img.shape[0]) ) # Position in img pixels
self.pos[eye_index][0] = pos[0]+self.drag_offset[eye_index][0]
self.pos[eye_index][1] = pos[1]+self.drag_offset[eye_index][1]
else:
self.video_size = [round(self.eye_frames[eye_index].width*self.eye_scale_factor), round(self.eye_frames[eye_index].height*self.eye_scale_factor)]
#3. keep in image bounds, do this even when not dragging because the image video_sizes could change.
self.pos[eye_index][1] = min(frame.img.shape[0]-self.video_size[1],max(self.pos[eye_index][1],0)) #frame.img.shape[0] is height, frame.img.shape[1] is width of screen
self.pos[eye_index][0] = min(frame.img.shape[1]-self.video_size[0],max(self.pos[eye_index][0],0))
#4. flipping images, converting to greyscale
eye_gray = cv2.cvtColor(self.eye_frames[eye_index].img,cv2.COLOR_BGR2GRAY) #auto gray scaling
eyeimage = cv2.resize(eye_gray,(0,0),fx=self.eye_scale_factor, fy=self.eye_scale_factor)
if self.mirror[str(eye_index)]:
eyeimage = np.fliplr(eyeimage)
if self.flip[str(eye_index)]:
eyeimage = np.flipud(eyeimage)
#5. finally overlay the image
x,y = int(self.pos[eye_index][0]),int(self.pos[eye_index][1])
transparent_image_overlay((x,y),cv2.cvtColor(eyeimage,cv2.COLOR_GRAY2BGR),frame.img,self.alpha)
def update(self,frame,events):
for eye_index in self.showeyes:
requested_eye_frame_idx = self.eye_world_frame_map[eye_index][frame.index]
#1. do we need a new frame?
if requested_eye_frame_idx != self.eye_frames[eye_index].index:
# do we need to seek?
if requested_eye_frame_idx == self.eye_cap[eye_index].get_frame_index()+1:
# if we just need to seek by one frame, its faster to just read one and and throw it away.
_ = self.eye_cap[eye_index].get_frame()
if requested_eye_frame_idx != self.eye_cap[eye_index].get_frame_index():
# only now do I need to seek
self.eye_cap[eye_index].seek_to_frame(requested_eye_frame_idx)
# reading the new eye frame frame
try:
self.eye_frames[eye_index] = self.eye_cap[eye_index].get_frame()
except EndofVideoFileError:
logger.warning("Reached the end of the eye video for eye video %s."%eye_index)
else:
#our old frame is still valid because we are doing upsampling
pass
#2. dragging image
if self.drag_offset[eye_index] is not None:
pos = glfwGetCursorPos(glfwGetCurrentContext())
pos = normalize(pos,glfwGetWindowSize(glfwGetCurrentContext()))
pos = denormalize(pos,(frame.img.shape[1],frame.img.shape[0]) ) # Position in img pixels
self.pos[eye_index][0] = pos[0]+self.drag_offset[eye_index][0]
self.pos[eye_index][1] = pos[1]+self.drag_offset[eye_index][1]
else:
self.video_size = [round(self.eye_frames[eye_index].width*self.eye_scale_factor), round(self.eye_frames[eye_index].height*self.eye_scale_factor)]
#3. keep in image bounds, do this even when not dragging because the image video_sizes could change.
self.pos[eye_index][1] = min(frame.img.shape[0]-self.video_size[1],max(self.pos[eye_index][1],0)) #frame.img.shape[0] is height, frame.img.shape[1] is width of screen
self.pos[eye_index][0] = min(frame.img.shape[1]-self.video_size[0],max(self.pos[eye_index][0],0))
#4. flipping images, converting to greyscale
eye_gray = cv2.cvtColor(self.eye_frames[eye_index].img,cv2.COLOR_BGR2GRAY) #auto gray scaling
eyeimage = cv2.resize(eye_gray,(0,0),fx=self.eye_scale_factor, fy=self.eye_scale_factor)
if self.mirror[str(eye_index)]:
eyeimage = np.fliplr(eyeimage)
if self.flip[str(eye_index)]:
eyeimage = np.flipud(eyeimage)
#5. finally overlay the image
x,y = int(self.pos[eye_index][0]),int(self.pos[eye_index][1])
transparent_image_overlay((x,y),cv2.cvtColor(eyeimage,cv2.COLOR_GRAY2BGR),frame.img,self.alpha)
def recent_events(self, events):
frame = events.get('frame')
if not frame:
return
if frame.index == self.out_mark or frame.index == self.in_mark:
self.g_pool.play = False
if self.drag_in:
x, y = glfwGetCursorPos(glfwGetCurrentContext())
x, _ = self.screen_to_bar_space((x, y))
self.in_mark = x
elif self.drag_out:
x, y = glfwGetCursorPos(glfwGetCurrentContext())
x, _ = self.screen_to_bar_space((x, y))
self.out_mark = x
def current_mouse_pos(window, camera_render_size, frame_size):
hdpi_fac = getHDPIFactor(window)
x, y = glfwGetCursorPos(glfwGetCurrentContext())
pos = x * hdpi_fac, y * hdpi_fac
pos = normalize(pos, camera_render_size)
# Position in img pixels
pos = denormalize(pos, frame_size)
return (int(pos[0]), int(pos[1]))
def on_resize(window, w, h):
if not g_pool.iconified:
active_window = glfw.glfwGetCurrentContext()
glfw.glfwMakeContextCurrent(window)
g_pool.gui.update_window(w, h)
graph.adjust_size(w, h)
adjust_gl_view(w, h)
glfw.glfwMakeContextCurrent(active_window)
def on_resize(window,w, h):
if not g_pool.iconified:
active_window = glfw.glfwGetCurrentContext()
glfw.glfwMakeContextCurrent(window)
g_pool.gui.update_window(w,h)
graph.adjust_size(w,h)
adjust_gl_view(w,h)
glfw.glfwMakeContextCurrent(active_window)
def current_mouse_pos(window, camera_render_size, frame_size):
hdpi_fac = getHDPIFactor(window)
x, y = glfwGetCursorPos(glfwGetCurrentContext())
pos = x * hdpi_fac, y * hdpi_fac
pos = normalize(pos, camera_render_size)
# Position in img pixels
pos = denormalize(pos, frame_size)
return (int(pos[0]), int(pos[1]))
def current_mouse_pos(window, camera_render_size, frame_size):
content_scale = get_content_scale(window)
x, y = glfwGetCursorPos(glfwGetCurrentContext())
pos = x * content_scale, y * content_scale
pos = normalize(pos, camera_render_size)
# Position in img pixels
pos = denormalize(pos, frame_size)
return (int(pos[0]), int(pos[1]))
def recent_events(self, events):
frame = events.get('frame')
if not frame:
return
if frame.index == self.out_mark or frame.index == self.in_mark:
self.g_pool.play = False
if self.drag_in:
x,y = glfwGetCursorPos(glfwGetCurrentContext())
x,_ = self.screen_to_bar_space((x,y))
self.in_mark = x
elif self.drag_out:
x,y = glfwGetCursorPos(glfwGetCurrentContext())
x,_ = self.screen_to_bar_space((x,y))
self.out_mark = x
def gl_display(self):
if hasattr(self,'glfont'):
from glfw import glfwGetFramebufferSize,glfwGetCurrentContext
self.window_size = glfwGetFramebufferSize(glfwGetCurrentContext())
self.glfont.set_color_float((0.,0.,0.,1.))
self.glfont.set_size(int(self.window_size[0]/30.))
self.glfont.set_blur(5.)
self.glfont.draw_limited_text(self.window_size[0]/2.,self.window_size[1]/2.,self.info_text,self.window_size[0]*0.8)
self.glfont.set_blur(0.96)
self.glfont.set_color_float((1.,1.,1.,1.0))
self.glfont.draw_limited_text(self.window_size[0]/2.,self.window_size[1]/2.,self.info_text,self.window_size[0]*0.8)
def init_ui(self):
self.glfont = fontstash.Context()
self.glfont.add_font('opensans',get_opensans_font_path())
self.glfont.set_size(32)
self.glfont.set_color_float((0.2,0.5,0.9,1.0))
self.glfont.set_align_string(v_align='center',h_align='middle')
self.window_size = glfwGetFramebufferSize(glfwGetCurrentContext())
self.tex = Render_Target(*self.window_size)
self._socket = zmq_tools.Msg_Receiver(self.g_pool.zmq_ctx,self.g_pool.ipc_sub_url,('logging',))
def gl_display(self):
if hasattr(self,'glfont'):
from glfw import glfwGetFramebufferSize,glfwGetCurrentContext
self.window_size = glfwGetFramebufferSize(glfwGetCurrentContext())
self.glfont.set_color_float((0.,0.,0.,1.))
self.glfont.set_size(int(self.window_size[0]/30.))
self.glfont.set_blur(5.)
self.glfont.draw_limited_text(self.window_size[0]/2.,self.window_size[1]/2.,self.info_text,self.window_size[0]*0.8)
self.glfont.set_blur(0.96)
self.glfont.set_color_float((1.,1.,1.,1.0))
self.glfont.draw_limited_text(self.window_size[0]/2.,self.window_size[1]/2.,self.info_text,self.window_size[0]*0.8)
def on_click(self,img_pos,button,action):
"""
gets called when the user clicks in the window screen
"""
hdpi_factor = float(glfwGetFramebufferSize(glfwGetCurrentContext())[0]/glfwGetWindowSize(glfwGetCurrentContext())[0])
pos = glfwGetCursorPos(glfwGetCurrentContext())
pos = pos[0]*hdpi_factor,pos[1]*hdpi_factor
#drag the seek point
if action == GLFW_PRESS:
screen_in_mark_pos = self.bar_space_to_screen((self.in_mark,0))
screen_out_mark_pos = self.bar_space_to_screen((self.out_mark,0))
#in mark
dist = abs(pos[0]-screen_in_mark_pos[0])+abs(pos[1]-screen_in_mark_pos[1])
if dist < 10:
if self.distance_in_pix(self.in_mark,self.capture.get_frame_index()) > 20:
self.drag_in=True
return
#out mark
dist = abs(pos[0]-screen_out_mark_pos[0])+abs(pos[1]-screen_out_mark_pos[1])
if dist < 10:
if self.distance_in_pix(self.out_mark,self.capture.get_frame_index()) > 20:
self.drag_out=True
elif action == GLFW_RELEASE:
if self.drag_out or self.drag_in:
logger.info("Section: "+self.get_string())
self.drag_out = False
self.drag_in = False
# would be great to expand the click area horizontally for big sections
for s in self.sections:
if s is not self.sections[self.focus]:
midsec = self.mid_sections[self.sections.index(s)]
screen_midsec_pos = self.bar_space_to_screen((midsec,0))
dist = abs(pos[0]-screen_midsec_pos[0])+abs(pos[1]-screen_midsec_pos[1])
if dist < 10:
if self.distance_in_pix(midsec,self.capture.get_frame_index()) > 20:
self.focus = self.sections.index(s)
break
def update(self, frame, recent_pupil_positions, events):
self.current_frame_index = frame.index
self.norm_seek_pos = self.current_frame_index / float(self.frame_count)
if self.drag_mode:
pos = glfwGetCursorPos(glfwGetCurrentContext())
norm_seek_pos, _ = self.screen_to_seek_bar(pos)
norm_seek_pos = min(1, max(0, norm_seek_pos))
if abs(norm_seek_pos - self.norm_seek_pos) >= .01:
seek_pos = int(norm_seek_pos * self.frame_count)
self.cap.seek_to_frame(seek_pos)
self.g_pool.new_seek = True
def update(self,frame,recent_pupil_positions,events):
self.current_frame_index = frame.index
self.norm_seek_pos = self.current_frame_index/float(self.frame_count)
if self.drag_mode:
pos = glfwGetCursorPos(glfwGetCurrentContext())
norm_seek_pos, _ = self.screen_to_seek_bar(pos)
norm_seek_pos = min(1,max(0,norm_seek_pos))
if abs(norm_seek_pos-self.norm_seek_pos) >=.01:
seek_pos = int(norm_seek_pos*self.frame_count)
self.cap.seek_to_frame(seek_pos)
self.g_pool.new_seek = True
def init_gui(self):
#lets make a menu entry in the sidebar
self.menu = ui.Scrolling_Menu('view add edit annotations')
self.g_pool.gui.append(self.menu)
#add a button to close the plugin
self.menu.append(ui.Button('close',self.close))
self.menu.append(ui.Text_Input('new_annotation_name',self))
self.menu.append(ui.Text_Input('new_annotation_hotkey',self))
self.menu.append(ui.Button('add annotation type',self.add_annotation))
self.sub_menu = ui.Growing_Menu('Events - click to remove')
self.menu.append(self.sub_menu)
self.update_buttons()
self.on_window_resize(glfwGetCurrentContext(),*glfwGetWindowSize(glfwGetCurrentContext()))
self.glfont = fontstash.Context()
self.glfont.add_font('opensans',get_opensans_font_path())
self.glfont.set_size(24)
#self.glfont.set_color_float((0.2,0.5,0.9,1.0))
self.glfont.set_align_string(v_align='center',h_align='middle')
def init_gui(self):
#lets make a menu entry in the sidebar
self.menu = ui.Scrolling_Menu('view add edit annotations')
self.g_pool.gui.append(self.menu)
#add a button to close the plugin
self.menu.append(ui.Button('close',self.close))
self.menu.append(ui.Text_Input('new_annotation_name',self))
self.menu.append(ui.Text_Input('new_annotation_hotkey',self))
self.menu.append(ui.Button('add annotation type',self.add_annotation))
self.sub_menu = ui.Growing_Menu('Events - click to remove')
self.menu.append(self.sub_menu)
self.update_buttons()
self.on_window_resize(glfwGetCurrentContext(),*glfwGetWindowSize(glfwGetCurrentContext()))
self.glfont = fontstash.Context()
self.glfont.add_font('opensans',get_opensans_font_path())
self.glfont.set_size(24)
#self.glfont.set_color_float((0.2,0.5,0.9,1.0))
self.glfont.set_align_string(v_align='center',h_align='middle')
def on_resize(window, w, h):
if is_window_visible(window):
active_window = glfw.glfwGetCurrentContext()
glfw.glfwMakeContextCurrent(window)
hdpi_factor = float(glfw.glfwGetFramebufferSize(window)[0] / glfw.glfwGetWindowSize(window)[0])
g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor
g_pool.gui.update_window(w, h)
g_pool.gui.collect_menus()
for g in g_pool.graphs:
g.scale = hdpi_factor
g.adjust_window_size(w, h)
adjust_gl_view(w, h)
glfw.glfwMakeContextCurrent(active_window)
def on_resize(window, w, h):
if is_window_visible(window):
active_window = glfw.glfwGetCurrentContext()
glfw.glfwMakeContextCurrent(window)
hdpi_factor = float(glfw.glfwGetFramebufferSize(window)[0] / glfw.glfwGetWindowSize(window)[0])
g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor
g_pool.gui.update_window(w, h)
g_pool.gui.collect_menus()
for g in g_pool.graphs:
g.scale = hdpi_factor
g.adjust_window_size(w, h)
adjust_gl_view(w, h)
glfw.glfwMakeContextCurrent(active_window)
def update(self, frame, events):
if self.drag_offset is not None:
pos = glfwGetCursorPos(glfwGetCurrentContext())
pos = normalize(pos, glfwGetWindowSize(glfwGetCurrentContext()))
pos = denormalize(pos,
(frame.img.shape[1],
frame.img.shape[0])) # Position in img pixels
self.pos[0] = pos[0] + self.drag_offset[0]
self.pos[1] = pos[1] + self.drag_offset[1]
if self.watermark is not None:
#keep in image bounds, do this even when not dragging because the image sizes could change.
self.pos[1] = min(frame.img.shape[0] - self.watermark.shape[0],
max(self.pos[1], 0))
self.pos[0] = min(frame.img.shape[1] - self.watermark.shape[1],
max(self.pos[0], 0))
pos = int(self.pos[0]), int(self.pos[1])
img = frame.img
roi = slice(pos[1], pos[1] + self.watermark.shape[0]), slice(
pos[0], pos[0] + self.watermark.shape[1])
img[roi] = self.watermark * self.alpha_mask + img[roi] * (
1 - self.alpha_mask)
def recent_events(self,events):
if 'frame' in events:
frame = events['frame']
self.img_shape = frame.height,frame.width,3
if self.running:
gray = frame.gray
# hack "self.markers" instead "self.screens" is keeped for inheritence compatibility
self.markers = detect_screens(gray)
if self.mode == "Show marker IDs":
draw_markers(frame.img,self.markers)
events['frame'] = frame
# locate surfaces, map gaze
for s in self.surfaces:
s.locate(self.markers,self.camera_calibration,self.min_marker_perimeter,self.min_id_confidence, self.locate_3d)
if s.detected:
s.gaze_on_srf = s.map_data_to_surface(events.get('gaze_positions',[]),s.m_from_screen)
else:
s.gaze_on_srf =[]
events['surface'] = []
for s in self.surfaces:
if s.detected:
events['surface'].append({
'name':s.name,
'uid':s.uid,
'm_to_screen':s.m_to_screen.tolist(),
'm_from_screen':s.m_from_screen.tolist(),
'gaze_on_srf': s.gaze_on_srf,
'timestamp':frame.timestamp,
'camera_pose_3d':s.camera_pose_3d.tolist() if s.camera_pose_3d is not None else None
})
if self.running:
self.button.status_text = '{}/{}'.format(len([s for s in self.surfaces if s.detected]), len(self.surfaces))
else:
self.button.status_text = 'tracking paused'
if self.mode == 'Show Markers and Surfaces':
# edit surfaces by user
if self.edit_surf_verts:
window = glfwGetCurrentContext()
pos = glfwGetCursorPos(window)
pos = normalize(pos,glfwGetWindowSize(window),flip_y=True)
for s,v_idx in self.edit_surf_verts:
if s.detected:
new_pos = s.img_to_ref_surface(np.array(pos))
s.move_vertex(v_idx,new_pos)
def _on_gl_display(self):
if not self._window:
return
active_window = glfw.glfwGetCurrentContext()
glfw.glfwMakeContextCurrent(self._window)
self._init_3d_window()
self._trackball.push()
self._render()
self._trackball.pop()
glfw.glfwSwapBuffers(self._window)
glfw.glfwMakeContextCurrent(active_window)
glfw.glfwPollEvents()
def init_gui(self):
self.glfont = fontstash.Context()
self.glfont.add_font('opensans', get_opensans_font_path())
self.glfont.set_size(32)
self.glfont.set_color_float((0.2, 0.5, 0.9, 1.0))
self.glfont.set_align_string(v_align='center', h_align='middle')
self.window_size = glfwGetFramebufferSize(glfwGetCurrentContext())
self.tex = Render_Target(*self.window_size)
self.log_handler = Log_to_Callback(self.on_log)
logger = logging.getLogger()
logger.addHandler(self.log_handler)
self.log_handler.setLevel(logging.INFO)
def _on_gl_display(self):
if not self._window:
return
active_window = glfw.glfwGetCurrentContext()
glfw.glfwMakeContextCurrent(self._window)
self._init_3d_window()
self._trackball.push()
self._render()
self._trackball.pop()
glfw.glfwSwapBuffers(self._window)
glfw.glfwMakeContextCurrent(active_window)
glfw.glfwPollEvents()
def update(self,frame,events):
self.current_frame_index = frame.index
if self.drag_mode:
x,y = glfwGetCursorPos(glfwGetCurrentContext())
x,_ = self.screen_to_seek_bar((x,y))
seek_pos = min(self.frame_count,max(0,x))
if abs(seek_pos-self.current_frame_index) >=.002*self.frame_count:
seek_pos = int(min(seek_pos,self.frame_count-5)) #the last frames can be problematic to seek to
try:
self.cap.seek_to_frame(seek_pos)
self.current_frame_index = seek_pos
except:
pass
self.g_pool.new_seek = True
def init_gui(self):
self.glfont = fontstash.Context()
self.glfont.add_font('opensans',get_opensans_font_path())
self.glfont.set_size(32)
self.glfont.set_color_float((0.2,0.5,0.9,1.0))
self.glfont.set_align_string(v_align='center',h_align='middle')
self.window_size = glfwGetFramebufferSize(glfwGetCurrentContext())
self.tex = Render_Target(*self.window_size)
self.log_handler = Log_to_Callback(self.on_log)
logger = logging.getLogger()
logger.addHandler(self.log_handler)
self.log_handler.setLevel(logging.INFO)
def update(self,frame,events):
self.current_frame_index = frame.index
if self.drag_mode:
x,y = glfwGetCursorPos(glfwGetCurrentContext())
x,_ = self.screen_to_seek_bar((x,y))
seek_pos = min(self.frame_count,max(0,x))
seek_pos = int(min(seek_pos,self.frame_count-5)) #the last frames can be problematic to seek to
if self.current_frame_index-1 != seek_pos:
try:
# logger.info('seeking to {} form {}'.format(seek_pos,self.current_frame_index))
self.cap.seek_to_frame(seek_pos)
self.current_frame_index = self.cap.get_frame_index()
except:
pass
self.g_pool.new_seek = True
def on_resize(window, w, h):
nonlocal window_size
nonlocal camera_render_size
active_window = glfw.glfwGetCurrentContext()
glfw.glfwMakeContextCurrent(window)
hdpi_factor = float(glfw.glfwGetFramebufferSize(window)[0] / glfw.glfwGetWindowSize(window)[0])
g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor
window_size = w, h
camera_render_size = w-int(icon_bar_width*g_pool.gui.scale), h
g_pool.gui.update_window(w, h)
g_pool.gui.collect_menus()
for g in g_pool.graphs:
g.scale = hdpi_factor
g.adjust_window_size(w, h)
adjust_gl_view(w, h)
glfw.glfwMakeContextCurrent(active_window)
def on_resize(window, w, h):
nonlocal window_size
nonlocal camera_render_size
nonlocal hdpi_factor
active_window = glfw.glfwGetCurrentContext()
glfw.glfwMakeContextCurrent(window)
hdpi_factor = glfw.getHDPIFactor(window)
g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor
window_size = w, h
camera_render_size = w - int(icon_bar_width * g_pool.gui.scale), h
g_pool.gui.update_window(w, h)
g_pool.gui.collect_menus()
for g in g_pool.graphs:
g.scale = hdpi_factor
g.adjust_window_size(w, h)
adjust_gl_view(w, h)
glfw.glfwMakeContextCurrent(active_window)
def update(self,frame,events):
self.img_shape = frame.height,frame.width,3
if self.running:
gray = frame.gray
# hack "self.markers" instead "self.screens" is keeped for inheritence compatibility
self.markers = detect_screens(gray)
if self.mode == "Show marker IDs":
draw_markers(frame.img,self.markers)
events['surfaces'] = []
# locate surfaces
for s in self.surfaces:
s.locate(self.markers,self.camera_calibration,self.min_marker_perimeter, self.locate_3d)
if s.detected:
events['surfaces'].append({'name':s.name,'uid':s.uid,'m_to_screen':s.m_to_screen,'m_from_screen':s.m_from_screen, 'timestamp':frame.timestamp})
if self.running:
self.button.status_text = '%s/%s'%(len([s for s in self.surfaces if s.detected]),len(self.surfaces))
else:
self.button.status_text = 'tracking paused'
if self.mode == 'Show Markers and Surfaces':
# edit surfaces by user
if self.edit_surf_verts:
window = glfwGetCurrentContext()
pos = glfwGetCursorPos(window)
pos = normalize(pos,glfwGetWindowSize(window),flip_y=True)
for s,v_idx in self.edit_surf_verts:
if s.detected:
new_pos = s.img_to_ref_surface(np.array(pos))
s.move_vertex(v_idx,new_pos)
#map recent gaze onto detected surfaces used for pupil server
for s in self.surfaces:
if s.detected:
s.gaze_on_srf = []
for p in events.get('gaze_positions',[]):
gp_on_s = tuple(s.img_to_ref_surface(np.array(p['norm_pos'])))
p['realtime gaze on ' + s.name] = gp_on_s
s.gaze_on_srf.append(gp_on_s)
def init_gui(self):
self.glfont = fontstash.Context()
self.glfont.add_font('opensans',get_opensans_font_path())
self.glfont.set_size(32)
self.glfont.set_color_float((0.2,0.5,0.9,1.0))
self.glfont.set_align_string(v_align='center',h_align='middle')
self.window_size = glfwGetFramebufferSize(glfwGetCurrentContext())
self.tex = Render_Target(*self.window_size)
if self.g_pool.app == 'capture':
self._socket = zmq_tools.Msg_Receiver(self.g_pool.zmq_ctx,self.g_pool.ipc_sub_url,('logging',))
self._poller = zmq_tools.zmq.Poller()
self._poller.register(self._socket.socket)
else:
self._socket = None
self.log_handler = Log_to_Callback(self.on_log)
logger = logging.getLogger()
logger.addHandler(self.log_handler)
self.log_handler.setLevel(logging.INFO)
def gl_display_in_window(self, world_tex):
"""
here we map a selected surface onto a separate window.
"""
if self._window and self.surface.detected:
active_window = glfw.glfwGetCurrentContext()
glfw.glfwMakeContextCurrent(self._window)
gl_utils.clear_gl_screen()
# cv uses 3x3 gl uses 4x4 transformation matrices
width, height = self.tracker.camera_model.resolution
img_corners = np.array(
[(0, height), (width, height), (width, 0), (0, 0)], dtype=np.float32
)
denorm_trans = _get_norm_to_points_trans(img_corners)
trans_mat = self.surface.dist_img_to_surf_trans @ denorm_trans
trans_mat = gl_utils.cvmat_to_glmat(trans_mat)
gl.glMatrixMode(gl.GL_PROJECTION)
gl.glPushMatrix()
gl.glLoadIdentity()
gl.glOrtho(0, 1, 0, 1, -1, 1)
gl.glMatrixMode(gl.GL_MODELVIEW)
gl.glPushMatrix()
# apply trans_mat to our quad - this will stretch the quad such that the
# surface will span the window extends
gl.glLoadMatrixf(trans_mat)
world_tex.draw()
gl.glMatrixMode(gl.GL_PROJECTION)
gl.glPopMatrix()
gl.glMatrixMode(gl.GL_MODELVIEW)
gl.glPopMatrix()
self.draw_recent_pupil_positions()
glfw.glfwSwapBuffers(self._window)
glfw.glfwMakeContextCurrent(active_window)
def on_click(self,img_pos,button,action):
"""
gets called when the user clicks in the window screen
"""
pos = glfwGetCursorPos(glfwGetCurrentContext())
#drag the seek point
if action == GLFW_PRESS:
screen_seek_pos = self.seek_bar_to_screen((self.norm_seek_pos,0))
dist = abs(pos[0]-screen_seek_pos[0])+abs(pos[1]-screen_seek_pos[1])
if dist < 20:
self.drag_mode=True
self.was_playing = self.g_pool.play
self.g_pool.play = False
elif action == GLFW_RELEASE:
if self.drag_mode:
norm_seek_pos, _ = self.screen_to_seek_bar(pos)
norm_seek_pos = min(1,max(0,norm_seek_pos))
seek_pos = int(norm_seek_pos*self.frame_count)
self.cap.seek_to_frame(seek_pos)
self.g_pool.new_seek = True
self.drag_mode=False
self.g_pool.play = self.was_playing
