def gl_display_in_window(self,world_tex_id): """ here we map a selected surface onto a seperate window. """ if self._window and self.detected: active_window = glfwGetCurrentContext() glfwMakeContextCurrent(self._window) clear_gl_screen() # cv uses 3x3 gl uses 4x4 tranformation matricies m = cvmat_to_glmat(self.m_from_screen) glMatrixMode(GL_PROJECTION) glPushMatrix() glLoadIdentity() gluOrtho2D(0, 1, 0, 1) # gl coord convention glMatrixMode(GL_MODELVIEW) glPushMatrix() #apply m to our quad - this will stretch the quad such that the ref suface will span the window extends glLoadMatrixf(m) draw_named_texture(world_tex_id) glMatrixMode(GL_PROJECTION) glPopMatrix() glMatrixMode(GL_MODELVIEW) glPopMatrix() # now lets get recent pupil positions on this surface: draw_gl_points_norm(self.gaze_on_srf,color=(0.,8.,.5,.8), size=80) glfwSwapBuffers(self._window) glfwMakeContextCurrent(active_window)
def gl_display_in_window(self, world_tex_id): """ here we map a selected surface onto a seperate window. """ if self._window and self.detected: active_window = glfwGetCurrentContext() glfwMakeContextCurrent(self._window) clear_gl_screen() # cv uses 3x3 gl uses 4x4 tranformation matricies m = cvmat_to_glmat(self.m_from_screen) glMatrixMode(GL_PROJECTION) glPushMatrix() glLoadIdentity() gluOrtho2D(0, 1, 0, 1) # gl coord convention glMatrixMode(GL_MODELVIEW) glPushMatrix() # apply m to our quad - this will stretch the quad such that the ref suface will span the window extends glLoadMatrixf(m) draw_named_texture(world_tex_id) glMatrixMode(GL_PROJECTION) glPopMatrix() glMatrixMode(GL_MODELVIEW) glPopMatrix() # now lets get recent pupil positions on this surface: draw_gl_points_norm(self.gaze_on_srf, color=(0.0, 8.0, 0.5, 0.8), size=80) glfwSwapBuffers(self._window) glfwMakeContextCurrent(active_window)
def gl_display_in_window(self, surface): """ here we map a selected surface onto a seperate window. """ active_window = glfwGetCurrentContext() glfwMakeContextCurrent(self._window) clear_gl_screen() # cv uses 3x3 gl uses 4x4 tranformation matricies m = cvmat_to_glmat(surface.m_from_screen) glMatrixMode(GL_PROJECTION) glPushMatrix() glLoadIdentity() gluOrtho2D(0, 1, 0, 1) # gl coord convention glMatrixMode(GL_MODELVIEW) glPushMatrix() #apply m to our quad - this will stretch the quad such that the ref suface will span the window extends glLoadMatrixf(m) draw_named_texture(self.g_pool.image_tex) glMatrixMode(GL_PROJECTION) glPopMatrix() glMatrixMode(GL_MODELVIEW) glPopMatrix() #now lets get recent pupil positions on this surface: try: gaze_on_surface = [ p['realtime gaze on ' + surface.name] for p in self.recent_pupil_positions ] except KeyError: gaze_on_surface = [] draw_gl_points_norm(gaze_on_surface, color=(0., 8., .5, .8), size=80) glfwSwapBuffers(self._window) glfwMakeContextCurrent(active_window)
def gl_display_in_window(self,surface): """ here we map a selected surface onto a seperate window. """ active_window = glfwGetCurrentContext() glfwMakeContextCurrent(self._window) clear_gl_screen() # cv uses 3x3 gl uses 4x4 tranformation matricies m = cvmat_to_glmat(surface.m_from_screen) glMatrixMode(GL_PROJECTION) glPushMatrix() glLoadIdentity() gluOrtho2D(0, 1, 0, 1) # gl coord convention glMatrixMode(GL_MODELVIEW) glPushMatrix() #apply m to our quad - this will stretch the quad such that the ref suface will span the window extends glLoadMatrixf(m) draw_named_texture(self.g_pool.image_tex) glMatrixMode(GL_PROJECTION) glPopMatrix() glMatrixMode(GL_MODELVIEW) glPopMatrix() #now lets get recent pupil positions on this surface: try: gaze_on_surface = [p['realtime gaze on '+surface.name] for p in self.recent_pupil_positions] except KeyError: gaze_on_surface = [] draw_gl_points_norm(gaze_on_surface,color=(0.,8.,.5,.8), size=80) glfwSwapBuffers(self._window) glfwMakeContextCurrent(active_window)
def gl_display_heatmap(self): if self.heatmap_texture and self.detected: # cv uses 3x3 gl uses 4x4 tranformation matricies m = cvmat_to_glmat(self.m_to_screen) glMatrixMode(GL_PROJECTION) glPushMatrix() glLoadIdentity() gluOrtho2D(0, 1, 0, 1) # gl coord convention glMatrixMode(GL_MODELVIEW) glPushMatrix() #apply m to our quad - this will stretch the quad such that the ref suface will span the window extends glLoadMatrixf(m) draw_named_texture(self.heatmap_texture) glMatrixMode(GL_PROJECTION) glPopMatrix() glMatrixMode(GL_MODELVIEW) glPopMatrix()
def world(g_pool, cap_src, cap_size): """world Creates a window, gl context. Grabs images from a capture. Receives Pupil coordinates from g_pool.pupil_queue Can run various plug-ins. """ # Callback functions def on_resize(window, w, h): active_window = glfwGetCurrentContext() glfwMakeContextCurrent(window) norm_size = normalize((w, h), glfwGetWindowSize(window)) fb_size = denormalize(norm_size, glfwGetFramebufferSize(window)) atb.TwWindowSize(*map(int, fb_size)) adjust_gl_view(w, h, window) glfwMakeContextCurrent(active_window) for p in g_pool.plugins: p.on_window_resize(window, w, h) def on_iconify(window, iconfied): if not isinstance(cap, FakeCapture): g_pool.update_textures.value = not iconfied def on_key(window, key, scancode, action, mods): if not atb.TwEventKeyboardGLFW(key, action): if action == GLFW_PRESS: if key == GLFW_KEY_ESCAPE: on_close(window) def on_char(window, char): if not atb.TwEventCharGLFW(char, 1): pass def on_button(window, button, action, mods): if not atb.TwEventMouseButtonGLFW(button, action): pos = glfwGetCursorPos(window) pos = normalize(pos, glfwGetWindowSize(world_window)) pos = denormalize(pos, (frame.img.shape[1], frame.img.shape[0])) # Position in img pixels for p in g_pool.plugins: p.on_click(pos, button, action) def on_pos(window, x, y): norm_pos = normalize((x, y), glfwGetWindowSize(window)) fb_x, fb_y = denormalize(norm_pos, glfwGetFramebufferSize(window)) if atb.TwMouseMotion(int(fb_x), int(fb_y)): pass def on_scroll(window, x, y): if not atb.TwMouseWheel(int(x)): pass def on_close(window): g_pool.quit.value = True logger.info('Process closing from window') # load session persistent settings session_settings = Persistent_Dict( os.path.join(g_pool.user_dir, 'user_settings_world')) def load(var_name, default): return session_settings.get(var_name, default) def save(var_name, var): session_settings[var_name] = var # Initialize capture cap = autoCreateCapture(cap_src, cap_size, 24, timebase=g_pool.timebase) # Get an image from the grabber try: frame = cap.get_frame() except CameraCaptureError: logger.error("Could not retrieve image from capture") cap.close() return height, width = frame.img.shape[:2] # load last calibration data try: pt_cloud = np.load(os.path.join(g_pool.user_dir, 'cal_pt_cloud.npy')) logger.debug("Using calibration found in %s" % g_pool.user_dir) map_pupil = calibrate.get_map_from_cloud(pt_cloud, (width, height)) except: logger.debug("No calibration found.") def map_pupil(vector): """ 1 to 1 mapping """ return vector # any object we attach to the g_pool object *from now on* will only be visible to this process! # vars should be declared here to make them visible to the code reader. g_pool.plugins = [] g_pool.map_pupil = map_pupil g_pool.update_textures = c_bool(1) if isinstance(cap, FakeCapture): g_pool.update_textures.value = False g_pool.capture = cap g_pool.rec_name = recorder.get_auto_name() # helpers called by the main atb bar def update_fps(): old_time, bar.timestamp = bar.timestamp, time() dt = bar.timestamp - old_time if dt: bar.fps.value += .05 * (1. / dt - bar.fps.value) def set_window_size(mode, data): height, width = frame.img.shape[:2] ratio = (1, .75, .5, .25)[mode] w, h = int(width * ratio), int(height * ratio) glfwSetWindowSize(world_window, w, h) data.value = mode # update the bar.value def get_from_data(data): """ helper for atb getter and setter use """ return data.value def set_rec_name(val): if not val.value: g_pool.rec_name = recorder.get_auto_name() else: g_pool.rec_name = val.value def get_rec_name(): return create_string_buffer(g_pool.rec_name, 512) def open_calibration(selection, data): # prepare destruction of current ref_detector... and remove it for p in g_pool.plugins: if isinstance(p, calibration_routines.detector_by_index): p.alive = False g_pool.plugins = [p for p in g_pool.plugins if p.alive] new_ref_detector = calibration_routines.detector_by_index[selection]( g_pool, atb_pos=bar.next_atb_pos) g_pool.plugins.append(new_ref_detector) g_pool.plugins.sort(key=lambda p: p.order) # save the value for atb bar data.value = selection def toggle_record_video(): for p in g_pool.plugins: if isinstance(p, recorder.Recorder): p.alive = False return new_plugin = recorder.Recorder(g_pool, g_pool.rec_name, bar.fps.value, frame.img.shape, bar.record_eye.value, g_pool.eye_tx, bar.audio.value) g_pool.plugins.append(new_plugin) g_pool.plugins.sort(key=lambda p: p.order) def toggle_show_calib_result(): for p in g_pool.plugins: if isinstance(p, Show_Calibration): p.alive = False return new_plugin = Show_Calibration(g_pool, frame.img.shape) g_pool.plugins.append(new_plugin) g_pool.plugins.sort(key=lambda p: p.order) def toggle_server(): for p in g_pool.plugins: if isinstance(p, Pupil_Server): p.alive = False return new_plugin = Pupil_Server(g_pool, (10, 300)) g_pool.plugins.append(new_plugin) g_pool.plugins.sort(key=lambda p: p.order) def toggle_remote(): for p in g_pool.plugins: if isinstance(p, Pupil_Remote): p.alive = False return new_plugin = Pupil_Remote(g_pool, (10, 360), on_char) g_pool.plugins.append(new_plugin) g_pool.plugins.sort(key=lambda p: p.order) def toggle_ar(): for p in g_pool.plugins: if isinstance(p, Marker_Detector): p.alive = False return new_plugin = Marker_Detector(g_pool, (10, 400)) g_pool.plugins.append(new_plugin) g_pool.plugins.sort(key=lambda p: p.order) def reset_timebase(): #the last frame from worldcam will be t0 g_pool.timebase.value = g_pool.capure.get_now() logger.info( "New timebase set to %s all timestamps will count from here now." % g_pool.timebase.value) atb.init() # add main controls ATB bar bar = atb.Bar(name="World", label="Controls", help="Scene controls", color=(50, 50, 50), alpha=100, valueswidth=150, text='light', position=(10, 10), refresh=.3, size=(300, 200)) bar.next_atb_pos = (10, 220) bar.fps = c_float(0.0) bar.timestamp = time() bar.calibration_type = c_int(load("calibration_type", 0)) bar.record_eye = c_bool(load("record_eye", 0)) bar.audio = c_int(load("audio", -1)) bar.window_size = c_int(load("window_size", 0)) window_size_enum = atb.enum("Display Size", { "Full": 0, "Medium": 1, "Half": 2, "Mini": 3 }) calibrate_type_enum = atb.enum("Calibration Method", calibration_routines.index_by_name) audio_enum = atb.enum("Audio Input", dict(Audio_Input_List())) bar.version = create_string_buffer(g_pool.version, 512) bar.add_var( "fps", bar.fps, step=1., readonly=True, help= "Refresh speed of this process. Especially during recording it should not drop below the camera set frame rate." ) bar.add_var( "display size", vtype=window_size_enum, setter=set_window_size, getter=get_from_data, data=bar.window_size, help="Resize the world window. This has no effect on the actual image." ) bar.add_var("calibration method", setter=open_calibration, getter=get_from_data, data=bar.calibration_type, vtype=calibrate_type_enum, group="Calibration", help="Please choose your desired calibration method.") bar.add_button("show calibration result", toggle_show_calib_result, group="Calibration", help="Click to show calibration result.") bar.add_var("session name", create_string_buffer(512), getter=get_rec_name, setter=set_rec_name, group="Recording", help="Give your recording session a custom name.") bar.add_button("record", toggle_record_video, key="r", group="Recording", help="Start/Stop Recording") bar.add_var("record eye", bar.record_eye, group="Recording", help="check to save raw video of eye") bar.add_var("record audio", bar.audio, vtype=audio_enum, group="Recording", help="Select from audio recording options.") bar.add_button( "start/stop marker tracking", toggle_ar, key="x", help="find markers in scene to map gaze onto referace surfaces") bar.add_button( "start/stop server", toggle_server, key="s", help= "the server broadcasts pupil and gaze positions locally or via network" ) bar.add_button("start/stop remote", toggle_remote, key="w", help="remote allows seding commad to pupil via network") bar.add_button( "set timebase to now", reset_timebase, help="this button allows the timestamps to count from now on.", key="t") bar.add_var( "update screen", g_pool.update_textures, help= "if you dont need to see the camera image updated, you can turn this of to reduce CPU load." ) bar.add_separator("Sep1") bar.add_var("version", bar.version, readonly=True) bar.add_var("exit", g_pool.quit) # add uvc camera controls ATB bar cap.create_atb_bar(pos=(320, 10)) # Initialize glfw glfwInit() world_window = glfwCreateWindow(width, height, "World", None, None) glfwMakeContextCurrent(world_window) # Register callbacks world_window glfwSetWindowSizeCallback(world_window, on_resize) glfwSetWindowCloseCallback(world_window, on_close) glfwSetWindowIconifyCallback(world_window, on_iconify) glfwSetKeyCallback(world_window, on_key) glfwSetCharCallback(world_window, on_char) glfwSetMouseButtonCallback(world_window, on_button) glfwSetCursorPosCallback(world_window, on_pos) glfwSetScrollCallback(world_window, on_scroll) #set the last saved window size set_window_size(bar.window_size.value, bar.window_size) on_resize(world_window, *glfwGetWindowSize(world_window)) glfwSetWindowPos(world_window, 0, 0) # gl_state settings basic_gl_setup() g_pool.image_tex = create_named_texture(frame.img) # refresh speed settings glfwSwapInterval(0) #load calibration plugin open_calibration(bar.calibration_type.value, bar.calibration_type) #load gaze_display plugin g_pool.plugins.append(Display_Recent_Gaze(g_pool)) # Event loop while not g_pool.quit.value: # Get an image from the grabber try: frame = cap.get_frame() except CameraCaptureError: logger.error("Capture from Camera Failed. Stopping.") break except EndofVideoFileError: logger.warning("Video File is done. Stopping") break update_fps() #a container that allows plugins to post and read events events = [] #receive and map pupil positions recent_pupil_positions = [] while not g_pool.pupil_queue.empty(): p = g_pool.pupil_queue.get() if p['norm_pupil'] is None: p['norm_gaze'] = None else: p['norm_gaze'] = g_pool.map_pupil(p['norm_pupil']) recent_pupil_positions.append(p) # allow each Plugin to do its work. for p in g_pool.plugins: p.update(frame, recent_pupil_positions, events) #check if a plugin need to be destroyed g_pool.plugins = [p for p in g_pool.plugins if p.alive] # render camera image glfwMakeContextCurrent(world_window) make_coord_system_norm_based() if g_pool.update_textures.value: draw_named_texture(g_pool.image_tex, frame.img) else: draw_named_texture(g_pool.image_tex) make_coord_system_pixel_based(frame.img.shape) # render visual feedback from loaded plugins for p in g_pool.plugins: p.gl_display() atb.draw() glfwSwapBuffers(world_window) glfwPollEvents() # de-init all running plugins for p in g_pool.plugins: p.alive = False #reading p.alive actually runs plug-in cleanup _ = p.alive save('window_size', bar.window_size.value) save('calibration_type', bar.calibration_type.value) save('record_eye', bar.record_eye.value) save('audio', bar.audio.value) session_settings.close() cap.close() atb.terminate() glfwDestroyWindow(world_window) glfwTerminate() logger.debug("Process done")
def world(g_pool,cap_src,cap_size): """world Creates a window, gl context. Grabs images from a capture. Receives Pupil coordinates from g_pool.pupil_queue Can run various plug-ins. """ # Callback functions def on_resize(window,w, h): active_window = glfwGetCurrentContext() glfwMakeContextCurrent(window) norm_size = normalize((w,h),glfwGetWindowSize(window)) fb_size = denormalize(norm_size,glfwGetFramebufferSize(window)) atb.TwWindowSize(*map(int,fb_size)) adjust_gl_view(w,h,window) glfwMakeContextCurrent(active_window) def on_iconify(window,iconfied): if not isinstance(cap,FakeCapture): g_pool.update_textures.value = not iconfied def on_key(window, key, scancode, action, mods): if not atb.TwEventKeyboardGLFW(key,action): if action == GLFW_PRESS: if key == GLFW_KEY_ESCAPE: on_close(window) def on_char(window,char): if not atb.TwEventCharGLFW(char,1): pass def on_button(window,button, action, mods): if not atb.TwEventMouseButtonGLFW(button,action): pos = glfwGetCursorPos(window) pos = normalize(pos,glfwGetWindowSize(world_window)) pos = denormalize(pos,(frame.img.shape[1],frame.img.shape[0]) ) # Position in img pixels for p in g_pool.plugins: p.on_click(pos,button,action) def on_pos(window,x, y): norm_pos = normalize((x,y),glfwGetWindowSize(window)) fb_x,fb_y = denormalize(norm_pos,glfwGetFramebufferSize(window)) if atb.TwMouseMotion(int(fb_x),int(fb_y)): pass def on_scroll(window,x,y): if not atb.TwMouseWheel(int(x)): pass def on_close(window): g_pool.quit.value = True logger.info('Process closing from window') # load session persistent settings session_settings = shelve.open(os.path.join(g_pool.user_dir,'user_settings_world'),protocol=2) def load(var_name,default): return session_settings.get(var_name,default) def save(var_name,var): session_settings[var_name] = var # load last calibration data try: pt_cloud = np.load(os.path.join(g_pool.user_dir,'cal_pt_cloud.npy')) logger.info("Using calibration found in %s" %g_pool.user_dir) map_pupil = calibrate.get_map_from_cloud(pt_cloud,(width,height)) except: logger.info("No calibration found.") def map_pupil(vector): """ 1 to 1 mapping """ return vector # any object we attach to the g_pool object now will only be visible to this process! # vars should be declared here to make them visible to the reader. g_pool.plugins = [] g_pool.map_pupil = map_pupil g_pool.update_textures = c_bool(1) # Initialize capture cap = autoCreateCapture(cap_src, cap_size, 24, timebase=g_pool.timebase) if isinstance(cap,FakeCapture): g_pool.update_textures.value = False # Get an image from the grabber try: frame = cap.get_frame() except CameraCaptureError: logger.error("Could not retrieve image from capture") cap.close() return height,width = frame.img.shape[:2] g_pool.capture = cap # helpers called by the main atb bar def update_fps(): old_time, bar.timestamp = bar.timestamp, time() dt = bar.timestamp - old_time if dt: bar.fps.value += .05 * (1. / dt - bar.fps.value) def set_window_size(mode,data): height,width = frame.img.shape[:2] ratio = (1,.75,.5,.25)[mode] w,h = int(width*ratio),int(height*ratio) glfwSetWindowSize(world_window,w,h) data.value=mode # update the bar.value def get_from_data(data): """ helper for atb getter and setter use """ return data.value def open_calibration(selection,data): # prepare destruction of current ref_detector... and remove it for p in g_pool.plugins: if isinstance(p,calibration_routines.detector_by_index): p.alive = False g_pool.plugins = [p for p in g_pool.plugins if p.alive] new_ref_detector = calibration_routines.detector_by_index[selection](g_pool,atb_pos=bar.next_atb_pos) g_pool.plugins.append(new_ref_detector) g_pool.plugins.sort(key=lambda p: p.order) # save the value for atb bar data.value=selection def toggle_record_video(): for p in g_pool.plugins: if isinstance(p,recorder.Recorder): p.alive = False return # set up folder within recordings named by user input in atb if not bar.rec_name.value: bar.rec_name.value = recorder.get_auto_name() new_plugin = recorder.Recorder(g_pool,bar.rec_name.value, bar.fps.value, frame.img.shape, bar.record_eye.value, g_pool.eye_tx) g_pool.plugins.append(new_plugin) g_pool.plugins.sort(key=lambda p: p.order) def toggle_show_calib_result(): for p in g_pool.plugins: if isinstance(p,Show_Calibration): p.alive = False return new_plugin = Show_Calibration(g_pool,frame.img.shape) g_pool.plugins.append(new_plugin) g_pool.plugins.sort(key=lambda p: p.order) def toggle_server(): for p in g_pool.plugins: if isinstance(p,Pupil_Server): p.alive = False return new_plugin = Pupil_Server(g_pool,(10,300)) g_pool.plugins.append(new_plugin) g_pool.plugins.sort(key=lambda p: p.order) def toggle_ar(): for p in g_pool.plugins: if isinstance(p,Marker_Detector): p.alive = False return new_plugin = Marker_Detector(g_pool,(10,400)) g_pool.plugins.append(new_plugin) g_pool.plugins.sort(key=lambda p: p.order) def reset_timebase(): #the last frame from worldcam will be t0 g_pool.timebase.value = cap.get_now() logger.info("New timebase set to %s all timestamps will count from here now."%g_pool.timebase.value) atb.init() # add main controls ATB bar bar = atb.Bar(name = "World", label="Controls", help="Scene controls", color=(50, 50, 50), alpha=100,valueswidth=150, text='light', position=(10, 10),refresh=.3, size=(300, 200)) bar.next_atb_pos = (10,220) bar.fps = c_float(0.0) bar.timestamp = time() bar.calibration_type = c_int(load("calibration_type",0)) bar.record_eye = c_bool(load("record_eye",0)) bar.window_size = c_int(load("window_size",0)) window_size_enum = atb.enum("Display Size",{"Full":0, "Medium":1,"Half":2,"Mini":3}) calibrate_type_enum = atb.enum("Calibration Method",calibration_routines.index_by_name) bar.rec_name = create_string_buffer(512) bar.version = create_string_buffer(g_pool.version,512) bar.rec_name.value = recorder.get_auto_name() bar.add_var("fps", bar.fps, step=1., readonly=True, help="Refresh speed of this process. Especially during recording it should not drop below the camera set frame rate.") bar.add_var("display size", vtype=window_size_enum,setter=set_window_size,getter=get_from_data,data=bar.window_size,help="Resize the world window. This has no effect on the actual image.") bar.add_var("calibration method",setter=open_calibration,getter=get_from_data,data=bar.calibration_type, vtype=calibrate_type_enum,group="Calibration", help="Please choose your desired calibration method.") bar.add_button("show calibration result",toggle_show_calib_result, group="Calibration", help="Click to show calibration result.") bar.add_var("session name",bar.rec_name, group="Recording", help="Give your recording session a custom name.") bar.add_button("record", toggle_record_video, key="r", group="Recording", help="Start/Stop Recording") bar.add_var("record eye", bar.record_eye, group="Recording", help="check to save raw video of eye") bar.add_button("start/stop marker tracking",toggle_ar,key="x",help="find markers in scene to map gaze onto referace surfaces") bar.add_button("start/stop server",toggle_server,key="s",help="the server broadcasts pupil and gaze positions locally or via network") bar.add_button("set timebase to now",reset_timebase,help="this button allows the timestamps to count from now on.",key="t") bar.add_var("update screen", g_pool.update_textures,help="if you dont need to see the camera image updated, you can turn this of to reduce CPU load.") bar.add_separator("Sep1") bar.add_var("version",bar.version, readonly=True) bar.add_var("exit", g_pool.quit) # add uvc camera controls ATB bar cap.create_atb_bar(pos=(320,10)) # Initialize glfw glfwInit() world_window = glfwCreateWindow(width, height, "World", None, None) glfwMakeContextCurrent(world_window) # Register callbacks world_window glfwSetWindowSizeCallback(world_window,on_resize) glfwSetWindowCloseCallback(world_window,on_close) glfwSetWindowIconifyCallback(world_window,on_iconify) glfwSetKeyCallback(world_window,on_key) glfwSetCharCallback(world_window,on_char) glfwSetMouseButtonCallback(world_window,on_button) glfwSetCursorPosCallback(world_window,on_pos) glfwSetScrollCallback(world_window,on_scroll) #set the last saved window size set_window_size(bar.window_size.value,bar.window_size) on_resize(world_window, *glfwGetWindowSize(world_window)) glfwSetWindowPos(world_window,0,0) # gl_state settings basic_gl_setup() g_pool.image_tex = create_named_texture(frame.img) # refresh speed settings glfwSwapInterval(0) #load calibration plugin open_calibration(bar.calibration_type.value,bar.calibration_type) #load gaze_display plugin g_pool.plugins.append(Display_Recent_Gaze(g_pool)) # Event loop while not g_pool.quit.value: # Get an image from the grabber try: frame = cap.get_frame() except CameraCaptureError: logger.error("Capture from Camera Failed. Stopping.") break except EndofVideoFileError: logger.warning("Video File is done. Stopping") break update_fps() #a container that allows plugins to post and read events events = [] #receive and map pupil positions recent_pupil_positions = [] while not g_pool.pupil_queue.empty(): p = g_pool.pupil_queue.get() if p['norm_pupil'] is None: p['norm_gaze'] = None else: p['norm_gaze'] = g_pool.map_pupil(p['norm_pupil']) recent_pupil_positions.append(p) # allow each Plugin to do its work. for p in g_pool.plugins: p.update(frame,recent_pupil_positions,events) #check if a plugin need to be destroyed g_pool.plugins = [p for p in g_pool.plugins if p.alive] # render camera image glfwMakeContextCurrent(world_window) make_coord_system_norm_based() if g_pool.update_textures.value: draw_named_texture(g_pool.image_tex,frame.img) else: draw_named_texture(g_pool.image_tex) make_coord_system_pixel_based(frame.img.shape) # render visual feedback from loaded plugins for p in g_pool.plugins: p.gl_display() atb.draw() glfwSwapBuffers(world_window) glfwPollEvents() # de-init all running plugins for p in g_pool.plugins: p.alive = False #reading p.alive actually runs plug-in cleanup _ = p.alive save('window_size',bar.window_size.value) save('calibration_type',bar.calibration_type.value) save('record_eye',bar.record_eye.value) session_settings.close() cap.close() atb.terminate() glfwDestroyWindow(world_window) glfwTerminate() logger.debug("Process done")
def main(): # Callback functions def on_resize(window, w, h): active_window = glfwGetCurrentContext() glfwMakeContextCurrent(window) adjust_gl_view(w, h, window) norm_size = normalize((w, h), glfwGetWindowSize(window)) fb_size = denormalize(norm_size, glfwGetFramebufferSize(window)) atb.TwWindowSize(*map(int, fb_size)) glfwMakeContextCurrent(active_window) for p in g.plugins: p.on_window_resize(window, w, h) def on_key(window, key, scancode, action, mods): if not atb.TwEventKeyboardGLFW(key, action): if action == GLFW_PRESS: pass def on_char(window, char): if not atb.TwEventCharGLFW(char, 1): pass def on_button(window, button, action, mods): if not atb.TwEventMouseButtonGLFW(button, action): pos = glfwGetCursorPos(window) pos = normalize(pos, glfwGetWindowSize(main_window)) pos = denormalize(pos, (frame.img.shape[1], frame.img.shape[0])) # Position in img pixels for p in g.plugins: p.on_click(pos, button, action) def on_pos(window, x, y): norm_pos = normalize((x, y), glfwGetWindowSize(window)) fb_x, fb_y = denormalize(norm_pos, glfwGetFramebufferSize(window)) if atb.TwMouseMotion(int(fb_x), int(fb_y)): pass def on_scroll(window, x, y): if not atb.TwMouseWheel(int(x)): pass def on_close(window): glfwSetWindowShouldClose(main_window, True) logger.debug('Process closing from window') try: rec_dir = sys.argv[1] except: #for dev, supply hardcoded dir: rec_dir = '/Users/mkassner/Desktop/Marker_Tracking_Demo_Recording/' if os.path.isdir(rec_dir): logger.debug("Dev option: Using hadcoded data dir.") else: if getattr(sys, 'frozen', False): logger.warning( "You did not supply a data directory when you called this script! \ \nPlease drag a Pupil recoding directory onto the launch icon." ) else: logger.warning( "You did not supply a data directory when you called this script! \ \nPlease supply a Pupil recoding directory as first arg when calling Pupil Player." ) return if not is_pupil_rec_dir(rec_dir): logger.error( "You did not supply a dir with the required files inside.") return #backwards compatibility fn. patch_meta_info(rec_dir) #parse and load data folder info video_path = rec_dir + "/world.avi" timestamps_path = rec_dir + "/timestamps.npy" gaze_positions_path = rec_dir + "/gaze_positions.npy" meta_info_path = rec_dir + "/info.csv" #parse info.csv file with open(meta_info_path) as info: meta_info = dict( ((line.strip().split('\t')) for line in info.readlines())) rec_version = meta_info["Capture Software Version"] rec_version_float = int( filter(type(rec_version).isdigit, rec_version)[:3]) / 100. #(get major,minor,fix of version) logger.debug("Recording version: %s , %s" % (rec_version, rec_version_float)) #load gaze information gaze_list = np.load(gaze_positions_path) timestamps = np.load(timestamps_path) #correlate data positions_by_frame = correlate_gaze(gaze_list, timestamps) # load session persistent settings session_settings = Persistent_Dict(os.path.join(user_dir, "user_settings")) def load(var_name, default): return session_settings.get(var_name, default) def save(var_name, var): session_settings[var_name] = var # Initialize capture cap = autoCreateCapture(video_path, timestamps=timestamps_path) if isinstance(cap, FakeCapture): logger.error("could not start capture.") return width, height = cap.get_size() # Initialize glfw glfwInit() main_window = glfwCreateWindow( width, height, "Pupil Player: " + meta_info["Recording Name"] + " - " + rec_dir.split(os.path.sep)[-1], None, None) glfwMakeContextCurrent(main_window) # Register callbacks main_window glfwSetWindowSizeCallback(main_window, on_resize) glfwSetWindowCloseCallback(main_window, on_close) glfwSetKeyCallback(main_window, on_key) glfwSetCharCallback(main_window, on_char) glfwSetMouseButtonCallback(main_window, on_button) glfwSetCursorPosCallback(main_window, on_pos) glfwSetScrollCallback(main_window, on_scroll) # create container for globally scoped varfs (within world) g = Temp() g.plugins = [] g.play = False g.new_seek = True g.user_dir = user_dir g.rec_dir = rec_dir g.app = 'player' g.timestamps = timestamps g.positions_by_frame = positions_by_frame # helpers called by the main atb bar def update_fps(): old_time, bar.timestamp = bar.timestamp, time() dt = bar.timestamp - old_time if dt: bar.fps.value += .1 * (1. / dt - bar.fps.value) def set_window_size(mode, data): width, height = cap.get_size() ratio = (1, .75, .5, .25)[mode] w, h = int(width * ratio), int(height * ratio) glfwSetWindowSize(main_window, w, h) data.value = mode # update the bar.value def get_from_data(data): """ helper for atb getter and setter use """ return data.value def get_play(): return g.play def set_play(value): g.play = value def next_frame(): try: cap.seek_to_frame(cap.get_frame_index()) except FileSeekError: pass g.new_seek = True def prev_frame(): try: cap.seek_to_frame(cap.get_frame_index() - 2) except FileSeekError: pass g.new_seek = True def open_plugin(selection, data): if plugin_by_index[selection] not in additive_plugins: for p in g.plugins: if isinstance(p, plugin_by_index[selection]): return g.plugins = [p for p in g.plugins if p.alive] logger.debug('Open Plugin: %s' % name_by_index[selection]) new_plugin = plugin_by_index[selection](g) g.plugins.append(new_plugin) g.plugins.sort(key=lambda p: p.order) if hasattr(new_plugin, 'init_gui'): new_plugin.init_gui() # save the value for atb bar data.value = selection def get_from_data(data): """ helper for atb getter and setter use """ return data.value atb.init() # add main controls ATB bar bar = atb.Bar(name="Controls", label="Controls", help="Scene controls", color=(50, 50, 50), alpha=100, valueswidth=150, text='light', position=(10, 10), refresh=.1, size=(300, 160)) bar.next_atb_pos = (10, 220) bar.fps = c_float(0.0) bar.timestamp = time() bar.window_size = c_int(load("window_size", 0)) window_size_enum = atb.enum("Display Size", { "Full": 0, "Medium": 1, "Half": 2, "Mini": 3 }) bar.version = create_string_buffer(version, 512) bar.recording_version = create_string_buffer(rec_version, 512) bar.add_var("fps", bar.fps, step=1., readonly=True) bar._fps = c_float(cap.get_fps()) bar.add_var("recoding fps", bar._fps, readonly=True) bar.add_var("display size", vtype=window_size_enum, setter=set_window_size, getter=get_from_data, data=bar.window_size) bar.add_var("play", vtype=c_bool, getter=get_play, setter=set_play, key="space") bar.add_button('step next', next_frame, key='right') bar.add_button('step prev', prev_frame, key='left') bar.add_var("frame index", vtype=c_int, getter=lambda: cap.get_frame_index() - 1) bar.plugin_to_load = c_int(0) plugin_type_enum = atb.enum("Plug In", index_by_name) bar.add_var("plugin", setter=open_plugin, getter=get_from_data, data=bar.plugin_to_load, vtype=plugin_type_enum) bar.add_var( "version of recording", bar.recording_version, readonly=True, help="version of the capture software used to make this recording") bar.add_var("version of player", bar.version, readonly=True, help="version of the Pupil Player") bar.add_button("exit", on_close, data=main_window, key="esc") #set the last saved window size set_window_size(bar.window_size.value, bar.window_size) on_resize(main_window, *glfwGetWindowSize(main_window)) glfwSetWindowPos(main_window, 0, 0) #we always load these plugins g.plugins.append( Export_Launcher(g, data_dir=rec_dir, frame_count=len(timestamps))) g.plugins.append(Seek_Bar(g, capture=cap)) g.trim_marks = Trim_Marks(g, capture=cap) g.plugins.append(g.trim_marks) #these are loaded based on user settings for initializer in load('plugins', []): name, args = initializer logger.debug("Loading plugin: %s with settings %s" % (name, args)) try: p = plugin_by_name[name](g, **args) g.plugins.append(p) except: logger.warning("Plugin '%s' failed to load from settings file." % name) if load('plugins', "_") == "_": #lets load some default if we dont have presets g.plugins.append(Scan_Path(g)) g.plugins.append(Vis_Polyline(g)) g.plugins.append(Vis_Circle(g)) # g.plugins.append(Vis_Light_Points(g)) #sort by exec order g.plugins.sort(key=lambda p: p.order) #init gui for p in g.plugins: if hasattr(p, 'init_gui'): p.init_gui() # gl_state settings basic_gl_setup() g.image_tex = create_named_texture((height, width, 3)) while not glfwWindowShouldClose(main_window): update_fps() #grab new frame if g.play or g.new_seek: try: new_frame = cap.get_frame() except EndofVideoFileError: #end of video logic: pause at last frame. g.play = False if g.new_seek: display_time = new_frame.timestamp g.new_seek = False frame = new_frame.copy() #new positons and events we make a deepcopy just like the image is a copy. current_pupil_positions = deepcopy(positions_by_frame[frame.index]) events = [] # allow each Plugin to do its work. for p in g.plugins: p.update(frame, current_pupil_positions, events) #check if a plugin need to be destroyed g.plugins = [p for p in g.plugins if p.alive] # render camera image glfwMakeContextCurrent(main_window) make_coord_system_norm_based() draw_named_texture(g.image_tex, frame.img) make_coord_system_pixel_based(frame.img.shape) # render visual feedback from loaded plugins for p in g.plugins: p.gl_display() #present frames at appropriate speed wait_time = frame.timestamp - display_time display_time = frame.timestamp try: spent_time = time() - timestamp sleep(wait_time - spent_time) except: pass timestamp = time() atb.draw() glfwSwapBuffers(main_window) glfwPollEvents() plugin_save = [] for p in g.plugins: try: p_initializer = p.get_class_name(), p.get_init_dict() plugin_save.append(p_initializer) except AttributeError: #not all plugins need to be savable, they will not have the init dict. # any object without a get_init_dict method will throw this exception. pass # de-init all running plugins for p in g.plugins: p.alive = False #reading p.alive actually runs plug-in cleanup _ = p.alive save('plugins', plugin_save) save('window_size', bar.window_size.value) session_settings.close() cap.close() bar.destroy() glfwDestroyWindow(main_window) glfwTerminate() logger.debug("Process done")
def eye(g_pool,cap_src,cap_size): """ Creates a window, gl context. Grabs images from a capture. Streams Pupil coordinates into g_pool.pupil_queue """ # modify the root logger for this process logger = logging.getLogger() # remove inherited handlers logger.handlers = [] # create file handler which logs even debug messages fh = logging.FileHandler(os.path.join(g_pool.user_dir,'eye.log'),mode='w') fh.setLevel(logging.DEBUG) # create console handler with a higher log level ch = logging.StreamHandler() ch.setLevel(logging.WARNING) # create formatter and add it to the handlers formatter = logging.Formatter('EYE Process: %(asctime)s - %(name)s - %(levelname)s - %(message)s') fh.setFormatter(formatter) formatter = logging.Formatter('E Y E Process [%(levelname)s] %(name)s : %(message)s') ch.setFormatter(formatter) # add the handlers to the logger logger.addHandler(fh) logger.addHandler(ch) # create logger for the context of this function logger = logging.getLogger(__name__) # Callback functions def on_resize(window,w, h): adjust_gl_view(w,h,window) norm_size = normalize((w,h),glfwGetWindowSize(window)) fb_size = denormalize(norm_size,glfwGetFramebufferSize(window)) atb.TwWindowSize(*map(int,fb_size)) def on_key(window, key, scancode, action, mods): if not atb.TwEventKeyboardGLFW(key,int(action == GLFW_PRESS)): if action == GLFW_PRESS: if key == GLFW_KEY_ESCAPE: on_close(window) def on_char(window,char): if not atb.TwEventCharGLFW(char,1): pass def on_button(window,button, action, mods): if not atb.TwEventMouseButtonGLFW(button,int(action == GLFW_PRESS)): if action == GLFW_PRESS: pos = glfwGetCursorPos(window) pos = normalize(pos,glfwGetWindowSize(window)) pos = denormalize(pos,(frame.img.shape[1],frame.img.shape[0]) ) # pos in frame.img pixels u_r.setStart(pos) bar.draw_roi.value = 1 else: bar.draw_roi.value = 0 def on_pos(window,x, y): norm_pos = normalize((x,y),glfwGetWindowSize(window)) fb_x,fb_y = denormalize(norm_pos,glfwGetFramebufferSize(window)) if atb.TwMouseMotion(int(fb_x),int(fb_y)): pass if bar.draw_roi.value == 1: pos = denormalize(norm_pos,(frame.img.shape[1],frame.img.shape[0]) ) # pos in frame.img pixels u_r.setEnd(pos) def on_scroll(window,x,y): if not atb.TwMouseWheel(int(x)): pass def on_close(window): g_pool.quit.value = True logger.info('Process closing from window') # Helper functions called by the main atb bar def start_roi(): bar.display.value = 1 bar.draw_roi.value = 2 def update_fps(): old_time, bar.timestamp = bar.timestamp, time() dt = bar.timestamp - old_time if dt: bar.fps.value += .05 * (1. / dt - bar.fps.value) bar.dt.value = dt def get_from_data(data): """ helper for atb getter and setter use """ return data.value # load session persistent settings session_settings = shelve.open(os.path.join(g_pool.user_dir,'user_settings_eye'),protocol=2) def load(var_name,default): return session_settings.get(var_name,default) def save(var_name,var): session_settings[var_name] = var # Initialize capture cap = autoCreateCapture(cap_src, cap_size,timebase=g_pool.timebase) if cap is None: logger.error("Did not receive valid Capture") return # check if it works frame = cap.get_frame() if frame.img is None: logger.error("Could not retrieve image from capture") cap.close() return height,width = frame.img.shape[:2] u_r = Roi(frame.img.shape) u_r.set(load('roi',default=None)) writer = None pupil_detector = Canny_Detector(g_pool) atb.init() # Create main ATB Controls bar = atb.Bar(name = "Eye", label="Display", help="Scene controls", color=(50, 50, 50), alpha=100, text='light', position=(10, 10),refresh=.3, size=(200, 100)) bar.fps = c_float(0.0) bar.timestamp = time() bar.dt = c_float(0.0) bar.sleep = c_float(0.0) bar.display = c_int(load('bar.display',0)) bar.draw_pupil = c_bool(load('bar.draw_pupil',True)) bar.draw_roi = c_int(0) dispay_mode_enum = atb.enum("Mode",{"Camera Image":0, "Region of Interest":1, "Algorithm":2, "CPU Save": 3}) bar.add_var("FPS",bar.fps, step=1.,readonly=True) bar.add_var("Mode", bar.display,vtype=dispay_mode_enum, help="select the view-mode") bar.add_var("Show_Pupil_Point", bar.draw_pupil) bar.add_button("Draw_ROI", start_roi, help="drag on screen to select a region of interest") bar.add_var("SlowDown",bar.sleep, step=0.01,min=0.0) bar.add_var("SaveSettings&Exit", g_pool.quit) cap.create_atb_bar(pos=(220,10)) # create a bar for the detector pupil_detector.create_atb_bar(pos=(10,120)) glfwInit() window = glfwCreateWindow(width, height, "Eye", None, None) glfwMakeContextCurrent(window) # Register callbacks window glfwSetWindowSizeCallback(window,on_resize) glfwSetWindowCloseCallback(window,on_close) glfwSetKeyCallback(window,on_key) glfwSetCharCallback(window,on_char) glfwSetMouseButtonCallback(window,on_button) glfwSetCursorPosCallback(window,on_pos) glfwSetScrollCallback(window,on_scroll) glfwSetWindowPos(window,800,0) on_resize(window,width,height) # gl_state settings basic_gl_setup() g_pool.image_tex = create_named_texture(frame.img) # refresh speed settings glfwSwapInterval(0) # event loop while not g_pool.quit.value: # Get an image from the grabber try: frame = cap.get_frame() except CameraCaptureError: logger.error("Capture from Camera Failed. Stopping.") break except EndofVideoFileError: logger.warning("Video File is done. Stopping") break update_fps() sleep(bar.sleep.value) # for debugging only ### RECORDING of Eye Video (on demand) ### # Setup variables and lists for recording if g_pool.eye_rx.poll(): command = g_pool.eye_rx.recv() if command is not None: record_path = command logger.info("Will save eye video to: %s"%record_path) video_path = os.path.join(record_path, "eye.avi") timestamps_path = os.path.join(record_path, "eye_timestamps.npy") writer = cv2.VideoWriter(video_path, cv2.cv.CV_FOURCC(*'DIVX'), bar.fps.value, (frame.img.shape[1], frame.img.shape[0])) timestamps = [] else: logger.info("Done recording eye.") writer = None np.save(timestamps_path,np.asarray(timestamps)) del timestamps if writer: writer.write(frame.img) timestamps.append(frame.timestamp) # pupil ellipse detection result = pupil_detector.detect(frame,user_roi=u_r,visualize=bar.display.value == 2) # stream the result g_pool.pupil_queue.put(result) # VISUALIZATION direct visualizations on the frame.img data if bar.display.value == 1: # and a solid (white) frame around the user defined ROI r_img = frame.img[u_r.lY:u_r.uY,u_r.lX:u_r.uX] r_img[:,0] = 255,255,255 r_img[:,-1]= 255,255,255 r_img[0,:] = 255,255,255 r_img[-1,:]= 255,255,255 # GL-drawing clear_gl_screen() make_coord_system_norm_based() if bar.display.value != 3: draw_named_texture(g_pool.image_tex,frame.img) else: draw_named_texture(g_pool.image_tex) make_coord_system_pixel_based(frame.img.shape) if result['norm_pupil'] is not None and bar.draw_pupil.value: if result.has_key('axes'): pts = cv2.ellipse2Poly( (int(result['center'][0]),int(result['center'][1])), (int(result["axes"][0]/2),int(result["axes"][1]/2)), int(result["angle"]),0,360,15) draw_gl_polyline(pts,(1.,0,0,.5)) draw_gl_point_norm(result['norm_pupil'],color=(1.,0.,0.,0.5)) atb.draw() glfwSwapBuffers(window) glfwPollEvents() # END while running # in case eye reconding was still runnnig: Save&close if writer: logger.info("Done recording eye.") writer = None np.save(timestamps_path,np.asarray(timestamps)) # save session persistent settings save('roi',u_r.get()) save('bar.display',bar.display.value) save('bar.draw_pupil',bar.draw_pupil.value) session_settings.close() pupil_detector.cleanup() cap.close() atb.terminate() glfwDestroyWindow(window) glfwTerminate() #flushing queue incase world process did not exit gracefully while not g_pool.pupil_queue.empty(): g_pool.pupil_queue.get() g_pool.pupil_queue.close() logger.debug("Process done")
def eye(g_pool, cap_src, cap_size): """ Creates a window, gl context. Grabs images from a capture. Streams Pupil coordinates into g_pool.pupil_queue """ # modify the root logger for this process logger = logging.getLogger() # remove inherited handlers logger.handlers = [] # create file handler which logs even debug messages fh = logging.FileHandler(os.path.join(g_pool.user_dir, 'eye.log'), mode='w') fh.setLevel(logging.DEBUG) # create console handler with a higher log level ch = logging.StreamHandler() ch.setLevel(logging.WARNING) # create formatter and add it to the handlers formatter = logging.Formatter( 'EYE Process: %(asctime)s - %(name)s - %(levelname)s - %(message)s') fh.setFormatter(formatter) formatter = logging.Formatter( 'E Y E Process [%(levelname)s] %(name)s : %(message)s') ch.setFormatter(formatter) # add the handlers to the logger logger.addHandler(fh) logger.addHandler(ch) # create logger for the context of this function logger = logging.getLogger(__name__) # Callback functions def on_resize(window, w, h): adjust_gl_view(w, h, window) norm_size = normalize((w, h), glfwGetWindowSize(window)) fb_size = denormalize(norm_size, glfwGetFramebufferSize(window)) atb.TwWindowSize(*map(int, fb_size)) def on_key(window, key, scancode, action, mods): if not atb.TwEventKeyboardGLFW(key, int(action == GLFW_PRESS)): if action == GLFW_PRESS: if key == GLFW_KEY_ESCAPE: on_close(window) def on_char(window, char): if not atb.TwEventCharGLFW(char, 1): pass def on_button(window, button, action, mods): if not atb.TwEventMouseButtonGLFW(button, int(action == GLFW_PRESS)): if action == GLFW_PRESS: pos = glfwGetCursorPos(window) pos = normalize(pos, glfwGetWindowSize(window)) pos = denormalize( pos, (frame.img.shape[1], frame.img.shape[0])) # pos in frame.img pixels u_r.setStart(pos) bar.draw_roi.value = 1 else: bar.draw_roi.value = 0 def on_pos(window, x, y): norm_pos = normalize((x, y), glfwGetWindowSize(window)) fb_x, fb_y = denormalize(norm_pos, glfwGetFramebufferSize(window)) if atb.TwMouseMotion(int(fb_x), int(fb_y)): pass if bar.draw_roi.value == 1: pos = denormalize(norm_pos, (frame.img.shape[1], frame.img.shape[0])) # pos in frame.img pixels u_r.setEnd(pos) def on_scroll(window, x, y): if not atb.TwMouseWheel(int(x)): pass def on_close(window): g_pool.quit.value = True logger.info('Process closing from window') # Helper functions called by the main atb bar def start_roi(): bar.display.value = 1 bar.draw_roi.value = 2 def update_fps(): old_time, bar.timestamp = bar.timestamp, time() dt = bar.timestamp - old_time if dt: bar.fps.value += .05 * (1. / dt - bar.fps.value) bar.dt.value = dt def get_from_data(data): """ helper for atb getter and setter use """ return data.value # load session persistent settings session_settings = shelve.open(os.path.join(g_pool.user_dir, 'user_settings_eye'), protocol=2) def load(var_name, default): return session_settings.get(var_name, default) def save(var_name, var): session_settings[var_name] = var # Initialize capture cap = autoCreateCapture(cap_src, cap_size, timebase=g_pool.timebase) if cap is None: logger.error("Did not receive valid Capture") return # check if it works frame = cap.get_frame() if frame.img is None: logger.error("Could not retrieve image from capture") cap.close() return height, width = frame.img.shape[:2] u_r = Roi(frame.img.shape) u_r.set(load('roi', default=None)) writer = None pupil_detector = Canny_Detector(g_pool) atb.init() # Create main ATB Controls bar = atb.Bar(name="Eye", label="Display", help="Scene controls", color=(50, 50, 50), alpha=100, text='light', position=(10, 10), refresh=.3, size=(200, 100)) bar.fps = c_float(0.0) bar.timestamp = time() bar.dt = c_float(0.0) bar.sleep = c_float(0.0) bar.display = c_int(load('bar.display', 0)) bar.draw_pupil = c_bool(load('bar.draw_pupil', True)) bar.draw_roi = c_int(0) dispay_mode_enum = atb.enum("Mode", { "Camera Image": 0, "Region of Interest": 1, "Algorithm": 2, "CPU Save": 3 }) bar.add_var("FPS", bar.fps, step=1., readonly=True) bar.add_var("Mode", bar.display, vtype=dispay_mode_enum, help="select the view-mode") bar.add_var("Show_Pupil_Point", bar.draw_pupil) bar.add_button("Draw_ROI", start_roi, help="drag on screen to select a region of interest") bar.add_var("SlowDown", bar.sleep, step=0.01, min=0.0) bar.add_var("SaveSettings&Exit", g_pool.quit) cap.create_atb_bar(pos=(220, 10)) # create a bar for the detector pupil_detector.create_atb_bar(pos=(10, 120)) glfwInit() window = glfwCreateWindow(width, height, "Eye", None, None) glfwMakeContextCurrent(window) # Register callbacks window glfwSetWindowSizeCallback(window, on_resize) glfwSetWindowCloseCallback(window, on_close) glfwSetKeyCallback(window, on_key) glfwSetCharCallback(window, on_char) glfwSetMouseButtonCallback(window, on_button) glfwSetCursorPosCallback(window, on_pos) glfwSetScrollCallback(window, on_scroll) glfwSetWindowPos(window, 800, 0) on_resize(window, width, height) # gl_state settings basic_gl_setup() g_pool.image_tex = create_named_texture(frame.img) # refresh speed settings glfwSwapInterval(0) # event loop while not g_pool.quit.value: # Get an image from the grabber try: frame = cap.get_frame() except CameraCaptureError: logger.error("Capture from Camera Failed. Stopping.") break except EndofVideoFileError: logger.warning("Video File is done. Stopping") break update_fps() sleep(bar.sleep.value) # for debugging only ### RECORDING of Eye Video (on demand) ### # Setup variables and lists for recording if g_pool.eye_rx.poll(): command = g_pool.eye_rx.recv() if command is not None: record_path = command logger.info("Will save eye video to: %s" % record_path) video_path = os.path.join(record_path, "eye.avi") timestamps_path = os.path.join(record_path, "eye_timestamps.npy") writer = cv2.VideoWriter( video_path, cv2.cv.CV_FOURCC(*'DIVX'), bar.fps.value, (frame.img.shape[1], frame.img.shape[0])) timestamps = [] else: logger.info("Done recording eye.") writer = None np.save(timestamps_path, np.asarray(timestamps)) del timestamps if writer: writer.write(frame.img) timestamps.append(frame.timestamp) # pupil ellipse detection result = pupil_detector.detect(frame, user_roi=u_r, visualize=bar.display.value == 2) # stream the result g_pool.pupil_queue.put(result) # VISUALIZATION direct visualizations on the frame.img data if bar.display.value == 1: # and a solid (white) frame around the user defined ROI r_img = frame.img[u_r.lY:u_r.uY, u_r.lX:u_r.uX] r_img[:, 0] = 255, 255, 255 r_img[:, -1] = 255, 255, 255 r_img[0, :] = 255, 255, 255 r_img[-1, :] = 255, 255, 255 # GL-drawing clear_gl_screen() make_coord_system_norm_based() if bar.display.value != 3: draw_named_texture(g_pool.image_tex, frame.img) else: draw_named_texture(g_pool.image_tex) make_coord_system_pixel_based(frame.img.shape) if result['norm_pupil'] is not None and bar.draw_pupil.value: if result.has_key('axes'): pts = cv2.ellipse2Poly( (int(result['center'][0]), int(result['center'][1])), (int(result["axes"][0] / 2), int(result["axes"][1] / 2)), int(result["angle"]), 0, 360, 15) draw_gl_polyline(pts, (1., 0, 0, .5)) draw_gl_point_norm(result['norm_pupil'], color=(1., 0., 0., 0.5)) atb.draw() glfwSwapBuffers(window) glfwPollEvents() # END while running # in case eye reconding was still runnnig: Save&close if writer: logger.info("Done recording eye.") writer = None np.save(timestamps_path, np.asarray(timestamps)) # save session persistent settings save('roi', u_r.get()) save('bar.display', bar.display.value) save('bar.draw_pupil', bar.draw_pupil.value) session_settings.close() pupil_detector.cleanup() cap.close() atb.terminate() glfwDestroyWindow(window) glfwTerminate() #flushing queue incase world process did not exit gracefully while not g_pool.pupil_queue.empty(): g_pool.pupil_queue.get() g_pool.pupil_queue.close() logger.debug("Process done")
def main(): # Callback functions def on_resize(window,w, h): active_window = glfwGetCurrentContext() glfwMakeContextCurrent(window) adjust_gl_view(w,h,window) norm_size = normalize((w,h),glfwGetWindowSize(window)) fb_size = denormalize(norm_size,glfwGetFramebufferSize(window)) atb.TwWindowSize(*map(int,fb_size)) glfwMakeContextCurrent(active_window) for p in g.plugins: p.on_window_resize(window,w,h) def on_key(window, key, scancode, action, mods): if not atb.TwEventKeyboardGLFW(key,action): if action == GLFW_PRESS: pass def on_char(window,char): if not atb.TwEventCharGLFW(char,1): pass def on_button(window,button, action, mods): if not atb.TwEventMouseButtonGLFW(button,action): pos = glfwGetCursorPos(window) pos = normalize(pos,glfwGetWindowSize(main_window)) pos = denormalize(pos,(frame.img.shape[1],frame.img.shape[0]) ) # Position in img pixels for p in g.plugins: p.on_click(pos,button,action) def on_pos(window,x, y): norm_pos = normalize((x,y),glfwGetWindowSize(window)) fb_x,fb_y = denormalize(norm_pos,glfwGetFramebufferSize(window)) if atb.TwMouseMotion(int(fb_x),int(fb_y)): pass def on_scroll(window,x,y): if not atb.TwMouseWheel(int(x)): pass def on_close(window): glfwSetWindowShouldClose(main_window,True) logger.debug('Process closing from window') try: rec_dir = sys.argv[1] except: #for dev, supply hardcoded dir: rec_dir = '/Users/mkassner/Desktop/Marker_Tracking_Demo_Recording/' if os.path.isdir(rec_dir): logger.debug("Dev option: Using hadcoded data dir.") else: if getattr(sys, 'frozen', False): logger.warning("You did not supply a data directory when you called this script! \ \nPlease drag a Pupil recoding directory onto the launch icon.") else: logger.warning("You did not supply a data directory when you called this script! \ \nPlease supply a Pupil recoding directory as first arg when calling Pupil Player.") return if not is_pupil_rec_dir(rec_dir): logger.error("You did not supply a dir with the required files inside.") return #backwards compatibility fn. patch_meta_info(rec_dir) #parse and load data folder info video_path = rec_dir + "/world.avi" timestamps_path = rec_dir + "/timestamps.npy" gaze_positions_path = rec_dir + "/gaze_positions.npy" meta_info_path = rec_dir + "/info.csv" #parse info.csv file with open(meta_info_path) as info: meta_info = dict( ((line.strip().split('\t')) for line in info.readlines() ) ) rec_version = meta_info["Capture Software Version"] rec_version_float = int(filter(type(rec_version).isdigit, rec_version)[:3])/100. #(get major,minor,fix of version) logger.debug("Recording version: %s , %s"%(rec_version,rec_version_float)) #load gaze information gaze_list = np.load(gaze_positions_path) timestamps = np.load(timestamps_path) #correlate data positions_by_frame = correlate_gaze(gaze_list,timestamps) # load session persistent settings session_settings = Persistent_Dict(os.path.join(user_dir,"user_settings")) def load(var_name,default): return session_settings.get(var_name,default) def save(var_name,var): session_settings[var_name] = var # Initialize capture cap = autoCreateCapture(video_path,timestamps=timestamps_path) if isinstance(cap,FakeCapture): logger.error("could not start capture.") return width,height = cap.get_size() # Initialize glfw glfwInit() main_window = glfwCreateWindow(width, height, "Pupil Player: "+meta_info["Recording Name"]+" - "+ rec_dir.split(os.path.sep)[-1], None, None) glfwMakeContextCurrent(main_window) # Register callbacks main_window glfwSetWindowSizeCallback(main_window,on_resize) glfwSetWindowCloseCallback(main_window,on_close) glfwSetKeyCallback(main_window,on_key) glfwSetCharCallback(main_window,on_char) glfwSetMouseButtonCallback(main_window,on_button) glfwSetCursorPosCallback(main_window,on_pos) glfwSetScrollCallback(main_window,on_scroll) # create container for globally scoped varfs (within world) g = Temp() g.plugins = [] g.play = False g.new_seek = True g.user_dir = user_dir g.rec_dir = rec_dir g.app = 'player' g.timestamps = timestamps g.positions_by_frame = positions_by_frame # helpers called by the main atb bar def update_fps(): old_time, bar.timestamp = bar.timestamp, time() dt = bar.timestamp - old_time if dt: bar.fps.value += .1 * (1. / dt - bar.fps.value) def set_window_size(mode,data): width,height = cap.get_size() ratio = (1,.75,.5,.25)[mode] w,h = int(width*ratio),int(height*ratio) glfwSetWindowSize(main_window,w,h) data.value=mode # update the bar.value def get_from_data(data): """ helper for atb getter and setter use """ return data.value def get_play(): return g.play def set_play(value): g.play = value def next_frame(): try: cap.seek_to_frame(cap.get_frame_index()) except FileSeekError: pass g.new_seek = True def prev_frame(): try: cap.seek_to_frame(cap.get_frame_index()-2) except FileSeekError: pass g.new_seek = True def open_plugin(selection,data): if plugin_by_index[selection] not in additive_plugins: for p in g.plugins: if isinstance(p,plugin_by_index[selection]): return g.plugins = [p for p in g.plugins if p.alive] logger.debug('Open Plugin: %s'%name_by_index[selection]) new_plugin = plugin_by_index[selection](g) g.plugins.append(new_plugin) g.plugins.sort(key=lambda p: p.order) if hasattr(new_plugin,'init_gui'): new_plugin.init_gui() # save the value for atb bar data.value=selection def get_from_data(data): """ helper for atb getter and setter use """ return data.value atb.init() # add main controls ATB bar bar = atb.Bar(name = "Controls", label="Controls", help="Scene controls", color=(50, 50, 50), alpha=100,valueswidth=150, text='light', position=(10, 10),refresh=.1, size=(300, 160)) bar.next_atb_pos = (10,220) bar.fps = c_float(0.0) bar.timestamp = time() bar.window_size = c_int(load("window_size",0)) window_size_enum = atb.enum("Display Size",{"Full":0, "Medium":1,"Half":2,"Mini":3}) bar.version = create_string_buffer(version,512) bar.recording_version = create_string_buffer(rec_version,512) bar.add_var("fps", bar.fps, step=1., readonly=True) bar._fps = c_float(cap.get_fps()) bar.add_var("recoding fps",bar._fps,readonly=True) bar.add_var("display size", vtype=window_size_enum,setter=set_window_size,getter=get_from_data,data=bar.window_size) bar.add_var("play",vtype=c_bool,getter=get_play,setter=set_play,key="space") bar.add_button('step next',next_frame,key='right') bar.add_button('step prev',prev_frame,key='left') bar.add_var("frame index",vtype=c_int,getter=lambda:cap.get_frame_index()-1 ) bar.plugin_to_load = c_int(0) plugin_type_enum = atb.enum("Plug In",index_by_name) bar.add_var("plugin",setter=open_plugin,getter=get_from_data,data=bar.plugin_to_load, vtype=plugin_type_enum) bar.add_var("version of recording",bar.recording_version, readonly=True, help="version of the capture software used to make this recording") bar.add_var("version of player",bar.version, readonly=True, help="version of the Pupil Player") bar.add_button("exit", on_close,data=main_window,key="esc") #set the last saved window size set_window_size(bar.window_size.value,bar.window_size) on_resize(main_window, *glfwGetWindowSize(main_window)) glfwSetWindowPos(main_window,0,0) #we always load these plugins g.plugins.append(Export_Launcher(g,data_dir=rec_dir,frame_count=len(timestamps))) g.plugins.append(Seek_Bar(g,capture=cap)) g.trim_marks = Trim_Marks(g,capture=cap) g.plugins.append(g.trim_marks) #these are loaded based on user settings for initializer in load('plugins',[]): name, args = initializer logger.debug("Loading plugin: %s with settings %s"%(name, args)) try: p = plugin_by_name[name](g,**args) g.plugins.append(p) except: logger.warning("Plugin '%s' failed to load from settings file." %name) if load('plugins',"_") == "_": #lets load some default if we dont have presets g.plugins.append(Scan_Path(g)) g.plugins.append(Vis_Polyline(g)) g.plugins.append(Vis_Circle(g)) # g.plugins.append(Vis_Light_Points(g)) #sort by exec order g.plugins.sort(key=lambda p: p.order) #init gui for p in g.plugins: if hasattr(p,'init_gui'): p.init_gui() # gl_state settings basic_gl_setup() g.image_tex = create_named_texture((height,width,3)) while not glfwWindowShouldClose(main_window): update_fps() #grab new frame if g.play or g.new_seek: try: new_frame = cap.get_frame() except EndofVideoFileError: #end of video logic: pause at last frame. g.play=False if g.new_seek: display_time = new_frame.timestamp g.new_seek = False frame = new_frame.copy() #new positons and events we make a deepcopy just like the image is a copy. current_pupil_positions = deepcopy(positions_by_frame[frame.index]) events = [] # allow each Plugin to do its work. for p in g.plugins: p.update(frame,current_pupil_positions,events) #check if a plugin need to be destroyed g.plugins = [p for p in g.plugins if p.alive] # render camera image glfwMakeContextCurrent(main_window) make_coord_system_norm_based() draw_named_texture(g.image_tex,frame.img) make_coord_system_pixel_based(frame.img.shape) # render visual feedback from loaded plugins for p in g.plugins: p.gl_display() #present frames at appropriate speed wait_time = frame.timestamp - display_time display_time = frame.timestamp try: spent_time = time()-timestamp sleep(wait_time-spent_time) except: pass timestamp = time() atb.draw() glfwSwapBuffers(main_window) glfwPollEvents() plugin_save = [] for p in g.plugins: try: p_initializer = p.get_class_name(),p.get_init_dict() plugin_save.append(p_initializer) except AttributeError: #not all plugins need to be savable, they will not have the init dict. # any object without a get_init_dict method will throw this exception. pass # de-init all running plugins for p in g.plugins: p.alive = False #reading p.alive actually runs plug-in cleanup _ = p.alive save('plugins',plugin_save) save('window_size',bar.window_size.value) session_settings.close() cap.close() bar.destroy() glfwDestroyWindow(main_window) glfwTerminate() logger.debug("Process done")