def world(g_pool): """world """ # Callback functions def on_resize(w, h): atb.TwWindowSize(w, h); adjust_gl_view(w,h) def on_key(key, pressed): if not atb.TwEventKeyboardGLFW(key,pressed): if pressed: if key == GLFW_KEY_ESC: on_close() def on_char(char, pressed): if not atb.TwEventCharGLFW(char,pressed): pass def on_button(button, pressed): if not atb.TwEventMouseButtonGLFW(button,pressed): if pressed: pos = glfwGetMousePos() pos = normalize(pos,glfwGetWindowSize()) pos = denormalize(pos,(frame.img.shape[1],frame.img.shape[0]) ) # Position in img pixels for p in g.plugins: p.on_click(pos) def on_pos(x, y): if atb.TwMouseMotion(x,y): pass def on_scroll(pos): if not atb.TwMouseWheel(pos): pass def on_close(): g_pool.quit.value = True print "WORLD Process closing from window" # load session persistent settings session_settings = shelve.open('user_settings_world',protocol=2) def load(var_name,default): try: return session_settings[var_name] except: return default def save(var_name,var): session_settings[var_name] = var # gaze object gaze = Temp() gaze.map_coords = (0., 0.) gaze.image_coords = (0., 0.) # Initialize capture, check if it works cap = autoCreateCapture(g_pool.world_src, g_pool.world_size,24) if cap is None: print "WORLD: Error could not create Capture" return frame = cap.get_frame() if frame.img is None: print "WORLD: Error could not get image" return height,width = frame.img.shape[:2] # helpers called by the main atb bar def update_fps(): old_time, bar.timestamp = bar.timestamp, time() dt = bar.timestamp - old_time if dt: bar.fps.value += .05 * (1 / dt - bar.fps.value) def set_window_size(mode,data): height,width = frame.img.shape[:2] ratio = (1,.75,.5,.25)[mode] w,h = int(width*ratio),int(height*ratio) glfwSetWindowSize(w,h) data.value=mode # update the bar.value def get_from_data(data): """ helper for atb getter and setter use """ return data.value def open_calibration(selection,data): # prepare destruction of old ref_detector. if g.current_ref_detector: g.current_ref_detector.alive = False # remove old ref detector from list of plugins g.plugins = [p for p in g.plugins if p.alive] print "selected: ",reference_detectors.name_by_index[selection] g.current_ref_detector = reference_detectors.detector_by_index[selection](global_calibrate=g_pool.calibrate, shared_pos=g_pool.ref, screen_marker_pos = g_pool.marker, screen_marker_state = g_pool.marker_state, atb_pos=bar.next_atb_pos) g.plugins.append(g.current_ref_detector) # save the value for atb bar data.value=selection def toggle_record_video(): if any([True for p in g.plugins if isinstance(p,recorder.Recorder)]): for p in g.plugins: if isinstance(p,recorder.Recorder): p.alive = False else: # set up folder within recordings named by user input in atb if not bar.rec_name.value: bar.rec_name.value = recorder.get_auto_name() recorder_instance = recorder.Recorder(bar.rec_name.value, bar.fps.value, frame.img.shape, g_pool.pos_record, g_pool.eye_tx) g.plugins.append(recorder_instance) def toggle_show_calib_result(): if any([True for p in g.plugins if isinstance(p,Show_Calibration)]): for p in g.plugins: if isinstance(p,Show_Calibration): p.alive = False else: calib = Show_Calibration(frame.img.shape) g.plugins.append(calib) def show_calib_result(): # kill old if any if any([True for p in g.plugins if isinstance(p,Show_Calibration)]): for p in g.plugins: if isinstance(p,Show_Calibration): p.alive = False g.plugins = [p for p in g.plugins if p.alive] # make new calib = Show_Calibration(frame.img.shape) g.plugins.append(calib) def hide_calib_result(): if any([True for p in g.plugins if isinstance(p,Show_Calibration)]): for p in g.plugins: if isinstance(p,Show_Calibration): p.alive = False # Initialize ant tweak bar - inherits from atb.Bar atb.init() bar = atb.Bar(name = "World", label="Controls", help="Scene controls", color=(50, 50, 50), alpha=100,valueswidth=150, text='light', position=(10, 10),refresh=.3, size=(300, 200)) bar.next_atb_pos = (10,220) bar.fps = c_float(0.0) bar.timestamp = time() bar.calibration_type = c_int(load("calibration_type",0)) bar.show_calib_result = c_bool(0) bar.record_video = c_bool(0) bar.record_running = c_bool(0) bar.play = g_pool.play bar.window_size = c_int(load("window_size",0)) window_size_enum = atb.enum("Display Size",{"Full":0, "Medium":1,"Half":2,"Mini":3}) bar.calibrate_type_enum = atb.enum("Calibration Method",reference_detectors.index_by_name) bar.rec_name = create_string_buffer(512) bar.rec_name.value = recorder.get_auto_name() # play and record can be tied together via pointers to the objects # bar.play = bar.record_video bar.add_var("fps", bar.fps, step=1., readonly=True) bar.add_var("display size", vtype=window_size_enum,setter=set_window_size,getter=get_from_data,data=bar.window_size) bar.add_var("calibration method",setter=open_calibration,getter=get_from_data,data=bar.calibration_type, vtype=bar.calibrate_type_enum,group="Calibration", help="Please choose your desired calibration method.") bar.add_button("show calibration result",toggle_show_calib_result, group="Calibration", help="Click to show calibration result.") bar.add_var("session name",bar.rec_name, group="Recording", help="creates folder Data_Name_XXX, where xxx is an increasing number") bar.add_button("record", toggle_record_video, key="r", group="Recording", help="Start/Stop Recording") bar.add_separator("Sep1") bar.add_var("play video", bar.play, help="play a video in the Player window") bar.add_var("exit", g_pool.quit) # add uvc camera controls to a seperate ATB bar cap.create_atb_bar(pos=(320,10)) # create container for globally scoped vars (within world) g = Temp() g.plugins = [] g.current_ref_detector = None open_calibration(bar.calibration_type.value,bar.calibration_type) # Initialize glfw glfwInit() height,width = frame.img.shape[:2] glfwOpenWindow(width, height, 0, 0, 0, 8, 0, 0, GLFW_WINDOW) glfwSetWindowTitle("World") glfwSetWindowPos(0,0) #set the last saved window size set_window_size(bar.window_size.value,bar.window_size) # Register callbacks glfwSetWindowSizeCallback(on_resize) glfwSetWindowCloseCallback(on_close) glfwSetKeyCallback(on_key) glfwSetCharCallback(on_char) glfwSetMouseButtonCallback(on_button) glfwSetMousePosCallback(on_pos) glfwSetMouseWheelCallback(on_scroll) # gl_state settings import OpenGL.GL as gl gl.glEnable(gl.GL_POINT_SMOOTH) gl.glBlendFunc(gl.GL_SRC_ALPHA, gl.GL_ONE_MINUS_SRC_ALPHA) gl.glEnable(gl.GL_BLEND) del gl # Event loop while glfwGetWindowParam(GLFW_OPENED) and not g_pool.quit.value: # Get input characters entered in player if g_pool.player_input.value: player_input = g_pool.player_input.value g_pool.player_input.value = 0 on_char(player_input,True) # Get an image from the grabber frame = cap.get_frame() update_fps() for p in g.plugins: p.update(frame) g.plugins = [p for p in g.plugins if p.alive] g_pool.player_refresh.set() # render the screen clear_gl_screen() draw_gl_texture(frame.img) # render visual feedback from loaded plugins for p in g.plugins: p.gl_display() # update gaze point from shared variable pool and draw on screen. If both coords are 0: no pupil pos was detected. if not g_pool.gaze[:] == [0.,0.]: draw_gl_point_norm(g_pool.gaze[:],color=(1.,0.,0.,0.5)) atb.draw() glfwSwapBuffers() # end while running and clean-up # de-init all running plugins for p in g.plugins: p.alive = False g.plugins = [p for p in g.plugins if p.alive] save('window_size',bar.window_size.value) save('calibration_type',bar.calibration_type.value) session_settings.close() cap.close() glfwCloseWindow() glfwTerminate() print "WORLD Process closed"
def world(g_pool): """world """ ###Callback funtions def on_resize(w, h): atb.TwWindowSize(w, h); adjust_gl_view(w,h) def on_key(key, pressed): if not atb.TwEventKeyboardGLFW(key,pressed): if pressed: if key == GLFW_KEY_ESC: on_close() def on_char(char, pressed): if not atb.TwEventCharGLFW(char,pressed): pass def on_button(button, pressed): if not atb.TwEventMouseButtonGLFW(button,pressed): if pressed: pos = glfwGetMousePos() pos = normalize(pos,glfwGetWindowSize()) pos = denormalize(pos,(img.shape[1],img.shape[0]) ) #pos in img pixels ref.detector.new_ref(pos) def on_pos(x, y): if atb.TwMouseMotion(x,y): pass def on_scroll(pos): if not atb.TwMouseWheel(pos): pass def on_close(): g_pool.quit.value = True print "WORLD Process closing from window" ref = Temp() ref.detector = no_Detector(g_pool.calibrate,g_pool.ref_x,g_pool.ref_y) ###objects as variable containers # pattern object pattern = Temp() pattern.centers = None pattern.obj_grid = gen_pattern_grid((4, 11)) # calib grid pattern.obj_points = [] pattern.img_points = [] pattern.map = (0, 2, 7, 16, 21, 23, 39, 40, 42) pattern.board_centers = None # gaze object gaze = Temp() gaze.map_coords = (0., 0.) gaze.image_coords = (0., 0.) # record object record = Temp() record.writer = None record.path_parent = os.path.dirname(os.path.abspath(sys.argv[0])) record.path = None record.counter = 0 # initialize capture, check if it works cap = autoCreateCapture(g_pool.world_src, g_pool.world_size) if cap is None: print "WORLD: Error could not create Capture" return s, img = cap.read_RGB() if not s: print "WORLD: Error could not get image" return height,width = img.shape[:2] ###helpers called by the main atb bar def update_fps(): old_time, bar.timestamp = bar.timestamp, time() dt = bar.timestamp - old_time if dt: bar.fps.value += .05 * (1 / dt - bar.fps.value) def set_window_size(mode,data): height,width = img.shape[:2] ratio = (1,.75,.5,.25)[mode] w,h = int(width*ratio),int(height*ratio) glfwSetWindowSize(w,h) data.value=mode #update the bar.value def get_from_data(data): """ helper for atb getter and setter use """ return data.value def start_calibration(): c_type = bar.calibration_type.value if c_type == cal_type["Directed 9-Point"]: print 'WORLD: Starting Directed 9-Point calibration.' ref.detector = Nine_Point_Detector(global_calibrate=g_pool.calibrate, shared_x=g_pool.ref_x, shared_y=g_pool.ref_y, shared_stage=g_pool.cal9_stage, shared_step=g_pool.cal9_step, shared_cal9_active=g_pool.cal9, shared_circle_id=g_pool.cal9_circle_id, auto_advance=False) elif c_type == cal_type["Automated 9-Point"]: print 'WORLD: Starting Automated 9-Point calibration.' ref.detector = Nine_Point_Detector(global_calibrate=g_pool.calibrate, shared_x=g_pool.ref_x, shared_y=g_pool.ref_y, shared_stage=g_pool.cal9_stage, shared_step=g_pool.cal9_step, shared_cal9_active=g_pool.cal9, shared_circle_id=g_pool.cal9_circle_id, auto_advance=True) elif c_type == cal_type["Natural Features"]: print 'WORLD: Starting Natural Features calibration.' ref.detector = Natural_Features_Detector(global_calibrate=g_pool.calibrate, shared_x=g_pool.ref_x, shared_y=g_pool.ref_y) elif c_type == cal_type["Black Dot"]: print 'WORLD: Starting Black Dot calibration.' ref.detector = Black_Dot_Detector(global_calibrate=g_pool.calibrate, shared_x=g_pool.ref_x, shared_y=g_pool.ref_y) def advance_calibration(): ref.detector.advance() def stop_calibration(): ref.detector = no_Detector(global_calibrate=g_pool.calibrate, shared_x=g_pool.ref_x, shared_y=g_pool.ref_y) ### Initialize ant tweak bar inherits from atb.Bar atb.init() bar = atb.Bar(name = "World", label="Controls", help="Scene controls", color=(50, 50, 50), alpha=100, text='light', position=(10, 10),refresh=.3, size=(200, 200)) bar.fps = c_float(0.0) bar.timestamp = time() bar.calibration_type = c_int(1) bar.show_calib_result = c_bool(0) bar.calibration_images = False bar.record_video = c_bool(0) bar.record_running = c_bool(0) bar.play = g_pool.play bar.window_size = c_int(0) window_size_enum = atb.enum("Display Size",{"Full":0, "Medium":1,"Half":2,"Mini":3}) cal_type = {"Directed 9-Point":0,"Automated 9-Point":1,"Natural Features":3,"Black Dot":4}#"Manual 9-Point":2 calibrate_type_enum = atb.enum("Calibration Method",cal_type) bar.rec_name = create_string_buffer(512) # play and record can be tied together via pointers to the objects # bar.play = bar.record_video bar.add_var("FPS", bar.fps, step=1., readonly=True) bar.add_var("Display_Size", vtype=window_size_enum,setter=set_window_size,getter=get_from_data,data=bar.window_size) bar.add_var("Cal/Calibration_Method",bar.calibration_type, vtype=calibrate_type_enum) bar.add_button("Cal/Start_Calibration",start_calibration, key='c') bar.add_button("Cal/Next_Point",advance_calibration,key="SPACE", help="Hit space to calibrate on next dot") bar.add_button("Cal/Stop_Calibration",stop_calibration, key='d') bar.add_var("Cal/show_calibration_result",bar.show_calib_result, help="yellow lines indecate fit error, red outline shows the calibrated area.") bar.add_var("Rec/rec_name",bar.rec_name) bar.add_var("Rec/Record_Video", bar.record_video, key="r", help="Start/Stop Recording") bar.add_separator("Sep1") bar.add_var("Play Source Video", bar.play) bar.add_var("Exit", g_pool.quit) #add 4vl2 camera controls to a seperate ATB bar if cap.controls is not None: c_bar = atb.Bar(name="Camera_Controls", label=cap.name, help="UVC Camera Controls", color=(50,50,50), alpha=100, text='light',position=(220, 10),refresh=2., size=(200, 200)) # c_bar.add_var("auto_refresher",vtype=atb.TW_TYPE_BOOL8,getter=cap.uvc_refresh_all,setter=None,readonly=True) # c_bar.define(definition='visible=0', varname="auto_refresher") sorted_controls = [c for c in cap.controls.itervalues()] sorted_controls.sort(key=lambda c: c.order) for control in sorted_controls: name = control.atb_name if control.type=="bool": c_bar.add_var(name,vtype=atb.TW_TYPE_BOOL8,getter=control.get_val,setter=control.set_val) elif control.type=='int': c_bar.add_var(name,vtype=atb.TW_TYPE_INT32,getter=control.get_val,setter=control.set_val) c_bar.define(definition='min='+str(control.min), varname=name) c_bar.define(definition='max='+str(control.max), varname=name) c_bar.define(definition='step='+str(control.step), varname=name) elif control.type=="menu": if control.menu is None: vtype = None else: vtype= atb.enum(name,control.menu) c_bar.add_var(name,vtype=vtype,getter=control.get_val,setter=control.set_val) if control.menu is None: c_bar.define(definition='min='+str(control.min), varname=name) c_bar.define(definition='max='+str(control.max), varname=name) c_bar.define(definition='step='+str(control.step), varname=name) else: pass if control.flags == "inactive": pass # c_bar.define(definition='readonly=1',varname=control.name) c_bar.add_button("refresh",cap.update_from_device) c_bar.add_button("load defaults",cap.load_defaults) else: c_bar = None ### Initialize glfw glfwInit() height,width = img.shape[:2] glfwOpenWindow(width, height, 0, 0, 0, 8, 0, 0, GLFW_WINDOW) glfwSetWindowTitle("World") glfwSetWindowPos(0,0) #register callbacks glfwSetWindowSizeCallback(on_resize) glfwSetWindowCloseCallback(on_close) glfwSetKeyCallback(on_key) glfwSetCharCallback(on_char) glfwSetMouseButtonCallback(on_button) glfwSetMousePosCallback(on_pos) glfwSetMouseWheelCallback(on_scroll) #gl_state settings import OpenGL.GL as gl gl.glEnable(gl.GL_POINT_SMOOTH) gl.glPointSize(20) gl.glBlendFunc(gl.GL_SRC_ALPHA, gl.GL_ONE_MINUS_SRC_ALPHA) gl.glEnable(gl.GL_BLEND) ###event loop while glfwGetWindowParam(GLFW_OPENED) and not g_pool.quit.value: update_fps() # get an image from the grabber s, img = cap.read() ref.detector.detect(img) if ref.detector.is_done(): stop_calibration() g_pool.player_refresh.set() # #gather pattern centers and find cam intrisics # if bar.screen_shot and pattern.centers is not None: # bar.screen_shot = False # # calibrate the camera intrinsics if the board is found # # append list of circle grid center points to pattern.img_points # # append generic list of circle grid pattern type to pattern.obj_points # pattern.centers = circle_grid(img) # pattern.img_points.append(pattern.centers) # pattern.obj_points.append(pattern.obj_grid) # print "Number of Patterns Captured:", len(pattern.img_points) # #if pattern.img_points.shape[0] > 10: # if len(pattern.img_points) > 10: # camera_matrix, dist_coefs = calibrate_camera(np.asarray(pattern.img_points), # np.asarray(pattern.obj_points), # (img.shape[1], img.shape[0])) # np.save("camera_matrix.npy", camera_matrix) # np.save("dist_coefs.npy", dist_coefs) # pattern.img_points = [] # bar.find_pattern.value = False ### Setup recording process if bar.record_video and not bar.record_running: record.path = os.path.join(record.path_parent, "data%03d/" % record.counter) while True: try: os.mkdir(record.path) break except: print "We dont want to overwrite data, incrementing counter & trying to make new data folder" record.counter += 1 record.path = os.path.join(record.path_parent, "data%03d/" % record.counter) #video video_path = os.path.join(record.path, "world.avi") #FFV1 -- good speed lossless big file #DIVX -- good speed good compression medium file record.writer = cv2.VideoWriter(video_path, cv2.cv.CV_FOURCC(*'DIVX'), bar.fps.value, (img.shape[1], img.shape[0])) # positions data to eye process g_pool.pos_record.value = True g_pool.eye_tx.send(record.path) bar.record_running = 1 g_pool.frame_count_record.value = 0 # While Recording... if bar.record_video and bar.record_running: # Save image frames to video writer # increment the frame_count_record value # Eye positions can be associated with frames of recording even if different framerates are used record.writer.write(img) g_pool.frame_count_record.value += 1 # Finish all recordings, clean up. if not bar.record_video and bar.record_running: # for conviniece: copy camera intrinsics into each data folder at the end of a recording. try: camera_matrix = np.load("camera_matrix.npy") dist_coefs = np.load("dist_coefs.npy") cam_path = os.path.join(record.path, "camera_matrix.npy") dist_path = os.path.join(record.path, "dist_coefs.npy") np.save(cam_path, camera_matrix) np.save(dist_path, dist_coefs) except: print "no camera intrinsics found, will not copy them into data folder" g_pool.pos_record.value = 0 del record.writer bar.record_running = 0 ###render the screen clear_gl_screen() cv2.cvtColor(img, cv2.COLOR_BGR2RGB,img) draw_gl_texture(img) ###render calibration results: if bar.show_calib_result.value: cal_pt_cloud = np.load("cal_pt_cloud.npy") pX,pY,wX,wY = cal_pt_cloud.transpose() map_fn = get_map_from_cloud(cal_pt_cloud,(width,height)) modelled_world_pts = map_fn((pX,pY)) pts = np.array(modelled_world_pts,dtype=np.float32).transpose() calib_bounds = cv2.convexHull(pts)[:,0] for observed,modelled in zip(zip(wX,wY),np.array(modelled_world_pts).transpose()): draw_gl_polyline_norm((modelled,observed),(1.,0.5,0.,.5)) draw_gl_polyline_norm(calib_bounds,(1.0,0,0,.5)) #render visual feedback from detector ref.detector.display(img) # render detector point if ref.detector.pos[0] or ref.detector.pos[1]: draw_gl_point_norm(ref.detector.pos,(0.,1.,0.,0.5)) # update gaze point from shared variable pool and draw on screen. If both coords are 0: no pupil pos was detected. if g_pool.gaze_x.value !=0 or g_pool.gaze_y.value !=0: draw_gl_point_norm((g_pool.gaze_x.value, g_pool.gaze_y.value),(1.,0.,0.,0.5)) atb.draw() glfwSwapBuffers() ###end while running clean-up print "WORLD Process closed" glfwCloseWindow() glfwTerminate()