def save_settings(self): settings_eye0 = Persistent_Dict(os.path.join(self.rec_dir,'user_settings_eye0') ) settings_eye1 = Persistent_Dict(os.path.join(self.rec_dir,'user_settings_eye1') ) settings_roi = Persistent_Dict(os.path.join(self.rec_dir,'user_settings_roi') ) settings_eye0.update(self.current_settings_0) settings_eye1.update(self.current_settings_1) settings_roi['roi0'] = self.u_r[0].get() settings_roi['roi1'] = self.u_r[1].get() settings_eye0.close() settings_eye1.close() settings_roi.close()
class Vis_Eye_Video_Overlay(Plugin): """docstring This plugin allows the user to overlay the eye recording on the recording of his field of vision Features: flip video across horiz/vert axes, click and drag around interface, scale video size from 20% to 100%, show only 1 or 2 or both eyes features updated by Andrew June 2015 """ def __init__(self,g_pool,alpha=0.6,eye_scale_factor=.5,move_around=0,mirror={'0':False,'1':False}, flip={'0':False,'1':False},pos=[(640,10),(10,10)], show_ellipses=True): super().__init__(g_pool) self.order = .6 self.menu = None # user controls self.alpha = alpha #opacity level of eyes self.eye_scale_factor = eye_scale_factor #scale self.showeyes = 0,1 #modes: any text containg both means both eye is present, on 'only eye1' if only one eye recording self.move_around = move_around #boolean whether allow to move clip around screen or not self.video_size = [0,0] #video_size of recording (bc scaling) self.show_ellipses = show_ellipses self.detect_3D = 0 self.algorithm = 0 self.persistent_settings_0 = Persistent_Dict(os.path.join(g_pool.user_dir,'user_settings_eye0') ) self.persistent_settings_1 = Persistent_Dict(os.path.join(g_pool.user_dir,'user_settings_eye1') ) self.persistent_settings_roi = Persistent_Dict(os.path.join(g_pool.user_dir,'user_settings_roi') ) self.current_settings_0 = {} self.current_settings_1 = {} self.gPool0 = Global_Container() self.gPool1 = Global_Container() self.gPool0.pupil_queue = Queue() self.gPool1.pupil_queue = Queue() self.center = [0,0] self.mouse_img = [[0,0],[0,0]] self.g_pool = g_pool #below is a fckng mess self.pupil_size_min = 40 self.pupil_size_max = 150 self.intensity_range = 17 self.model_sensitivity = 0.997 self.ellipse_roundness_ratio = 0.1 self.coarse_filter_min = 150 self.coarse_filter_max = 300 self.initial_ellipse_fit_treshhold = 1.8 self.canny_treshold = 200 self.canny_ration = 3 self.pupil_size_min1 = 40 self.pupil_size_max1 = 150 self.intensity_range1 = 17 self.model_sensitivity1 = 0.997 self.ellipse_roundness_ratio1 = 0.1 self.coarse_filter_min1 = 150 self.coarse_filter_max1 = 300 self.initial_ellipse_fit_treshhold1 = 1.8 self.canny_treshold1 = 200 self.canny_ration1 = 3 uroi0 = self.persistent_settings_roi.get('roi0', False) uroi1 = self.persistent_settings_roi.get('roi1', False) self.u_r = [UIRoi((480, 640)), UIRoi((480, 640))] if uroi0 != False: self.u_r[0].lX = uroi0[0] self.u_r[0].lY = uroi0[1] self.u_r[0].uX = uroi0[2] self.u_r[0].uY = uroi0[3] if uroi1 != False: self.u_r[1].lX = uroi1[0] self.u_r[1].lY = uroi1[1] self.u_r[1].uX = uroi1[2] self.u_r[1].uY = uroi1[3] self.urActive = False self.rec_dir = g_pool.rec_dir #variables specific to each eye self.eye_frames = [] self.eye_world_frame_map = [] self.eye_cap = [] self.eye_cap_3D = [] self.mirror = mirror #do we horiz flip first eye self.flip = flip #do we vert flip first eye self.pos = [list(pos[0]),list(pos[1])] #positions of 2 eyes self.drag_offset = [None,None] pupil_detector_eye0 = Detector_2D(g_pool = self.gPool0) pupil_detector_eye1 = Detector_2D(g_pool = self.gPool1) pupil_detector_eye0_3D = Detector_3D(g_pool = self.gPool0) pupil_detector_eye1_3D = Detector_3D(g_pool = self.gPool1) self.pupil_detectors2D = [pupil_detector_eye0,pupil_detector_eye1] self.pupil_detectors3D = [pupil_detector_eye0_3D,pupil_detector_eye1_3D] self.pupil_detectors = self.pupil_detectors2D self.glint_settings = {} self.glint_dist = 3.0 self.glint_thres = 5 self.glint_min = 50 self.glint_max = 750 self.dilate = 0 self.erode = 0 self.glint_dist1 = 3.0 self.glint_thres1 = 5 self.glint_min1 = 50 self.glint_max1 = 750 self.dilate1 = 0 self.erode1 = 0 self.recalculating = 0 self.pupil_settings = 'default' self.msg = "" glint_detector0 = Glint_Detector(g_pool, self.glint_settings) glint_detector1 = Glint_Detector(g_pool, self.glint_settings) self.glint_detectors = [glint_detector0, glint_detector1] # load eye videos and eye timestamps if VersionFormat(self.g_pool.meta_info['Capture Software Version'][1:]) < VersionFormat('0.4'): eye_video_path = os.path.join(g_pool.rec_dir,'eye.avi'),'None' self.eye_timestamps_path = os.path.join(g_pool.rec_dir,'eye_timestamps.npy'),'None' else: eye_video_path = os.path.join(g_pool.rec_dir,'eye0.*'),os.path.join(g_pool.rec_dir,'eye1.*') self.eye_timestamps_path = os.path.join(g_pool.rec_dir,'eye0_timestamps.npy'),os.path.join(g_pool.rec_dir,'eye1_timestamps.npy') #try to load eye video and ts for each eye. self.eye_ts = [] for video,ts in zip(eye_video_path,self.eye_timestamps_path): try: self.eye_cap.append(File_Source(self.g_pool,source_path=glob(video)[0],timestamps=np.load(ts))) self.eye_cap_3D.append(File_Source(self.g_pool,source_path=glob(video)[0],timestamps=np.load(ts))) except(IndexError,FileCaptureError): pass else: self.eye_frames.append(self.eye_cap[-1].get_frame()) try: eye_timestamps = list(np.load(ts)) except: pass else: self.eye_world_frame_map.append(correlate_eye_world(eye_timestamps,g_pool.timestamps)) if len(self.eye_cap) == 2: logger.debug("Loaded binocular eye video data.") elif len(self.eye_cap) == 1: logger.debug("Loaded monocular eye video data") self.showeyes = (0,) else: logger.error("Could not load eye video.") self.alive = False return self.setKeys(self.persistent_settings_0, 0) self.setKeys(self.persistent_settings_1, 1) def unset_alive(self): self.alive = False def setKeys(self, pupil_settings_new, eye = 0): keys = list(self.__dict__) for key in keys: try: if eye == 0: self.__dict__[key] = pupil_settings_new[key] else: self.__dict__[key + "1"] = pupil_settings_new[key] logger.info("set key with the name '%s'" %key) except: pass #logger.info("no key with the name '%s' in settings" %key) def init_gui(self): # initialize the menu self.menu = ui.Scrolling_Menu('Eye Video Overlay') self.update_gui() self.g_pool.gui.append(self.menu) def set_pupil_settings(self, new_settings): self.pupil_settings = new_settings if not new_settings == "default": try: path = os.path.join(self.g_pool.user_dir,'pupil_settings_' + new_settings + '.json') with open(path, 'r') as fp: json_str = fp.read() pupil_settings_new = json.loads(json_str) except: logger.error("Settings don't exist") self.setKeys(pupil_settings_new, 0) self.setKeys(pupil_settings_new, 1) def save_settings(self): settings_eye0 = Persistent_Dict(os.path.join(self.rec_dir,'user_settings_eye0') ) settings_eye1 = Persistent_Dict(os.path.join(self.rec_dir,'user_settings_eye1') ) settings_roi = Persistent_Dict(os.path.join(self.rec_dir,'user_settings_roi') ) settings_eye0.update(self.current_settings_0) settings_eye1.update(self.current_settings_1) settings_roi['roi0'] = self.u_r[0].get() settings_roi['roi1'] = self.u_r[1].get() settings_eye0.close() settings_eye1.close() settings_roi.close() def update_gui(self): self.menu.elements[:] = [] self.menu.append(ui.Button('Close',self.unset_alive)) self.menu.append(ui.Switch('detect_3D',self,label="3D detection")) self.menu.append(ui.Switch('algorithm',self,label="Algorithm view")) self.menu.append(ui.Button("Save settings", self.save_settings)) self.menu.append(ui.Switch('show_ellipses', self, label="Visualize Recorded Gaze")) self.menu.append(ui.Selector('pupil_settings',self,setter=self.set_pupil_settings,selection=['default','indoors','outdoors_sunny', 'outdoors_cloudy', 'vanilla'], labels=['Default', 'Indoors', 'Outdoors Sunny', 'Outdoors Cloudy', 'Vanilla'], label="Pupil settings") ) pupil0_menu = ui.Growing_Menu('Pupil0') pupil0_menu.collapsed = True pupil0_menu.append(ui.Slider('pupil_size_min',self,min=0,step=1,max=250,label='Pupil min size')) pupil0_menu.append(ui.Slider('pupil_size_max' ,self,min=0,step=1,max=400,label='Pupil max size')) pupil0_menu.append(ui.Slider('intensity_range',self,min=0,step=1,max=60,label='Pupil intensity range')) pupil0_menu.append(ui.Slider('model_sensitivity',self,min=0.0,step=0.0001,max=1.0,label='Model sensitivity')) pupil0_menu[-1].display_format = '%0.4f' pupil0_menu.append(ui.Slider('ellipse_roundness_ratio',self,min=0.01,step=0.01,max=1.0,label='ellipse_roundness_ratio')) pupil0_menu.append(ui.Slider('coarse_filter_min',self,min=10,step=1,max=500,label='coarse_filter_min')) pupil0_menu.append(ui.Slider('coarse_filter_max',self,min=100,step=1,max=1000,label='coarse_filter_max')) pupil0_menu.append(ui.Slider('canny_treshold',self,min=50,step=1,max=500,label='canny_treshold')) pupil0_menu.append(ui.Slider('canny_ration',self,min=1,step=1,max=20,label='canny_ration')) pupil0_menu.append(ui.Button('Reset 3D model', self.reset_3D_Model_eye0 )) self.menu.append(pupil0_menu) pupil1_menu = ui.Growing_Menu('Pupil1') pupil1_menu.collapsed = True pupil1_menu.append(ui.Slider('pupil_size_min1',self,min=0,step=1,max=250,label='Pupil min size')) pupil1_menu.append(ui.Slider('pupil_size_max1' ,self,min=0,step=1,max=400,label='Pupil max size')) pupil1_menu.append(ui.Slider('intensity_range1',self,min=0,step=1,max=60,label='Pupil intensity range')) pupil1_menu.append(ui.Slider('model_sensitivity1',self,min=0.0, step=0.0001, max=1.0, label='Model sensitivity')) pupil1_menu[-1].display_format = '%0.4f' pupil1_menu.append(ui.Slider('ellipse_roundness_ratio1',self,min=0.01,step=0.01,max=1.0,label='ellipse_roundness_ratio')) pupil1_menu.append(ui.Slider('coarse_filter_min1',self,min=10,step=1,max=500,label='coarse_filter_min')) pupil1_menu.append(ui.Slider('coarse_filter_max1',self,min=100,step=1,max=1000,label='coarse_filter_max')) pupil1_menu.append(ui.Slider('canny_treshold1',self,min=50,step=1,max=500,label='canny_treshold')) pupil1_menu.append(ui.Slider('canny_ration1',self,min=1,step=1,max=20,label='canny_ration')) pupil1_menu.append(ui.Button('Reset 3D model', self.reset_3D_Model_eye1 )) self.menu.append(pupil1_menu) glint_menu = ui.Growing_Menu('Glint0') glint_menu.collapsed = True glint_menu.append(ui.Slider('glint_dist',self,label='Distance from pupil',min=0,max=5,step=0.25)) glint_menu.append(ui.Slider('glint_thres',self,label='Intensity offset',min=0,max=255,step=1)) glint_menu.append(ui.Slider('glint_min',self,label='Min size',min=1,max=250,step=1)) glint_menu.append(ui.Slider('glint_max',self,label='Max size',min=50,max=1000,step=5)) glint_menu.append(ui.Slider('dilate',self,label='Dilate',min=0,max=2,step=1)) self.menu.append(glint_menu) glint_menu1 = ui.Growing_Menu('Glint1') glint_menu1.collapsed = True glint_menu1.append(ui.Slider('glint_dist1',self,label='Distance from pupil',min=0,max=5,step=0.25)) glint_menu1.append(ui.Slider('glint_thres1',self,label='Intensity offset',min=0,max=255,step=1)) glint_menu1.append(ui.Slider('glint_min1',self,label='Min size',min=1,max=250,step=1)) glint_menu1.append(ui.Slider('glint_max1',self,label='Max size',min=50,max=1000,step=5)) glint_menu1.append(ui.Slider('dilate1',self,label='Dilate',min=0,max=2,step=1)) self.menu.append(glint_menu1) self.menu.append(ui.Button("Recalculate pupils", self.recalculate)) self.menu.append(ui.Info_Text('Show the eye video overlaid on top of the world video. Eye1 is usually the right eye')) self.menu.append(ui.Slider('alpha',self,min=0.0,step=0.05,max=1.0,label='Opacity')) self.menu.append(ui.Slider('eye_scale_factor',self,min=0.2,step=0.1,max=1.0,label='Video Scale')) self.menu.append(ui.Switch('move_around',self,label="Move Overlay")) if len(self.eye_cap) == 2: self.menu.append(ui.Selector('showeyes',self,label='Show',selection=[(0,),(1,),(0,1)],labels= ['Eye 0','Eye 1','both'],setter=self.set_showeyes)) if 0 in self.showeyes: self.menu.append(ui.Switch('0',self.mirror,label="Eye 0: Horiz. Flip")) self.menu.append(ui.Switch('0',self.flip,label="Eye 0: Vert. Flip")) if 1 in self.showeyes: self.menu.append(ui.Switch('1',self.mirror,label="Eye 1: Horiz Flip")) self.menu.append(ui.Switch('1',self.flip,label="Eye 1: Vert Flip")) def reset_3D_Model_eye0(self): self.pupil_detectors3D[0].reset_3D_Model() def reset_3D_Model_eye1(self): self.pupil_detectors3D[1].reset_3D_Model() def setPupilDetectors(self): if self.detect_3D == 1: self.pupil_detectors = self.pupil_detectors3D else: self.pupil_detectors = self.pupil_detectors2D def setSettings(self, eye_index, settings, glint_settings): if eye_index == 0: settings["intensity_range"] = self.intensity_range settings["pupil_size_min"] = self.pupil_size_min settings["pupil_size_max"] = self.pupil_size_max settings['model_sensitivity'] = self.model_sensitivity settings["ellipse_roundness_ratio"] = self.ellipse_roundness_ratio settings["coarse_filter_min"] = self.coarse_filter_min settings["coarse_filter_max"] = self.coarse_filter_max settings["canny_treshold"] = self.canny_treshold settings["canny_ration"] = self.canny_ration if self.detect_3D: settings_copy = dict(settings) del settings_copy['2D_Settings'] del settings_copy['3D_Settings'] settings['2D_Settings'].update(settings_copy) settings['3D_Settings'].update(settings_copy) glint_settings['glint_dist'] = self.glint_dist glint_settings['glint_thres'] = self.glint_thres glint_settings['glint_min'] = self.glint_min glint_settings['glint_max'] = self.glint_max glint_settings['dilate'] = self.dilate glint_settings['erode'] = self.erode self.current_settings_0.update(settings) if eye_index == 1: settings["intensity_range"] = self.intensity_range1 settings["pupil_size_min"] = self.pupil_size_min1 settings["pupil_size_max"] = self.pupil_size_max1 settings['model_sensitivity'] = self.model_sensitivity1 settings["ellipse_roundness_ratio"] = self.ellipse_roundness_ratio1 settings["coarse_filter_min"] = self.coarse_filter_min1 settings["coarse_filter_max"] = self.coarse_filter_max1 settings["canny_treshold"] = self.canny_treshold1 settings["canny_ration"] = self.canny_ration1 if self.detect_3D: settings_copy = dict(settings) del settings_copy['2D_Settings'] del settings_copy['3D_Settings'] settings['2D_Settings'].update(settings_copy) settings['3D_Settings'].update(settings_copy) glint_settings['glint_dist'] = self.glint_dist1 glint_settings['glint_thres'] = self.glint_thres1 glint_settings['glint_min'] = self.glint_min1 glint_settings['glint_max'] = self.glint_max1 glint_settings['dilate'] = self.dilate1 glint_settings['erode'] = self.erode1 self.current_settings_1.update(settings) def calculate_pupil(self,eye_index, ts): self.recalculating += 1 pupil_detector = self.pupil_detectors[eye_index] glint_detector = self.glint_detectors[eye_index] settings = pupil_detector.get_settings() glint_settings = glint_detector.settings() self.setSettings(eye_index, settings, glint_settings) glint_detector.update() self.gPool0.pupil_queue = Queue() self.gPool1.pupil_queue = Queue() using3D = self.detect_3D model_birth_timestamp = -1 calibTime = float("inf") updating = False inPast = False present = 0 lastCalibTime = float("inf") lastCalibFrame = float("inf") try: fp = os.path.join(self.rec_dir,"/../calibTimes.npy") logger.info(fp) calibTimes = np.load(fp).tolist() calibTime = calibTimes.pop(0) logger.info(calibTimes) except: calibTimes = [0] calibTime = calibTimes.pop(0) logger.info("No calibration times") pass timestamps = np.load(ts) data = {'pupil_positions':[]} if not using3D: eye_cap = self.eye_cap[eye_index] else: eye_cap = self.eye_cap_3D[eye_index] eye_cap.seek_to_frame(0) i = 0 while True: if i == timestamps.size: break if i % 1000 == 0: logger.info("eye %d: %d frames processed, 3D: %s" % (eye_index, i, bool(using3D))) t = timestamps[i] image = eye_cap.get_frame() result,roi = pupil_detector.detect(image, self.u_r[eye_index], False) glints = [[0,0,0,0,0,0], [0,0,0,0,0,1]] #glint_detector.glint(image, eye_index, u_roi=self.u_r, pupil=result, roi=roi) result['glints'] = glints result['id'] = eye_index result['timestamp'] = t result['updating'] = updating data['pupil_positions'].append(result) center = result['ellipse']['center'] center = [int(x) for x in center] if eye_index == 0: self.gPool0.pupil_queue.put(result) else: self.gPool1.pupil_queue.put(result) if using3D and model_birth_timestamp != result['model_birth_timestamp']: model_birth_timestamp = result['model_birth_timestamp'] lastCalibTime = result['model_birth_timestamp'] if not updating: updating = True lastCalibFrame = i logger.info("eye %d: 3D Model update at frame %d" % (eye_index, i)) if inPast and t >= present: inPast = False if using3D and (t > calibTime or (result['projected_sphere']['axes'][0] < 300 and t > lastCalibTime + 20)) and not inPast: pupil_detector.reset_3D_Model() if len(calibTimes) > 0: calibTime = calibTimes.pop(0) else: calibTime = float("inf") logger.info("eye %d: 3D Model manual reset at frame %d" % (eye_index, i)) if t > (lastCalibTime + 30) or (i == timestamps.size -1 and updating == True): i = lastCalibFrame eye_cap.seek_to_frame(i) updating = False inPast = True present = t lastCalibTime = float("inf") lastCalibFrame = float("inf") logger.info("eye %d: returning to the past" % eye_index) else: i += 1 settings = pupil_detector.get_settings() settings['roi'] = self.u_r[eye_index].get() data['pupil_detector_settings'] = settings if using3D: save_object(data,os.path.join(self.rec_dir,"recalculated_pupil_3D_" + str(eye_index))) else: save_object(data,os.path.join(self.rec_dir,"recalculated_pupil_" + str(eye_index))) logger.info("eye %d finished" % eye_index) self.recalculating -= 1 #self.threads[eye_index].join() def recalculate(self): if self.recalculating > 0: logger.warning("Already recalculating") else: self.setPupilDetectors() self.threads = [[],[]] for eye_index, ts in zip(self.showeyes, self.eye_timestamps_path): self.threads[eye_index] = _thread.start_new_thread(self.calculate_pupil, (eye_index, ts) ) def set_showeyes(self,new_mode): #everytime we choose eye setting (either use eye 1, 2, or both, updates the gui menu to remove certain options from list) self.showeyes = new_mode self.update_gui() def deinit_gui(self): if self.menu: self.g_pool.gui.remove(self.menu) self.menu = None def update(self, frame, events): for eye_index in self.showeyes: requested_eye_frame_idx = self.eye_world_frame_map[eye_index][frame.index] #1. do we need a new frame? if (requested_eye_frame_idx != self.eye_frames[eye_index].index or self.urActive) and self.recalculating == 0: # do we need to seek? if requested_eye_frame_idx == self.eye_cap[eye_index].get_frame_index()+1: # if we just need to seek by one frame, its faster to just read one and and throw it away. _ = self.eye_cap[eye_index].get_frame() if requested_eye_frame_idx != self.eye_cap[eye_index].get_frame_index(): # only now do I need to seek self.eye_cap[eye_index].seek_to_frame(requested_eye_frame_idx) # reading the new eye frame frame try: self.eye_frames[eye_index] = self.eye_cap[eye_index].get_frame() except EndofVideoFileError: logger.warning("Reached the end of the eye video for eye video {}.".format(eye_index)) else: #our old frame is still valid because we are doing upsampling pass #2. dragging image if self.drag_offset[eye_index] is not None: pos = glfwGetCursorPos(glfwGetCurrentContext()) pos = normalize(pos,glfwGetWindowSize(glfwGetCurrentContext())) pos = denormalize(pos,(frame.img.shape[1],frame.img.shape[0]) ) # Position in img pixels self.pos[eye_index][0] = pos[0]+self.drag_offset[eye_index][0] self.pos[eye_index][1] = pos[1]+self.drag_offset[eye_index][1] else: self.video_size = [round(self.eye_frames[eye_index].width*self.eye_scale_factor), round(self.eye_frames[eye_index].height*self.eye_scale_factor)] pos = glfwGetCursorPos(glfwGetCurrentContext()) pos = normalize(pos,glfwGetWindowSize(glfwGetCurrentContext())) pos = denormalize(pos,(frame.img.shape[1],frame.img.shape[0]) ) pos = (int(pos[0] - self.pos[eye_index][0]), int(pos[1] - self.pos[eye_index][1])) self.mouse_img[eye_index] = pos if self.urActive == eye_index + 1: self.u_r[eye_index].move_vertex(self.u_r[eye_index].active_pt_idx,pos) if self.recalculating == 0: self.setPupilDetectors() pupil_detector = self.pupil_detectors[eye_index] glint_detector = self.glint_detectors[eye_index] settings = pupil_detector.get_settings() glint_settings = glint_detector.settings() self.setSettings(eye_index, settings, glint_settings) glint_detector.update() new_frame = self.eye_frames[eye_index] if self.algorithm == 1: view = "algorithm" else: view = False result, roi = pupil_detector.detect(new_frame, self.u_r[eye_index], view) glints = [[0,0,0,0,0,0], [0,0,0,0,0,1]] #glint_detector.glint(new_frame, eye_index, u_roi=self.u_r, pupil=result, roi=roi) if eye_index == 0: self.gPool0.pupil_queue.put(result) else: self.gPool1.pupil_queue.put(result) #3. keep in image bounds, do this even when not dragging because the image video_sizes could change. #self.pos[eye_index][1] = min(frame.img.shape[0]-self.video_size[1],max(self.pos[eye_index][1],0)) #frame.img.shape[0] is height, frame.img.shape[1] is width of screen #self.pos[eye_index][0] = min(frame.img.shape[1]-self.video_size[0],max(self.pos[eye_index][0],0)) #4. flipping images, converting to greyscale #eye_gray = cv2.cvtColor(self.eye_frames[eye_index].img,cv2.COLOR_BGR2GRAY) #auto gray scaling pts = cv2.ellipse2Poly( (int(result['ellipse']['center'][0]),int(result['ellipse']['center'][1])), (int(result['ellipse']['axes'][0]/2),int(result['ellipse']['axes'][1]/2)), int(result['ellipse']['angle']),0,360,15) cv2.polylines(self.eye_frames[eye_index].img, [pts], 1, (0,255,0)) center = result['ellipse']['center'] center = [int(x) for x in center] cv2.circle(self.eye_frames[eye_index].img, tuple(center), True, (0,255,0), thickness=15) #img = 255 - np.mean(new_frame.img, axis=2) #img = (img*255).astype(np.uint8) #ret = pyelse.run(img, 10) #x = ret.center.x #y = ret.center.y #cv2.circle(self.eye_frames[eye_index].img, (int(x), int(y)), True, (255,0,0), thickness=10) glints = np.array(glints) #if len(glints)>0 and glints[0][3]: # for g in glints: # cv2.circle(self.eye_frames[eye_index].img, (int(g[1]),int(g[2])), True,(255,0,0),thickness=5) if result['method'] == '3d c++': eye_ball = result['projected_sphere'] try: pts = cv2.ellipse2Poly( (int(eye_ball['center'][0]),int(eye_ball['center'][1])), (int(eye_ball['axes'][0]/2),int(eye_ball['axes'][1]/2)), int(eye_ball['angle']),0,360,8) except ValueError as e: pass else: cv2.polylines(self.eye_frames[eye_index].img, [pts], 1, (255,0,0)) if self.show_ellipses and events['pupil_positions'] and self.recalculating == 0: for pd in events['pupil_positions']: if pd['id'] == eye_index and pd['timestamp'] == self.eye_frames[eye_index].timestamp: break if pd['method'] == '3d c++': eye_ball = pd['projected_sphere'] try: pts = cv2.ellipse2Poly( (int(eye_ball['center'][0]),int(eye_ball['center'][1])), (int(eye_ball['axes'][0]/2),int(eye_ball['axes'][1]/2)), int(eye_ball['angle']),0,360,8) except ValueError as e: pass else: cv2.polylines(self.eye_frames[eye_index].img, [pts], 1, (0,255,0)) el = pd['ellipse'] conf = int(pd.get('model_confidence', pd.get('confidence', 0.1)) * 255) center = list(map(lambda val: int(val), el['center'])) el['axes'] = tuple(map(lambda val: int(val/2), el['axes'])) el['angle'] = int(el['angle']) el_points = cv2.ellipse2Poly(tuple(center), el['axes'], el['angle'], 0, 360, 1) cv2.polylines(self.eye_frames[eye_index].img, [np.asarray(el_points)], True, (0, 0, 255, conf), thickness=math.ceil(2)) cv2.circle(self.eye_frames[eye_index].img, tuple(center), int(5*self.eye_scale_factor), (0, 0, 255, conf), thickness=-1) rect = np.asarray(self.u_r[eye_index].rect) #rect[:,[0, 1]] = rect[:,[1, 0]] cv2.polylines(self.eye_frames[eye_index].img, [rect],True, (0, 255, 255), thickness=math.ceil(2)) for corner in rect: cv2.circle(self.eye_frames[eye_index].img, (int(corner[0]), int(corner[1])), int(5*self.eye_scale_factor), (0, 255, 255), thickness=-1) #3. keep in image bounds, do this even when not dragging because the image video_sizes could change. self.pos[eye_index][1] = min(frame.img.shape[0]-self.video_size[1],max(self.pos[eye_index][1],0)) #frame.img.shape[0] is height, frame.img.shape[1] is width of screen self.pos[eye_index][0] = min(frame.img.shape[1]-self.video_size[0],max(self.pos[eye_index][0],0)) #4. flipping images, converting to greyscale #eye_gray = cv2.cvtColor(self.eye_frames[eye_index].img,cv2.COLOR_BGR2GRAY) #auto gray scaling eyeimage = cv2.resize(self.eye_frames[eye_index].img,(0,0),fx=self.eye_scale_factor, fy=self.eye_scale_factor) if self.mirror[str(eye_index)]: eyeimage = np.fliplr(eyeimage) if self.flip[str(eye_index)]: eyeimage = np.flipud(eyeimage) #eyeimage = cv2.cvtColor(eyeimage, cv2.COLOR_GRAY2BGR) # 5. finally overlay the image x, y = int(self.pos[eye_index][0]), int(self.pos[eye_index][1]) transparent_image_overlay((x, y), eyeimage, frame.img, self.alpha) def on_click(self,pos,button,action): if action == 1: if self.urActive == False: for eye_index in self.showeyes: if self.pos[eye_index][0] < pos[0] < self.pos[eye_index][0]+self.video_size[0] and self.pos[eye_index][1] < pos[1] < self.pos[eye_index][1] + self.video_size[1]: img_pos = self.mouse_img[eye_index] if self.u_r[eye_index].mouse_over_edit_pt(img_pos, self.u_r[eye_index].handle_size + 20,self.u_r[eye_index].handle_size + 20): self.urActive = eye_index + 1 else: self.urActive = False if self.move_around == 1 and action == 1: for eye_index in self.showeyes: if self.pos[eye_index][0] < pos[0] < self.pos[eye_index][0]+self.video_size[0] and self.pos[eye_index][1] < pos[1] < self.pos[eye_index][1] + self.video_size[1]: self.drag_offset[eye_index] = self.pos[eye_index][0]-pos[0],self.pos[eye_index][1]-pos[1] return else: self.drag_offset = [None,None] def get_init_dict(self): return {'alpha':self.alpha,'eye_scale_factor':self.eye_scale_factor,'move_around':self.move_around,'mirror':self.mirror,'flip':self.flip,'pos':self.pos,'move_around':self.move_around, 'show_ellipses': self.show_ellipses} def cleanup(self): """ called when the plugin gets terminated. This happens either voluntarily or forced. if you have a GUI or glfw window destroy it here. """ self.persistent_settings_0.update(self.current_settings_0) self.persistent_settings_1.update(self.current_settings_1) self.persistent_settings_roi['roi0'] = self.u_r[0].get() self.persistent_settings_roi['roi1'] = self.u_r[1].get() self.persistent_settings_0.close() self.persistent_settings_1.close() self.persistent_settings_roi.close() self.deinit_gui()