def __init__(self,g_pool,annotations=None): if annotations: super().__init__(g_pool,annotations) else: super().__init__(g_pool) from player_methods import correlate_data self.frame_count = len(self.g_pool.timestamps) #display layout self.padding = 20. #in sceen pixel self.window_size = 0,0 #first we try to load annoations previously saved with pupil player try: annotations_list = load_object(os.path.join(self.g_pool.rec_dir, "annotations")) except IOError as e: #if that fails we assume this is the first time this recording is played and we load annotations from pupil_data try: notifications_list = load_object(os.path.join(self.g_pool.rec_dir, "pupil_data"))['notifications'] annotations_list = [n for n in notifications_list if n['subject']=='annotation'] except (KeyError,IOError) as e: annotations_list = [] logger.debug('No annotations found in pupil_data file.') else: logger.debug('loaded {} annotations from pupil_data file'.format(len(annotations_list))) else: logger.debug('loaded {} annotations from annotations file'.format(len(annotations_list))) self.annotations_by_frame = correlate_data(annotations_list, self.g_pool.timestamps) self.annotations_list = annotations_list
def stop(self): logger.info("Stopping Touchup") self.smooth_pos = 0.,0. self.sample_site = -2,-2 self.counter = 0 self.active = False self.button.status_text = '' offset_pt_clound = calibrate.preprocess_2d_data_monocular(calibrate.closest_matches_monocular(self.ref_list,self.gaze_list) ) if len(offset_pt_clound)<3: logger.error('Did not sample enough data for touchup please retry.') return #Calulate the offset for gaze to target offset_pt_clound = np.array(offset_pt_clound) offset = offset_pt_clound[:,:2]-offset_pt_clound[:,2:] mean_offset = np.mean(offset,axis=0) user_calibration = load_object(os.path.join(self.g_pool.user_dir, "user_calibration_data")) self.pupil_list = user_calibration['pupil_list'] self.ref_list = user_calibration['ref_list'] calibration_method = user_calibration['calibration_method'] if '3d' in calibration_method: logger.error('adjust calibration is not supported for 3d calibration.') return for r in self.ref_list: r['norm_pos'] = [ r['norm_pos'][0]-mean_offset[0],r['norm_pos'][1]-mean_offset[1] ] finish_calibration(self.g_pool,self.pupil_list,self.ref_list)
def load_camera_calibration(g_pool): if g_pool.app == 'capture': try: camera_calibration = load_object(os.path.join(g_pool.user_dir,'camera_calibration')) except: camera_calibration = None else: same_name = camera_calibration['camera_name'] == g_pool.capture.name same_resolution = camera_calibration['resolution'] == g_pool.capture.frame_size if not (same_name and same_resolution): logger.warning('Loaded camera calibration but camera name and/or resolution has changed.') camera_calibration = None else: logger.info("Loaded user calibrated calibration for %s@%s."%(g_pool.capture.name,g_pool.capture.frame_size)) if not camera_calibration: logger.debug("Trying to load pre recorded calibration.") try: camera_calibration = pre_recorded_calibrations[g_pool.capture.name][g_pool.capture.frame_size] except KeyError: logger.info("Pre recorded calibration for %s@%s not found."%(g_pool.capture.name,g_pool.capture.frame_size)) else: logger.info("Loaded pre recorded calibration for %s@%s."%(g_pool.capture.name,g_pool.capture.frame_size)) if not camera_calibration: logger.warning("Camera calibration not found please run Camera_Intrinsics_Estimation or Chessboard_Calibration to calibrate camera.") return camera_calibration else: raise NotImplementedError()
def compare_test_py(): global test_file_Folder test_file_Folder += 'py/' # Iterate every frame frameNumber = 0 while True: # Get an image from the grabber try: frame = cap.get_frame() frameNumber += 1 except CameraCaptureError: print "Capture from Camera Failed. Stopping." break except EndofVideoFileError: print "Video File is done." break # send to detector result = detector_py.detect(frame,user_roi=u_r,visualize=False) #load corresponding test files reference_result = load_object( test_file_Folder + 'result_frame_py{}'.format(frameNumber)) compare_dict(reference_result, result ) print "Frame {}".format(frameNumber) print "Finished compare test py."
def __init__(self, g_pool,manual_ref_edit_mode=False): super().__init__(g_pool) self.manual_ref_edit_mode = manual_ref_edit_mode self.menu = None self.process_pipe = None self.result_dir = os.path.join(g_pool.rec_dir, 'offline_data') os.makedirs(self.result_dir, exist_ok=True) try: session_data = load_object(os.path.join(self.result_dir, 'offline_calibration_gaze')) if session_data['version'] != self.session_data_version: logger.warning("Session data from old version. Will not use this.") assert False except Exception as e: map_range = [0, len(self.g_pool.timestamps)] calib_range = [len(self.g_pool.timestamps)//10, len(self.g_pool.timestamps)//2] session_data = {} session_data['sections'] = [make_section_dict(calib_range, map_range), ] session_data['circle_marker_positions'] = [] session_data['manual_ref_positions'] = [] self.sections = session_data['sections'] self.circle_marker_positions = session_data['circle_marker_positions'] self.manual_ref_positions = session_data['manual_ref_positions'] if self.circle_marker_positions: self.detection_progress = 100.0 for s in self.sections: self.calibrate_section(s) self.correlate_and_publish() else: self.detection_progress = 0.0 self.start_detection_task()
def __init__( self, g_pool, source_path=None, frame_size=None, frame_rate=None, name="Fake Source", *args, **kwargs ): super().__init__(g_pool, *args, **kwargs) if self.timing == "external": self.recent_events = self.recent_events_external_timing else: self.recent_events = self.recent_events_own_timing if source_path: meta = load_object(source_path) frame_size = meta["frame_size"] frame_rate = meta["frame_rate"] self.timestamps = np.load( os.path.splitext(source_path)[0] + "_timestamps.npy" ) else: self.timestamps = None self.fps = frame_rate self._name = name self.make_img(tuple(frame_size)) self.source_path = source_path self.current_frame_idx = 0 self.target_frame_idx = 0
def update_recording_v086_to_v087(rec_dir): logger.info("Updating recording from v0.8.6 format to v0.8.7 format") pupil_data = load_object(os.path.join(rec_dir, "pupil_data")) meta_info_path = os.path.join(rec_dir, "info.csv") def _clamp_norm_point(pos): """realisitic numbers for norm pos should be in this range. Grossly bigger or smaller numbers are results bad exrapolation and can cause overflow erorr when denormalized and cast as int32. """ return min(100, max(-100, pos[0])), min(100, max(-100, pos[1])) for g in pupil_data["gaze_positions"]: if "topic" not in g: # we missed this in one gaze mapper g["topic"] = "gaze" g["norm_pos"] = _clamp_norm_point(g["norm_pos"]) save_object(pupil_data, os.path.join(rec_dir, "pupil_data")) with open(meta_info_path, "r", encoding="utf-8") as csvfile: meta_info = csv_utils.read_key_value_file(csvfile) meta_info["Capture Software Version"] = "v0.8.7" with open(meta_info_path, "w", newline="") as csvfile: csv_utils.write_key_value_file(csvfile, meta_info)
def set_pupil_settings(new_settings): g_pool.pupil_settings = new_settings if not new_settings == "default": try: pupil_settings_new = load_object(os.path.join(g_pool.user_dir,'pupil_settings_' + new_settings)) except: logger.error("Settings don't exist") pupil_settings = g_pool.pupil_detector.get_settings() controls = cap.capture.controls controls_dict = dict([(c.display_name,c) for c in controls]) try: cap.frame_rate = pupil_settings_new['frame_rate'] cap.frame_size = pupil_settings_new['frame_size'] except: logger.info("no frame rate and frame size in camera settings") for key in controls_dict: try: controls_dict[key].value = pupil_settings_new[key] except: logger.info("no key with the name '%s' in camera settings" %key) for key in pupil_settings.keys(): try: pupil_settings[key] = pupil_settings_new[key] except: logger.info("no key with the name '%s' in pupil settings" %key)
def update_recording_bytes_to_unicode(rec_dir): logger.info("Updating recording from bytes to unicode.") def convert(data): if isinstance(data, bytes): return data.decode() elif isinstance(data, str) or isinstance(data, np.ndarray): return data elif isinstance(data, collections.Mapping): return dict(map(convert, data.items())) elif isinstance(data, collections.Iterable): return type(data)(map(convert, data)) else: return data for file in os.listdir(rec_dir): if file.startswith('.') or os.path.splitext(file)[1] in ('.mp4', '.avi'): continue rec_file = os.path.join(rec_dir, file) try: rec_object = load_object(rec_file) converted_object = convert(rec_object) if converted_object != rec_object: logger.info('Converted `{}` from bytes to unicode'.format(file)) save_object(converted_object, rec_file) except (UnpicklingError, IsADirectoryError): continue # manually convert k v dicts. meta_info_path = os.path.join(rec_dir, "info.csv") with open(meta_info_path, 'r', encoding='utf-8') as csvfile: meta_info = csv_utils.read_key_value_file(csvfile) with open(meta_info_path, 'w', newline='') as csvfile: csv_utils.write_key_value_file(csvfile, meta_info)
def __init__(self, g_pool): super().__init__(g_pool) zmq_ctx = zmq.Context() self.data_sub = zmq_tools.Msg_Receiver(zmq_ctx, g_pool.ipc_sub_url, topics=('pupil',)) self.data_dir = os.path.join(g_pool.rec_dir, 'offline_data') os.makedirs(self.data_dir , exist_ok=True) try: session_data = load_object(os.path.join(self.data_dir , 'offline_pupil_data')) assert session_data.get('version') != self.session_data_version except: session_data = {} session_data["detection_method"]='3d' session_data['pupil_positions'] = [] session_data['detection_progress'] = [0.,0.] session_data['detection_status'] = ["unknown","unknown"] self.detection_method = session_data["detection_method"] self.pupil_positions = session_data['pupil_positions'] self.eye_processes = [None, None] self.detection_progress = session_data['detection_progress'] self.detection_status = session_data['detection_status'] self.menu = None # start processes if self.detection_progress[0] < 100: self.start_eye_process(0) if self.detection_progress[1] < 100: self.start_eye_process(1) # either we did not start them or they failed to start (mono setup etc) # either way we are done and can publish if self.eye_processes == [None, None]: self.correlate_publish()
def update_recording_v086_to_v087(rec_dir): logger.info("Updating recording from v0.8.6 format to v0.8.7 format") pupil_data = load_object(os.path.join(rec_dir, "pupil_data")) meta_info_path = os.path.join(rec_dir,"info.csv") def _clamp_norm_point(pos): '''realisitic numbers for norm pos should be in this range. Grossly bigger or smaller numbers are results bad exrapolation and can cause overflow erorr when denormalized and cast as int32. ''' return min(100.,max(-100.,pos[0])),min(100.,max(-100.,pos[1])) for g in pupil_data.get('gaze_positions', []): if 'topic' not in g: # we missed this in one gaze mapper g['topic'] = 'gaze' g['norm_pos'] = _clamp_norm_point(g['norm_pos']) save_object(pupil_data,os.path.join(rec_dir, "pupil_data")) with open(meta_info_path,'r',encoding='utf-8') as csvfile: meta_info = csv_utils.read_key_value_file(csvfile) meta_info['Data Format Version'] = 'v0.8.7' update_meta_info(rec_dir, meta_info)
def save_intrinsics(directory, cam_name, resolution, intrinsics): """ Saves camera intrinsics calibration to a file. For each unique camera name we maintain a single file containing all calibrations associated with this camera name. :param directory: Directory to which the intrinsics file will be written :param cam_name: Name of the camera, e.g. 'Pupil Cam 1 ID2' :param resolution: Camera resolution given as a tuple. This needs to match the resolution the calibration has been computed with. :param intrinsics: The camera intrinsics dictionary. :return: """ # Try to load previous camera calibrations save_path = os.path.join( directory, "{}.intrinsics".format(cam_name.replace(" ", "_")) ) try: calib_dict = load_object(save_path, allow_legacy=False) except: calib_dict = {} calib_dict["version"] = __version__ calib_dict[str(resolution)] = intrinsics save_object(calib_dict, save_path) logger.info( "Calibration for camera {} at resolution {} saved to {}".format( cam_name, resolution, save_path ) )
def convert_pupil_mobile_recording_to_v094(rec_dir): logger.info("Converting Pupil Mobile recording to v0.9.4 format") # convert time files and rename corresponding videos time_pattern = os.path.join(rec_dir, '*.time') for time_loc in glob.glob(time_pattern): time_file_name = os.path.split(time_loc)[1] time_name, time_ext = os.path.splitext(time_file_name) potential_locs = [os.path.join(rec_dir, time_name+ext) for ext in ('.mjpeg', '.mp4','.m4a')] existing_locs = [loc for loc in potential_locs if os.path.exists(loc)] if not existing_locs: continue else: video_loc = existing_locs[0] if time_name in ('Pupil Cam1 ID0', 'Pupil Cam1 ID1'): time_name = 'eye'+time_name[-1] # rename eye files elif time_name in ('Pupil Cam1 ID2', 'Logitech Webcam C930e'): cam_calib_loc = os.path.join(rec_dir, 'camera_calibration') try: camera_calibration = load_object(cam_calib_loc) except: # no camera calibration found video = av.open(video_loc, 'r') frame_size = video.streams.video[0].format.width, video.streams.video[0].format.height del video try: camera_calibration = pre_recorded_calibrations[time_name][frame_size] except KeyError: camera_calibration = idealized_camera_calibration(frame_size) logger.warning('Camera calibration not found. Will assume idealized camera.') save_object(camera_calibration, cam_calib_loc) time_name = 'world' # assume world file elif time_name.startswith('audio_'): time_name = 'audio' timestamps = np.fromfile(time_loc, dtype='>f8') timestamp_loc = os.path.join(rec_dir, '{}_timestamps.npy'.format(time_name)) logger.info('Creating "{}"'.format(os.path.split(timestamp_loc)[1])) np.save(timestamp_loc, timestamps) if time_name == 'audio': video_dst = os.path.join(rec_dir, time_name) + '.mp4' logger.info('Renaming "{}" to "{}"'.format(os.path.split(video_loc)[1], os.path.split(video_dst)[1])) os.rename(video_loc, video_dst) else: video_dst = os.path.join(rec_dir, time_name) + os.path.splitext(video_loc)[1] logger.info('Renaming "{}" to "{}"'.format(os.path.split(video_loc)[1], os.path.split(video_dst)[1])) os.rename(video_loc, video_dst) pupil_data_loc = os.path.join(rec_dir, 'pupil_data') if not os.path.exists(pupil_data_loc): logger.info('Creating "pupil_data"') save_object({'pupil_positions': [], 'gaze_positions': [], 'notifications': []}, pupil_data_loc)
def __init__(self, g_pool, mode="Show markers and frames", min_marker_perimeter=40): super(Marker_Detector, self).__init__(g_pool) self.order = 0.2 # all markers that are detected in the most recent frame self.markers = [] # load camera intrinsics try: camera_calibration = load_object(os.path.join(self.g_pool.user_dir, "camera_calibration")) except: self.camera_intrinsics = None else: same_name = camera_calibration["camera_name"] == self.g_pool.capture.name same_resolution = camera_calibration["resolution"] == self.g_pool.capture.frame_size if same_name and same_resolution: logger.info("Loaded camera calibration. 3D marker tracking enabled.") K = camera_calibration["camera_matrix"] dist_coefs = camera_calibration["dist_coefs"] resolution = camera_calibration["resolution"] self.camera_intrinsics = K, dist_coefs, resolution else: logger.info( "Loaded camera calibration but camera name and/or resolution has changed. Please re-calibrate." ) self.camera_intrinsics = None # all registered surfaces self.surface_definitions = Persistent_Dict(os.path.join(g_pool.user_dir, "surface_definitions")) self.surfaces = [ Reference_Surface(saved_definition=d) for d in self.surface_definitions.get("realtime_square_marker_surfaces", []) if isinstance(d, dict) ] # edit surfaces self.edit_surfaces = [] # plugin state self.mode = mode self.running = True self.robust_detection = 1 self.aperture = 11 self.min_marker_perimeter = min_marker_perimeter self.locate_3d = False # debug vars self.draw_markers = 0 self.show_surface_idx = 0 self.img_shape = None self.menu = None self.button = None self.add_button = None
def update_recording_v05_to_v074(rec_dir): logger.info("Updating recording from v0.5x/v0.6x/v0.7x format to v0.7.4 format") pupil_data = load_object(os.path.join(rec_dir, "pupil_data")) save_object(pupil_data,os.path.join(rec_dir, "pupil_data_old")) for p in pupil_data['pupil_positions']: p['method'] = '2d python' try: save_object(pupil_data,os.path.join(rec_dir, "pupil_data")) except IOError: pass
def load_camera_calibration(g_pool): if g_pool.app != 'player': try: camera_calibration = load_object(os.path.join(g_pool.user_dir,'camera_calibration'),allow_legacy=False) camera_calibration['camera_name'] except (KeyError,ValueError): camera_calibration = None logger.warning('Invalid or Deprecated camera calibration found. Please recalibrate camera.') except: camera_calibration = None else: same_name = camera_calibration['camera_name'] == g_pool.capture.name same_resolution = tuple(camera_calibration['resolution']) == g_pool.capture.frame_size if not (same_name and same_resolution): logger.warning('Loaded camera calibration but camera name and/or resolution has changed.') camera_calibration = None else: logger.info("Loaded user calibrated calibration for {}@{}.".format(g_pool.capture.name,g_pool.capture.frame_size)) if not camera_calibration: logger.debug("Trying to load pre recorded calibration.") try: camera_calibration = pre_recorded_calibrations[g_pool.capture.name][g_pool.capture.frame_size] except KeyError: logger.info("Pre recorded calibration for {}@{} not found.".format(g_pool.capture.name,g_pool.capture.frame_size)) else: logger.info("Loaded pre recorded calibration for {}@{}.".format(g_pool.capture.name,g_pool.capture.frame_size)) if not camera_calibration: camera_calibration = idealized_camera_calibration(g_pool.capture.frame_size) logger.warning("Camera calibration not found. Will assume idealized camera. Please calibrate your cameras. Using camera 'Camera_Intrinsics_Estimation'.") else: try: camera_calibration = load_object(os.path.join(g_pool.rec_dir,'camera_calibration')) except: camera_calibration = idealized_camera_calibration(g_pool.capture.frame_size) logger.warning("Camera calibration not found. Will assume idealized camera. Please calibrate your cameras before your next recording.") else: logger.info("Loaded Camera calibration from file.") return camera_calibration
def __init__(self,g_pool): super(Show_Calibration, self).__init__(g_pool) self.menu=None logger.error("This will be implemented as part of gaze mapper soon.") self.alive= False return width,height = self.g_pool.capture.frame_size if g_pool.app == 'capture': cal_pt_path = os.path.join(g_pool.user_dir,"user_calibration_data") else: cal_pt_path = os.path.join(g_pool.rec_dir,"user_calibration_data") try: user_calibration_data = load_object(cal_pt_path) except: logger.warning("Please calibrate first") self.close() return if self.g_pool.binocular: fn_input_eye1 = cal_pt_cloud[:,2:4].transpose() cal_pt_cloud[:,0:2] = np.array(map_fn(fn_input_eye0, fn_input_eye1)).transpose() cal_pt_cloud[:,2:4] = cal_pt_cloud[:,4:6] else: fn_input = cal_pt_cloud[:,0:2].transpose() cal_pt_cloud[:,0:2] = np.array(map_fn(fn_input)).transpose() ref_pts = cal_pt_cloud[inlier_map][:,np.newaxis,2:4] ref_pts = np.array(ref_pts,dtype=np.float32) logger.debug("calibration ref_pts %s"%ref_pts) if len(ref_pts)== 0: logger.warning("Calibration is bad. Please re-calibrate") self.close() return self.calib_bounds = cv2.convexHull(ref_pts) # create a list [[px1,py1],[wx1,wy1],[px2,py2],[wx2,wy2]...] of outliers and inliers for gl_lines self.outliers = np.concatenate((cal_pt_cloud[~inlier_map][:,0:2],cal_pt_cloud[~inlier_map][:,2:4])).reshape(-1,2) self.inliers = np.concatenate((cal_pt_cloud[inlier_map][:,0:2],cal_pt_cloud[inlier_map][:,2:4]),axis=1).reshape(-1,2) self.inlier_ratio = cal_pt_cloud[inlier_map].shape[0]/float(cal_pt_cloud.shape[0]) self.inlier_count = cal_pt_cloud[inlier_map].shape[0] # hull = cv2.approxPolyDP(self.calib_bounds, 0.001,closed=True) full_screen_area = 1. logger.debug("calibration bounds %s"%self.calib_bounds) self.calib_area_ratio = cv2.contourArea(self.calib_bounds)/full_screen_area
def _load_data_from_file(self, filepath): try: dict_representation = fm.load_object(filepath) except FileNotFoundError: return None if dict_representation.get("version", None) != self._item_class.version: logger.warning( "Data in {} is in old file format. Will not load these!".format( filepath ) ) return None return dict_representation.get("data", None)
def update_recording_v073_to_v074(rec_dir): logger.info("Updating recording from v0.7x format to v0.7.4 format") pupil_data = load_object(os.path.join(rec_dir, "pupil_data")) modified = False for p in pupil_data["pupil_positions"]: if p["method"] == "3D c++": p["method"] = "3d c++" try: p["projected_sphere"] = p.pop("projectedSphere") except: p["projected_sphere"] = {"center": (0, 0), "angle": 0, "axes": (0, 0)} p["model_confidence"] = p.pop("modelConfidence") p["model_id"] = p.pop("modelID") p["circle_3d"] = p.pop("circle3D") p["diameter_3d"] = p.pop("diameter_3D") modified = True if modified: save_object(load_object(os.path.join(rec_dir, "pupil_data")), os.path.join(rec_dir, "pupil_data_old")) try: save_object(pupil_data, os.path.join(rec_dir, "pupil_data")) except IOError: pass
def update_recording_v073_to_v074(rec_dir): logger.info("Updating recording from v0.7x format to v0.7.4 format") pupil_data = load_object(os.path.join(rec_dir, "pupil_data")) modified = False for p in pupil_data['pupil_positions']: if p['method'] == "3D c++": p['method'] = "3d c++" try: p['projected_sphere'] = p.pop('projectedSphere') except: p['projected_sphere'] = {'center':(0,0),'angle':0,'axes':(0,0)} p['model_confidence'] = p.pop('modelConfidence') p['model_id'] = p.pop('modelID') p['circle_3d'] = p.pop('circle3D') p['diameter_3d'] = p.pop('diameter_3D') modified = True if modified: save_object(load_object(os.path.join(rec_dir, "pupil_data")),os.path.join(rec_dir, "pupil_data_old")) try: save_object(pupil_data,os.path.join(rec_dir, "pupil_data")) except IOError: pass
def calibrate_from_user_calibration_data_file(self): user_calibration = load_object(os.path.join(self.g_pool.user_dir, "user_calibration_data")) self.pupil_list = user_calibration['pupil_list'] self.ref_list = user_calibration['ref_list'] calibration_method = user_calibration['calibration_method'] if '3d' in calibration_method: logger.error('adjust calibration is not supported for 3d calibration.') return finish_calibration.finish_calibration(self.g_pool,self.pupil_list,self.ref_list)
def load_intrinsics(directory, cam_name, resolution): """ Loads a pre-recorded intrinsics calibration for the given camera and resolution. If no pre-recorded calibration is available we fall back on default values. :param directory: The directory in which to look for the intrinsincs file :param cam_name: Name of the camera, e.g. 'Pupil Cam 1 ID2' :param resolution: Camera resolution given as a tuple. :return: Camera Model Object """ file_path = os.path.join( directory, "{}.intrinsics".format(cam_name.replace(" ", "_")) ) try: calib_dict = load_object(file_path, allow_legacy=False) if calib_dict["version"] < __version__: logger.warning("Deprecated camera calibration found.") logger.info( "Please recalibrate using the Camera Intrinsics Estimation calibration." ) os.rename( file_path, "{}.deprecated.v{}".format(file_path, calib_dict["version"]) ) intrinsics = calib_dict[str(resolution)] logger.info("Previously recorded calibration found and loaded!") except Exception: logger.info( "No user calibration found for camera {} at resolution {}".format( cam_name, resolution ) ) if ( cam_name in pre_recorded_calibrations and str(resolution) in pre_recorded_calibrations[cam_name] ): logger.info("Loading pre-recorded calibration") intrinsics = pre_recorded_calibrations[cam_name][str(resolution)] else: logger.info("No pre-recorded calibration available") logger.warning("Loading dummy calibration") return Dummy_Camera(resolution, cam_name) if intrinsics["cam_type"] == "fisheye": return Fisheye_Dist_Camera( intrinsics["camera_matrix"], intrinsics["dist_coefs"], resolution, cam_name ) else: return Radial_Dist_Camera( intrinsics["camera_matrix"], intrinsics["dist_coefs"], resolution, cam_name )
def load_camera_calibration(g_pool): if g_pool.app == 'capture': try: camera_calibration = load_object(os.path.join(g_pool.user_dir,'camera_calibration')) except: camera_calibration = None else: same_name = camera_calibration['camera_name'] == g_pool.capture.name same_resolution = camera_calibration['resolution'] == g_pool.capture.frame_size if not (same_name and same_resolution): logger.warning('Loaded camera calibration but camera name and/or resolution has changed.') camera_calibration = None else: logger.info("Loaded user calibrated calibration for %s@%s."%(g_pool.capture.name,g_pool.capture.frame_size)) if not camera_calibration: logger.debug("Trying to load pre recorded calibration.") try: camera_calibration = pre_recorded_calibrations[g_pool.capture.name][g_pool.capture.frame_size] except KeyError: logger.info("Pre recorded calibration for %s@%s not found."%(g_pool.capture.name,g_pool.capture.frame_size)) else: logger.info("Loaded pre recorded calibration for %s@%s."%(g_pool.capture.name,g_pool.capture.frame_size)) if not camera_calibration: camera_calibration = idealized_camera_calibration(g_pool.capture.frame_size) logger.warning("Camera calibration not found. Will assume idealized camera. Please calibrate your cameras. Using camera 'Camera_Intrinsics_Estimation'.") else: try: camera_calibration = load_object(os.path.join(g_pool.rec_dir,'camera_calibration')) except: camera_calibration = idealized_camera_calibration(g_pool.capture.frame_size) logger.warning("Camera calibration not found. Will assume idealized camera. Please calibrate your cameras before your next recording.") else: logger.info("Loaded Camera calibration from file.") return camera_calibration
def update_recording_v091_to_v093(rec_dir): logger.info("Updating recording from v0.9.1 format to v0.9.3 format") meta_info_path = os.path.join(rec_dir,"info.csv") pupil_data = load_object(os.path.join(rec_dir, "pupil_data")) for g in pupil_data.get('gaze_positions', []): # fixing recordings made with bug https://github.com/pupil-labs/pupil/issues/598 g['norm_pos'] = float(g['norm_pos'][0]), float(g['norm_pos'][1]) save_object(pupil_data, os.path.join(rec_dir, "pupil_data")) with open(meta_info_path, 'r', encoding='utf-8') as csvfile: meta_info = csv_utils.read_key_value_file(csvfile) meta_info['Data Format Version'] = 'v0.9.3' update_meta_info(rec_dir, meta_info)
def __init__(self,g_pool,fullscreen = False): super(Camera_Intrinsics_Estimation, self).__init__(g_pool) self.collect_new = False self.calculated = False self.obj_grid = _gen_pattern_grid((4, 11)) self.img_points = [] self.obj_points = [] self.count = 10 self.display_grid = _make_grid() self._window = None self.menu = None self.button = None self.clicks_to_close = 5 self.window_should_close = False self.fullscreen = fullscreen self.monitor_idx = 0 self.glfont = fontstash.Context() self.glfont.add_font('opensans',get_opensans_font_path()) self.glfont.set_size(32) self.glfont.set_color_float((0.2,0.5,0.9,1.0)) self.glfont.set_align_string(v_align='center') self.undist_img = None self.show_undistortion = False self.show_undistortion_switch = None try: camera_calibration = load_object(os.path.join(self.g_pool.user_dir,'camera_calibration')) except: self.camera_intrinsics = None else: logger.info('Loaded camera calibration. Click show undistortion to verify.') logger.info('Hint: Lines in the real world should be straigt in the image.') same_name = camera_calibration['camera_name'] == self.g_pool.capture.name same_resolution = camera_calibration['resolution'] == self.g_pool.capture.frame_size if not (same_name and same_resolution): logger.warning('Loaded camera calibration but camera name and/or resolution has changed. Please re-calibrate.') K = camera_calibration['camera_matrix'] dist_coefs = camera_calibration['dist_coefs'] resolution = camera_calibration['resolution'] self.camera_intrinsics = K,dist_coefs,resolution
def update_recording_v093_to_v094(rec_dir): logger.info("Updating recording from v0.9.3 to v0.9.4.") meta_info_path = os.path.join(rec_dir, "info.csv") for file in os.listdir(rec_dir): if file.startswith('.') or os.path.splitext(file)[1] in ('.mp4', '.avi'): continue rec_file = os.path.join(rec_dir, file) try: rec_object = load_object(rec_file,allow_legacy=False) save_object(rec_object, rec_file) except: try: rec_object = load_object(rec_file,allow_legacy=True) save_object(rec_object, rec_file) logger.info('Converted `{}` from pickle to msgpack'.format(file)) except: logger.warning("did not convert {}".format(rec_file)) with open(meta_info_path, 'r', encoding='utf-8') as csvfile: meta_info = csv_utils.read_key_value_file(csvfile) meta_info['Data Format Version'] = 'v0.9.4' update_meta_info(rec_dir, meta_info)
def __init__(self, g_pool): super().__init__(g_pool) zmq_ctx = zmq.Context() self.data_sub = zmq_tools.Msg_Receiver( zmq_ctx, g_pool.ipc_sub_url, topics=("pupil", "notify.file_source.video_finished"), hwm=100_000, ) self.data_dir = os.path.join(g_pool.rec_dir, "offline_data") os.makedirs(self.data_dir, exist_ok=True) try: session_meta_data = fm.load_object( os.path.join(self.data_dir, self.session_data_name + ".meta")) assert session_meta_data.get( "version") == self.session_data_version except (AssertionError, FileNotFoundError): session_meta_data = {} session_meta_data["detection_method"] = "3d" session_meta_data["detection_status"] = ["unknown", "unknown"] self.detection_method = session_meta_data["detection_method"] self.detection_status = session_meta_data["detection_status"] pupil = fm.load_pldata_file(self.data_dir, self.session_data_name) ts_data_zip = zip(pupil.timestamps, pupil.data) ts_topic_zip = zip(pupil.timestamps, pupil.topics) self.pupil_positions = collections.OrderedDict(ts_data_zip) self.id_topics = collections.OrderedDict(ts_topic_zip) self.eye_video_loc = [None, None] self.eye_frame_num = [0, 0] for topic in self.id_topics.values(): eye_id = int(topic[-1]) self.eye_frame_num[eye_id] += 1 self.pause_switch = None self.detection_paused = False # start processes for eye_id in range(2): if self.detection_status[eye_id] != "complete": self.start_eye_process(eye_id) # either we did not start them or they failed to start (mono setup etc) # either way we are done and can publish if self.eye_video_loc == [None, None]: self.correlate_publish()
def __init__(self, g_pool): super().__init__(g_pool) zmq_ctx = zmq.Context() self.data_sub = zmq_tools.Msg_Receiver( zmq_ctx, g_pool.ipc_sub_url, topics=("pupil", "notify.file_source.video_finished"), hwm=100_000, ) self.data_dir = os.path.join(g_pool.rec_dir, "offline_data") os.makedirs(self.data_dir, exist_ok=True) try: session_meta_data = fm.load_object( os.path.join(self.data_dir, self.session_data_name + ".meta") ) assert session_meta_data.get("version") == self.session_data_version except (AssertionError, FileNotFoundError): session_meta_data = {} session_meta_data["detection_method"] = "3d" session_meta_data["detection_status"] = ["unknown", "unknown"] self.detection_method = session_meta_data["detection_method"] self.detection_status = session_meta_data["detection_status"] pupil = fm.load_pldata_file(self.data_dir, self.session_data_name) ts_data_zip = zip(pupil.timestamps, pupil.data) ts_topic_zip = zip(pupil.timestamps, pupil.topics) self.pupil_positions = collections.OrderedDict(ts_data_zip) self.id_topics = collections.OrderedDict(ts_topic_zip) self.eye_video_loc = [None, None] self.eye_frame_num = [0, 0] for topic in self.id_topics.values(): eye_id = int(topic[-1]) self.eye_frame_num[eye_id] += 1 self.pause_switch = None self.detection_paused = False # start processes for eye_id in range(2): if self.detection_status[eye_id] != "complete": self.start_eye_process(eye_id) # either we did not start them or they failed to start (mono setup etc) # either way we are done and can publish if self.eye_video_loc == [None, None]: self.correlate_publish()
def load_intrinsics(directory, cam_name, resolution): """ Loads a pre-recorded intrinsics calibration for the given camera and resolution. If no pre-recorded calibration is available we fall back on default values. :param directory: The directory in which to look for the intrinsincs file :param cam_name: Name of the camera, e.g. 'Pupil Cam 1 ID2' :param resolution: Camera resolution given as a tuple. :return: Camera Model Object """ file_path = os.path.join( directory, '{}.intrinsics'.format(cam_name.replace(" ", "_"))) try: calib_dict = load_object(file_path, allow_legacy=False) if calib_dict['version'] < __version__: logger.warning('Deprecated camera calibration found.') logger.info( 'Please recalibrate using the Camera Intrinsics Estimation calibration.' ) os.rename( file_path, '{}.deprecated.v{}'.format(file_path, calib_dict['version'])) intrinsics = calib_dict[str(resolution)] logger.info("Previously recorded calibration found and loaded!") except Exception as e: logger.info( "No user calibration found for camera {} at resolution {}".format( cam_name, resolution)) if cam_name in pre_recorded_calibrations and str( resolution) in pre_recorded_calibrations[cam_name]: logger.info("Loading pre-recorded calibration") intrinsics = pre_recorded_calibrations[cam_name][str(resolution)] else: logger.info("No pre-recorded calibration available") logger.warning("Loading dummy calibration") intrinsics = {'cam_type': 'dummy'} if intrinsics['cam_type'] == 'dummy': return Dummy_Camera(resolution, cam_name) elif intrinsics['cam_type'] == 'fisheye': return Fisheye_Dist_Camera(intrinsics['camera_matrix'], intrinsics['dist_coefs'], resolution, cam_name) elif intrinsics['cam_type'] == 'radial': return Radial_Dist_Camera(intrinsics['camera_matrix'], intrinsics['dist_coefs'], resolution, cam_name)
def update_recording_v082_to_v083(rec_dir): logger.info("Updating recording from v0.8.2 format to v0.8.3 format") pupil_data = load_object(os.path.join(rec_dir, "pupil_data")) meta_info_path = os.path.join(rec_dir,"info.csv") for d in pupil_data['gaze_positions']: if 'base' in d: d['base_data'] = d.pop('base') save_object(pupil_data,os.path.join(rec_dir, "pupil_data")) with open(meta_info_path,'r',encoding='utf-8') as csvfile: meta_info = csv_utils.read_key_value_file(csvfile) meta_info['Data Format Version'] = 'v0.8.3' update_meta_info(rec_dir, meta_info)
def update_recording_v083_to_v086(rec_dir): logger.info("Updating recording from v0.8.3 format to v0.8.6 format") pupil_data = fm.load_object(os.path.join(rec_dir, "pupil_data")) meta_info_path = os.path.join(rec_dir, "info.csv") for topic in pupil_data.keys(): for d in pupil_data[topic]: d['topic'] = topic fm.save_object(pupil_data, os.path.join(rec_dir, "pupil_data")) with open(meta_info_path, 'r', encoding='utf-8') as csvfile: meta_info = csv_utils.read_key_value_file(csvfile) meta_info['Data Format Version'] = 'v0.8.6' update_meta_info(rec_dir, meta_info)
def update_recording_v082_to_v083(rec_dir): logger.info("Updating recording from v0.8.2 format to v0.8.3 format") pupil_data = fm.load_object(os.path.join(rec_dir, "pupil_data")) meta_info_path = os.path.join(rec_dir, "info.csv") for d in pupil_data['gaze_positions']: if 'base' in d: d['base_data'] = d.pop('base') fm.save_object(pupil_data, os.path.join(rec_dir, "pupil_data")) with open(meta_info_path, 'r', encoding='utf-8') as csvfile: meta_info = csv_utils.read_key_value_file(csvfile) meta_info['Data Format Version'] = 'v0.8.3' update_meta_info(rec_dir, meta_info)
def update_recording_v083_to_v086(rec_dir): logger.info("Updating recording from v0.8.3 format to v0.8.6 format") pupil_data = load_object(os.path.join(rec_dir, "pupil_data")) meta_info_path = os.path.join(rec_dir,"info.csv") for topic in pupil_data.keys(): for d in pupil_data[topic]: d['topic'] = topic save_object(pupil_data,os.path.join(rec_dir, "pupil_data")) with open(meta_info_path,'r',encoding='utf-8') as csvfile: meta_info = csv_utils.read_key_value_file(csvfile) meta_info['Data Format Version'] = 'v0.8.6' update_meta_info(rec_dir, meta_info)
def update_recording_v082_to_v083(rec_dir): logger.info("Updating recording from v0.8.2 format to v0.8.3 format") pupil_data = fm.load_object(os.path.join(rec_dir, "pupil_data")) meta_info_path = os.path.join(rec_dir, "info.csv") for d in pupil_data["gaze_positions"]: if "base" in d: d["base_data"] = d.pop("base") fm.save_object(pupil_data, os.path.join(rec_dir, "pupil_data")) with open(meta_info_path, "r", encoding="utf-8") as csvfile: meta_info = csv_utils.read_key_value_file(csvfile) meta_info["Data Format Version"] = "v0.8.3" update_meta_info(rec_dir, meta_info)
def update_recording_v082_to_v083(rec_dir): logger.info("Updating recording from v0.8.2 format to v0.8.3 format") pupil_data = load_object(os.path.join(rec_dir, "pupil_data")) meta_info_path = os.path.join(rec_dir, "info.csv") for d in pupil_data['gaze_positions']: if 'base' in d: d['base_data'] = d.pop('base') save_object(pupil_data, os.path.join(rec_dir, "pupil_data")) with open(meta_info_path) as csvfile: meta_info = csv_utils.read_key_value_file(csvfile) meta_info['Capture Software Version'] = 'v0.8.3' with open(meta_info_path, 'w') as csvfile: csv_utils.write_key_value_file(csvfile, meta_info)
def __init__(self, g_pool, source_path=None, frame_size=None, frame_rate=None, name='Fake Source', *args, **kwargs): super().__init__(g_pool, *args, **kwargs) if source_path: meta = load_object(source_path) frame_size = meta['frame_size'] frame_rate = meta['frame_rate'] self.timestamps = np.load(os.path.splitext(source_path)[0] + '_timestamps.npy') else: self.timestamps = None self.fps = frame_rate self._name = name self.make_img(tuple(frame_size)) self.source_path = source_path self.current_frame_idx = 0 self.target_frame_idx = 0
def update_recording_v083_to_v086(rec_dir): logger.info("Updating recording from v0.8.3 format to v0.8.6 format") pupil_data = load_object(os.path.join(rec_dir, "pupil_data")) meta_info_path = os.path.join(rec_dir,"info.csv") for topic in pupil_data.keys(): for d in pupil_data[topic]: d['topic'] = topic save_object(pupil_data,os.path.join(rec_dir, "pupil_data")) with open(meta_info_path) as csvfile: meta_info = csv_utils.read_key_value_file(csvfile) meta_info['Capture Software Version'] = 'v0.8.6' with open(meta_info_path,'w') as csvfile: csv_utils.write_key_value_file(csvfile,meta_info)
def update_recording_v083_to_v086(rec_dir): logger.info("Updating recording from v0.8.3 format to v0.8.6 format") pupil_data = load_object(os.path.join(rec_dir, "pupil_data")) meta_info_path = os.path.join(rec_dir, "info.csv") for topic in pupil_data.keys(): for d in pupil_data[topic]: d['topic'] = topic save_object(pupil_data, os.path.join(rec_dir, "pupil_data")) with open(meta_info_path) as csvfile: meta_info = csv_utils.read_key_value_file(csvfile) meta_info['Capture Software Version'] = 'v0.8.6' with open(meta_info_path, 'w') as csvfile: csv_utils.write_key_value_file(csvfile, meta_info)
def __init__(self, g_pool): super().__init__(g_pool) self.data_dir = os.path.join(g_pool.rec_dir, 'offline_data') os.makedirs(self.data_dir, exist_ok=True) try: session_data = load_object( os.path.join(self.data_dir, 'offline_pupil_data')) assert session_data.get('version') != self.session_data_version except: session_data = {} session_data["detection_method"] = '3d' session_data['pupil_positions'] = [] session_data['detection_progress'] = [0., 0.] session_data['detection_status'] = ["unknown", "unknown"] self.detection_method = session_data["detection_method"] self.pupil_positions = session_data['pupil_positions'] self.eye_processes = [None, None] self.detection_progress = session_data['detection_progress'] self.detection_status = session_data['detection_status'] # Pupil Offline Detection self.eyes_are_alive = Value(c_bool, 0), Value(c_bool, 0) logger.debug('Starting eye process communication channel...') self.ipc_pub_url, self.ipc_sub_url, self.ipc_push_url = self.initialize_ipc( ) sleep(0.2) self.data_sub = zmq_tools.Msg_Receiver(self.zmq_ctx, self.ipc_sub_url, topics=('', )) self.eye_control = zmq_tools.Msg_Dispatcher(self.zmq_ctx, self.ipc_push_url) self.menu = None #start processes if self.detection_progress[0] < 100: self.start_eye_process(0) if self.detection_progress[1] < 100: self.start_eye_process(1) #either we did not start them or they failed to start (mono setup etc) #either way we are done and can publish if self.eye_processes == [None, None]: self.correlate_publish()
def _load_from_file(self): file_path = self._plmodel_file_path try: dict_representation = fm.load_object(file_path) except FileNotFoundError: return None, None if dict_representation.get("version", None) != self.version: logger.warning( "Data in {} is in old file format. Will not load these!".format( file_path ) ) return None, None return ( dict_representation.get("recording_uuid", None), dict_representation.get("data", None), )
def load_camera_calibration(g_pool): if g_pool.app == 'capture': try: camera_calibration = load_object( os.path.join(g_pool.user_dir, 'camera_calibration')) except: camera_calibration = None else: same_name = camera_calibration[ 'camera_name'] == g_pool.capture.name same_resolution = camera_calibration[ 'resolution'] == g_pool.capture.frame_size if not (same_name and same_resolution): logger.warning( 'Loaded camera calibration but camera name and/or resolution has changed.' ) camera_calibration = None else: logger.info("Loaded user calibrated calibration for %s@%s." % (g_pool.capture.name, g_pool.capture.frame_size)) if not camera_calibration: logger.debug("Trying to load pre recorded calibration.") try: camera_calibration = pre_recorded_calibrations[ g_pool.capture.name][g_pool.capture.frame_size] except KeyError: logger.info("Pre recorded calibration for %s@%s not found." % (g_pool.capture.name, g_pool.capture.frame_size)) else: logger.info("Loaded pre recorded calibration for %s@%s." % (g_pool.capture.name, g_pool.capture.frame_size)) if not camera_calibration: logger.warning( "Camera calibration not found please run Camera_Intrinsics_Estimation to calibrate camera." ) return camera_calibration else: raise NotImplementedError()
def stop(self): logger.info("Stopping Touchup") self.smooth_pos = 0., 0. self.sample_site = -2, -2 self.counter = 0 self.active = False self.button.status_text = '' offset_pt_clound = calibrate.preprocess_2d_data_monocular( calibrate.closest_matches_monocular(self.ref_list, self.gaze_list)) if len(offset_pt_clound) < 3: logger.error( 'Did not sample enough data for touchup please retry.') return #Calulate the offset for gaze to target offset_pt_clound = np.array(offset_pt_clound) offset = offset_pt_clound[:, :2] - offset_pt_clound[:, 2:] mean_offset = np.mean(offset, axis=0) user_calibration = load_object( os.path.join(self.g_pool.user_dir, "user_calibration_data")) self.pupil_list = user_calibration['pupil_list'] self.ref_list = user_calibration['ref_list'] calibration_method = user_calibration['calibration_method'] if '3d' in calibration_method: logger.error( 'adjust calibration is not supported for 3d calibration.') return for r in self.ref_list: r['norm_pos'] = [ r['norm_pos'][0] - mean_offset[0], r['norm_pos'][1] - mean_offset[1] ] finish_calibration(self.g_pool, self.pupil_list, self.ref_list, force='2d')
def update_recording_v086_to_v087(rec_dir): logger.info("Updating recording from v0.8.6 format to v0.8.7 format") pupil_data = fm.load_object(os.path.join(rec_dir, "pupil_data")) def _clamp_norm_point(pos): """realisitic numbers for norm pos should be in this range. Grossly bigger or smaller numbers are results bad exrapolation and can cause overflow erorr when denormalized and cast as int32. """ return min(100.0, max(-100.0, pos[0])), min(100.0, max(-100.0, pos[1])) for g in pupil_data.get("gaze_positions", []): if "topic" not in g: # we missed this in one gaze mapper g["topic"] = "gaze" g["norm_pos"] = _clamp_norm_point(g["norm_pos"]) fm.save_object(pupil_data, os.path.join(rec_dir, "pupil_data")) _update_info_version_to("v0.8.7", rec_dir)
def save_intrinsics(directory, cam_name, resolution, intrinsics): """ Saves camera intrinsics calibration to a file. For each unique camera name we maintain a single file containing all calibrations associated with this camera name. :param directory: Directory to which the intrinsics file will be written :param cam_name: Name of the camera, e.g. 'Pupil Cam 1 ID2' :param resolution: Camera resolution given as a tuple. This needs to match the resolution the calibration has been computed with. :param intrinsics: The camera intrinsics dictionary. :return: """ # Try to load previous camera calibrations save_path = os.path.join(directory, '{}.intrinsics'.format(cam_name.replace(" ", "_"))) try: calib_dict = load_object(save_path, allow_legacy=False) except: calib_dict = {} calib_dict['version'] = __version__ calib_dict[str(resolution)] = intrinsics save_object(calib_dict, save_path) logger.info("Calibration for camera {} at resolution {} saved to {}".format(cam_name, resolution, save_path))
def __init__(self, g_pool): super().__init__(g_pool) self._detection_paused = False zmq_ctx = zmq.Context() self.data_sub = zmq_tools.Msg_Receiver( zmq_ctx, g_pool.ipc_sub_url, topics=("pupil", "notify.file_source"), hwm=100_000, ) self.data_dir = os.path.join(g_pool.rec_dir, "offline_data") os.makedirs(self.data_dir, exist_ok=True) try: session_meta_data = fm.load_object( os.path.join(self.data_dir, self.session_data_name + ".meta") ) assert session_meta_data.get("version") == self.session_data_version except (AssertionError, FileNotFoundError): session_meta_data = {} session_meta_data["detection_status"] = ["unknown", "unknown"] self.detection_status = session_meta_data["detection_status"] self._pupil_data_store = pm.PupilDataCollector() pupil_data_from_cache = pm.PupilDataBisector.load_from_file( self.data_dir, self.session_data_name ) self.publish_existing(pupil_data_from_cache) # Start offline pupil detection if not complete yet: self.eye_video_loc = [None, None] self.eye_frame_num = [0, 0] self.eye_frame_idx = [-1, -1] # start processes for eye_id in range(2): if self.detection_status[eye_id] != "complete": self.start_eye_process(eye_id)
def __init__(self, g_pool, manual_ref_edit_mode=False): super().__init__(g_pool) self.timeline_line_height = 16 self.manual_ref_edit_mode = manual_ref_edit_mode self.menu = None self.process_pipe = None self.result_dir = os.path.join(g_pool.rec_dir, 'offline_data') os.makedirs(self.result_dir, exist_ok=True) try: session_data = load_object( os.path.join(self.result_dir, 'offline_calibration_gaze')) if session_data['version'] != self.session_data_version: logger.warning( "Session data from old version. Will not use this.") assert False except Exception as e: map_range = [0, len(self.g_pool.timestamps)] calib_range = [ len(self.g_pool.timestamps) // 10, len(self.g_pool.timestamps) // 2 ] session_data = {} session_data['sections'] = [ make_section_dict(calib_range, map_range), ] session_data['circle_marker_positions'] = [] session_data['manual_ref_positions'] = [] self.sections = session_data['sections'] self.circle_marker_positions = session_data['circle_marker_positions'] self.manual_ref_positions = session_data['manual_ref_positions'] if self.circle_marker_positions: self.detection_progress = 100.0 for s in self.sections: self.calibrate_section(s) self.correlate_and_publish() else: self.detection_progress = 0.0 self.start_detection_task()
def __init__(self, g_pool): super().__init__(g_pool) zmq_ctx = zmq.Context() self.data_sub = zmq_tools.Msg_Receiver( zmq_ctx, g_pool.ipc_sub_url, topics=('pupil', 'notify.file_source.video_finished')) self.data_dir = os.path.join(g_pool.rec_dir, 'offline_data') os.makedirs(self.data_dir, exist_ok=True) try: session_data = load_object( os.path.join(self.data_dir, 'offline_pupil_data')) assert session_data.get('version') != self.session_data_version except: session_data = {} session_data["detection_method"] = '3d' session_data['pupil_positions'] = [] session_data['detection_progress'] = [0., 0.] session_data['detection_status'] = ["unknown", "unknown"] self.detection_method = session_data["detection_method"] self.pupil_positions = session_data['pupil_positions'] self.eye_processes = [None, None] self.detection_progress = session_data['detection_progress'] self.detection_status = session_data['detection_status'] self.pause_switch = None self.detection_paused = False # start processes if self.detection_progress[0] < 100: self.start_eye_process(0) if self.detection_progress[1] < 100: self.start_eye_process(1) # either we did not start them or they failed to start (mono setup etc) # either way we are done and can publish if self.eye_processes == [None, None]: self.correlate_publish()
def update_recording_bytes_to_unicode(rec_dir): logger.info("Updating recording from bytes to unicode.") # update to python 3 meta_info_path = os.path.join(rec_dir, "info.csv") def convert(data): if isinstance(data, bytes): return data.decode() elif isinstance(data, str) or isinstance(data, np.ndarray): return data elif isinstance(data, collections.Mapping): return dict(map(convert, data.items())) elif isinstance(data, collections.Iterable): return type(data)(map(convert, data)) else: return data for file in os.listdir(rec_dir): rec_file = os.path.join(rec_dir, file) try: rec_object = load_object(rec_file) converted_object = convert(rec_object) if converted_object != rec_object: logger.info( 'Converted `{}` from bytes to unicode'.format(file)) save_object(rec_object, rec_file) except (ValueError, IsADirectoryError): continue # except TypeError: # logger.error('TypeError when parsing `{}`'.format(file)) # continue with open(meta_info_path, 'r', encoding='utf-8') as csvfile: meta_info = csv_utils.read_key_value_file(csvfile) meta_info['Capture Software Version'] = 'v0.8.8' with open(meta_info_path, 'w', newline='') as csvfile: csv_utils.write_key_value_file(csvfile, meta_info)
def from_file( directory: str, cam_name: str, resolution: T.Tuple[int] ) -> "Camera_Model": """ Loads recorded intrinsics for the given camera and resolution. If no recorded intrinsics are available we fall back to default values. If no default values are available, we use dummy intrinsics. :param directory: The directory in which to look for the intrinsincs file. :param cam_name: Name of the camera, e.g. 'Pupil Cam 1 ID2'. :param resolution: Camera resolution. """ file_path = os.path.join( directory, "{}.intrinsics".format(cam_name.replace(" ", "_")) ) try: intrinsics_dict = load_object(file_path, allow_legacy=False) if intrinsics_dict["version"] < __version__: logger.warning("Deprecated camera intrinsics found.") logger.info( "Please recalculate the camera intrinsics using the Camera" " Intrinsics Estimation." ) os.rename( file_path, "{}.deprecated.v{}".format(file_path, intrinsics_dict["version"]), ) intrinsics = intrinsics_dict[str(resolution)] logger.info("Loading previously recorded intrinsics...") return Camera_Model._from_raw_intrinsics(cam_name, resolution, intrinsics) except Exception: logger.debug( f"No recorded intrinsics found for camera {cam_name} at resolution" f" {resolution}" ) return Camera_Model.from_default(cam_name, resolution)
def rectify_gaze_data(path, K, D, rect_camera_matrix): #if not os.path.exists(path + 'pupil_data_original'): # data = load_object(path + 'pupil_data') # save_object(data, path + 'pupil_data_original') #else: # data = load_object(path + 'pupil_data_original') data = load_object(path + 'pupil_data') if not 'gaze_positions' in data: print("no gaze_positions", data.keys()) return gazes = data['gaze_positions'] for g in gazes: gaze = denormalize(g['norm_pos'], 1280, 720) gaze = np.float32(gaze).reshape(-1, 1, 2) gaze = cv2.fisheye.undistortPoints(gaze, K, D, P=rect_camera_matrix).reshape(2) gaze = normalize(gaze, 1280, 720) g['norm_pos'] = gaze save_object(data, path + 'pupil_data_corrected')
def update_offline_calibrations_to_latest_version(cls, rec_dir): calib_dir = cls._calibration_directory_from_recording(rec_dir) if not calib_dir.exists(): return if not calib_dir.is_dir(): return # TODO: Raise exception - "calibrations" must be a directory for calib_path in sorted(calib_dir.glob("[!.]*.plcal")): calib_dict = fm.load_object(calib_path) version = calib_dict.get("version", None) data = calib_dict.get("data", None) if version == Calibration.version: # Calibration already at current version continue # No-op elif version > Calibration.version: # Calibration at newer version than current model continue # No-op elif data is not None: try: if version == CalibrationV1.version == 1: cls.__update_and_save_calibration_v1_as_latest_version( rec_dir, data ) continue # Success except Exception as err: logger.warning(str(err)) # Failed to update logger.warning( f'Unable to update calibration "{calib_path.name}" from version {version} to the current version {Calibration.version}' )
def update_recording_bytes_to_unicode(rec_dir): logger.info("Updating recording from bytes to unicode.") def convert(data): if isinstance(data, bytes): return data.decode() elif isinstance(data, str) or isinstance(data, np.ndarray): return data elif isinstance(data, collections.Mapping): return dict(map(convert, data.items())) elif isinstance(data, collections.Iterable): return type(data)(map(convert, data)) else: return data for file in os.listdir(rec_dir): if file.startswith('.') or os.path.splitext(file)[1] in ('.mp4', '.avi'): continue rec_file = os.path.join(rec_dir, file) try: rec_object = fm.load_object(rec_file) converted_object = convert(rec_object) if converted_object != rec_object: logger.info( 'Converted `{}` from bytes to unicode'.format(file)) fm.save_object(converted_object, rec_file) except (fm.UnpicklingError, IsADirectoryError): continue # manually convert k v dicts. meta_info_path = os.path.join(rec_dir, "info.csv") with open(meta_info_path, 'r', encoding='utf-8') as csvfile: meta_info = csv_utils.read_key_value_file(csvfile) with open(meta_info_path, 'w', newline='') as csvfile: csv_utils.write_key_value_file(csvfile, meta_info)
def export(should_terminate, frames_to_export, current_frame, rec_dir, user_dir, start_frame=None, end_frame=None, plugin_initializers=[], out_file_path=None): logger = logging.getLogger(__name__ + ' with pid: ' + str(os.getpid())) #parse info.csv file meta_info_path = os.path.join(rec_dir, "info.csv") with open(meta_info_path) as info: meta_info = dict( ((line.strip().split('\t')) for line in info.readlines())) video_path = glob(os.path.join(rec_dir, "world.*"))[0] timestamps_path = os.path.join(rec_dir, "world_timestamps.npy") pupil_data_path = os.path.join(rec_dir, "pupil_data") rec_version = read_rec_version(meta_info) if rec_version >= VersionFormat('0.5'): pass elif rec_version >= VersionFormat('0.4'): update_recording_0v4_to_current(rec_dir) elif rec_version >= VersionFormat('0.3'): update_recording_0v3_to_current(rec_dir) timestamps_path = os.path.join(rec_dir, "timestamps.npy") else: logger.Error("This recording is to old. Sorry.") return timestamps = np.load(timestamps_path) cap = File_Capture(video_path, timestamps=timestamps_path) #Out file path verification, we do this before but if one uses a seperate tool, this will kick in. if out_file_path is None: out_file_path = os.path.join(rec_dir, "world_viz.mp4") else: file_name = os.path.basename(out_file_path) dir_name = os.path.dirname(out_file_path) if not dir_name: dir_name = rec_dir if not file_name: file_name = 'world_viz.mp4' out_file_path = os.path.expanduser(os.path.join(dir_name, file_name)) if os.path.isfile(out_file_path): logger.warning("Video out file already exsists. I will overwrite!") os.remove(out_file_path) logger.debug("Saving Video to %s" % out_file_path) #Trim mark verification #make sure the trim marks (start frame, endframe) make sense: We define them like python list slices,thus we can test them like such. trimmed_timestamps = timestamps[start_frame:end_frame] if len(trimmed_timestamps) == 0: logger.warn( "Start and end frames are set such that no video will be exported." ) return False if start_frame == None: start_frame = 0 #these two vars are shared with the lauching process and give a job length and progress report. frames_to_export.value = len(trimmed_timestamps) current_frame.value = 0 logger.debug( "Will export from frame %s to frame %s. This means I will export %s frames." % (start_frame, start_frame + frames_to_export.value, frames_to_export.value)) #setup of writer writer = AV_Writer(out_file_path) cap.seek_to_frame(start_frame) start_time = time() g = Global_Container() g.app = 'exporter' g.capture = cap g.rec_dir = rec_dir g.user_dir = user_dir g.rec_version = rec_version g.timestamps = timestamps # load pupil_positions, gaze_positions pupil_data = load_object(pupil_data_path) pupil_list = pupil_data['pupil_positions'] gaze_list = pupil_data['gaze_positions'] g.pupil_positions_by_frame = correlate_data(pupil_list, g.timestamps) g.gaze_positions_by_frame = correlate_data(gaze_list, g.timestamps) g.fixations_by_frame = [[] for x in g.timestamps ] #populated by the fixation detector plugin #add plugins g.plugins = Plugin_List(g, plugin_by_name, plugin_initializers) while frames_to_export.value - current_frame.value > 0: if should_terminate.value: logger.warning("User aborted export. Exported %s frames to %s." % (current_frame.value, out_file_path)) #explicit release of VideoWriter writer.close() writer = None return False try: frame = cap.get_frame_nowait() except EndofVideoFileError: break events = {} #new positons and events events['gaze_positions'] = g.gaze_positions_by_frame[frame.index] events['pupil_positions'] = g.pupil_positions_by_frame[frame.index] # allow each Plugin to do its work. for p in g.plugins: p.update(frame, events) writer.write_video_frame(frame) current_frame.value += 1 writer.close() writer = None duration = time() - start_time effective_fps = float(current_frame.value) / duration logger.info( "Export done: Exported %s frames to %s. This took %s seconds. Exporter ran at %s frames per second" % (current_frame.value, out_file_path, duration, effective_fps)) return True
#cd /home/gazetracker/pupil/pupil_v0.9.14-7_linux_x64/pupil/pupil_src/shared_modules from file_methods import save_object, load_object import os import numpy ##check dictionary pupil_data = load_object( os.path.join("/home/gazetracker/recordings/2018_02_17/006_95%", "pupil_data")) #print(pupil_data.keys()) #dict_keys(['gaze_positions', 'fixations', 'notifications', 'pupil_positions']) gaze = pupil_data['gaze_positions'] #print(gaze[0].keys()) #dict_keys(['norm_pos', 'gaze_point_3d', 'topic', 'base_data', 'timestamp', 'gaze_normals_3d', 'eye_centers_3d', 'confidence']) ##extract & save gaze_pose from dictionary norm = [] i = 0 time = [] while (i < len(gaze)): time.append(gaze[i]['timestamp']) norm.append(gaze[i]['norm_pos']) i = i + 1 numpy.save("/home/gazetracker/recordings/2018_02_17/006_95%/norm_pos.npy", norm) numpy.save("/home/gazetracker/recordings/2018_02_17/006_95%/pos_timestamp.npy", time) ##load gaze_pose norm_pos = numpy.load( "/home/gazetracker/recordings/2018_02_17/006_95%/norm_pos.npy") #(6812, 2)
def check_for_worldless_recording(rec_dir): logger.info("Checking for world-less recording") valid_ext = (".mp4", ".mkv", ".avi", ".h264", ".mjpeg") world_video_exists = any( ( os.path.splitext(f)[1] in valid_ext for f in glob.glob(os.path.join(rec_dir, "world.*")) ) ) if not world_video_exists: fake_world_version = 1 fake_world_path = os.path.join(rec_dir, "world.fake") if os.path.exists(fake_world_path): fake_world = fm.load_object(fake_world_path) if fake_world["version"] == fake_world_version: return logger.warning("No world video found. Constructing an artificial replacement.") eye_ts_files = glob.glob(os.path.join(rec_dir, "eye*_timestamps.npy")) min_ts = np.inf max_ts = -np.inf for f in eye_ts_files: try: eye_ts = np.load(f) assert len(eye_ts.shape) == 1 assert eye_ts.shape[0] > 1 min_ts = min(min_ts, eye_ts[0]) max_ts = max(max_ts, eye_ts[-1]) except AssertionError: logger.debug( ( "Ignoring {} since it does not conform with the expected format" " (one-dimensional list with at least two entries)".format(f) ) ) assert -np.inf < min_ts < max_ts < np.inf, ( "This recording is invalid because it does not contain any valid eye timestamp" " files from which artifical world timestamps could be generated from." ) frame_rate = 30 timestamps = np.arange(min_ts, max_ts, 1 / frame_rate) np.save(os.path.join(rec_dir, "world_timestamps.npy"), timestamps) fm.save_object( { "frame_rate": frame_rate, "frame_size": (1280, 720), "version": fake_world_version, }, os.path.join(rec_dir, "world.fake"), ) lookup_entry = np.dtype( [ ("container_idx", "<i8"), ("container_frame_idx", "<i8"), ("timestamp", "<f8"), ] ) lookup = np.empty(timestamps.size, dtype=lookup_entry).view(np.recarray) lookup.timestamp = timestamps lookup.container_idx = -1 np.save(os.path.join(rec_dir, "world_lookup.npy"), lookup)
def export(should_terminate, frames_to_export, current_frame, rec_dir, user_dir, min_data_confidence, start_frame=None, end_frame=None, plugin_initializers=(), out_file_path=None): vis_plugins = sorted([ Vis_Circle, Vis_Cross, Vis_Polyline, Vis_Light_Points, Vis_Watermark, Scan_Path ], key=lambda x: x.__name__) analysis_plugins = sorted([ Manual_Gaze_Correction, Eye_Video_Overlay, Pupil_Angle_3D_Fixation_Detector, Gaze_Position_2D_Fixation_Detector ], key=lambda x: x.__name__) user_plugins = sorted(import_runtime_plugins( os.path.join(user_dir, 'plugins')), key=lambda x: x.__name__) available_plugins = vis_plugins + analysis_plugins + user_plugins name_by_index = [p.__name__ for p in available_plugins] index_by_name = dict(zip(name_by_index, range(len(name_by_index)))) plugin_by_name = dict(zip(name_by_index, available_plugins)) logger = logging.getLogger(__name__ + ' with pid: ' + str(os.getpid())) update_recording_to_recent(rec_dir) video_path = [ f for f in glob(os.path.join(rec_dir, "world.*")) if f[-3:] in ('mp4', 'mkv', 'avi') ][0] timestamps_path = os.path.join(rec_dir, "world_timestamps.npy") pupil_data_path = os.path.join(rec_dir, "pupil_data") meta_info = load_meta_info(rec_dir) rec_version = read_rec_version(meta_info) g_pool = Global_Container() g_pool.app = 'exporter' g_pool.min_data_confidence = min_data_confidence timestamps = np.load(timestamps_path) cap = File_Source(g_pool, video_path, timestamps=timestamps) #Out file path verification, we do this before but if one uses a seperate tool, this will kick in. if out_file_path is None: out_file_path = os.path.join(rec_dir, "world_viz.mp4") else: file_name = os.path.basename(out_file_path) dir_name = os.path.dirname(out_file_path) if not dir_name: dir_name = rec_dir if not file_name: file_name = 'world_viz.mp4' out_file_path = os.path.expanduser(os.path.join(dir_name, file_name)) if os.path.isfile(out_file_path): logger.warning("Video out file already exsists. I will overwrite!") os.remove(out_file_path) logger.debug("Saving Video to %s" % out_file_path) #Trim mark verification #make sure the trim marks (start frame, endframe) make sense: We define them like python list slices,thus we can test them like such. trimmed_timestamps = timestamps[start_frame:end_frame] if len(trimmed_timestamps) == 0: logger.warn( "Start and end frames are set such that no video will be exported." ) return False if start_frame == None: start_frame = 0 #these two vars are shared with the lauching process and give a job length and progress report. frames_to_export.value = len(trimmed_timestamps) current_frame.value = 0 logger.debug( "Will export from frame %s to frame %s. This means I will export %s frames." % (start_frame, start_frame + frames_to_export.value, frames_to_export.value)) #setup of writer writer = AV_Writer(out_file_path, fps=cap.frame_rate, use_timestamps=True) cap.seek_to_frame(start_frame) start_time = time() g_pool.capture = cap g_pool.rec_dir = rec_dir g_pool.user_dir = user_dir g_pool.rec_version = rec_version g_pool.timestamps = timestamps g_pool.delayed_notifications = {} g_pool.notifications = [] # load pupil_positions, gaze_positions pupil_data = load_object(pupil_data_path) pupil_list = pupil_data['pupil_positions'] gaze_list = pupil_data['gaze_positions'] g_pool.pupil_positions_by_frame = correlate_data(pupil_list, g_pool.timestamps) g_pool.gaze_positions_by_frame = correlate_data(gaze_list, g_pool.timestamps) g_pool.fixations_by_frame = [[] for x in g_pool.timestamps ] #populated by the fixation detector plugin #add plugins g_pool.plugins = Plugin_List(g_pool, plugin_by_name, plugin_initializers) while frames_to_export.value > current_frame.value: if should_terminate.value: logger.warning("User aborted export. Exported %s frames to %s." % (current_frame.value, out_file_path)) #explicit release of VideoWriter writer.close() writer = None return False try: frame = cap.get_frame_nowait() except EndofVideoFileError: break events = {} #new positons and events events['gaze_positions'] = g_pool.gaze_positions_by_frame[frame.index] events['pupil_positions'] = g_pool.pupil_positions_by_frame[ frame.index] # publish delayed notifiactions when their time has come. for n in g_pool.delayed_notifications.values(): if n['_notify_time_'] < time(): del n['_notify_time_'] del g_pool.delayed_notifications[n['subject']] g_pool.notifications.append(n) # notify each plugin if there are new notifactions: while g_pool.notifications: n = g_pool.notifications.pop(0) for p in g_pool.plugins: p.on_notify(n) # allow each Plugin to do its work. for p in g_pool.plugins: p.update(frame, events) writer.write_video_frame(frame) current_frame.value += 1 writer.close() writer = None duration = time() - start_time effective_fps = float(current_frame.value) / duration logger.info( "Export done: Exported %s frames to %s. This took %s seconds. Exporter ran at %s frames per second" % (current_frame.value, out_file_path, duration, effective_fps)) return True
def start(self): self.data = { 'pupil_positions': [], 'gaze_positions': [], 'notifications': [] } self.frame_count = 0 self.running = True self.menu.read_only = True self.start_time = time() session = os.path.join(self.rec_dir, self.session_name) try: os.makedirs(session) logger.debug( "Created new recordings session dir {}".format(session)) except: logger.debug( "Recordings session dir {} already exists, using it.".format( session)) if self.rec_path is None: # set up self incrementing folder within session folder counter = 0 while True: self.rec_path = os.path.join(session, "{:03d}/".format(counter)) try: os.mkdir(self.rec_path) logger.debug("Created new recording dir {}".format( self.rec_path)) break except: logger.debug( "We dont want to overwrite data, incrementing counter & trying to make new data folder" ) counter += 1 self.meta_info_path = os.path.join(self.rec_path, "info.csv") with open(self.meta_info_path, 'w', newline='') as csvfile: csv_utils.write_key_value_file( csvfile, { 'Recording Name': self.session_name, 'Start Date': strftime("%d.%m.%Y", localtime(self.start_time)), 'Start Time': strftime("%H:%M:%S", localtime(self.start_time)) }) self.video_path = os.path.join(self.rec_path, "world.mp4") if self.raw_jpeg and self.g_pool.capture.jpeg_support: self.writer = JPEG_Writer(self.video_path, self.g_pool.capture.frame_rate) elif hasattr(self.g_pool.capture._recent_frame, 'h264_buffer'): self.writer = H264Writer(self.video_path, self.g_pool.capture.frame_size[0], self.g_pool.capture.frame_size[1], int(self.g_pool.capture.frame_rate)) else: self.writer = AV_Writer(self.video_path, fps=self.g_pool.capture.frame_rate) try: cal_pt_path = os.path.join(self.g_pool.user_dir, "user_calibration_data") cal_data = load_object(cal_pt_path) notification = { 'subject': 'calibration.calibration_data', 'record': True } notification.update(cal_data) self.data['notifications'].append(notification) except: pass if self.show_info_menu: self.open_info_menu() logger.info("Started Recording.") self.notify_all({ 'subject': 'recording.started', 'rec_path': self.rec_path, 'session_name': self.session_name, 'record_eye': self.record_eye, 'compression': self.raw_jpeg })
def start(self): self.start_time = time() start_time_synced = self.g_pool.get_timestamp() if isinstance(self.g_pool.capture, NDSI_Source): # If the user did not enable TimeSync, the timestamps will be way off and # the recording code will crash. We check the difference between the last # frame's time and the start_time_synced and if this does not match, we stop # the recording and show a warning instead. TIMESTAMP_ERROR_THRESHOLD = 5.0 frame = self.g_pool.capture._recent_frame if frame is None: logger.error( "Your connection does not seem to be stable enough for " "recording Pupil Mobile via WiFi. We recommend recording " "on the phone.") return if abs(frame.timestamp - start_time_synced) > TIMESTAMP_ERROR_THRESHOLD: logger.error( "Pupil Mobile stream is not in sync. Aborting recording." " Enable the Time Sync plugin and try again.") return session = os.path.join(self.rec_root_dir, self.session_name) try: os.makedirs(session, exist_ok=True) logger.debug( "Created new recordings session dir {}".format(session)) except OSError: logger.error( "Could not start recording. Session dir {} not writable.". format(session)) return self.pldata_writers = {} self.frame_count = 0 self.running = True self.menu.read_only = True recording_uuid = uuid.uuid4() # set up self incrementing folder within session folder counter = 0 while True: self.rec_path = os.path.join(session, "{:03d}/".format(counter)) try: os.mkdir(self.rec_path) logger.debug("Created new recording dir {}".format( self.rec_path)) break except FileExistsError: logger.debug( "We dont want to overwrite data, incrementing counter & trying to make new data folder" ) counter += 1 self.meta_info = RecordingInfoFile.create_empty_file(self.rec_path) self.meta_info.recording_software_name = ( RecordingInfoFile.RECORDING_SOFTWARE_NAME_PUPIL_CAPTURE) self.meta_info.recording_software_version = str(self.g_pool.version) self.meta_info.recording_name = self.session_name self.meta_info.start_time_synced_s = start_time_synced self.meta_info.start_time_system_s = self.start_time self.meta_info.recording_uuid = recording_uuid self.meta_info.system_info = get_system_info() self.video_path = os.path.join(self.rec_path, "world.mp4") if self.raw_jpeg and self.g_pool.capture.jpeg_support: self.writer = JPEG_Writer(self.video_path, start_time_synced) elif hasattr(self.g_pool.capture._recent_frame, "h264_buffer"): self.writer = H264Writer( self.video_path, self.g_pool.capture.frame_size[0], self.g_pool.capture.frame_size[1], int(self.g_pool.capture.frame_rate), ) else: self.writer = MPEG_Writer(self.video_path, start_time_synced) calibration_data_notification_classes = [ CalibrationSetupNotification, CalibrationResultNotification, ] writer = PLData_Writer(self.rec_path, "notify") for note_class in calibration_data_notification_classes: try: file_path = os.path.join(self.g_pool.user_dir, note_class.file_name()) note = note_class.from_dict(load_object(file_path)) note_dict = note.as_dict() note_dict["topic"] = "notify." + note_dict["subject"] writer.append(note_dict) except FileNotFoundError: continue self.pldata_writers["notify"] = writer if self.show_info_menu: self.open_info_menu() logger.info("Started Recording.") self.notify_all({ "subject": "recording.started", "rec_path": self.rec_path, "session_name": self.session_name, "record_eye": self.record_eye, "compression": self.raw_jpeg, "start_time_synced": float(start_time_synced), })
for p in points: cam1_points.append(p) noise = 0 #randomize point in eye space pr = p #+ np.array( (uniform(-noise,+noise),uniform(-noise,+noise),uniform(-noise,+noise)) ) p2 = toEye(pr) # to cam2 coordinate system # print p,p2,toWorld(p2) # p2 *= 1.2,1.0,1.0 cam2_points.append(p2) cam1_observation = [p / np.linalg.norm(p) for p in cam1_points] cam2_observation = [p / np.linalg.norm(p) for p in cam2_points] else: #load real data: import file_methods cam1_observation, cam2_observation = file_methods.load_object( 'testdata') #inital guess though rigid transform initial_R, initial_t = find_rigid_transform(np.array(cam2_observation), np.array(cam1_observation)) initial_rotation_quaternion = math_helper.quaternion_from_rotation_matrix( initial_R) initial_translation = np.array(initial_t).reshape(3) if synth_data: initial_translation *= np.linalg.norm(cam2_center) / np.linalg.norm( initial_translation) # setup bundle adjustment o1 = {
def player(rec_dir, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir, app_version): # general imports from time import sleep import logging import errno from glob import glob from time import time # networking import zmq import zmq_tools import numpy as np # zmq ipc setup zmq_ctx = zmq.Context() ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url) notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=('notify', )) # log setup logging.getLogger("OpenGL").setLevel(logging.ERROR) logger = logging.getLogger() logger.handlers = [] logger.setLevel(logging.INFO) logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url)) # create logger for the context of this function logger = logging.getLogger(__name__) try: # imports from file_methods import Persistent_Dict, load_object # display import glfw # check versions for our own depedencies as they are fast-changing from pyglui import __version__ as pyglui_version from pyglui import ui, cygl from pyglui.cygl.utils import Named_Texture, RGBA import gl_utils # capture from video_capture import File_Source, EndofVideoFileError # helpers/utils from version_utils import VersionFormat from methods import normalize, denormalize, delta_t, get_system_info from player_methods import correlate_data, is_pupil_rec_dir, load_meta_info # Plug-ins from plugin import Plugin, Plugin_List, import_runtime_plugins from plugin_manager import Plugin_Manager from vis_circle import Vis_Circle from vis_cross import Vis_Cross from vis_polyline import Vis_Polyline from vis_light_points import Vis_Light_Points from vis_watermark import Vis_Watermark from vis_fixation import Vis_Fixation from vis_scan_path import Vis_Scan_Path from vis_eye_video_overlay import Vis_Eye_Video_Overlay from seek_control import Seek_Control from video_export_launcher import Video_Export_Launcher from offline_surface_tracker import Offline_Surface_Tracker # from marker_auto_trim_marks import Marker_Auto_Trim_Marks from fixation_detector import Offline_Fixation_Detector from batch_exporter import Batch_Exporter, Batch_Export from log_display import Log_Display from annotations import Annotation_Player from raw_data_exporter import Raw_Data_Exporter from log_history import Log_History from pupil_producers import Pupil_From_Recording, Offline_Pupil_Detection from gaze_producers import Gaze_From_Recording, Offline_Calibration from system_graphs import System_Graphs from system_timelines import System_Timelines from blink_detection import Offline_Blink_Detection assert VersionFormat(pyglui_version) >= VersionFormat( '1.17'), 'pyglui out of date, please upgrade to newest version' runtime_plugins = import_runtime_plugins( os.path.join(user_dir, 'plugins')) system_plugins = [ Log_Display, Seek_Control, Plugin_Manager, System_Graphs, Batch_Export, System_Timelines ] user_plugins = [ Vis_Circle, Vis_Fixation, Vis_Polyline, Vis_Light_Points, Vis_Cross, Vis_Watermark, Vis_Eye_Video_Overlay, Vis_Scan_Path, Offline_Fixation_Detector, Offline_Blink_Detection, Batch_Exporter, Video_Export_Launcher, Offline_Surface_Tracker, Raw_Data_Exporter, Annotation_Player, Log_History, Pupil_From_Recording, Offline_Pupil_Detection, Gaze_From_Recording, Offline_Calibration ] + runtime_plugins plugins = system_plugins + user_plugins # Callback functions def on_resize(window, w, h): nonlocal window_size nonlocal hdpi_factor hdpi_factor = float( glfw.glfwGetFramebufferSize(window)[0] / glfw.glfwGetWindowSize(window)[0]) g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor window_size = w, h g_pool.camera_render_size = w - int( icon_bar_width * g_pool.gui.scale), h g_pool.gui.update_window(*window_size) g_pool.gui.collect_menus() for p in g_pool.plugins: p.on_window_resize(window, *g_pool.camera_render_size) def on_window_key(window, key, scancode, action, mods): g_pool.gui.update_key(key, scancode, action, mods) def on_window_char(window, char): g_pool.gui.update_char(char) def on_window_mouse_button(window, button, action, mods): g_pool.gui.update_button(button, action, mods) def on_pos(window, x, y): x, y = x * hdpi_factor, y * hdpi_factor g_pool.gui.update_mouse(x, y) pos = x, y pos = normalize(pos, g_pool.camera_render_size) # Position in img pixels pos = denormalize(pos, g_pool.capture.frame_size) for p in g_pool.plugins: p.on_pos(pos) def on_scroll(window, x, y): g_pool.gui.update_scroll(x, y * scroll_factor) def on_drop(window, count, paths): for x in range(count): new_rec_dir = paths[x].decode('utf-8') if is_pupil_rec_dir(new_rec_dir): logger.debug( "Starting new session with '{}'".format(new_rec_dir)) ipc_pub.notify({ "subject": "player_drop_process.should_start", "rec_dir": new_rec_dir }) glfw.glfwSetWindowShouldClose(window, True) else: logger.error("'{}' is not a valid pupil recording".format( new_rec_dir)) tick = delta_t() def get_dt(): return next(tick) video_path = [ f for f in glob(os.path.join(rec_dir, "world.*")) if os.path.splitext(f)[1] in ('.mp4', '.mkv', '.avi', '.h264', '.mjpeg') ][0] pupil_data_path = os.path.join(rec_dir, "pupil_data") meta_info = load_meta_info(rec_dir) # log info about Pupil Platform and Platform in player.log logger.info('Application Version: {}'.format(app_version)) logger.info('System Info: {}'.format(get_system_info())) icon_bar_width = 50 window_size = None hdpi_factor = 1.0 # create container for globally scoped vars g_pool = Global_Container() g_pool.app = 'player' g_pool.zmq_ctx = zmq_ctx g_pool.ipc_pub = ipc_pub g_pool.ipc_pub_url = ipc_pub_url g_pool.ipc_sub_url = ipc_sub_url g_pool.ipc_push_url = ipc_push_url g_pool.plugin_by_name = {p.__name__: p for p in plugins} g_pool.camera_render_size = None # sets itself to g_pool.capture File_Source(g_pool, video_path) # load session persistent settings session_settings = Persistent_Dict( os.path.join(user_dir, "user_settings_player")) if VersionFormat(session_settings.get("version", '0.0')) != app_version: logger.info( "Session setting are a different version of this app. I will not use those." ) session_settings.clear() g_pool.capture.playback_speed = session_settings.get( 'playback_speed', 1.) width, height = session_settings.get('window_size', g_pool.capture.frame_size) window_pos = session_settings.get('window_position', window_position_default) glfw.glfwInit() main_window = glfw.glfwCreateWindow( width, height, "Pupil Player: " + meta_info["Recording Name"] + " - " + rec_dir.split(os.path.sep)[-1], None, None) glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1]) glfw.glfwMakeContextCurrent(main_window) cygl.utils.init() g_pool.main_window = main_window def set_scale(new_scale): g_pool.gui_user_scale = new_scale window_size = ( g_pool.camera_render_size[0] + int(icon_bar_width * g_pool.gui_user_scale * hdpi_factor), glfw.glfwGetFramebufferSize(main_window)[1]) logger.warning(icon_bar_width * g_pool.gui_user_scale * hdpi_factor) glfw.glfwSetWindowSize(main_window, *window_size) # load pupil_positions, gaze_positions g_pool.pupil_data = load_object(pupil_data_path) g_pool.binocular = meta_info.get('Eye Mode', 'monocular') == 'binocular' g_pool.version = app_version g_pool.timestamps = g_pool.capture.timestamps g_pool.get_timestamp = lambda: 0. g_pool.new_seek = True g_pool.user_dir = user_dir g_pool.rec_dir = rec_dir g_pool.meta_info = meta_info g_pool.min_data_confidence = session_settings.get( 'min_data_confidence', 0.6) g_pool.pupil_positions = [] g_pool.gaze_positions = [] g_pool.fixations = [] g_pool.notifications_by_frame = correlate_data( g_pool.pupil_data['notifications'], g_pool.timestamps) g_pool.pupil_positions_by_frame = [[] for x in g_pool.timestamps ] # populated by producer` g_pool.gaze_positions_by_frame = [[] for x in g_pool.timestamps ] # populated by producer g_pool.fixations_by_frame = [ [] for x in g_pool.timestamps ] # populated by the fixation detector plugin def set_data_confidence(new_confidence): g_pool.min_data_confidence = new_confidence notification = {'subject': 'min_data_confidence_changed'} notification['_notify_time_'] = time() + .8 g_pool.ipc_pub.notify(notification) def open_plugin(plugin): if plugin == "Select to load": return g_pool.plugins.add(plugin) def purge_plugins(): for p in g_pool.plugins: if p.__class__ in user_plugins: p.alive = False g_pool.plugins.clean() def do_export(_): export_range = g_pool.seek_control.trim_left, g_pool.seek_control.trim_right export_dir = os.path.join(g_pool.rec_dir, 'exports', '{}-{}'.format(*export_range)) try: os.makedirs(export_dir) except OSError as e: if e.errno != errno.EEXIST: logger.error("Could not create export dir") raise e else: overwrite_warning = "Previous export for range [{}-{}] already exists - overwriting." logger.warning(overwrite_warning.format(*export_range)) else: logger.info('Created export dir at "{}"'.format(export_dir)) notification = { 'subject': 'should_export', 'range': export_range, 'export_dir': export_dir } g_pool.ipc_pub.notify(notification) def reset_restart(): logger.warning("Resetting all settings and restarting Player.") glfw.glfwSetWindowShouldClose(main_window, True) ipc_pub.notify({'subject': 'clear_settings_process.should_start'}) ipc_pub.notify({ 'subject': 'player_process.should_start', 'rec_dir': rec_dir, 'delay': 2. }) def toggle_general_settings(collapsed): # this is the menu toggle logic. # Only one menu can be open. # If no menu is open the menubar should collapse. g_pool.menubar.collapsed = collapsed for m in g_pool.menubar.elements: m.collapsed = True general_settings.collapsed = collapsed g_pool.gui = ui.UI() g_pool.gui_user_scale = session_settings.get('gui_scale', 1.) g_pool.menubar = ui.Scrolling_Menu("Settings", pos=(-500, 0), size=(-icon_bar_width, 0), header_pos='left') g_pool.iconbar = ui.Scrolling_Menu("Icons", pos=(-icon_bar_width, 0), size=(0, 0), header_pos='hidden') g_pool.timelines = ui.Container((0, 0), (0, 0), (0, 0)) g_pool.timelines.horizontal_constraint = g_pool.menubar g_pool.user_timelines = ui.Timeline_Menu('User Timelines', pos=(0., -150.), size=(0., 0.), header_pos='headline') g_pool.user_timelines.color = RGBA(a=0.) g_pool.user_timelines.collapsed = True # add container that constaints itself to the seekbar height vert_constr = ui.Container((0, 0), (0, -50.), (0, 0)) vert_constr.append(g_pool.user_timelines) g_pool.timelines.append(vert_constr) general_settings = ui.Growing_Menu('General', header_pos='headline') general_settings.append( ui.Button( 'Reset window size', lambda: glfw.glfwSetWindowSize( main_window, g_pool.capture.frame_size[0], g_pool.capture. frame_size[1]))) general_settings.append( ui.Selector('gui_user_scale', g_pool, setter=set_scale, selection=[.8, .9, 1., 1.1, 1.2] + list(np.arange(1.5, 5.1, .5)), label='Interface Size')) general_settings.append( ui.Info_Text('Player Version: {}'.format(g_pool.version))) general_settings.append( ui.Info_Text('Capture Version: {}'.format( meta_info['Capture Software Version']))) general_settings.append( ui.Info_Text('Data Format Version: {}'.format( meta_info['Data Format Version']))) general_settings.append( ui.Slider('min_data_confidence', g_pool, setter=set_data_confidence, step=.05, min=0.0, max=1.0, label='Confidence threshold')) general_settings.append( ui.Button('Restart with default settings', reset_restart)) g_pool.menubar.append(general_settings) icon = ui.Icon('collapsed', general_settings, label=chr(0xe8b8), on_val=False, off_val=True, setter=toggle_general_settings, label_font='pupil_icons') icon.tooltip = 'General Settings' g_pool.iconbar.append(icon) user_plugin_separator = ui.Separator() user_plugin_separator.order = 0.35 g_pool.iconbar.append(user_plugin_separator) g_pool.quickbar = ui.Stretching_Menu('Quick Bar', (0, 100), (100, -100)) g_pool.export_button = ui.Thumb('export', label=chr(0xe2c5), getter=lambda: False, setter=do_export, hotkey='e', label_font='pupil_icons') g_pool.quickbar.extend([g_pool.export_button]) g_pool.gui.append(g_pool.menubar) g_pool.gui.append(g_pool.timelines) g_pool.gui.append(g_pool.iconbar) g_pool.gui.append(g_pool.quickbar) # we always load these plugins default_plugins = [('Plugin_Manager', {}), ('Seek_Control', {}), ('Log_Display', {}), ('Raw_Data_Exporter', {}), ('Vis_Polyline', {}), ('Vis_Circle', {}), ('System_Graphs', {}), ('System_Timelines', {}), ('Video_Export_Launcher', {}), ('Pupil_From_Recording', {}), ('Gaze_From_Recording', {})] g_pool.plugins = Plugin_List( g_pool, session_settings.get('loaded_plugins', default_plugins)) # Register callbacks main_window glfw.glfwSetFramebufferSizeCallback(main_window, on_resize) glfw.glfwSetKeyCallback(main_window, on_window_key) glfw.glfwSetCharCallback(main_window, on_window_char) glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button) glfw.glfwSetCursorPosCallback(main_window, on_pos) glfw.glfwSetScrollCallback(main_window, on_scroll) glfw.glfwSetDropCallback(main_window, on_drop) toggle_general_settings(True) g_pool.gui.configuration = session_settings.get('ui_config', {}) # gl_state settings gl_utils.basic_gl_setup() g_pool.image_tex = Named_Texture() # trigger on_resize on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window)) def handle_notifications(n): subject = n['subject'] if subject == 'start_plugin': g_pool.plugins.add(g_pool.plugin_by_name[n['name']], args=n.get('args', {})) elif subject.startswith('meta.should_doc'): ipc_pub.notify({ 'subject': 'meta.doc', 'actor': g_pool.app, 'doc': player.__doc__ }) for p in g_pool.plugins: if (p.on_notify.__doc__ and p.__class__.on_notify != Plugin.on_notify): ipc_pub.notify({ 'subject': 'meta.doc', 'actor': p.class_name, 'doc': p.on_notify.__doc__ }) while not glfw.glfwWindowShouldClose(main_window): # fetch newest notifications new_notifications = [] while notify_sub.new_data: t, n = notify_sub.recv() new_notifications.append(n) # notify each plugin if there are new notifications: for n in new_notifications: handle_notifications(n) for p in g_pool.plugins: p.on_notify(n) # grab new frame if g_pool.capture.play or g_pool.new_seek: g_pool.new_seek = False try: new_frame = g_pool.capture.get_frame() except EndofVideoFileError: # end of video logic: pause at last frame. g_pool.capture.play = False logger.warning("end of video") frame = new_frame.copy() events = {} events['frame'] = frame # report time between now and the last loop interation events['dt'] = get_dt() # pupil and gaze positions are added by their respective producer plugins events['pupil_positions'] = [] events['gaze_positions'] = [] # allow each Plugin to do its work. for p in g_pool.plugins: p.recent_events(events) # check if a plugin need to be destroyed g_pool.plugins.clean() glfw.glfwMakeContextCurrent(main_window) # render visual feedback from loaded plugins if gl_utils.is_window_visible(main_window): gl_utils.glViewport(0, 0, *g_pool.camera_render_size) g_pool.capture._recent_frame = frame g_pool.capture.gl_display() for p in g_pool.plugins: p.gl_display() gl_utils.glViewport(0, 0, *window_size) try: clipboard = glfw.glfwGetClipboardString( main_window).decode() except AttributeError: # clipbaord is None, might happen on startup clipboard = '' g_pool.gui.update_clipboard(clipboard) user_input = g_pool.gui.update() if user_input.clipboard and user_input.clipboard != clipboard: # only write to clipboard if content changed glfw.glfwSetClipboardString(main_window, user_input.clipboard.encode()) for b in user_input.buttons: button, action, mods = b x, y = glfw.glfwGetCursorPos(main_window) pos = x * hdpi_factor, y * hdpi_factor pos = normalize(pos, g_pool.camera_render_size) pos = denormalize(pos, g_pool.capture.frame_size) for p in g_pool.plugins: p.on_click(pos, button, action) for key, scancode, action, mods in user_input.keys: for p in g_pool.plugins: p.on_key(key, scancode, action, mods) for char_ in user_input.chars: for p in g_pool.plugins: p.on_char(char_) glfw.glfwSwapBuffers(main_window) # present frames at appropriate speed g_pool.capture.wait(frame) glfw.glfwPollEvents() session_settings['playback_speed'] = g_pool.capture.playback_speed session_settings['loaded_plugins'] = g_pool.plugins.get_initializers() session_settings['min_data_confidence'] = g_pool.min_data_confidence session_settings['gui_scale'] = g_pool.gui_user_scale session_settings['ui_config'] = g_pool.gui.configuration session_settings['window_size'] = glfw.glfwGetWindowSize(main_window) session_settings['window_position'] = glfw.glfwGetWindowPos( main_window) session_settings['version'] = str(g_pool.version) session_settings.close() # de-init all running plugins for p in g_pool.plugins: p.alive = False g_pool.plugins.clean() g_pool.capture.cleanup() g_pool.gui.terminate() glfw.glfwDestroyWindow(main_window) except: import traceback trace = traceback.format_exc() logger.error('Process Player crashed with trace:\n{}'.format(trace)) finally: logger.info("Process shutting down.") ipc_pub.notify({'subject': 'player_process.stopped'}) sleep(1.0)