def create_atb_bar(self, pos):
# add uvc camera controls to a separate ATB bar
size = (200, 200)
self.bar = atb.Bar(
name="Camera",
label=self.name,
help="UVC Camera Controls",
color=(50, 50, 50),
alpha=100,
text="light",
position=pos,
refresh=2.0,
size=size,
)
cameras_enum = atb.enum("Capture", dict([(c.name, c.src_id) for c in Camera_List()]))
self.bar.add_var("Capture", vtype=cameras_enum, getter=lambda: self.src_id, setter=self.re_init_cam_by_src_id)
self.bar.add_var(
"framerate",
vtype=atb.enum("framerate", self.capture.rates_menu),
getter=lambda: self.capture.current_rate_idx,
setter=self.capture.set_rate_idx,
)
sorted_controls = [c for c in self.controls.itervalues()]
sorted_controls.sort(key=lambda c: c.order)
for control in sorted_controls:
name = control.atb_name
if control.type == "bool":
self.bar.add_var(name, vtype=atb.TW_TYPE_BOOL8, getter=control.get_val, setter=control.set_val)
elif control.type == "int":
self.bar.add_var(name, vtype=atb.TW_TYPE_INT32, getter=control.get_val, setter=control.set_val)
self.bar.define(definition="min=" + str(control.min), varname=name)
self.bar.define(definition="max=" + str(control.max), varname=name)
self.bar.define(definition="step=" + str(control.step), varname=name)
elif control.type == "menu":
if control.menu is None:
vtype = None
else:
vtype = atb.enum(name, control.menu)
self.bar.add_var(name, vtype=vtype, getter=control.get_val, setter=control.set_val)
if control.menu is None:
self.bar.define(definition="min=" + str(control.min), varname=name)
self.bar.define(definition="max=" + str(control.max), varname=name)
self.bar.define(definition="step=" + str(control.step), varname=name)
else:
pass
if control.flags == "inactive":
pass
# self.bar.define(definition='readonly=1',varname=control.name)
self.bar.add_button("refresh", self.controls.update_from_device)
self.bar.add_button("load defaults", self.controls.load_defaults)
return size
def create_atb_bar(self,pos):
# add uvc camera controls to a separate ATB bar
size = (200,200)
self.bar = atb.Bar(name="Camera", label=self.name,
help="UVC Camera Controls", color=(50,50,50), alpha=100,
text='light',position=pos,refresh=2., size=size)
cameras_enum = atb.enum("Capture",dict([(c.name,c.src_id) for c in Camera_List()]) )
self.bar.add_var("Capture",vtype=cameras_enum,getter=lambda:self.src_id, setter=self.re_init_cam_by_src_id)
self.bar.add_var('framerate', vtype = atb.enum('framerate',self.capture.rates_menu), getter = lambda:self.capture.current_rate_idx, setter=self.capture.set_rate_idx )
self.bar.add_var('exposure', vtype=ctypes.c_float, getter = self.capture.get_exposure_time, setter=self.capture.set_exposure_time)
self.bar.add_var('gain', vtype = ctypes.c_int, getter = self.capture.get_hardware_gain, setter=self.capture.set_hardware_gain)
self.bar.add_var('contrast', vtype = ctypes.c_int, getter = self.capture.get_contrast, setter=self.capture.set_contrast)
self.bar.add_var('brightness', vtype = ctypes.c_int, getter = self.capture.get_brightness, setter=self.capture.set_brightness)
self.bar.add_var('auto gain', vtype = atb.TW_TYPE_BOOL8, getter = self.capture.getAutoGain, setter=self.capture.setAutoGain )
self.bar.add_var('auto exposure', vtype = atb.TW_TYPE_BOOL8, getter = self.capture.getAutoExposure, setter=self.capture.setAutoExposure )
self.capture.set_rate_idx(self.capture.current_rate_idx)
# self.capture.SetFrameRate(30)
# sorted_controls = [c for c in self.controls.itervalues()]
# sorted_controls.sort(key=lambda c: c.order)
# for control in sorted_controls:
# name = control.atb_name
# if control.type=="bool":
# self.bar.add_var(name,vtype=atb.TW_TYPE_BOOL8,getter=control.get_val,setter=control.set_val)
# elif control.type=='int':
# self.bar.add_var(name,vtype=atb.TW_TYPE_INT32,getter=control.get_val,setter=control.set_val)
# self.bar.define(definition='min='+str(control.min), varname=name)
# self.bar.define(definition='max='+str(control.max), varname=name)
# self.bar.define(definition='step='+str(control.step), varname=name)
# elif control.type=="menu":
# if control.menu is None:
# vtype = None
# else:
# vtype= atb.enum(name,control.menu)
# self.bar.add_var(name,vtype=vtype,getter=control.get_val,setter=control.set_val)
# if control.menu is None:
# self.bar.define(definition='min='+str(control.min), varname=name)
# self.bar.define(definition='max='+str(control.max), varname=name)
# self.bar.define(definition='step='+str(control.step), varname=name)
# else:
# pass
# if control.flags == "inactive":
# pass
# if control.name == 'exposure_auto_priority':
# # the controll should always be off. we set it to 0 on init (see above)
# self.bar.define(definition='readonly=1',varname=control.name)
#self.bar.add_button("refresh",self.controls.update_from_device)
#self.bar.add_button("load defaults",self.controls.load_defaults)
return size
def create_atb_bar(self,pos):
# add uvc camera controls to a separate ATB bar
size = (200,200)
self.bar = atb.Bar(name="Camera", label=self.name,
help="UVC Camera Controls", color=(50,50,50), alpha=100,
text='light',position=pos,refresh=2., size=size)
cameras_enum = atb.enum("Capture",dict([(c.name,c.src_id) for c in Camera_List()]) )
self.bar.add_var("Capture",vtype=cameras_enum,getter=lambda:self.src_id, setter=self.re_init_cam_by_src_id)
self.bar.add_var('framerate', vtype = atb.enum('framerate',self.capture.rates_menu), getter = lambda:self.capture.current_rate_idx, setter=self.capture.set_rate_idx )
self.bar.add_var('hardware timestamps',vtype=atb.TW_TYPE_BOOL8,getter=lambda:self.use_hw_ts)
sorted_controls = [c for c in self.controls.itervalues()]
sorted_controls.sort(key=lambda c: c.order)
for control in sorted_controls:
name = control.atb_name
if control.type=="bool":
self.bar.add_var(name,vtype=atb.TW_TYPE_BOOL8,getter=control.get_val,setter=control.set_val)
elif control.type=='int':
self.bar.add_var(name,vtype=atb.TW_TYPE_INT32,getter=control.get_val,setter=control.set_val)
self.bar.define(definition='min='+str(control.min), varname=name)
self.bar.define(definition='max='+str(control.max), varname=name)
self.bar.define(definition='step='+str(control.step), varname=name)
elif control.type=="menu":
if control.menu is None:
vtype = None
else:
vtype= atb.enum(name,control.menu)
self.bar.add_var(name,vtype=vtype,getter=control.get_val,setter=control.set_val)
if control.menu is None:
self.bar.define(definition='min='+str(control.min), varname=name)
self.bar.define(definition='max='+str(control.max), varname=name)
self.bar.define(definition='step='+str(control.step), varname=name)
else:
pass
if control.flags == "inactive":
pass
if control.name == 'exposure_auto_priority':
# the controll should always be off. we set it to 0 on init (see above)
self.bar.define(definition='readonly=1',varname=control.name)
self.bar.add_button("refresh",self.controls.update_from_device)
self.bar.add_button("load defaults",self.controls.load_defaults)
return size
def create_atb_bar(self,pos):
# add uvc camera controls to a separate ATB bar
size = (200,200)
self.bar = atb.Bar(name="Camera", label=self.name,
help="UVC Camera Controls", color=(50,50,50), alpha=100,
text='light',position=pos,refresh=2., size=size)
cameras_enum = atb.enum("Capture",dict([(c.name,c.src_id) for c in Camera_List()]) )
self.bar.add_var("Capture",vtype=cameras_enum,getter=lambda:self.src_id, setter=self.re_init_cam_by_src_id)
self.bar.add_var('framerate', vtype = atb.enum('framerate',self.capture.rates_menu), getter = lambda:self.capture.current_rate_idx, setter=self.capture.set_rate_idx )
self.bar.add_var('hardware timestamps',vtype=atb.TW_TYPE_BOOL8,getter=lambda:self.use_hw_ts)
sorted_controls = [c for c in self.controls.itervalues()]
sorted_controls.sort(key=lambda c: c.order)
for control in sorted_controls:
name = control.atb_name
if control.type=="bool":
self.bar.add_var(name,vtype=atb.TW_TYPE_BOOL8,getter=control.get_val,setter=control.set_val)
elif control.type=='int':
self.bar.add_var(name,vtype=atb.TW_TYPE_INT32,getter=control.get_val,setter=control.set_val)
self.bar.define(definition='min='+str(control.min), varname=name)
self.bar.define(definition='max='+str(control.max), varname=name)
self.bar.define(definition='step='+str(control.step), varname=name)
elif control.type=="menu":
if control.menu is None:
vtype = None
else:
vtype= atb.enum(name,control.menu)
self.bar.add_var(name,vtype=vtype,getter=control.get_val,setter=control.set_val)
if control.menu is None:
self.bar.define(definition='min='+str(control.min), varname=name)
self.bar.define(definition='max='+str(control.max), varname=name)
self.bar.define(definition='step='+str(control.step), varname=name)
else:
pass
if control.flags == "inactive":
pass
if control.name == 'exposure_auto_priority':
# the controll should always be off. we set it to 0 on init (see above)
self.bar.define(definition='readonly=1',varname=control.name)
self.bar.add_button("refresh",self.controls.update_from_device)
self.bar.add_button("load defaults",self.controls.load_defaults)
return size
def __init__(self,g_pool,atb_pos=(0,0)):
Plugin.__init__(self)
self.collect_new = False
self.calculated = False
self.obj_grid = _gen_pattern_grid((4, 11))
self.img_points = []
self.obj_points = []
self.count = 10
self.img_shape = None
self.display_grid = _make_grid()
self.window_should_open = False
self.window_should_close = False
self._window = None
self.fullscreen = c_bool(0)
self.monitor_idx = c_int(0)
self.monitor_handles = glfwGetMonitors()
self.monitor_names = [glfwGetMonitorName(m) for m in self.monitor_handles]
monitor_enum = atb.enum("Monitor",dict(((key,val) for val,key in enumerate(self.monitor_names))))
#primary_monitor = glfwGetPrimaryMonitor()
atb_label = "estimate camera instrinsics"
# Creating an ATB Bar is required. Show at least some info about the Ref_Detector
self._bar = atb.Bar(name =self.__class__.__name__, label=atb_label,
help="ref detection parameters", color=(50, 50, 50), alpha=100,
text='light', position=atb_pos,refresh=.3, size=(300, 100))
self._bar.add_var("monitor",self.monitor_idx, vtype=monitor_enum)
self._bar.add_var("fullscreen", self.fullscreen)
self._bar.add_button(" show pattern ", self.do_open, key='c')
self._bar.add_button(" Capture Pattern", self.advance, key="SPACE")
self._bar.add_var("patterns to capture", getter=self.get_count)
def __init__(self, name, g_pool, bar_defs):
super(Bar, self).__init__(name, **bar_defs)
self.fps = c_float(0.0)
self.timestamp = time()
self.dt = c_float(0.0)
self.sleep = c_float(0.0)
self.display = c_int(1)
self.draw_pupil = c_bool(1)
self.draw_roi = c_int(0)
self.bin_thresh = c_int(60)
self.blur = c_int(3)
self.pupil_ratio = c_float(1.0)
self.pupil_angle = c_float(0.0)
self.pupil_size = c_float(80.0)
self.pupil_size_tolerance = c_float(40.0)
self.canny_aperture = c_int(5)
self.canny_thresh = c_int(200)
self.canny_ratio = c_int(2)
self.record_eye = c_bool(0)
# add class field here and it will become session persistant
self.session_save = {
"display": self.display,
"draw_pupil": self.draw_pupil,
"bin_thresh": self.bin_thresh,
"pupil_ratio": self.pupil_ratio,
"pupil_size": self.pupil_size,
"mean_blur": self.blur,
"canny_aperture": self.canny_aperture,
"canny_thresh": self.canny_thresh,
"canny_ratio": self.canny_ratio,
}
self.load()
dispay_mode_enum = atb.enum(
"Mode", {"Camera Image": 0, "Region of Interest": 1, "Egdes": 2, "Corse Pupil Region": 3}
)
self.add_var("Display/FPS", self.fps, step=1.0, readonly=True)
self.add_var("Display/SlowDown", self.sleep, step=0.01, min=0.0)
self.add_var("Display/Mode", self.display, vtype=dispay_mode_enum, help="select the view-mode")
self.add_var("Display/Show_Pupil_Point", self.draw_pupil)
self.add_button("Draw_ROI", self.roi, help="drag on screen to select a region of interest", Group="Display")
self.add_var("Pupil/Shade", self.bin_thresh, readonly=True)
self.add_var("Pupil/Ratio", self.pupil_ratio, readonly=True)
self.add_var("Pupil/Angle", self.pupil_angle, step=0.1, readonly=True)
self.add_var("Pupil/Size", self.pupil_size, readonly=True)
self.add_var("Pupil/Size_Tolerance", self.pupil_size_tolerance, step=1, min=0)
self.add_var("Canny/MeanBlur", self.blur, step=2, max=7, min=1)
self.add_var("Canny/Aperture", self.canny_aperture, step=2, max=7, min=3)
self.add_var("Canny/Lower_Threshold", self.canny_thresh, step=1, min=1)
self.add_var(
"Canny/LowerUpperRatio",
self.canny_ratio,
step=1,
min=0,
help="Canny recommended a ratio between 3/1 and 2/1",
)
self.add_var("record_eye_video", self.record_eye, help="when recording also save the eye video stream")
self.add_var("SaveSettings&Exit", g_pool.quit)
def __init__(self, g_pool, atb_pos=(0, 0)):
Plugin.__init__(self)
self.g_pool = g_pool
self.active = False
self.detected = False
self.screen_marker_state = 0
self.screen_marker_max = 70 # maximum bound for state
self.active_site = 0
self.sites = []
self.display_pos = None
self.on_position = False
self.candidate_ellipses = []
self.pos = None
self.show_edges = c_bool(0)
self.dist_threshold = c_int(5)
self.area_threshold = c_int(20)
self.world_size = None
self._window = None
self.window_should_close = False
self.window_should_open = False
self.fullscreen = c_bool(1)
self.monitor_idx = c_int(0)
monitor_handles = glfwGetMonitors()
self.monitor_names = [glfwGetMonitorName(m) for m in monitor_handles]
monitor_enum = atb.enum(
"Monitor",
dict(((key, val) for val, key in enumerate(self.monitor_names))))
#primary_monitor = glfwGetPrimaryMonitor()
self.frame = None
self.timebase = 0
atb_label = "calibrate on screen"
# Creating an ATB Bar is required. Show at least some info about the Ref_Detector
self._bar = atb.Bar(name=self.__class__.__name__,
label=atb_label,
help="ref detection parameters",
color=(50, 50, 50),
alpha=100,
text='light',
position=atb_pos,
refresh=.3,
size=(300, 90))
self._bar.add_var("monitor", self.monitor_idx, vtype=monitor_enum)
self._bar.add_var("fullscreen", self.fullscreen)
self._bar.add_button(" start calibrating ", self.start, key='c')
self._bar.add_var("show edges",
self.show_edges,
group="Detector Variables")
self._bar.add_var("area threshold",
self.area_threshold,
group="Detector Variables")
self._bar.add_var("eccetricity threshold",
self.dist_threshold,
group="Detector Variables")
def __init__(self,g_pool,atb_pos=(10,320)):
Plugin.__init__(self)
self.g_pool = g_pool
self.window_should_open = False
self.window_should_close = False
self._window = None
self.pupil_display_list = []
self.nb_frame = 0 #to include a little delay between 2 changement of state
self.nb_blink = 0 #number of consecutive blink
self.time_eye_c = 0 #time when the eye is closed
self.time_eye_o = 0 #time when the eye is opened
self.eye_open = False #1 if the eye is opened, else 0
#To get the state Value
def get_state(data):
return data.value
#ATB variables
self.state_enum = atb.enum("State",{"Stop":self.STOP, "Backward":self.BACKWARD,"Normal Speed":self.NORMAL_SPEED,"High Speed":self.HIGH_SPEED,"Left":self.LEFT,"Right":self.RIGHT})
self.control_mode_enum = atb.enum("Control Mode",{"OFF":self.OFF, "ON":self.ON})
self.gaze_x = c_float(0.0)
self.gaze_y = c_float(0.0)
self.state = c_int(0)
self.control_mode = c_int(0)
# Creating an ATB Bar.
atb_label = "drEYEve"
self._bar = atb.Bar(name =self.__class__.__name__, label=atb_label,
help="state of drEYEve project", color=(50, 50, 50), alpha=100,
text='light', position=atb_pos,refresh=.3, size=(300, 100))
self._bar.add_var("Current state", vtype=self.state_enum, getter=get_state, data=self.state, readonly=True)
self._bar.add_var("Control mode", vtype=self.control_mode_enum, getter=get_state, data=self.control_mode, readonly=True)
self._bar.add_var("Gaze X", self.gaze_x, readonly=True)
self._bar.add_var("Gaze Y", self.gaze_y, readonly=True)
def __init__(self,g_pool,atb_pos=(0,0)):
super(Marker_Detector, self).__init__()
# all markers that are detected in the most recent frame
self.markers = []
# all registered surfaces
self.surface_definitions = shelve.open(os.path.join(g_pool.user_dir,'surface_definitions'),protocol=2)
self.surfaces = [Reference_Surface(saved_definition=d) for d in self.load('square_marker_surfaces',[]) if isinstance(d,dict)]
# edit surfaces
self.surface_edit_mode = c_bool(0)
self.edit_surfaces = []
#detector vars
self.robust_detection = c_bool(1)
self.aperture = c_int(11)
self.min_marker_perimeter = 80
#debug vars
self.draw_markers = c_bool(0)
self.show_surface_idx = c_int(0)
self.recent_pupil_positions = []
self.img_shape = None
#multi monitor setup
self.window_should_open = False
self.window_should_close = False
self._window = None
self.fullscreen = c_bool(0)
self.monitor_idx = c_int(0)
self.monitor_handles = glfwGetMonitors()
self.monitor_names = [glfwGetMonitorName(m) for m in self.monitor_handles]
monitor_enum = atb.enum("Monitor",dict(((key,val) for val,key in enumerate(self.monitor_names))))
#primary_monitor = glfwGetPrimaryMonitor()
atb_label = "marker detection"
self._bar = atb.Bar(name =self.__class__.__name__, label=atb_label,
help="marker detection parameters", color=(50, 50, 50), alpha=100,
text='light', position=atb_pos,refresh=.3, size=(300, 100))
self._bar.add_var("monitor",self.monitor_idx, vtype=monitor_enum,group="Window",)
self._bar.add_var("fullscreen", self.fullscreen,group="Window")
self._bar.add_button(" open Window ", self.do_open, key='m',group="Window")
self._bar.add_var("surface to show",self.show_surface_idx, step=1,min=0,group="Window")
self._bar.add_var('robust_detection',self.robust_detection,group="Detector")
self._bar.add_var("draw markers",self.draw_markers,group="Detector")
atb_pos = atb_pos[0],atb_pos[1]+110
self._bar_markers = atb.Bar(name =self.__class__.__name__+'markers', label='registered surfaces',
help="list of registered ref surfaces", color=(50, 100, 50), alpha=100,
text='light', position=atb_pos,refresh=.3, size=(300, 120))
self.update_bar_markers()
def __init__(self, g_pool, atb_pos=(0,0)):
Plugin.__init__(self)
self.g_pool = g_pool
self.active = False
self.detected = False
self.screen_marker_state = 0
self.screen_marker_max = 70 # maximum bound for state
self.active_site = 0
self.sites = []
self.display_pos = None
self.on_position = False
self.candidate_ellipses = []
self.pos = None
self.show_edges = c_bool(0)
self.dist_threshold = c_int(5)
self.area_threshold = c_int(20)
self.world_size = None
self._window = None
self.window_should_close = False
self.window_should_open = False
self.fullscreen = c_bool(1)
self.monitor_idx = c_int(0)
monitor_handles = glfwGetMonitors()
self.monitor_names = [glfwGetMonitorName(m) for m in monitor_handles]
monitor_enum = atb.enum("Monitor",dict(((key,val) for val,key in enumerate(self.monitor_names))))
#primary_monitor = glfwGetPrimaryMonitor()
atb_label = "calibrate on screen"
# Creating an ATB Bar is required. Show at least some info about the Ref_Detector
self._bar = atb.Bar(name = self.__class__.__name__, label=atb_label,
help="ref detection parameters", color=(50, 50, 50), alpha=100,
text='light', position=atb_pos,refresh=.3, size=(300, 100))
self._bar.add_var("monitor",self.monitor_idx, vtype=monitor_enum)
self._bar.add_var("fullscreen", self.fullscreen)
self._bar.add_button(" start calibrating ", self.start, key='c')
self._bar.add_separator("Sep1")
self._bar.add_var("show edges",self.show_edges)
self._bar.add_var("area threshold", self.area_threshold)
self._bar.add_var("eccetricity threshold", self.dist_threshold)
def update_bar_markers(self):
self._bar.clear()
self._bar.add_button('close',self.unset_alive)
self._bar.add_button(" add surface ", self.add_surface, key='a')
# when cache is updated, when surface is edited, when trimmarks are changed.
# dropdown menue: markers and surface, surface edit mode, heatmaps, metrics
self._bar.mode_enum = atb.enum("Mode",{"Show Markers and Frames":0,"Show Marker Id's":4, "Surface edit mode":1,"Show Heatmaps":2,"Show Metrics":3})
self._bar.add_var("Mode",self.mode,vtype=self._bar.mode_enum)
self._bar.add_button(" (re)-calculate gaze distributions ", self.recalculate)
self._bar.add_button(" Export Gaze and Surface Data ", self.save_surface_statsics_to_file)
for s,i in zip(self.surfaces,range(len(self.surfaces)))[::-1]:
self._bar.add_var("%s_name"%i,create_string_buffer(512),getter=s.atb_get_name,setter=s.atb_set_name,group=str(i),label='name')
self._bar.add_var("%s_markers"%i,create_string_buffer(512), getter=s.atb_marker_status,group=str(i),label='found/registered markers' )
self._bar.add_var("%s_x_scale"%i,vtype=c_float, getter=s.atb_get_scale_x, min=1,setter=s.atb_set_scale_x,group=str(i),label='real width', help='this scale factor is used to adjust the coordinate space for your needs (think photo pixels or mm or whatever)' )
self._bar.add_var("%s_y_scale"%i,vtype=c_float, getter=s.atb_get_scale_y,min=1,setter=s.atb_set_scale_y,group=str(i),label='real height',help='defining x and y scale factor you atumatically set the correct aspect ratio.' )
self._bar.add_var("%s_window"%i,setter=s.toggle_window,getter=s.window_open,group=str(i),label='open in window')
# self._bar.add_button("%s_hm"%i, s.generate_heatmap, label='generate_heatmap',group=str(i))
# self._bar.add_button("%s_export"%i, self.save_surface_positions_to_file,data=i, label='export surface data',group=str(i))
self._bar.add_button("%s_remove"%i, self.remove_surface,data=i,label='remove',group=str(i))
def create_atb_bar(self,pos):
# add uvc camera controls to a separate ATB bar
size = (200,200)
self.bar = atb.Bar(name="Camera_Controls", label=self.name,
help="UVC Camera Controls", color=(50,50,50), alpha=100,
text='light',position=pos,refresh=2., size=size)
sorted_controls = [c for c in self.controls.itervalues()]
sorted_controls.sort(key=lambda c: c.order)
for control in sorted_controls:
name = control.atb_name
if control.type=="bool":
self.bar.add_var(name,vtype=atb.TW_TYPE_BOOL8,getter=control.get_val,setter=control.set_val)
elif control.type=='int':
self.bar.add_var(name,vtype=atb.TW_TYPE_INT32,getter=control.get_val,setter=control.set_val)
self.bar.define(definition='min='+str(control.min), varname=name)
self.bar.define(definition='max='+str(control.max), varname=name)
self.bar.define(definition='step='+str(control.step), varname=name)
elif control.type=="menu":
if control.menu is None:
vtype = None
else:
vtype= atb.enum(name,control.menu)
self.bar.add_var(name,vtype=vtype,getter=control.get_val,setter=control.set_val)
if control.menu is None:
self.bar.define(definition='min='+str(control.min), varname=name)
self.bar.define(definition='max='+str(control.max), varname=name)
self.bar.define(definition='step='+str(control.step), varname=name)
else:
pass
if control.flags == "inactive":
pass
# self.bar.define(definition='readonly=1',varname=control.name)
self.bar.add_button("refresh",self.controls.update_from_device)
self.bar.add_button("load defaults",self.controls.load_defaults)
return size
def __init__(self, g_pool, atb_pos=(0,0)):
Plugin.__init__(self)
self.g_pool = g_pool
self.active = False
self.detected = False
self.screen_marker_state = 0
self.screen_marker_max = 70 # maximum bound for state
self.active_site = 0
self.sites = []
self.display_pos = None
self.on_position = False
self.candidate_ellipses = []
self.pos = None
#result calculation variables:
self.fow = c_float(90.) #taken from c930e specsheet, confirmed though mesurement within ~10deg.
self.res = c_float(np.sqrt(1280**2 + 720**2))
self.outlier_thresh = c_float(5.)
self.accuray = c_float(0)
self.percision = c_float(0)
try:
self.pt_cloud = np.load(os.path.join(self.g_pool.user_dir,'accuray_test_pt_cloud.npy'))
gaze,ref = self.pt_cloud[:,0:2],self.pt_cloud[:,2:4]
error_lines = np.array([[g,r] for g,r in zip(gaze,ref)])
self.error_lines = error_lines.reshape(-1,2)
except Exception:
self.error_lines = None
self.pt_cloud = None
self.show_edges = c_bool(0)
self.dist_threshold = c_int(5)
self.area_threshold = c_int(20)
self.world_size = None
self._window = None
self.window_should_close = False
self.window_should_open = False
self.fullscreen = c_bool(1)
self.monitor_idx = c_int(0)
monitor_handles = glfwGetMonitors()
self.monitor_names = [glfwGetMonitorName(m) for m in monitor_handles]
monitor_enum = atb.enum("Monitor",dict(((key,val) for val,key in enumerate(self.monitor_names))))
#primary_monitor = glfwGetPrimaryMonitor()
atb_label = "screen marker based accuracy test"
# Creating an ATB Bar is required. Show at least some info about the Ref_Detector
self._bar = atb.Bar(name = self.__class__.__name__, label=atb_label,
help="ref detection parameters", color=(50, 50, 50), alpha=100,
text='light', position=atb_pos,refresh=.3, size=(300, 100))
self._bar.add_var("monitor",self.monitor_idx, vtype=monitor_enum)
self._bar.add_var("fullscreen", self.fullscreen)
self._bar.add_button(" start test ", self.start, key='c')
self._bar.add_var('diagonal FOV',self.fow)
self._bar.add_var('diagonal resolution',self.res,readonly= True)
self._bar.add_var('outlier threshold deg',self.outlier_thresh)
self._bar.add_var('angular accuray',self.accuray,readonly=True)
self._bar.add_var('angular percision',self.percision,readonly=True)
self._bar.add_button('calculate result',self.calc_result)
self._bar.add_separator("Sep1")
self._bar.add_var("show edges",self.show_edges)
self._bar.add_var("area threshold", self.area_threshold)
self._bar.add_var("eccetricity threshold", self.dist_threshold)
"""
helper for atb getter and setter use
"""
return data.value
# Initialize ant tweak bar - inherits from atb.Bar
atb.init()
bar = atb.Bar(name = "Browser", label="Controls",
help="Scene controls", color=(50, 50, 50), alpha=100,valueswidth=150,
text='light', position=(10, 10),refresh=.3, size=(300, 200))
bar.next_atb_pos = (10,220)
bar.fps = c_float(0.0)
bar.timestamp = time()
bar.window_size = c_int(0)
window_size_enum = atb.enum("Display Size",{"Full":0, "Medium":1,"Half":2,"Mini":3})
bar.add_var("fps", bar.fps, step=1., readonly=True)
bar.add_var("display size", vtype=window_size_enum,setter=set_window_size,getter=get_from_data,data=bar.window_size)
# bar.add_var("exit", g_pool.quit)
# Initialize glfw
glfwInit()
width, height = 640,480
glfwOpenWindow(width, height, 0, 0, 0, 8, 0, 0, GLFW_WINDOW)
glfwSetWindowTitle("Browser")
glfwSetWindowPos(0,0)
# Register callbacks
def __init__(self,g_pool,atb_pos=(320,220)):
super(Marker_Detector, self).__init__()
self.g_pool = g_pool
self.order = .2
# all markers that are detected in the most recent frame
self.markers = []
# all registered surfaces
if g_pool.app == 'capture':
self.surface_definitions = shelve.open(os.path.join(g_pool.user_dir,'surface_definitions'),protocol=2)
self.surfaces = [Reference_Surface(saved_definition=d) for d in self.load('realtime_square_marker_surfaces',[]) if isinstance(d,dict)]
elif g_pool.app == 'player':
#in player we load from the rec_dir: but we have a couple options:
self.surface_definitions = shelve.open(os.path.join(g_pool.rec_dir,'surface_definitions'),protocol=2)
if self.load('offline_square_marker_surfaces',[]) != []:
logger.debug("Found ref surfaces defined or copied in previous session.")
self.surfaces = [Reference_Surface(saved_definition=d) for d in self.load('offline_square_marker_surfaces',[]) if isinstance(d,dict)]
elif self.load('offline_square_marker_surfaces',[]) != []:
logger.debug("Did not find ref surfaces def created or used by the user in player from earlier session. Loading surfaces defined during capture.")
self.surfaces = [Reference_Surface(saved_definition=d) for d in self.load('realtime_square_marker_surfaces',[]) if isinstance(d,dict)]
else:
logger.debug("No surface defs found. Please define using GUI.")
self.surfaces = []
# edit surfaces
self.surface_edit_mode = c_bool(0)
self.edit_surfaces = []
#detector vars
self.robust_detection = c_bool(1)
self.aperture = c_int(11)
self.min_marker_perimeter = 80
#debug vars
self.draw_markers = c_bool(0)
self.show_surface_idx = c_int(0)
self.recent_pupil_positions = []
self.img_shape = None
#multi monitor setup
self.window_should_open = False
self.window_should_close = False
self._window = None
self.fullscreen = c_bool(0)
self.monitor_idx = c_int(0)
monitor_handles = glfwGetMonitors()
self.monitor_names = [glfwGetMonitorName(m) for m in monitor_handles]
monitor_enum = atb.enum("Monitor",dict(((key,val) for val,key in enumerate(self.monitor_names))))
#primary_monitor = glfwGetPrimaryMonitor()
atb_label = "marker detection"
self._bar = atb.Bar(name =self.__class__.__name__, label=atb_label,
help="marker detection parameters", color=(50, 50, 50), alpha=100,
text='light', position=atb_pos,refresh=.3, size=(300, 100))
self._bar.add_var("monitor",self.monitor_idx, vtype=monitor_enum,group="Window",)
self._bar.add_var("fullscreen", self.fullscreen,group="Window")
self._bar.add_button(" open Window ", self.do_open, key='m',group="Window")
self._bar.add_var("surface to show",self.show_surface_idx, step=1,min=0,group="Window")
self._bar.add_var('robust_detection',self.robust_detection,group="Detector")
self._bar.add_var("draw markers",self.draw_markers,group="Detector")
self._bar.add_button('close',self.unset_alive)
atb_pos = atb_pos[0],atb_pos[1]+110
self._bar_markers = atb.Bar(name =self.__class__.__name__+'markers', label='registered surfaces',
help="list of registered ref surfaces", color=(50, 100, 50), alpha=100,
text='light', position=atb_pos,refresh=.3, size=(300, 120))
self.update_bar_markers()
def __init__(self, g_pool, atb_pos=(0,0)):
Plugin.__init__(self)
self.g_pool = g_pool
self.active = False
self.detected = False
self.screen_marker_state = 0
self.screen_marker_max = 70 # maximum bound for state
self.active_site = 0
self.sites = []
self.display_pos = None
self.on_position = False
self.candidate_ellipses = []
self.pos = None
#result calculation variables:
self.world_size = None
self.fov = c_float(90.) #taken from c930e specsheet, confirmed though mesurement within ~10deg.
self.res = c_float(1.)
self.outlier_thresh = c_float(5.)
self.accuray = c_float(0)
self.percision = c_float(0)
try:
self.pt_cloud = np.load(os.path.join(self.g_pool.user_dir,'accuray_test_pt_cloud.npy'))
gaze,ref = self.pt_cloud[:,0:2],self.pt_cloud[:,2:4]
error_lines = np.array([[g,r] for g,r in zip(gaze,ref)])
self.error_lines = error_lines.reshape(-1,2)
except Exception:
self.error_lines = None
self.pt_cloud = None
self.show_edges = c_bool(0)
self.dist_threshold = c_int(5)
self.area_threshold = c_int(20)
self._window = None
self.window_should_close = False
self.window_should_open = False
self.fullscreen = c_bool(1)
self.monitor_idx = c_int(0)
monitor_handles = glfwGetMonitors()
self.monitor_names = [glfwGetMonitorName(m) for m in monitor_handles]
monitor_enum = atb.enum("Monitor",dict(((key,val) for val,key in enumerate(self.monitor_names))))
#primary_monitor = glfwGetPrimaryMonitor()
atb_label = "screen marker based accuracy test"
# Creating an ATB Bar is required. Show at least some info about the Ref_Detector
self._bar = atb.Bar(name = self.__class__.__name__, label=atb_label,
help="ref detection parameters", color=(50, 50, 50), alpha=100,
text='light', position=atb_pos,refresh=.3, size=(300, 200))
self._bar.add_var("monitor",self.monitor_idx, vtype=monitor_enum)
self._bar.add_var("fullscreen", self.fullscreen)
self._bar.add_button(" start test ", self.start, key='c')
accuray_help ='''Accuracy is calculated as the average angular
offset (distance) (in degrees of visual angle)
between fixations locations and the corresponding
locations of the fixation targets.'''.replace("\n"," ").replace(" ",'')
percision_help = '''Precision is calculated as the Root Mean Square (RMS)
of the angular distance (in degrees of visual angle)
between successive samples during a fixation.'''.replace("\n"," ").replace(" ",'')
self._bar.add_var('diagonal FOV',self.fov,help="set the camera FOV here.",group='Error Calculation')
self._bar.add_var('diagonal resolution',self.res,readonly= True ,group='Error Calculation')
self._bar.add_var('outlier threshold deg',self.outlier_thresh ,group='Error Calculation')
self._bar.add_var('angular accuray',self.accuray,readonly=True ,group='Error Calculation',help=accuray_help)
self._bar.add_var('angular percision',self.percision,readonly=True ,group='Error Calculation',help=percision_help)
self._bar.add_button('calculate result',self.calc_result ,group='Error Calculation')
self._bar.add_var("show edges",self.show_edges, group="Detector Variables")
self._bar.add_var("area threshold", self.area_threshold ,group="Detector Variables")
self._bar.add_var("eccetricity threshold", self.dist_threshold, group="Detector Variables" )
def world(g_pool,cap_src,cap_size):
"""world
Creates a window, gl context.
Grabs images from a capture.
Receives Pupil coordinates from g_pool.pupil_queue
Can run various plug-ins.
"""
# Callback functions
def on_resize(window,w, h):
active_window = glfwGetCurrentContext()
glfwMakeContextCurrent(window)
adjust_gl_view(w,h)
atb.TwWindowSize(w, h)
glfwMakeContextCurrent(active_window)
def on_key(window, key, scancode, action, mods):
if not atb.TwEventKeyboardGLFW(key,action):
if action == GLFW_PRESS:
if key == GLFW_KEY_ESCAPE:
on_close(window)
def on_char(window,char):
if not atb.TwEventCharGLFW(char,1):
pass
def on_button(window,button, action, mods):
if not atb.TwEventMouseButtonGLFW(button,action):
pos = glfwGetCursorPos(window)
pos = normalize(pos,glfwGetWindowSize(world_window))
pos = denormalize(pos,(frame.img.shape[1],frame.img.shape[0]) ) # Position in img pixels
for p in g.plugins:
p.on_click(pos,button,action)
def on_pos(window,x, y):
if atb.TwMouseMotion(int(x),int(y)):
pass
def on_scroll(window,x,y):
if not atb.TwMouseWheel(int(x)):
pass
def on_close(window):
g_pool.quit.value = True
logger.info('Process closing from window')
# load session persistent settings
session_settings = shelve.open(os.path.join(g_pool.user_dir,'user_settings_world'),protocol=2)
def load(var_name,default):
return session_settings.get(var_name,default)
def save(var_name,var):
session_settings[var_name] = var
# Initialize capture, check if it works
cap = autoCreateCapture(cap_src, cap_size, 24)
if cap is None:
logger.error("Did not receive valid Capture")
return
frame = cap.get_frame()
if frame.img is None:
logger.error("Could not retrieve image from capture")
cap.close()
return
height,width = frame.img.shape[:2]
# helpers called by the main atb bar
def update_fps():
old_time, bar.timestamp = bar.timestamp, time()
dt = bar.timestamp - old_time
if dt:
bar.fps.value += .05 * (1. / dt - bar.fps.value)
def set_window_size(mode,data):
height,width = frame.img.shape[:2]
ratio = (1,.75,.5,.25)[mode]
w,h = int(width*ratio),int(height*ratio)
glfwSetWindowSize(world_window,w,h)
data.value=mode # update the bar.value
def get_from_data(data):
"""
helper for atb getter and setter use
"""
return data.value
def open_calibration(selection,data):
# prepare destruction of current ref_detector... and remove it
for p in g.plugins:
if isinstance(p,calibration_routines.detector_by_index):
p.alive = False
g.plugins = [p for p in g.plugins if p.alive]
new_ref_detector = calibration_routines.detector_by_index[selection](g_pool,atb_pos=bar.next_atb_pos)
g.plugins.append(new_ref_detector)
g.plugins.sort(key=lambda p: p.order)
# save the value for atb bar
data.value=selection
def toggle_record_video():
for p in g.plugins:
if isinstance(p,recorder.Recorder):
p.alive = False
return
# set up folder within recordings named by user input in atb
if not bar.rec_name.value:
bar.rec_name.value = recorder.get_auto_name()
new_plugin = recorder.Recorder(g_pool,bar.rec_name.value, bar.fps.value, frame.img.shape, bar.record_eye.value, g_pool.eye_tx)
g.plugins.append(new_plugin)
g.plugins.sort(key=lambda p: p.order)
def toggle_show_calib_result():
for p in g.plugins:
if isinstance(p,Show_Calibration):
p.alive = False
return
new_plugin = Show_Calibration(g_pool,frame.img.shape)
g.plugins.append(new_plugin)
g.plugins.sort(key=lambda p: p.order)
def toggle_server():
for p in g.plugins:
if isinstance(p,Pupil_Server):
p.alive = False
return
new_plugin = Pupil_Server(g_pool,(10,300))
g.plugins.append(new_plugin)
g.plugins.sort(key=lambda p: p.order)
def toggle_ar():
for p in g.plugins:
if isinstance(p,Marker_Detector):
p.alive = False
return
new_plugin = Marker_Detector(g_pool,(10,400))
g.plugins.append(new_plugin)
g.plugins.sort(key=lambda p: p.order)
atb.init()
# add main controls ATB bar
bar = atb.Bar(name = "World", label="Controls",
help="Scene controls", color=(50, 50, 50), alpha=100,valueswidth=150,
text='light', position=(10, 10),refresh=.3, size=(300, 200))
bar.next_atb_pos = (10,220)
bar.fps = c_float(0.0)
bar.timestamp = time()
bar.calibration_type = c_int(load("calibration_type",0))
bar.record_eye = c_bool(load("record_eye",0))
bar.window_size = c_int(load("window_size",0))
window_size_enum = atb.enum("Display Size",{"Full":0, "Medium":1,"Half":2,"Mini":3})
calibrate_type_enum = atb.enum("Calibration Method",calibration_routines.index_by_name)
bar.rec_name = create_string_buffer(512)
bar.version = create_string_buffer(g_pool.version,512)
bar.rec_name.value = recorder.get_auto_name()
bar.add_var("fps", bar.fps, step=1., readonly=True)
bar.add_var("display size", vtype=window_size_enum,setter=set_window_size,getter=get_from_data,data=bar.window_size)
bar.add_var("calibration method",setter=open_calibration,getter=get_from_data,data=bar.calibration_type, vtype=calibrate_type_enum,group="Calibration", help="Please choose your desired calibration method.")
bar.add_button("show calibration result",toggle_show_calib_result, group="Calibration", help="Click to show calibration result.")
bar.add_var("session name",bar.rec_name, group="Recording", help="creates folder Data_Name_XXX, where xxx is an increasing number")
bar.add_button("record", toggle_record_video, key="r", group="Recording", help="Start/Stop Recording")
bar.add_var("record eye", bar.record_eye, group="Recording", help="check to save raw video of eye")
bar.add_button("start/stop marker tracking",toggle_ar,key="x",help="find markers in scene to map gaze onto referace surfaces")
bar.add_button("start/stop server",toggle_server,key="s",help="the server broadcasts pupil and gaze positions locally or via network")
bar.add_separator("Sep1")
bar.add_var("version",bar.version, readonly=True)
bar.add_var("exit", g_pool.quit)
# add uvc camera controls ATB bar
cap.create_atb_bar(pos=(320,10))
# Initialize glfw
glfwInit()
world_window = glfwCreateWindow(width, height, "World", None, None)
glfwMakeContextCurrent(world_window)
# Register callbacks world_window
glfwSetWindowSizeCallback(world_window,on_resize)
glfwSetWindowCloseCallback(world_window,on_close)
glfwSetKeyCallback(world_window,on_key)
glfwSetCharCallback(world_window,on_char)
glfwSetMouseButtonCallback(world_window,on_button)
glfwSetCursorPosCallback(world_window,on_pos)
glfwSetScrollCallback(world_window,on_scroll)
#set the last saved window size
set_window_size(bar.window_size.value,bar.window_size)
on_resize(world_window, *glfwGetFramebufferSize(world_window))
glfwSetWindowPos(world_window,0,0)
# gl_state settings
basic_gl_setup()
# refresh speed settings
glfwSwapInterval(0)
# load last calibration data
try:
pt_cloud = np.load(os.path.join(g_pool.user_dir,'cal_pt_cloud.npy'))
logger.info("Using calibration found in %s" %g_pool.user_dir)
map_pupil = calibrate.get_map_from_cloud(pt_cloud,(width,height))
except:
logger.info("No calibration found.")
def map_pupil(vector):
""" 1 to 1 mapping
"""
return vector
# create container for globally scoped vars (within world)
g = Temp()
g.plugins = []
g_pool.map_pupil = map_pupil
#load calibration plugin
open_calibration(bar.calibration_type.value,bar.calibration_type)
#load gaze_display plugin
g.plugins.append(Display_Recent_Gaze(g_pool))
# Event loop
while not g_pool.quit.value:
# Get an image from the grabber
frame = cap.get_frame()
update_fps()
#a container that allows plugins to post and read events
events = []
#receive and map pupil positions
recent_pupil_positions = []
while not g_pool.pupil_queue.empty():
p = g_pool.pupil_queue.get()
if p['norm_pupil'] is None:
p['norm_gaze'] = None
else:
p['norm_gaze'] = g_pool.map_pupil(p['norm_pupil'])
recent_pupil_positions.append(p)
# allow each Plugin to do its work.
for p in g.plugins:
p.update(frame,recent_pupil_positions,events)
#check if a plugin need to be destroyed
g.plugins = [p for p in g.plugins if p.alive]
# render camera image
glfwMakeContextCurrent(world_window)
draw_gl_texture(frame.img)
# render visual feedback from loaded plugins
for p in g.plugins:
p.gl_display()
atb.draw()
glfwSwapBuffers(world_window)
glfwPollEvents()
# de-init all running plugins
for p in g.plugins:
p.alive = False
#reading p.alive actually runs plug-in cleanup
_ = p.alive
save('window_size',bar.window_size.value)
save('calibration_type',bar.calibration_type.value)
save('record_eye',bar.record_eye.value)
session_settings.close()
cap.close()
glfwDestroyWindow(world_window)
glfwTerminate()
logger.debug("Process done")
def eye(g_pool, cap_src, cap_size):
"""
Creates a window, gl context.
Grabs images from a capture.
Streams Pupil coordinates into g_pool.pupil_queue
"""
# modify the root logger for this process
logger = logging.getLogger()
# remove inherited handlers
logger.handlers = []
# create file handler which logs even debug messages
fh = logging.FileHandler(os.path.join(g_pool.user_dir, 'eye.log'),
mode='w')
fh.setLevel(logging.INFO)
# create console handler with a higher log level
ch = logging.StreamHandler()
ch.setLevel(logging.WARNING)
# create formatter and add it to the handlers
formatter = logging.Formatter(
'EYE Process: %(asctime)s - %(name)s - %(levelname)s - %(message)s')
fh.setFormatter(formatter)
formatter = logging.Formatter(
'E Y E Process [%(levelname)s] %(name)s : %(message)s')
ch.setFormatter(formatter)
# add the handlers to the logger
logger.addHandler(fh)
logger.addHandler(ch)
# create logger for the context of this function
logger = logging.getLogger(__name__)
# Callback functions
def on_resize(window, w, h):
adjust_gl_view(w, h)
atb.TwWindowSize(w, h)
def on_key(window, key, scancode, action, mods):
if not atb.TwEventKeyboardGLFW(key, int(action == GLFW_PRESS)):
if action == GLFW_PRESS:
if key == GLFW_KEY_ESCAPE:
on_close(window)
def on_char(window, char):
if not atb.TwEventCharGLFW(char, 1):
pass
def on_button(window, button, action, mods):
if not atb.TwEventMouseButtonGLFW(button, int(action == GLFW_PRESS)):
if action == GLFW_PRESS:
pos = glfwGetCursorPos(window)
pos = normalize(pos, glfwGetWindowSize(window))
pos = denormalize(
pos, (frame.img.shape[1],
frame.img.shape[0])) # pos in frame.img pixels
u_r.setStart(pos)
bar.draw_roi.value = 1
else:
bar.draw_roi.value = 0
def on_pos(window, x, y):
if atb.TwMouseMotion(int(x), int(y)):
pass
if bar.draw_roi.value == 1:
pos = x, y
pos = normalize(pos, glfwGetWindowSize(window))
pos = denormalize(pos,
(frame.img.shape[1],
frame.img.shape[0])) # pos in frame.img pixels
u_r.setEnd(pos)
def on_scroll(window, x, y):
if not atb.TwMouseWheel(int(x)):
pass
def on_close(window):
g_pool.quit.value = True
logger.info('Process closing from window')
# Helper functions called by the main atb bar
def start_roi():
bar.display.value = 1
bar.draw_roi.value = 2
def update_fps():
old_time, bar.timestamp = bar.timestamp, time()
dt = bar.timestamp - old_time
if dt:
bar.fps.value += .05 * (1. / dt - bar.fps.value)
bar.dt.value = dt
def get_from_data(data):
"""
helper for atb getter and setter use
"""
return data.value
# load session persistent settings
session_settings = shelve.open(os.path.join(g_pool.user_dir,
'user_settings_eye'),
protocol=2)
def load(var_name, default):
return session_settings.get(var_name, default)
def save(var_name, var):
session_settings[var_name] = var
# Initialize capture
cap = autoCreateCapture(cap_src, cap_size)
if cap is None:
logger.error("Did not receive valid Capture")
return
# check if it works
frame = cap.get_frame()
if frame.img is None:
logger.error("Could not retrieve image from capture")
cap.close()
return
height, width = frame.img.shape[:2]
cap.auto_rewind = False
u_r = Roi(frame.img.shape)
u_r.set(load('roi', default=None))
writer = None
pupil_detector = Canny_Detector(g_pool)
atb.init()
# Create main ATB Controls
bar = atb.Bar(name="Eye",
label="Display",
help="Scene controls",
color=(50, 50, 50),
alpha=100,
text='light',
position=(10, 10),
refresh=.3,
size=(200, 100))
bar.fps = c_float(0.0)
bar.timestamp = time()
bar.dt = c_float(0.0)
bar.sleep = c_float(0.0)
bar.display = c_int(load('bar.display', 0))
bar.draw_pupil = c_bool(load('bar.draw_pupil', True))
bar.draw_roi = c_int(0)
dispay_mode_enum = atb.enum("Mode", {
"Camera Image": 0,
"Region of Interest": 1,
"Algorithm": 2
})
bar.add_var("FPS", bar.fps, step=1., readonly=True)
bar.add_var("Mode",
bar.display,
vtype=dispay_mode_enum,
help="select the view-mode")
bar.add_var("Show_Pupil_Point", bar.draw_pupil)
bar.add_button("Draw_ROI",
start_roi,
help="drag on screen to select a region of interest")
bar.add_var("SlowDown", bar.sleep, step=0.01, min=0.0)
bar.add_var("SaveSettings&Exit", g_pool.quit)
cap.create_atb_bar(pos=(220, 10))
# create a bar for the detector
pupil_detector.create_atb_bar(pos=(10, 120))
glfwInit()
window = glfwCreateWindow(width, height, "Eye", None, None)
glfwMakeContextCurrent(window)
# Register callbacks window
glfwSetWindowSizeCallback(window, on_resize)
glfwSetWindowCloseCallback(window, on_close)
glfwSetKeyCallback(window, on_key)
glfwSetCharCallback(window, on_char)
glfwSetMouseButtonCallback(window, on_button)
glfwSetCursorPosCallback(window, on_pos)
glfwSetScrollCallback(window, on_scroll)
glfwSetWindowPos(window, 800, 0)
on_resize(window, width, height)
# gl_state settings
basic_gl_setup()
# refresh speed settings
glfwSwapInterval(0)
# event loop
while not g_pool.quit.value:
frame = cap.get_frame()
if frame.img is None:
break
update_fps()
sleep(bar.sleep.value) # for debugging only
if pupil_detector.should_sleep:
sleep(16)
pupil_detector.should_sleep = False
### RECORDING of Eye Video (on demand) ###
# Setup variables and lists for recording
if g_pool.eye_rx.poll():
command = g_pool.eye_rx.recv()
if command is not None:
record_path = command
logger.info("Will save eye video to: %(record_path)s")
video_path = os.path.join(record_path, "eye.avi")
timestamps_path = os.path.join(record_path,
"eye_timestamps.npy")
writer = cv2.VideoWriter(
video_path, cv2.cv.CV_FOURCC(*'DIVX'), bar.fps.value,
(frame.img.shape[1], frame.img.shape[0]))
timestamps = []
else:
logger.info("Done recording eye.")
writer = None
np.save(timestamps_path, np.asarray(timestamps))
del timestamps
if writer:
writer.write(frame.img)
timestamps.append(frame.timestamp)
# pupil ellipse detection
result = pupil_detector.detect(frame,
user_roi=u_r,
visualize=bar.display.value == 2)
# stream the result
g_pool.pupil_queue.put(result)
# VISUALIZATION direct visualizations on the frame.img data
if bar.display.value == 1:
# and a solid (white) frame around the user defined ROI
r_img = frame.img[u_r.lY:u_r.uY, u_r.lX:u_r.uX]
r_img[:, 0] = 255, 255, 255
r_img[:, -1] = 255, 255, 255
r_img[0, :] = 255, 255, 255
r_img[-1, :] = 255, 255, 255
# GL-drawing
clear_gl_screen()
draw_gl_texture(frame.img)
if result['norm_pupil'] is not None and bar.draw_pupil.value:
if result.has_key('axes'):
pts = cv2.ellipse2Poly(
(int(result['center'][0]), int(result['center'][1])),
(int(result["axes"][0] / 2), int(result["axes"][1] / 2)),
int(result["angle"]), 0, 360, 15)
draw_gl_polyline(pts, (1., 0, 0, .5))
draw_gl_point_norm(result['norm_pupil'], color=(1., 0., 0., 0.5))
atb.draw()
glfwSwapBuffers(window)
glfwPollEvents()
# END while running
# in case eye reconding was still runnnig: Save&close
if writer:
logger.info("Done recording eye.")
writer = None
np.save(timestamps_path, np.asarray(timestamps))
# save session persistent settings
save('roi', u_r.get())
save('bar.display', bar.display.value)
save('bar.draw_pupil', bar.draw_pupil.value)
session_settings.close()
pupil_detector.cleanup()
cap.close()
atb.terminate()
glfwDestroyWindow(window)
glfwTerminate()
#flushing queue incase world process did not exit gracefully
while not g_pool.pupil_queue.empty():
g_pool.pupil_queue.get()
g_pool.pupil_queue.close()
logger.debug("Process done")
def __init__(self, g_pool, atb_pos=(320, 220)):
super(Marker_Detector, self).__init__()
self.g_pool = g_pool
self.order = .2
# all markers that are detected in the most recent frame
self.markers = []
# all registered surfaces
if g_pool.app == 'capture':
self.surface_definitions = shelve.open(os.path.join(
g_pool.user_dir, 'surface_definitions'),
protocol=2)
self.surfaces = [
Reference_Surface(saved_definition=d)
for d in self.load('realtime_square_marker_surfaces', [])
if isinstance(d, dict)
]
elif g_pool.app == 'player':
#in player we load from the rec_dir: but we have a couple options:
self.surface_definitions = shelve.open(os.path.join(
g_pool.rec_dir, 'surface_definitions'),
protocol=2)
if self.load('offline_square_marker_surfaces', []) != []:
logger.debug(
"Found ref surfaces defined or copied in previous session."
)
self.surfaces = [
Reference_Surface(saved_definition=d)
for d in self.load('offline_square_marker_surfaces', [])
if isinstance(d, dict)
]
elif self.load('offline_square_marker_surfaces', []) != []:
logger.debug(
"Did not find ref surfaces def created or used by the user in player from earlier session. Loading surfaces defined during capture."
)
self.surfaces = [
Reference_Surface(saved_definition=d)
for d in self.load('realtime_square_marker_surfaces', [])
if isinstance(d, dict)
]
else:
logger.debug("No surface defs found. Please define using GUI.")
self.surfaces = []
# edit surfaces
self.surface_edit_mode = c_bool(0)
self.edit_surfaces = []
#detector vars
self.robust_detection = c_bool(1)
self.aperture = c_int(11)
self.min_marker_perimeter = 80
#debug vars
self.draw_markers = c_bool(0)
self.show_surface_idx = c_int(0)
self.recent_pupil_positions = []
self.img_shape = None
#multi monitor setup
self.window_should_open = False
self.window_should_close = False
self._window = None
self.fullscreen = c_bool(0)
self.monitor_idx = c_int(0)
monitor_handles = glfwGetMonitors()
self.monitor_names = [glfwGetMonitorName(m) for m in monitor_handles]
monitor_enum = atb.enum(
"Monitor",
dict(((key, val) for val, key in enumerate(self.monitor_names))))
#primary_monitor = glfwGetPrimaryMonitor()
atb_label = "marker detection"
self._bar = atb.Bar(name=self.__class__.__name__,
label=atb_label,
help="marker detection parameters",
color=(50, 50, 50),
alpha=100,
text='light',
position=atb_pos,
refresh=.3,
size=(300, 100))
self._bar.add_var(
"monitor",
self.monitor_idx,
vtype=monitor_enum,
group="Window",
)
self._bar.add_var("fullscreen", self.fullscreen, group="Window")
self._bar.add_button(" open Window ",
self.do_open,
key='m',
group="Window")
self._bar.add_var("surface to show",
self.show_surface_idx,
step=1,
min=0,
group="Window")
self._bar.add_var('robust_detection',
self.robust_detection,
group="Detector")
self._bar.add_var("draw markers", self.draw_markers, group="Detector")
self._bar.add_button('close', self.unset_alive)
atb_pos = atb_pos[0], atb_pos[1] + 110
self._bar_markers = atb.Bar(name=self.__class__.__name__ + 'markers',
label='registered surfaces',
help="list of registered ref surfaces",
color=(50, 100, 50),
alpha=100,
text='light',
position=atb_pos,
refresh=.3,
size=(300, 120))
self.update_bar_markers()
def world(g_pool,cap_src,cap_size):
"""world
Creates a window, gl context.
Grabs images from a capture.
Receives Pupil coordinates from g_pool.pupil_queue
Can run various plug-ins.
"""
# Callback functions
def on_resize(window,w, h):
active_window = glfwGetCurrentContext()
glfwMakeContextCurrent(window)
norm_size = normalize((w,h),glfwGetWindowSize(window))
fb_size = denormalize(norm_size,glfwGetFramebufferSize(window))
atb.TwWindowSize(*map(int,fb_size))
adjust_gl_view(w,h,window)
glfwMakeContextCurrent(active_window)
def on_iconify(window,iconfied):
if not isinstance(cap,FakeCapture):
g_pool.update_textures.value = not iconfied
def on_key(window, key, scancode, action, mods):
if not atb.TwEventKeyboardGLFW(key,action):
if action == GLFW_PRESS:
if key == GLFW_KEY_ESCAPE:
on_close(window)
def on_char(window,char):
if not atb.TwEventCharGLFW(char,1):
pass
def on_button(window,button, action, mods):
if not atb.TwEventMouseButtonGLFW(button,action):
pos = glfwGetCursorPos(window)
pos = normalize(pos,glfwGetWindowSize(world_window))
pos = denormalize(pos,(frame.img.shape[1],frame.img.shape[0]) ) # Position in img pixels
for p in g_pool.plugins:
p.on_click(pos,button,action)
def on_pos(window,x, y):
norm_pos = normalize((x,y),glfwGetWindowSize(window))
fb_x,fb_y = denormalize(norm_pos,glfwGetFramebufferSize(window))
if atb.TwMouseMotion(int(fb_x),int(fb_y)):
pass
def on_scroll(window,x,y):
if not atb.TwMouseWheel(int(x)):
pass
def on_close(window):
g_pool.quit.value = True
logger.info('Process closing from window')
# load session persistent settings
session_settings = shelve.open(os.path.join(g_pool.user_dir,'user_settings_world'),protocol=2)
def load(var_name,default):
return session_settings.get(var_name,default)
def save(var_name,var):
session_settings[var_name] = var
# load last calibration data
try:
pt_cloud = np.load(os.path.join(g_pool.user_dir,'cal_pt_cloud.npy'))
logger.info("Using calibration found in %s" %g_pool.user_dir)
map_pupil = calibrate.get_map_from_cloud(pt_cloud,(width,height))
except:
logger.info("No calibration found.")
def map_pupil(vector):
""" 1 to 1 mapping
"""
return vector
# any object we attach to the g_pool object now will only be visible to this process!
# vars should be declared here to make them visible to the reader.
g_pool.plugins = []
g_pool.map_pupil = map_pupil
g_pool.update_textures = c_bool(1)
# Initialize capture
cap = autoCreateCapture(cap_src, cap_size, 24, timebase=g_pool.timebase)
if isinstance(cap,FakeCapture):
g_pool.update_textures.value = False
# Get an image from the grabber
try:
frame = cap.get_frame()
except CameraCaptureError:
logger.error("Could not retrieve image from capture")
cap.close()
return
height,width = frame.img.shape[:2]
g_pool.capture = cap
# helpers called by the main atb bar
def update_fps():
old_time, bar.timestamp = bar.timestamp, time()
dt = bar.timestamp - old_time
if dt:
bar.fps.value += .05 * (1. / dt - bar.fps.value)
def set_window_size(mode,data):
height,width = frame.img.shape[:2]
ratio = (1,.75,.5,.25)[mode]
w,h = int(width*ratio),int(height*ratio)
glfwSetWindowSize(world_window,w,h)
data.value=mode # update the bar.value
def get_from_data(data):
"""
helper for atb getter and setter use
"""
return data.value
def open_calibration(selection,data):
# prepare destruction of current ref_detector... and remove it
for p in g_pool.plugins:
if isinstance(p,calibration_routines.detector_by_index):
p.alive = False
g_pool.plugins = [p for p in g_pool.plugins if p.alive]
new_ref_detector = calibration_routines.detector_by_index[selection](g_pool,atb_pos=bar.next_atb_pos)
g_pool.plugins.append(new_ref_detector)
g_pool.plugins.sort(key=lambda p: p.order)
# save the value for atb bar
data.value=selection
def toggle_record_video():
for p in g_pool.plugins:
if isinstance(p,recorder.Recorder):
p.alive = False
return
# set up folder within recordings named by user input in atb
if not bar.rec_name.value:
bar.rec_name.value = recorder.get_auto_name()
new_plugin = recorder.Recorder(g_pool,bar.rec_name.value, bar.fps.value, frame.img.shape, bar.record_eye.value, g_pool.eye_tx)
g_pool.plugins.append(new_plugin)
g_pool.plugins.sort(key=lambda p: p.order)
def toggle_show_calib_result():
for p in g_pool.plugins:
if isinstance(p,Show_Calibration):
p.alive = False
return
new_plugin = Show_Calibration(g_pool,frame.img.shape)
g_pool.plugins.append(new_plugin)
g_pool.plugins.sort(key=lambda p: p.order)
def toggle_server():
for p in g_pool.plugins:
if isinstance(p,Pupil_Server):
p.alive = False
return
new_plugin = Pupil_Server(g_pool,(10,300))
g_pool.plugins.append(new_plugin)
g_pool.plugins.sort(key=lambda p: p.order)
def toggle_ar():
for p in g_pool.plugins:
if isinstance(p,Marker_Detector):
p.alive = False
return
new_plugin = Marker_Detector(g_pool,(10,400))
g_pool.plugins.append(new_plugin)
g_pool.plugins.sort(key=lambda p: p.order)
def reset_timebase():
#the last frame from worldcam will be t0
g_pool.timebase.value = cap.get_now()
logger.info("New timebase set to %s all timestamps will count from here now."%g_pool.timebase.value)
atb.init()
# add main controls ATB bar
bar = atb.Bar(name = "World", label="Controls",
help="Scene controls", color=(50, 50, 50), alpha=100,valueswidth=150,
text='light', position=(10, 10),refresh=.3, size=(300, 200))
bar.next_atb_pos = (10,220)
bar.fps = c_float(0.0)
bar.timestamp = time()
bar.calibration_type = c_int(load("calibration_type",0))
bar.record_eye = c_bool(load("record_eye",0))
bar.window_size = c_int(load("window_size",0))
window_size_enum = atb.enum("Display Size",{"Full":0, "Medium":1,"Half":2,"Mini":3})
calibrate_type_enum = atb.enum("Calibration Method",calibration_routines.index_by_name)
bar.rec_name = create_string_buffer(512)
bar.version = create_string_buffer(g_pool.version,512)
bar.rec_name.value = recorder.get_auto_name()
bar.add_var("fps", bar.fps, step=1., readonly=True, help="Refresh speed of this process. Especially during recording it should not drop below the camera set frame rate.")
bar.add_var("display size", vtype=window_size_enum,setter=set_window_size,getter=get_from_data,data=bar.window_size,help="Resize the world window. This has no effect on the actual image.")
bar.add_var("calibration method",setter=open_calibration,getter=get_from_data,data=bar.calibration_type, vtype=calibrate_type_enum,group="Calibration", help="Please choose your desired calibration method.")
bar.add_button("show calibration result",toggle_show_calib_result, group="Calibration", help="Click to show calibration result.")
bar.add_var("session name",bar.rec_name, group="Recording", help="Give your recording session a custom name.")
bar.add_button("record", toggle_record_video, key="r", group="Recording", help="Start/Stop Recording")
bar.add_var("record eye", bar.record_eye, group="Recording", help="check to save raw video of eye")
bar.add_button("start/stop marker tracking",toggle_ar,key="x",help="find markers in scene to map gaze onto referace surfaces")
bar.add_button("start/stop server",toggle_server,key="s",help="the server broadcasts pupil and gaze positions locally or via network")
bar.add_button("set timebase to now",reset_timebase,help="this button allows the timestamps to count from now on.",key="t")
bar.add_var("update screen", g_pool.update_textures,help="if you dont need to see the camera image updated, you can turn this of to reduce CPU load.")
bar.add_separator("Sep1")
bar.add_var("version",bar.version, readonly=True)
bar.add_var("exit", g_pool.quit)
# add uvc camera controls ATB bar
cap.create_atb_bar(pos=(320,10))
# Initialize glfw
glfwInit()
world_window = glfwCreateWindow(width, height, "World", None, None)
glfwMakeContextCurrent(world_window)
# Register callbacks world_window
glfwSetWindowSizeCallback(world_window,on_resize)
glfwSetWindowCloseCallback(world_window,on_close)
glfwSetWindowIconifyCallback(world_window,on_iconify)
glfwSetKeyCallback(world_window,on_key)
glfwSetCharCallback(world_window,on_char)
glfwSetMouseButtonCallback(world_window,on_button)
glfwSetCursorPosCallback(world_window,on_pos)
glfwSetScrollCallback(world_window,on_scroll)
#set the last saved window size
set_window_size(bar.window_size.value,bar.window_size)
on_resize(world_window, *glfwGetWindowSize(world_window))
glfwSetWindowPos(world_window,0,0)
# gl_state settings
basic_gl_setup()
g_pool.image_tex = create_named_texture(frame.img)
# refresh speed settings
glfwSwapInterval(0)
#load calibration plugin
open_calibration(bar.calibration_type.value,bar.calibration_type)
#load gaze_display plugin
g_pool.plugins.append(Display_Recent_Gaze(g_pool))
# Event loop
while not g_pool.quit.value:
# Get an image from the grabber
try:
frame = cap.get_frame()
except CameraCaptureError:
logger.error("Capture from Camera Failed. Stopping.")
break
except EndofVideoFileError:
logger.warning("Video File is done. Stopping")
break
update_fps()
#a container that allows plugins to post and read events
events = []
#receive and map pupil positions
recent_pupil_positions = []
while not g_pool.pupil_queue.empty():
p = g_pool.pupil_queue.get()
if p['norm_pupil'] is None:
p['norm_gaze'] = None
else:
p['norm_gaze'] = g_pool.map_pupil(p['norm_pupil'])
recent_pupil_positions.append(p)
# allow each Plugin to do its work.
for p in g_pool.plugins:
p.update(frame,recent_pupil_positions,events)
#check if a plugin need to be destroyed
g_pool.plugins = [p for p in g_pool.plugins if p.alive]
# render camera image
glfwMakeContextCurrent(world_window)
make_coord_system_norm_based()
if g_pool.update_textures.value:
draw_named_texture(g_pool.image_tex,frame.img)
else:
draw_named_texture(g_pool.image_tex)
make_coord_system_pixel_based(frame.img.shape)
# render visual feedback from loaded plugins
for p in g_pool.plugins:
p.gl_display()
atb.draw()
glfwSwapBuffers(world_window)
glfwPollEvents()
# de-init all running plugins
for p in g_pool.plugins:
p.alive = False
#reading p.alive actually runs plug-in cleanup
_ = p.alive
save('window_size',bar.window_size.value)
save('calibration_type',bar.calibration_type.value)
save('record_eye',bar.record_eye.value)
session_settings.close()
cap.close()
atb.terminate()
glfwDestroyWindow(world_window)
glfwTerminate()
logger.debug("Process done")
def eye(g_pool,cap_src,cap_size):
"""
Creates a window, gl context.
Grabs images from a capture.
Streams Pupil coordinates into g_pool.pupil_queue
"""
# modify the root logger for this process
logger = logging.getLogger()
# remove inherited handlers
logger.handlers = []
# create file handler which logs even debug messages
fh = logging.FileHandler(os.path.join(g_pool.user_dir,'eye.log'),mode='w')
fh.setLevel(logging.DEBUG)
# create console handler with a higher log level
ch = logging.StreamHandler()
ch.setLevel(logging.WARNING)
# create formatter and add it to the handlers
formatter = logging.Formatter('EYE Process: %(asctime)s - %(name)s - %(levelname)s - %(message)s')
fh.setFormatter(formatter)
formatter = logging.Formatter('E Y E Process [%(levelname)s] %(name)s : %(message)s')
ch.setFormatter(formatter)
# add the handlers to the logger
logger.addHandler(fh)
logger.addHandler(ch)
# create logger for the context of this function
logger = logging.getLogger(__name__)
# Callback functions
def on_resize(window,w, h):
adjust_gl_view(w,h,window)
norm_size = normalize((w,h),glfwGetWindowSize(window))
fb_size = denormalize(norm_size,glfwGetFramebufferSize(window))
atb.TwWindowSize(*map(int,fb_size))
def on_key(window, key, scancode, action, mods):
if not atb.TwEventKeyboardGLFW(key,int(action == GLFW_PRESS)):
if action == GLFW_PRESS:
if key == GLFW_KEY_ESCAPE:
on_close(window)
def on_char(window,char):
if not atb.TwEventCharGLFW(char,1):
pass
def on_button(window,button, action, mods):
if not atb.TwEventMouseButtonGLFW(button,int(action == GLFW_PRESS)):
if action == GLFW_PRESS:
pos = glfwGetCursorPos(window)
pos = normalize(pos,glfwGetWindowSize(window))
pos = denormalize(pos,(frame.img.shape[1],frame.img.shape[0]) ) # pos in frame.img pixels
u_r.setStart(pos)
bar.draw_roi.value = 1
else:
bar.draw_roi.value = 0
def on_pos(window,x, y):
norm_pos = normalize((x,y),glfwGetWindowSize(window))
fb_x,fb_y = denormalize(norm_pos,glfwGetFramebufferSize(window))
if atb.TwMouseMotion(int(fb_x),int(fb_y)):
pass
if bar.draw_roi.value == 1:
pos = denormalize(norm_pos,(frame.img.shape[1],frame.img.shape[0]) ) # pos in frame.img pixels
u_r.setEnd(pos)
def on_scroll(window,x,y):
if not atb.TwMouseWheel(int(x)):
pass
def on_close(window):
g_pool.quit.value = True
logger.info('Process closing from window')
# Helper functions called by the main atb bar
def start_roi():
bar.display.value = 1
bar.draw_roi.value = 2
def update_fps():
old_time, bar.timestamp = bar.timestamp, time()
dt = bar.timestamp - old_time
if dt:
bar.fps.value += .05 * (1. / dt - bar.fps.value)
bar.dt.value = dt
def get_from_data(data):
"""
helper for atb getter and setter use
"""
return data.value
# load session persistent settings
session_settings = shelve.open(os.path.join(g_pool.user_dir,'user_settings_eye'),protocol=2)
def load(var_name,default):
return session_settings.get(var_name,default)
def save(var_name,var):
session_settings[var_name] = var
# Initialize capture
cap = autoCreateCapture(cap_src, cap_size,timebase=g_pool.timebase)
if cap is None:
logger.error("Did not receive valid Capture")
return
# check if it works
frame = cap.get_frame()
if frame.img is None:
logger.error("Could not retrieve image from capture")
cap.close()
return
height,width = frame.img.shape[:2]
u_r = Roi(frame.img.shape)
u_r.set(load('roi',default=None))
writer = None
pupil_detector = Canny_Detector(g_pool)
atb.init()
# Create main ATB Controls
bar = atb.Bar(name = "Eye", label="Display",
help="Scene controls", color=(50, 50, 50), alpha=100,
text='light', position=(10, 10),refresh=.3, size=(200, 100))
bar.fps = c_float(0.0)
bar.timestamp = time()
bar.dt = c_float(0.0)
bar.sleep = c_float(0.0)
bar.display = c_int(load('bar.display',0))
bar.draw_pupil = c_bool(load('bar.draw_pupil',True))
bar.draw_roi = c_int(0)
dispay_mode_enum = atb.enum("Mode",{"Camera Image":0,
"Region of Interest":1,
"Algorithm":2,
"CPU Save": 3})
bar.add_var("FPS",bar.fps, step=1.,readonly=True)
bar.add_var("Mode", bar.display,vtype=dispay_mode_enum, help="select the view-mode")
bar.add_var("Show_Pupil_Point", bar.draw_pupil)
bar.add_button("Draw_ROI", start_roi, help="drag on screen to select a region of interest")
bar.add_var("SlowDown",bar.sleep, step=0.01,min=0.0)
bar.add_var("SaveSettings&Exit", g_pool.quit)
cap.create_atb_bar(pos=(220,10))
# create a bar for the detector
pupil_detector.create_atb_bar(pos=(10,120))
glfwInit()
window = glfwCreateWindow(width, height, "Eye", None, None)
glfwMakeContextCurrent(window)
# Register callbacks window
glfwSetWindowSizeCallback(window,on_resize)
glfwSetWindowCloseCallback(window,on_close)
glfwSetKeyCallback(window,on_key)
glfwSetCharCallback(window,on_char)
glfwSetMouseButtonCallback(window,on_button)
glfwSetCursorPosCallback(window,on_pos)
glfwSetScrollCallback(window,on_scroll)
glfwSetWindowPos(window,800,0)
on_resize(window,width,height)
# gl_state settings
basic_gl_setup()
# refresh speed settings
glfwSwapInterval(0)
# event loop
while not g_pool.quit.value:
# Get an image from the grabber
try:
frame = cap.get_frame()
except CameraCaptureError:
logger.error("Capture from Camera Failed. Stopping.")
break
except EndofVideoFileError:
logger.warning("Video File is done. Stopping")
break
update_fps()
sleep(bar.sleep.value) # for debugging only
### RECORDING of Eye Video (on demand) ###
# Setup variables and lists for recording
if g_pool.eye_rx.poll():
command = g_pool.eye_rx.recv()
if command is not None:
record_path = command
logger.info("Will save eye video to: %s"%record_path)
video_path = os.path.join(record_path, "eye.avi")
timestamps_path = os.path.join(record_path, "eye_timestamps.npy")
writer = cv2.VideoWriter(video_path, cv2.cv.CV_FOURCC(*'DIVX'), bar.fps.value, (frame.img.shape[1], frame.img.shape[0]))
timestamps = []
else:
logger.info("Done recording eye.")
writer = None
np.save(timestamps_path,np.asarray(timestamps))
del timestamps
if writer:
writer.write(frame.img)
timestamps.append(frame.timestamp)
# pupil ellipse detection
result = pupil_detector.detect(frame,user_roi=u_r,visualize=bar.display.value == 2)
# stream the result
g_pool.pupil_queue.put(result)
# VISUALIZATION direct visualizations on the frame.img data
if bar.display.value == 1:
# and a solid (white) frame around the user defined ROI
r_img = frame.img[u_r.lY:u_r.uY,u_r.lX:u_r.uX]
r_img[:,0] = 255,255,255
r_img[:,-1]= 255,255,255
r_img[0,:] = 255,255,255
r_img[-1,:]= 255,255,255
# GL-drawing
clear_gl_screen()
draw_gl_texture(frame.img,update=bar.display.value != 3)
if result['norm_pupil'] is not None and bar.draw_pupil.value:
if result.has_key('axes'):
pts = cv2.ellipse2Poly( (int(result['center'][0]),int(result['center'][1])),
(int(result["axes"][0]/2),int(result["axes"][1]/2)),
int(result["angle"]),0,360,15)
draw_gl_polyline(pts,(1.,0,0,.5))
draw_gl_point_norm(result['norm_pupil'],color=(1.,0.,0.,0.5))
atb.draw()
glfwSwapBuffers(window)
glfwPollEvents()
# END while running
# in case eye reconding was still runnnig: Save&close
if writer:
logger.info("Done recording eye.")
writer = None
np.save(timestamps_path,np.asarray(timestamps))
# save session persistent settings
save('roi',u_r.get())
save('bar.display',bar.display.value)
save('bar.draw_pupil',bar.draw_pupil.value)
session_settings.close()
pupil_detector.cleanup()
cap.close()
atb.terminate()
glfwDestroyWindow(window)
glfwTerminate()
#flushing queue incase world process did not exit gracefully
while not g_pool.pupil_queue.empty():
g_pool.pupil_queue.get()
g_pool.pupil_queue.close()
logger.debug("Process done")
def __init__(self,
g_pool,
atb_pos=(0, 0),
numTrees=10,
numChecks=50,
knnk=2,
trainingProducts=[]):
super(Shopping_Detector, self).__init__()
print 'network'
self.context = zmq.Context()
self.socket = self.context.socket(zmq.PUB)
self.address = create_string_buffer('tcp://127.0.0.1:5000', 512)
print 'acquired network connection'
self.set_server(self.address)
help_str = "Shopping Message server: Using ZMQ and the *Publish-Subscribe* scheme"
self.g_pool = g_pool
self.loadedProducts = False
self.trained = False
# mapper init
print 'sift created'
self.detector = cv2.SIFT()
FLANN_INDEX_KDTREE = 6
FLANN_INDEX_KDTREE = 0
self.index_params = dict(algorithm=FLANN_INDEX_KDTREE,
trees=numTrees,
table_number=6,
key_size=12,
multi_probe_level=2)
self.index_params = dict(algorithm=FLANN_INDEX_KDTREE, trees=numTrees)
self.search_params = dict(checks=numChecks)
self.matcher = cv2.FlannBasedMatcher(self.index_params,
self.search_params)
self.trainingProducts = trainingProducts
self.create_training_products()
'''
self.trainingProducts.append(Product('/Users/mobileexperiencelab/Downloads/digboston.jpg'))
self.trainingProducts.append(Product('/Users/mobileexperiencelab/Downloads/jobs.png'))
self.trainingProducts.append(Product('/Users/mobileexperiencelab/Downloads/espn.jpg'))
self.trainingProducts.append(Product('/Users/mobileexperiencelab/Downloads/a.jpg'))
'''
self.generate_training_data()
self.frameCount = 0
self.mappedAreas = []
self.knnk = c_int(knnk)
self.minGoodMatches = c_int(4)
self.maxAvgDistance = c_int(150)
self.homographyThreshold = c_float(10.0)
self.distanceWeight = .7
self.recentGazePoints = collections.deque([], 3)
self.recent_products_queue = TimedCounter(7.5)
self.reco_thread = None
#multi monitor setup
self.window_should_open = False
self.window_should_close = False
self._window = None
self.fullscreen = c_bool(0)
self.monitor_idx = c_int(0)
self.monitor_handles = glfwGetMonitors()
self.monitor_names = [
glfwGetMonitorName(m) for m in self.monitor_handles
]
monitor_enum = atb.enum(
"Monitor",
dict(((key, val) for val, key in enumerate(self.monitor_names))))
#primary_monitor = glfwGetPrimaryMonitor()
atb_label = "Product Detection"
self._bar = atb.Bar(name=self.__class__.__name__,
label=atb_label,
help="product detection parameters",
color=(50, 50, 50),
alpha=100,
text='light',
position=atb_pos,
refresh=.3,
size=(300, 100))
self._bar.add_var(
"monitor",
self.monitor_idx,
vtype=monitor_enum,
group="Window",
)
self._bar.add_var("fullscreen", self.fullscreen, group="Window")
self._bar.add_button(" open Window ",
self.do_open,
key='m',
group="Window")
self._bar.add_var("min good matches",
self.minGoodMatches,
step=1,
min=0,
group="Detector")
self._bar.add_var("k-nearest-neighbors",
self.knnk,
step=1,
min=1,
group="Detector")
self._bar.add_var("homogoraphy threshold",
self.homographyThreshold,
step=.1,
min=1,
group="Detector")
self._bar.add_var("max distance of good matches",
self.maxAvgDistance,
step=1,
min=1,
max=500,
group="Detector")
self._bar.define("valueswidth=170")
self._bar.add_var("server address",
self.address,
getter=lambda: self.address,
setter=self.set_server)
self._bar.add_button("close", self.close)
atb_pos = atb_pos[0], atb_pos[1] + 110
self._bar_markers = atb.Bar(name=self.__class__.__name__ + 'markers',
label='registered surfaces',
help="list of registered ref surfaces",
color=(50, 100, 50),
alpha=100,
text='light',
position=atb_pos,
refresh=.3,
size=(300, 120))
def eye(g_pool):
"""
this process needs a docstring
"""
# # callback functions
def on_resize(w, h):
atb.TwWindowSize(w, h)
adjust_gl_view(w, h)
def on_key(key, pressed):
if not atb.TwEventKeyboardGLFW(key, pressed):
if pressed:
if key == GLFW_KEY_ESC:
on_close()
def on_char(char, pressed):
if not atb.TwEventCharGLFW(char, pressed):
pass
def on_button(button, pressed):
if not atb.TwEventMouseButtonGLFW(button, pressed):
if bar.draw_roi.value:
if pressed:
pos = glfwGetMousePos()
pos = normalize(pos, glfwGetWindowSize())
pos = denormalize(pos, (img.shape[1], img.shape[0])) # pos in img pixels
r.setStart(pos)
bar.draw_roi.value = 1
else:
bar.draw_roi.value = 0
def on_pos(x, y):
if atb.TwMouseMotion(x, y):
pass
if bar.draw_roi.value == 1:
pos = glfwGetMousePos()
pos = normalize(pos, glfwGetWindowSize())
pos = denormalize(pos, (img.shape[1], img.shape[0])) # pos in img pixels
r.setEnd(pos)
def on_scroll(pos):
if not atb.TwMouseWheel(pos):
pass
def on_close():
g_pool.quit.value = True
print "EYE Process closing from window"
# initialize capture, check if it works
cap = autoCreateCapture(g_pool.eye_src, g_pool.eye_size)
if cap is None:
print "EYE: Error could not create Capture"
return
s, img = cap.read_RGB()
if not s:
print "EYE: Error could not get image"
return
height, width = img.shape[:2]
# pupil object
pupil = Temp()
pupil.norm_coords = (0.0, 0.0)
pupil.image_coords = (0.0, 0.0)
pupil.ellipse = None
pupil.gaze_coords = (0.0, 0.0)
try:
pupil.pt_cloud = np.load("cal_pt_cloud.npy")
map_pupil = get_map_from_cloud(pupil.pt_cloud, g_pool.world_size) ###world video size here
except:
pupil.pt_cloud = None
def map_pupil(vector):
return vector
r = Roi(img.shape)
p_r = Roi(img.shape)
# local object
l_pool = Temp()
l_pool.calib_running = False
l_pool.record_running = False
l_pool.record_positions = []
l_pool.record_path = None
l_pool.writer = None
l_pool.region_r = 20
atb.init()
bar = Bar(
"Eye",
g_pool,
dict(
label="Controls",
help="eye detection controls",
color=(50, 50, 50),
alpha=100,
text="light",
position=(10, 10),
refresh=0.1,
size=(200, 300),
),
)
# add 4vl2 camera controls to a seperate ATB bar
if cap.controls is not None:
c_bar = atb.Bar(
name="Camera_Controls",
label=cap.name,
help="UVC Camera Controls",
color=(50, 50, 50),
alpha=100,
text="light",
position=(220, 10),
refresh=2.0,
size=(200, 200),
)
# c_bar.add_var("auto_refresher",vtype=atb.TW_TYPE_BOOL8,getter=cap.uvc_refresh_all,setter=None,readonly=True)
# c_bar.define(definition='visible=0', varname="auto_refresher")
sorted_controls = [c for c in cap.controls.itervalues()]
sorted_controls.sort(key=lambda c: c.order)
for control in sorted_controls:
name = control.atb_name
if control.type == "bool":
c_bar.add_var(name, vtype=atb.TW_TYPE_BOOL8, getter=control.get_val, setter=control.set_val)
elif control.type == "int":
c_bar.add_var(name, vtype=atb.TW_TYPE_INT32, getter=control.get_val, setter=control.set_val)
c_bar.define(definition="min=" + str(control.min), varname=name)
c_bar.define(definition="max=" + str(control.max), varname=name)
c_bar.define(definition="step=" + str(control.step), varname=name)
elif control.type == "menu":
if control.menu is None:
vtype = None
else:
vtype = atb.enum(name, control.menu)
c_bar.add_var(name, vtype=vtype, getter=control.get_val, setter=control.set_val)
if control.menu is None:
c_bar.define(definition="min=" + str(control.min), varname=name)
c_bar.define(definition="max=" + str(control.max), varname=name)
c_bar.define(definition="step=" + str(control.step), varname=name)
else:
pass
if control.flags == "inactive":
pass
# c_bar.define(definition='readonly=1',varname=control.name)
c_bar.add_button("refresh", cap.update_from_device)
c_bar.add_button("load defaults", cap.load_defaults)
else:
c_bar = None
# Initialize glfw
glfwInit()
glfwOpenWindow(width, height, 0, 0, 0, 8, 0, 0, GLFW_WINDOW)
glfwSetWindowTitle("Eye")
glfwSetWindowPos(800, 0)
if isinstance(g_pool.eye_src, str):
glfwSwapInterval(0) # turn of v-sync when using video as src for benchmarking
# register callbacks
glfwSetWindowSizeCallback(on_resize)
glfwSetWindowCloseCallback(on_close)
glfwSetKeyCallback(on_key)
glfwSetCharCallback(on_char)
glfwSetMouseButtonCallback(on_button)
glfwSetMousePosCallback(on_pos)
glfwSetMouseWheelCallback(on_scroll)
# gl_state settings
import OpenGL.GL as gl
gl.glEnable(gl.GL_POINT_SMOOTH)
gl.glPointSize(20)
gl.glBlendFunc(gl.GL_SRC_ALPHA, gl.GL_ONE_MINUS_SRC_ALPHA)
gl.glEnable(gl.GL_BLEND)
del gl
# event loop
while glfwGetWindowParam(GLFW_OPENED) and not g_pool.quit.value:
bar.update_fps()
s, img = cap.read_RGB()
sleep(bar.sleep.value) # for debugging only
###IMAGE PROCESSING
gray_img = grayscale(img[r.lY : r.uY, r.lX : r.uX])
integral = cv2.integral(gray_img)
integral = np.array(integral, dtype=c_float)
x, y, w = eye_filter(integral)
if w > 0:
p_r.set((y, x, y + w, x + w))
else:
p_r.set((0, 0, -1, -1))
# create view into the gray_img with the bounds of the rough pupil estimation
pupil_img = gray_img[p_r.lY : p_r.uY, p_r.lX : p_r.uX]
# pupil_img = cv2.morphologyEx(pupil_img, cv2.MORPH_OPEN, cv2.getStructuringElement(cv2.MORPH_RECT,(5,5)),iterations=2)
if True:
hist = cv2.calcHist(
[pupil_img], [0], None, [256], [0, 256]
) # (images, channels, mask, histSize, ranges[, hist[, accumulate]])
bins = np.arange(hist.shape[0])
spikes = bins[hist[:, 0] > 40] # every color seen in more than 40 pixels
if spikes.shape[0] > 0:
lowest_spike = spikes.min()
offset = 40
##display the histogram
sx, sy = 100, 1
colors = ((255, 0, 0), (0, 0, 255), (0, 255, 255))
h, w, chan = img.shape
# normalize
hist *= 1.0 / hist.max()
for i, h in zip(bins, hist[:, 0]):
c = colors[1]
cv2.line(img, (w, int(i * sy)), (w - int(h * sx), int(i * sy)), c)
cv2.line(img, (w, int(lowest_spike * sy)), (int(w - 0.5 * sx), int(lowest_spike * sy)), colors[0])
cv2.line(
img,
(w, int((lowest_spike + offset) * sy)),
(int(w - 0.5 * sx), int((lowest_spike + offset) * sy)),
colors[2],
)
# # k-means on the histogram finds peaks but thats no good for us...
# term_crit = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 10, 1.0)
# compactness, bestLabels, centers = cv2.kmeans(data=hist, K=2, criteria=term_crit, attempts=10, flags=cv2.KMEANS_RANDOM_CENTERS)
# cv2.line(img,(0,1),(int(compactness),1),(0,0,0))
# good_cluster = np.argmax(centers)
# # A = hist[bestLabels.ravel() == good_cluster]
# # B = hist[bestLabels.ravel() != good_cluster]
# bins = np.arange(hist.shape[0])
# good_bins = bins[bestLabels.ravel() == good_cluster]
# good_bins_mean = good_bins.sum()/good_bins.shape[0]
# good_bins_min = good_bins.min()
# h,w,chan = img.shape
# for h, i, label in zip(hist[:,0],range(hist.shape[0]), bestLabels.ravel()):
# c = colors[label]
# cv2.line(img,(w,int(i*sy)),(w-int(h*sx),int(i*sy)),c)
else:
# direct k-means on the image is best but expensive
Z = pupil_img[:: w / 30 + 1, :: w / 30 + 1].reshape((-1, 1))
Z = np.float32(Z)
# define criteria, number of clusters(K) and apply kmeans()
criteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 20, 2.0)
K = 5
ret, label, center = cv2.kmeans(Z, K, criteria, 10, cv2.KMEANS_RANDOM_CENTERS)
offset = 0
center.sort(axis=0)
lowest_spike = int(center[1])
# # Now convert back into uint8, and make original image
# center = np.uint8(center)
# res = center[label.flatten()]
# binary_img = res.reshape((pupil_img.shape))
# binary_img = bin_thresholding(binary_img,image_upper=res.min()+1)
# bar.bin_thresh.value = res.min()+1
bar.bin_thresh.value = lowest_spike
binary_img = bin_thresholding(pupil_img, image_upper=lowest_spike + offset)
kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (7, 7))
cv2.dilate(binary_img, kernel, binary_img, iterations=2)
spec_mask = bin_thresholding(pupil_img, image_upper=250)
cv2.erode(spec_mask, kernel, spec_mask, iterations=1)
if bar.blur.value > 1:
pupil_img = cv2.medianBlur(pupil_img, bar.blur.value)
# create contours using Canny edge dectetion
contours = cv2.Canny(
pupil_img,
bar.canny_thresh.value,
bar.canny_thresh.value * bar.canny_ratio.value,
apertureSize=bar.canny_aperture.value,
)
# remove contours in areas not dark enough and where the glint (spectral refelction from IR leds)
contours = cv2.min(contours, spec_mask)
contours = cv2.min(contours, binary_img)
# Ellipse fitting from countours
result = fit_ellipse(
img[r.lY : r.uY, r.lX : r.uX][p_r.lY : p_r.uY, p_r.lX : p_r.uX],
contours,
binary_img,
target_size=bar.pupil_size.value,
size_tolerance=bar.pupil_size_tolerance.value,
)
# # Vizualizations
overlay = cv2.cvtColor(pupil_img, cv2.COLOR_GRAY2RGB) # create an RGB view onto the gray pupil ROI
overlay[:, :, 0] = cv2.max(pupil_img, contours) # green channel
overlay[:, :, 2] = cv2.max(pupil_img, binary_img) # blue channel
overlay[:, :, 1] = cv2.min(pupil_img, spec_mask) # red channel
# draw a blue dotted frame around the automatic pupil ROI in overlay...
overlay[::2, 0] = 0, 0, 255
overlay[::2, -1] = 0, 0, 255
overlay[0, ::2] = 0, 0, 255
overlay[-1, ::2] = 0, 0, 255
# and a solid (white) frame around the user defined ROI
gray_img[:, 0] = 255
gray_img[:, -1] = 255
gray_img[0, :] = 255
gray_img[-1, :] = 255
if bar.display.value == 0:
img = img
elif bar.display.value == 1:
img[r.lY : r.uY, r.lX : r.uX] = cv2.cvtColor(gray_img, cv2.COLOR_GRAY2RGB)
elif bar.display.value == 2:
img[r.lY : r.uY, r.lX : r.uX] = cv2.cvtColor(gray_img, cv2.COLOR_GRAY2RGB)
img[r.lY : r.uY, r.lX : r.uX][p_r.lY : p_r.uY, p_r.lX : p_r.uX] = overlay
elif bar.display.value == 3:
img = cv2.cvtColor(pupil_img, cv2.COLOR_GRAY2RGB)
else:
pass
if result is not None:
pupil.ellipse, others = result
pupil.image_coords = r.add_vector(p_r.add_vector(pupil.ellipse["center"]))
# update pupil size,angle and ratio for the ellipse filter algorithm
bar.pupil_size.value = bar.pupil_size.value + 0.5 * (pupil.ellipse["major"] - bar.pupil_size.value)
bar.pupil_ratio.value = bar.pupil_ratio.value + 0.7 * (pupil.ellipse["ratio"] - bar.pupil_ratio.value)
bar.pupil_angle.value = bar.pupil_angle.value + 1.0 * (pupil.ellipse["angle"] - bar.pupil_angle.value)
# if pupil found tighten the size tolerance
bar.pupil_size_tolerance.value -= 1
bar.pupil_size_tolerance.value = max(10, min(50, bar.pupil_size_tolerance.value))
# clamp pupil size
bar.pupil_size.value = max(20, min(300, bar.pupil_size.value))
# normalize
pupil.norm_coords = normalize(pupil.image_coords, (img.shape[1], img.shape[0]), flip_y=True)
# from pupil to gaze
pupil.gaze_coords = map_pupil(pupil.norm_coords)
g_pool.gaze_x.value, g_pool.gaze_y.value = pupil.gaze_coords
else:
pupil.ellipse = None
g_pool.gaze_x.value, g_pool.gaze_y.value = 0.0, 0.0
pupil.gaze_coords = None # whithout this line the last know pupil position is recorded if none is found
bar.pupil_size_tolerance.value += 1
###CALIBRATION###
# Initialize Calibration (setup variables and lists)
if g_pool.calibrate.value and not l_pool.calib_running:
l_pool.calib_running = True
pupil.pt_cloud = []
# While Calibrating...
if l_pool.calib_running and ((g_pool.ref_x.value != 0) or (g_pool.ref_y.value != 0)) and pupil.ellipse:
pupil.pt_cloud.append([pupil.norm_coords[0], pupil.norm_coords[1], g_pool.ref_x.value, g_pool.ref_y.value])
# Calculate mapping coefs
if not g_pool.calibrate.value and l_pool.calib_running:
l_pool.calib_running = 0
if pupil.pt_cloud: # some data was actually collected
print "Calibrating with", len(pupil.pt_cloud), "collected data points."
map_pupil = get_map_from_cloud(np.array(pupil.pt_cloud), g_pool.world_size, verbose=True)
np.save("cal_pt_cloud.npy", np.array(pupil.pt_cloud))
###RECORDING###
# Setup variables and lists for recording
if g_pool.pos_record.value and not l_pool.record_running:
l_pool.record_path = g_pool.eye_rx.recv()
print "l_pool.record_path: ", l_pool.record_path
video_path = path.join(l_pool.record_path, "eye.avi")
# FFV1 -- good speed lossless big file
# DIVX -- good speed good compression medium file
if bar.record_eye.value:
l_pool.writer = cv2.VideoWriter(
video_path, cv2.cv.CV_FOURCC(*"DIVX"), bar.fps.value, (img.shape[1], img.shape[0])
)
l_pool.record_positions = []
l_pool.record_running = True
# While recording...
if l_pool.record_running:
if pupil.gaze_coords is not None:
l_pool.record_positions.append(
[
pupil.gaze_coords[0],
pupil.gaze_coords[1],
pupil.norm_coords[0],
pupil.norm_coords[1],
bar.dt,
g_pool.frame_count_record.value,
]
)
if l_pool.writer is not None:
l_pool.writer.write(cv2.cvtColor(img, cv2.cv.COLOR_BGR2RGB))
# Done Recording: Save values and flip switch to off for recording
if not g_pool.pos_record.value and l_pool.record_running:
positions_path = path.join(l_pool.record_path, "gaze_positions.npy")
cal_pt_cloud_path = path.join(l_pool.record_path, "cal_pt_cloud.npy")
np.save(positions_path, np.asarray(l_pool.record_positions))
try:
np.save(cal_pt_cloud_path, np.asarray(pupil.pt_cloud))
except:
print "Warning: No calibration data associated with this recording."
l_pool.writer = None
l_pool.record_running = False
### GL-drawing
clear_gl_screen()
draw_gl_texture(img)
if bar.draw_pupil and pupil.ellipse:
pts = cv2.ellipse2Poly(
(int(pupil.image_coords[0]), int(pupil.image_coords[1])),
(int(pupil.ellipse["axes"][0] / 2), int(pupil.ellipse["axes"][1] / 2)),
int(pupil.ellipse["angle"]),
0,
360,
15,
)
draw_gl_polyline(pts, (1.0, 0, 0, 0.5))
draw_gl_point_norm(pupil.norm_coords, (1.0, 0.0, 0.0, 0.5))
atb.draw()
glfwSwapBuffers()
# end while running
print "EYE Process closed"
r.save()
bar.save()
atb.terminate()
glfwCloseWindow()
glfwTerminate()
def world(g_pool):
"""world
"""
###Callback funtions
def on_resize(w, h):
atb.TwWindowSize(w, h);
adjust_gl_view(w,h)
def on_key(key, pressed):
if not atb.TwEventKeyboardGLFW(key,pressed):
if pressed:
if key == GLFW_KEY_ESC:
on_close()
def on_char(char, pressed):
if not atb.TwEventCharGLFW(char,pressed):
pass
def on_button(button, pressed):
if not atb.TwEventMouseButtonGLFW(button,pressed):
if pressed:
pos = glfwGetMousePos()
pos = normalize(pos,glfwGetWindowSize())
pos = denormalize(pos,(img.shape[1],img.shape[0]) ) #pos in img pixels
ref.detector.new_ref(pos)
def on_pos(x, y):
if atb.TwMouseMotion(x,y):
pass
def on_scroll(pos):
if not atb.TwMouseWheel(pos):
pass
def on_close():
g_pool.quit.value = True
print "WORLD Process closing from window"
ref = Temp()
ref.detector = no_Detector(g_pool.calibrate,g_pool.ref_x,g_pool.ref_y)
###objects as variable containers
# pattern object
pattern = Temp()
pattern.centers = None
pattern.obj_grid = gen_pattern_grid((4, 11)) # calib grid
pattern.obj_points = []
pattern.img_points = []
pattern.map = (0, 2, 7, 16, 21, 23, 39, 40, 42)
pattern.board_centers = None
# gaze object
gaze = Temp()
gaze.map_coords = (0., 0.)
gaze.image_coords = (0., 0.)
# record object
record = Temp()
record.writer = None
record.path_parent = os.path.dirname(os.path.abspath(sys.argv[0]))
record.path = None
record.counter = 0
# initialize capture, check if it works
cap = autoCreateCapture(g_pool.world_src, g_pool.world_size)
if cap is None:
print "WORLD: Error could not create Capture"
return
s, img = cap.read_RGB()
if not s:
print "WORLD: Error could not get image"
return
height,width = img.shape[:2]
###helpers called by the main atb bar
def update_fps():
old_time, bar.timestamp = bar.timestamp, time()
dt = bar.timestamp - old_time
if dt:
bar.fps.value += .05 * (1 / dt - bar.fps.value)
def set_window_size(mode,data):
height,width = img.shape[:2]
ratio = (1,.75,.5,.25)[mode]
w,h = int(width*ratio),int(height*ratio)
glfwSetWindowSize(w,h)
data.value=mode #update the bar.value
def get_from_data(data):
"""
helper for atb getter and setter use
"""
return data.value
def start_calibration():
c_type = bar.calibration_type.value
if c_type == cal_type["Directed 9-Point"]:
print 'WORLD: Starting Directed 9-Point calibration.'
ref.detector = Nine_Point_Detector(global_calibrate=g_pool.calibrate,
shared_x=g_pool.ref_x,
shared_y=g_pool.ref_y,
shared_stage=g_pool.cal9_stage,
shared_step=g_pool.cal9_step,
shared_cal9_active=g_pool.cal9,
shared_circle_id=g_pool.cal9_circle_id,
auto_advance=False)
elif c_type == cal_type["Automated 9-Point"]:
print 'WORLD: Starting Automated 9-Point calibration.'
ref.detector = Nine_Point_Detector(global_calibrate=g_pool.calibrate,
shared_x=g_pool.ref_x,
shared_y=g_pool.ref_y,
shared_stage=g_pool.cal9_stage,
shared_step=g_pool.cal9_step,
shared_cal9_active=g_pool.cal9,
shared_circle_id=g_pool.cal9_circle_id,
auto_advance=True)
elif c_type == cal_type["Natural Features"]:
print 'WORLD: Starting Natural Features calibration.'
ref.detector = Natural_Features_Detector(global_calibrate=g_pool.calibrate,
shared_x=g_pool.ref_x,
shared_y=g_pool.ref_y)
elif c_type == cal_type["Black Dot"]:
print 'WORLD: Starting Black Dot calibration.'
ref.detector = Black_Dot_Detector(global_calibrate=g_pool.calibrate,
shared_x=g_pool.ref_x,
shared_y=g_pool.ref_y)
def advance_calibration():
ref.detector.advance()
def stop_calibration():
ref.detector = no_Detector(global_calibrate=g_pool.calibrate,
shared_x=g_pool.ref_x,
shared_y=g_pool.ref_y)
### Initialize ant tweak bar inherits from atb.Bar
atb.init()
bar = atb.Bar(name = "World", label="Controls",
help="Scene controls", color=(50, 50, 50), alpha=100,
text='light', position=(10, 10),refresh=.3, size=(200, 200))
bar.fps = c_float(0.0)
bar.timestamp = time()
bar.calibration_type = c_int(1)
bar.show_calib_result = c_bool(0)
bar.calibration_images = False
bar.record_video = c_bool(0)
bar.record_running = c_bool(0)
bar.play = g_pool.play
bar.window_size = c_int(0)
window_size_enum = atb.enum("Display Size",{"Full":0, "Medium":1,"Half":2,"Mini":3})
cal_type = {"Directed 9-Point":0,"Automated 9-Point":1,"Natural Features":3,"Black Dot":4}#"Manual 9-Point":2
calibrate_type_enum = atb.enum("Calibration Method",cal_type)
bar.rec_name = create_string_buffer(512)
# play and record can be tied together via pointers to the objects
# bar.play = bar.record_video
bar.add_var("FPS", bar.fps, step=1., readonly=True)
bar.add_var("Display_Size", vtype=window_size_enum,setter=set_window_size,getter=get_from_data,data=bar.window_size)
bar.add_var("Cal/Calibration_Method",bar.calibration_type, vtype=calibrate_type_enum)
bar.add_button("Cal/Start_Calibration",start_calibration, key='c')
bar.add_button("Cal/Next_Point",advance_calibration,key="SPACE", help="Hit space to calibrate on next dot")
bar.add_button("Cal/Stop_Calibration",stop_calibration, key='d')
bar.add_var("Cal/show_calibration_result",bar.show_calib_result, help="yellow lines indecate fit error, red outline shows the calibrated area.")
bar.add_var("Rec/rec_name",bar.rec_name)
bar.add_var("Rec/Record_Video", bar.record_video, key="r", help="Start/Stop Recording")
bar.add_separator("Sep1")
bar.add_var("Play Source Video", bar.play)
bar.add_var("Exit", g_pool.quit)
#add 4vl2 camera controls to a seperate ATB bar
if cap.controls is not None:
c_bar = atb.Bar(name="Camera_Controls", label=cap.name,
help="UVC Camera Controls", color=(50,50,50), alpha=100,
text='light',position=(220, 10),refresh=2., size=(200, 200))
# c_bar.add_var("auto_refresher",vtype=atb.TW_TYPE_BOOL8,getter=cap.uvc_refresh_all,setter=None,readonly=True)
# c_bar.define(definition='visible=0', varname="auto_refresher")
sorted_controls = [c for c in cap.controls.itervalues()]
sorted_controls.sort(key=lambda c: c.order)
for control in sorted_controls:
name = control.atb_name
if control.type=="bool":
c_bar.add_var(name,vtype=atb.TW_TYPE_BOOL8,getter=control.get_val,setter=control.set_val)
elif control.type=='int':
c_bar.add_var(name,vtype=atb.TW_TYPE_INT32,getter=control.get_val,setter=control.set_val)
c_bar.define(definition='min='+str(control.min), varname=name)
c_bar.define(definition='max='+str(control.max), varname=name)
c_bar.define(definition='step='+str(control.step), varname=name)
elif control.type=="menu":
if control.menu is None:
vtype = None
else:
vtype= atb.enum(name,control.menu)
c_bar.add_var(name,vtype=vtype,getter=control.get_val,setter=control.set_val)
if control.menu is None:
c_bar.define(definition='min='+str(control.min), varname=name)
c_bar.define(definition='max='+str(control.max), varname=name)
c_bar.define(definition='step='+str(control.step), varname=name)
else:
pass
if control.flags == "inactive":
pass
# c_bar.define(definition='readonly=1',varname=control.name)
c_bar.add_button("refresh",cap.update_from_device)
c_bar.add_button("load defaults",cap.load_defaults)
else:
c_bar = None
### Initialize glfw
glfwInit()
height,width = img.shape[:2]
glfwOpenWindow(width, height, 0, 0, 0, 8, 0, 0, GLFW_WINDOW)
glfwSetWindowTitle("World")
glfwSetWindowPos(0,0)
#register callbacks
glfwSetWindowSizeCallback(on_resize)
glfwSetWindowCloseCallback(on_close)
glfwSetKeyCallback(on_key)
glfwSetCharCallback(on_char)
glfwSetMouseButtonCallback(on_button)
glfwSetMousePosCallback(on_pos)
glfwSetMouseWheelCallback(on_scroll)
#gl_state settings
import OpenGL.GL as gl
gl.glEnable(gl.GL_POINT_SMOOTH)
gl.glPointSize(20)
gl.glBlendFunc(gl.GL_SRC_ALPHA, gl.GL_ONE_MINUS_SRC_ALPHA)
gl.glEnable(gl.GL_BLEND)
###event loop
while glfwGetWindowParam(GLFW_OPENED) and not g_pool.quit.value:
update_fps()
# get an image from the grabber
s, img = cap.read()
ref.detector.detect(img)
if ref.detector.is_done():
stop_calibration()
g_pool.player_refresh.set()
# #gather pattern centers and find cam intrisics
# if bar.screen_shot and pattern.centers is not None:
# bar.screen_shot = False
# # calibrate the camera intrinsics if the board is found
# # append list of circle grid center points to pattern.img_points
# # append generic list of circle grid pattern type to pattern.obj_points
# pattern.centers = circle_grid(img)
# pattern.img_points.append(pattern.centers)
# pattern.obj_points.append(pattern.obj_grid)
# print "Number of Patterns Captured:", len(pattern.img_points)
# #if pattern.img_points.shape[0] > 10:
# if len(pattern.img_points) > 10:
# camera_matrix, dist_coefs = calibrate_camera(np.asarray(pattern.img_points),
# np.asarray(pattern.obj_points),
# (img.shape[1], img.shape[0]))
# np.save("camera_matrix.npy", camera_matrix)
# np.save("dist_coefs.npy", dist_coefs)
# pattern.img_points = []
# bar.find_pattern.value = False
### Setup recording process
if bar.record_video and not bar.record_running:
record.path = os.path.join(record.path_parent, "data%03d/" % record.counter)
while True:
try:
os.mkdir(record.path)
break
except:
print "We dont want to overwrite data, incrementing counter & trying to make new data folder"
record.counter += 1
record.path = os.path.join(record.path_parent, "data%03d/" % record.counter)
#video
video_path = os.path.join(record.path, "world.avi")
#FFV1 -- good speed lossless big file
#DIVX -- good speed good compression medium file
record.writer = cv2.VideoWriter(video_path, cv2.cv.CV_FOURCC(*'DIVX'), bar.fps.value, (img.shape[1], img.shape[0]))
# positions data to eye process
g_pool.pos_record.value = True
g_pool.eye_tx.send(record.path)
bar.record_running = 1
g_pool.frame_count_record.value = 0
# While Recording...
if bar.record_video and bar.record_running:
# Save image frames to video writer
# increment the frame_count_record value
# Eye positions can be associated with frames of recording even if different framerates are used
record.writer.write(img)
g_pool.frame_count_record.value += 1
# Finish all recordings, clean up.
if not bar.record_video and bar.record_running:
# for conviniece: copy camera intrinsics into each data folder at the end of a recording.
try:
camera_matrix = np.load("camera_matrix.npy")
dist_coefs = np.load("dist_coefs.npy")
cam_path = os.path.join(record.path, "camera_matrix.npy")
dist_path = os.path.join(record.path, "dist_coefs.npy")
np.save(cam_path, camera_matrix)
np.save(dist_path, dist_coefs)
except:
print "no camera intrinsics found, will not copy them into data folder"
g_pool.pos_record.value = 0
del record.writer
bar.record_running = 0
###render the screen
clear_gl_screen()
cv2.cvtColor(img, cv2.COLOR_BGR2RGB,img)
draw_gl_texture(img)
###render calibration results:
if bar.show_calib_result.value:
cal_pt_cloud = np.load("cal_pt_cloud.npy")
pX,pY,wX,wY = cal_pt_cloud.transpose()
map_fn = get_map_from_cloud(cal_pt_cloud,(width,height))
modelled_world_pts = map_fn((pX,pY))
pts = np.array(modelled_world_pts,dtype=np.float32).transpose()
calib_bounds = cv2.convexHull(pts)[:,0]
for observed,modelled in zip(zip(wX,wY),np.array(modelled_world_pts).transpose()):
draw_gl_polyline_norm((modelled,observed),(1.,0.5,0.,.5))
draw_gl_polyline_norm(calib_bounds,(1.0,0,0,.5))
#render visual feedback from detector
ref.detector.display(img)
# render detector point
if ref.detector.pos[0] or ref.detector.pos[1]:
draw_gl_point_norm(ref.detector.pos,(0.,1.,0.,0.5))
# update gaze point from shared variable pool and draw on screen. If both coords are 0: no pupil pos was detected.
if g_pool.gaze_x.value !=0 or g_pool.gaze_y.value !=0:
draw_gl_point_norm((g_pool.gaze_x.value, g_pool.gaze_y.value),(1.,0.,0.,0.5))
atb.draw()
glfwSwapBuffers()
###end while running clean-up
print "WORLD Process closed"
glfwCloseWindow()
glfwTerminate()
def main():
# Callback functions
def on_resize(window,w, h):
active_window = glfwGetCurrentContext()
glfwMakeContextCurrent(window)
adjust_gl_view(w,h,window)
norm_size = normalize((w,h),glfwGetWindowSize(window))
fb_size = denormalize(norm_size,glfwGetFramebufferSize(window))
atb.TwWindowSize(*map(int,fb_size))
glfwMakeContextCurrent(active_window)
for p in g.plugins:
p.on_window_resize(window,w,h)
def on_key(window, key, scancode, action, mods):
if not atb.TwEventKeyboardGLFW(key,action):
if action == GLFW_PRESS:
pass
def on_char(window,char):
if not atb.TwEventCharGLFW(char,1):
pass
def on_button(window,button, action, mods):
if not atb.TwEventMouseButtonGLFW(button,action):
pos = glfwGetCursorPos(window)
pos = normalize(pos,glfwGetWindowSize(main_window))
pos = denormalize(pos,(frame.img.shape[1],frame.img.shape[0]) ) # Position in img pixels
for p in g.plugins:
p.on_click(pos,button,action)
def on_pos(window,x, y):
norm_pos = normalize((x,y),glfwGetWindowSize(window))
fb_x,fb_y = denormalize(norm_pos,glfwGetFramebufferSize(window))
if atb.TwMouseMotion(int(fb_x),int(fb_y)):
pass
def on_scroll(window,x,y):
if not atb.TwMouseWheel(int(x)):
pass
def on_close(window):
glfwSetWindowShouldClose(main_window,True)
logger.debug('Process closing from window')
try:
rec_dir = sys.argv[1]
except:
#for dev, supply hardcoded dir:
rec_dir = '/Users/mkassner/Desktop/Marker_Tracking_Demo_Recording/'
if os.path.isdir(rec_dir):
logger.debug("Dev option: Using hadcoded data dir.")
else:
if getattr(sys, 'frozen', False):
logger.warning("You did not supply a data directory when you called this script! \
\nPlease drag a Pupil recoding directory onto the launch icon.")
else:
logger.warning("You did not supply a data directory when you called this script! \
\nPlease supply a Pupil recoding directory as first arg when calling Pupil Player.")
return
if not is_pupil_rec_dir(rec_dir):
logger.error("You did not supply a dir with the required files inside.")
return
#backwards compatibility fn.
patch_meta_info(rec_dir)
#parse and load data folder info
video_path = rec_dir + "/world.avi"
timestamps_path = rec_dir + "/timestamps.npy"
gaze_positions_path = rec_dir + "/gaze_positions.npy"
meta_info_path = rec_dir + "/info.csv"
#parse info.csv file
with open(meta_info_path) as info:
meta_info = dict( ((line.strip().split('\t')) for line in info.readlines() ) )
rec_version = meta_info["Capture Software Version"]
rec_version_float = int(filter(type(rec_version).isdigit, rec_version)[:3])/100. #(get major,minor,fix of version)
logger.debug("Recording version: %s , %s"%(rec_version,rec_version_float))
#load gaze information
gaze_list = np.load(gaze_positions_path)
timestamps = np.load(timestamps_path)
#correlate data
positions_by_frame = correlate_gaze(gaze_list,timestamps)
# load session persistent settings
session_settings = Persistent_Dict(os.path.join(user_dir,"user_settings"))
def load(var_name,default):
return session_settings.get(var_name,default)
def save(var_name,var):
session_settings[var_name] = var
# Initialize capture
cap = autoCreateCapture(video_path,timestamps=timestamps_path)
if isinstance(cap,FakeCapture):
logger.error("could not start capture.")
return
width,height = cap.get_size()
# Initialize glfw
glfwInit()
main_window = glfwCreateWindow(width, height, "Pupil Player: "+meta_info["Recording Name"]+" - "+ rec_dir.split(os.path.sep)[-1], None, None)
glfwMakeContextCurrent(main_window)
# Register callbacks main_window
glfwSetWindowSizeCallback(main_window,on_resize)
glfwSetWindowCloseCallback(main_window,on_close)
glfwSetKeyCallback(main_window,on_key)
glfwSetCharCallback(main_window,on_char)
glfwSetMouseButtonCallback(main_window,on_button)
glfwSetCursorPosCallback(main_window,on_pos)
glfwSetScrollCallback(main_window,on_scroll)
# create container for globally scoped varfs (within world)
g = Temp()
g.plugins = []
g.play = False
g.new_seek = True
g.user_dir = user_dir
g.rec_dir = rec_dir
g.app = 'player'
g.timestamps = timestamps
g.positions_by_frame = positions_by_frame
# helpers called by the main atb bar
def update_fps():
old_time, bar.timestamp = bar.timestamp, time()
dt = bar.timestamp - old_time
if dt:
bar.fps.value += .1 * (1. / dt - bar.fps.value)
def set_window_size(mode,data):
width,height = cap.get_size()
ratio = (1,.75,.5,.25)[mode]
w,h = int(width*ratio),int(height*ratio)
glfwSetWindowSize(main_window,w,h)
data.value=mode # update the bar.value
def get_from_data(data):
"""
helper for atb getter and setter use
"""
return data.value
def get_play():
return g.play
def set_play(value):
g.play = value
def next_frame():
try:
cap.seek_to_frame(cap.get_frame_index())
except FileSeekError:
pass
g.new_seek = True
def prev_frame():
try:
cap.seek_to_frame(cap.get_frame_index()-2)
except FileSeekError:
pass
g.new_seek = True
def open_plugin(selection,data):
if plugin_by_index[selection] not in additive_plugins:
for p in g.plugins:
if isinstance(p,plugin_by_index[selection]):
return
g.plugins = [p for p in g.plugins if p.alive]
logger.debug('Open Plugin: %s'%name_by_index[selection])
new_plugin = plugin_by_index[selection](g)
g.plugins.append(new_plugin)
g.plugins.sort(key=lambda p: p.order)
if hasattr(new_plugin,'init_gui'):
new_plugin.init_gui()
# save the value for atb bar
data.value=selection
def get_from_data(data):
"""
helper for atb getter and setter use
"""
return data.value
atb.init()
# add main controls ATB bar
bar = atb.Bar(name = "Controls", label="Controls",
help="Scene controls", color=(50, 50, 50), alpha=100,valueswidth=150,
text='light', position=(10, 10),refresh=.1, size=(300, 160))
bar.next_atb_pos = (10,220)
bar.fps = c_float(0.0)
bar.timestamp = time()
bar.window_size = c_int(load("window_size",0))
window_size_enum = atb.enum("Display Size",{"Full":0, "Medium":1,"Half":2,"Mini":3})
bar.version = create_string_buffer(version,512)
bar.recording_version = create_string_buffer(rec_version,512)
bar.add_var("fps", bar.fps, step=1., readonly=True)
bar._fps = c_float(cap.get_fps())
bar.add_var("recoding fps",bar._fps,readonly=True)
bar.add_var("display size", vtype=window_size_enum,setter=set_window_size,getter=get_from_data,data=bar.window_size)
bar.add_var("play",vtype=c_bool,getter=get_play,setter=set_play,key="space")
bar.add_button('step next',next_frame,key='right')
bar.add_button('step prev',prev_frame,key='left')
bar.add_var("frame index",vtype=c_int,getter=lambda:cap.get_frame_index()-1 )
bar.plugin_to_load = c_int(0)
plugin_type_enum = atb.enum("Plug In",index_by_name)
bar.add_var("plugin",setter=open_plugin,getter=get_from_data,data=bar.plugin_to_load, vtype=plugin_type_enum)
bar.add_var("version of recording",bar.recording_version, readonly=True, help="version of the capture software used to make this recording")
bar.add_var("version of player",bar.version, readonly=True, help="version of the Pupil Player")
bar.add_button("exit", on_close,data=main_window,key="esc")
#set the last saved window size
set_window_size(bar.window_size.value,bar.window_size)
on_resize(main_window, *glfwGetWindowSize(main_window))
glfwSetWindowPos(main_window,0,0)
#we always load these plugins
g.plugins.append(Export_Launcher(g,data_dir=rec_dir,frame_count=len(timestamps)))
g.plugins.append(Seek_Bar(g,capture=cap))
g.trim_marks = Trim_Marks(g,capture=cap)
g.plugins.append(g.trim_marks)
#these are loaded based on user settings
for initializer in load('plugins',[]):
name, args = initializer
logger.debug("Loading plugin: %s with settings %s"%(name, args))
try:
p = plugin_by_name[name](g,**args)
g.plugins.append(p)
except:
logger.warning("Plugin '%s' failed to load from settings file." %name)
if load('plugins',"_") == "_":
#lets load some default if we dont have presets
g.plugins.append(Scan_Path(g))
g.plugins.append(Vis_Polyline(g))
g.plugins.append(Vis_Circle(g))
# g.plugins.append(Vis_Light_Points(g))
#sort by exec order
g.plugins.sort(key=lambda p: p.order)
#init gui
for p in g.plugins:
if hasattr(p,'init_gui'):
p.init_gui()
# gl_state settings
basic_gl_setup()
g.image_tex = create_named_texture((height,width,3))
while not glfwWindowShouldClose(main_window):
update_fps()
#grab new frame
if g.play or g.new_seek:
try:
new_frame = cap.get_frame()
except EndofVideoFileError:
#end of video logic: pause at last frame.
g.play=False
if g.new_seek:
display_time = new_frame.timestamp
g.new_seek = False
frame = new_frame.copy()
#new positons and events we make a deepcopy just like the image is a copy.
current_pupil_positions = deepcopy(positions_by_frame[frame.index])
events = []
# allow each Plugin to do its work.
for p in g.plugins:
p.update(frame,current_pupil_positions,events)
#check if a plugin need to be destroyed
g.plugins = [p for p in g.plugins if p.alive]
# render camera image
glfwMakeContextCurrent(main_window)
make_coord_system_norm_based()
draw_named_texture(g.image_tex,frame.img)
make_coord_system_pixel_based(frame.img.shape)
# render visual feedback from loaded plugins
for p in g.plugins:
p.gl_display()
#present frames at appropriate speed
wait_time = frame.timestamp - display_time
display_time = frame.timestamp
try:
spent_time = time()-timestamp
sleep(wait_time-spent_time)
except:
pass
timestamp = time()
atb.draw()
glfwSwapBuffers(main_window)
glfwPollEvents()
plugin_save = []
for p in g.plugins:
try:
p_initializer = p.get_class_name(),p.get_init_dict()
plugin_save.append(p_initializer)
except AttributeError:
#not all plugins need to be savable, they will not have the init dict.
# any object without a get_init_dict method will throw this exception.
pass
# de-init all running plugins
for p in g.plugins:
p.alive = False
#reading p.alive actually runs plug-in cleanup
_ = p.alive
save('plugins',plugin_save)
save('window_size',bar.window_size.value)
session_settings.close()
cap.close()
bar.destroy()
glfwDestroyWindow(main_window)
glfwTerminate()
logger.debug("Process done")
def create_atb_bar(self, pos):
''' create advanced tweak bar with setting for Mr. Gaze '''
self.bar = atb.Bar(name="Mr_Gaze_Detector",
label="Mr. Gaze Controls",
help="Mr. Gaze Params",
color=(50, 50, 50),
alpha=100,
text='light',
position=pos,
refresh=.3,
size=(200, 250))
# VIDEO section
self.bar.add_var("downsampling",
vtype=c_long,
setter=self.set_downsampling,
getter=self.get_downsampling,
min=1,
max=5)
# PUPILDETECT section
self.bar.add_var("pupil detect",
vtype=c_bool,
setter=self.set_pupildetect_enabled,
getter=self.get_pupildetect_enabled)
self.bar.add_var("specificity",
vtype=c_long,
setter=self.set_specificity,
getter=self.get_specificity,
min=0,
max=100)
self.bar.add_var("scalefactor",
vtype=c_float,
setter=self.set_scalefactor,
getter=self.get_scalefactor,
min=0,
max=2,
step=0.01)
# PUPILSEG section
self.bar.pupilseg_method_enum = atb.enum("pupilseg M", {
"otsu": 0,
"manual": 1
})
self.bar.add_var("Pupilseg M",
vtype=self.bar.pupilseg_method_enum,
setter=self.set_pupilseg_method,
getter=self.get_pupilseg_method,
help="select pupil seg method")
self.bar.add_var("pupildiameterperc",
vtype=c_float,
setter=self.set_pupildiameterperc,
getter=self.get_pupildiameterperc,
min=0,
max=100)
self.bar.add_var("glintdiameterperc",
vtype=c_float,
setter=self.set_glintdiameterperc,
getter=self.get_glintdiameterperc,
min=0,
max=100)
self.bar.add_var("pupilthresholdperc",
vtype=c_float,
setter=self.set_pupilthresholdperc,
getter=self.get_pupilthresholdperc,
min=0,
max=100)
# self.bar.add_var("pupilhigh", vtype=c_float, setter=self.set_pupilhigh, getter=self.get_pupilhigh, min=0, max=100)
# self.bar.add_var("glintlow", vtype=c_float, setter=self.set_glintlow, getter=self.get_glintlow, min=0, max=100)
# self.bar.add_var("sigma", vtype=c_float, setter=self.set_sigma, getter=self.get_sigma, min=0, max=5)
# PUPILFIT section -----------
method_dict = {
0: "ROBUST_LSQ",
1: "LSQ",
2: "RANSAC",
3: "RANSAC_SUPPORT"
}
rev_method_dict = {y: x for x, y in method_dict.iteritems()}
self.bar.pupilfit_method_enum = atb.enum("pupilfit M", rev_method_dict)
self.bar.add_var("Pupilfit M",
vtype=self.bar.pupilfit_method_enum,
setter=self.set_pupilfit_method,
getter=self.get_pupilfit_method,
help="select pupil fit method")
self.bar.add_var("maxiterations",
vtype=c_long,
setter=self.set_maxiterations,
getter=self.get_maxiterations,
min=0,
max=25)
self.bar.add_var("maxrefinements",
vtype=c_long,
setter=self.set_maxrefinements,
getter=self.get_maxrefinements,
min=0,
max=25)
self.bar.add_var("maxinlierperc",
vtype=c_float,
setter=self.set_maxinlierperc,
getter=self.get_maxinlierperc,
min=0,
max=100)
# PREPROC section -------------
self.bar.add_var("perclow",
vtype=c_float,
setter=self.set_perclow,
getter=self.get_perclow,
min=0,
max=100)
self.bar.add_var("perchigh",
vtype=c_float,
setter=self.set_perchigh,
getter=self.get_perchigh,
min=0,
max=100)
# # ARTIFACTS section ------------------
# self.bar.add_var("MR clean", vtype=c_bool, setter=self.set_mrclean_enabled, getter=self.get_mrclean_enabled)
# self.bar.add_var("zthresh", vtype=c_float, setter=self.set_zthresh, getter=self.get_zthresh, min=0, max=25)
# method_dict = {0:"highpass", 1:"knownfixations"}
# rev_method_dict = {y:x for x,y in method_dict.iteritems()}
# self.bar.motioncorr_method_enum = atb.enum("MoCo Method", rev_method_dict)
# self.bar.add_var("Motioncorr M", vtype=self.bar.motioncorr_method_enum, setter=self.set_motioncorr_method, getter=self.get_motioncorr_method, help="select motion correction method")
# self.bar.add_var("mocokernel", vtype=c_long, setter=self.set_mocokernel, getter=self.get_mocokernel, min=0, max=250)
# OUTPUT section ------------
self.bar.add_var("graphics",
vtype=c_bool,
setter=self.set_graphics,
getter=self.get_graphics)
self.bar.add_var("verbose",
vtype=c_bool,
setter=self.set_verbose,
getter=self.get_verbose)
def __init__(self,g_pool,atb_pos=(0,0), numTrees = 10, numChecks = 50, knnk = 2, trainingProducts = []):
super(Shopping_Detector, self).__init__()
print 'network'
self.context = zmq.Context()
self.socket = self.context.socket(zmq.PUB)
self.address = create_string_buffer('tcp://127.0.0.1:5000', 512)
print 'acquired network connection'
self.set_server(self.address)
help_str = "Shopping Message server: Using ZMQ and the *Publish-Subscribe* scheme"
self.g_pool = g_pool
self.loadedProducts = False
self.trained = False
# mapper init
print 'sift created'
self.detector = cv2.SIFT()
FLANN_INDEX_KDTREE = 6
FLANN_INDEX_KDTREE = 0
self.index_params = dict(algorithm = FLANN_INDEX_KDTREE, trees = numTrees, table_number = 6, key_size = 12, multi_probe_level = 2)
self.index_params = dict(algorithm = FLANN_INDEX_KDTREE, trees = numTrees)
self.search_params = dict(checks = numChecks)
self.matcher = cv2.FlannBasedMatcher(self.index_params, self.search_params)
self.trainingProducts = trainingProducts
self.create_training_products()
'''
self.trainingProducts.append(Product('/Users/mobileexperiencelab/Downloads/digboston.jpg'))
self.trainingProducts.append(Product('/Users/mobileexperiencelab/Downloads/jobs.png'))
self.trainingProducts.append(Product('/Users/mobileexperiencelab/Downloads/espn.jpg'))
self.trainingProducts.append(Product('/Users/mobileexperiencelab/Downloads/a.jpg'))
'''
self.generate_training_data()
self.frameCount = 0
self.mappedAreas = []
self.knnk = c_int(knnk)
self.minGoodMatches = c_int(4)
self.maxAvgDistance = c_int(150)
self.homographyThreshold = c_float(10.0)
self.distanceWeight = .7
self.recentGazePoints = collections.deque([], 3)
self.recent_products_queue = TimedCounter(7.5)
self.reco_thread = None
#multi monitor setup
self.window_should_open = False
self.window_should_close = False
self._window = None
self.fullscreen = c_bool(0)
self.monitor_idx = c_int(0)
self.monitor_handles = glfwGetMonitors()
self.monitor_names = [glfwGetMonitorName(m) for m in self.monitor_handles]
monitor_enum = atb.enum("Monitor",dict(((key,val) for val,key in enumerate(self.monitor_names))))
#primary_monitor = glfwGetPrimaryMonitor()
atb_label = "Product Detection"
self._bar = atb.Bar(name =self.__class__.__name__, label=atb_label,
help="product detection parameters", color=(50, 50, 50), alpha=100,
text='light', position=atb_pos,refresh=.3, size=(300, 100))
self._bar.add_var("monitor",self.monitor_idx, vtype=monitor_enum,group="Window",)
self._bar.add_var("fullscreen", self.fullscreen,group="Window")
self._bar.add_button(" open Window ", self.do_open, key='m',group="Window")
self._bar.add_var("min good matches",self.minGoodMatches, step=1,min=0,group="Detector")
self._bar.add_var("k-nearest-neighbors",self.knnk, step=1,min=1,group="Detector")
self._bar.add_var("homogoraphy threshold",self.homographyThreshold, step=.1,min=1,group="Detector")
self._bar.add_var("max distance of good matches",self.maxAvgDistance, step=1,min=1,max=500,group="Detector")
self._bar.define("valueswidth=170")
self._bar.add_var("server address",self.address, getter=lambda:self.address, setter=self.set_server)
self._bar.add_button("close", self.close)
atb_pos = atb_pos[0],atb_pos[1]+110
self._bar_markers = atb.Bar(name =self.__class__.__name__+'markers', label='registered surfaces',
help="list of registered ref surfaces", color=(50, 100, 50), alpha=100,
text='light', position=atb_pos,refresh=.3, size=(300, 120))
fill = c_bool(1)
color = (c_float * 3)(1.0,1.0,0.5)
shape = c_int(0)
lighted = c_bool(1)
bar.add_var("Trackball/Phi", step=0.5, getter=trackball._get_phi, setter=trackball._set_phi)
bar.add_var("Trackball/Theta", step=0.5, getter=trackball._get_theta, setter=trackball._set_theta)
bar.add_var("Trackball/Zoom", step=0.01, getter=trackball._get_zoom, setter=trackball._set_zoom)
bar.add_var("Light/State", lighted)
bar.add_var("Light/Position", position, vtype=atb.TW_TYPE_QUAT4F)
bar.add_var("Light/Diffuse", diffuse)
bar.add_var("Light/Ambient", ambient)
bar.add_var("Light/Specular", specular)
Shape = atb.enum("Shape", {'Cube':0, 'Torus':1, 'Teapot':2})
bar.add_var("Object/Shape", shape, vtype=Shape)
bar.add_var("Object/Fill", fill)
bar.add_var("Object/Color", color)
bar.add_separator("")
bar.add_button("Quit", quit, key="ESCAPE", help="Quit application")
gl.glEnable (gl.GL_BLEND)
gl.glEnable (gl.GL_COLOR_MATERIAL)
gl.glColorMaterial(gl.GL_FRONT_AND_BACK, gl.GL_AMBIENT_AND_DIFFUSE)
gl.glBlendFunc (gl.GL_SRC_ALPHA, gl.GL_ONE_MINUS_SRC_ALPHA)
glut.glutMouseFunc(on_mouse)
glut.glutReshapeFunc(on_reshape)
glut.glutDisplayFunc(on_draw)
glut.glutPassiveMotionFunc(on_motion)
def main():
# Callback functions
def on_resize(window, w, h):
active_window = glfwGetCurrentContext()
glfwMakeContextCurrent(window)
adjust_gl_view(w, h, window)
norm_size = normalize((w, h), glfwGetWindowSize(window))
fb_size = denormalize(norm_size, glfwGetFramebufferSize(window))
atb.TwWindowSize(*map(int, fb_size))
glfwMakeContextCurrent(active_window)
for p in g.plugins:
p.on_window_resize(window, w, h)
def on_key(window, key, scancode, action, mods):
if not atb.TwEventKeyboardGLFW(key, action):
if action == GLFW_PRESS:
pass
def on_char(window, char):
if not atb.TwEventCharGLFW(char, 1):
pass
def on_button(window, button, action, mods):
if not atb.TwEventMouseButtonGLFW(button, action):
pos = glfwGetCursorPos(window)
pos = normalize(pos, glfwGetWindowSize(main_window))
pos = denormalize(pos,
(frame.img.shape[1],
frame.img.shape[0])) # Position in img pixels
for p in g.plugins:
p.on_click(pos, button, action)
def on_pos(window, x, y):
norm_pos = normalize((x, y), glfwGetWindowSize(window))
fb_x, fb_y = denormalize(norm_pos, glfwGetFramebufferSize(window))
if atb.TwMouseMotion(int(fb_x), int(fb_y)):
pass
def on_scroll(window, x, y):
if not atb.TwMouseWheel(int(x)):
pass
def on_close(window):
glfwSetWindowShouldClose(main_window, True)
logger.debug('Process closing from window')
try:
rec_dir = sys.argv[1]
except:
#for dev, supply hardcoded dir:
rec_dir = '/Users/mkassner/Desktop/Marker_Tracking_Demo_Recording/'
if os.path.isdir(rec_dir):
logger.debug("Dev option: Using hadcoded data dir.")
else:
if getattr(sys, 'frozen', False):
logger.warning(
"You did not supply a data directory when you called this script! \
\nPlease drag a Pupil recoding directory onto the launch icon."
)
else:
logger.warning(
"You did not supply a data directory when you called this script! \
\nPlease supply a Pupil recoding directory as first arg when calling Pupil Player."
)
return
if not is_pupil_rec_dir(rec_dir):
logger.error(
"You did not supply a dir with the required files inside.")
return
#backwards compatibility fn.
patch_meta_info(rec_dir)
#parse and load data folder info
video_path = rec_dir + "/world.avi"
timestamps_path = rec_dir + "/timestamps.npy"
gaze_positions_path = rec_dir + "/gaze_positions.npy"
meta_info_path = rec_dir + "/info.csv"
#parse info.csv file
with open(meta_info_path) as info:
meta_info = dict(
((line.strip().split('\t')) for line in info.readlines()))
rec_version = meta_info["Capture Software Version"]
rec_version_float = int(
filter(type(rec_version).isdigit,
rec_version)[:3]) / 100. #(get major,minor,fix of version)
logger.debug("Recording version: %s , %s" %
(rec_version, rec_version_float))
#load gaze information
gaze_list = np.load(gaze_positions_path)
timestamps = np.load(timestamps_path)
#correlate data
positions_by_frame = correlate_gaze(gaze_list, timestamps)
# load session persistent settings
session_settings = Persistent_Dict(os.path.join(user_dir, "user_settings"))
def load(var_name, default):
return session_settings.get(var_name, default)
def save(var_name, var):
session_settings[var_name] = var
# Initialize capture
cap = autoCreateCapture(video_path, timestamps=timestamps_path)
if isinstance(cap, FakeCapture):
logger.error("could not start capture.")
return
width, height = cap.get_size()
# Initialize glfw
glfwInit()
main_window = glfwCreateWindow(
width, height, "Pupil Player: " + meta_info["Recording Name"] + " - " +
rec_dir.split(os.path.sep)[-1], None, None)
glfwMakeContextCurrent(main_window)
# Register callbacks main_window
glfwSetWindowSizeCallback(main_window, on_resize)
glfwSetWindowCloseCallback(main_window, on_close)
glfwSetKeyCallback(main_window, on_key)
glfwSetCharCallback(main_window, on_char)
glfwSetMouseButtonCallback(main_window, on_button)
glfwSetCursorPosCallback(main_window, on_pos)
glfwSetScrollCallback(main_window, on_scroll)
# create container for globally scoped varfs (within world)
g = Temp()
g.plugins = []
g.play = False
g.new_seek = True
g.user_dir = user_dir
g.rec_dir = rec_dir
g.app = 'player'
g.timestamps = timestamps
g.positions_by_frame = positions_by_frame
# helpers called by the main atb bar
def update_fps():
old_time, bar.timestamp = bar.timestamp, time()
dt = bar.timestamp - old_time
if dt:
bar.fps.value += .1 * (1. / dt - bar.fps.value)
def set_window_size(mode, data):
width, height = cap.get_size()
ratio = (1, .75, .5, .25)[mode]
w, h = int(width * ratio), int(height * ratio)
glfwSetWindowSize(main_window, w, h)
data.value = mode # update the bar.value
def get_from_data(data):
"""
helper for atb getter and setter use
"""
return data.value
def get_play():
return g.play
def set_play(value):
g.play = value
def next_frame():
try:
cap.seek_to_frame(cap.get_frame_index())
except FileSeekError:
pass
g.new_seek = True
def prev_frame():
try:
cap.seek_to_frame(cap.get_frame_index() - 2)
except FileSeekError:
pass
g.new_seek = True
def open_plugin(selection, data):
if plugin_by_index[selection] not in additive_plugins:
for p in g.plugins:
if isinstance(p, plugin_by_index[selection]):
return
g.plugins = [p for p in g.plugins if p.alive]
logger.debug('Open Plugin: %s' % name_by_index[selection])
new_plugin = plugin_by_index[selection](g)
g.plugins.append(new_plugin)
g.plugins.sort(key=lambda p: p.order)
if hasattr(new_plugin, 'init_gui'):
new_plugin.init_gui()
# save the value for atb bar
data.value = selection
def get_from_data(data):
"""
helper for atb getter and setter use
"""
return data.value
atb.init()
# add main controls ATB bar
bar = atb.Bar(name="Controls",
label="Controls",
help="Scene controls",
color=(50, 50, 50),
alpha=100,
valueswidth=150,
text='light',
position=(10, 10),
refresh=.1,
size=(300, 160))
bar.next_atb_pos = (10, 220)
bar.fps = c_float(0.0)
bar.timestamp = time()
bar.window_size = c_int(load("window_size", 0))
window_size_enum = atb.enum("Display Size", {
"Full": 0,
"Medium": 1,
"Half": 2,
"Mini": 3
})
bar.version = create_string_buffer(version, 512)
bar.recording_version = create_string_buffer(rec_version, 512)
bar.add_var("fps", bar.fps, step=1., readonly=True)
bar._fps = c_float(cap.get_fps())
bar.add_var("recoding fps", bar._fps, readonly=True)
bar.add_var("display size",
vtype=window_size_enum,
setter=set_window_size,
getter=get_from_data,
data=bar.window_size)
bar.add_var("play",
vtype=c_bool,
getter=get_play,
setter=set_play,
key="space")
bar.add_button('step next', next_frame, key='right')
bar.add_button('step prev', prev_frame, key='left')
bar.add_var("frame index",
vtype=c_int,
getter=lambda: cap.get_frame_index() - 1)
bar.plugin_to_load = c_int(0)
plugin_type_enum = atb.enum("Plug In", index_by_name)
bar.add_var("plugin",
setter=open_plugin,
getter=get_from_data,
data=bar.plugin_to_load,
vtype=plugin_type_enum)
bar.add_var(
"version of recording",
bar.recording_version,
readonly=True,
help="version of the capture software used to make this recording")
bar.add_var("version of player",
bar.version,
readonly=True,
help="version of the Pupil Player")
bar.add_button("exit", on_close, data=main_window, key="esc")
#set the last saved window size
set_window_size(bar.window_size.value, bar.window_size)
on_resize(main_window, *glfwGetWindowSize(main_window))
glfwSetWindowPos(main_window, 0, 0)
#we always load these plugins
g.plugins.append(
Export_Launcher(g, data_dir=rec_dir, frame_count=len(timestamps)))
g.plugins.append(Seek_Bar(g, capture=cap))
g.trim_marks = Trim_Marks(g, capture=cap)
g.plugins.append(g.trim_marks)
#these are loaded based on user settings
for initializer in load('plugins', []):
name, args = initializer
logger.debug("Loading plugin: %s with settings %s" % (name, args))
try:
p = plugin_by_name[name](g, **args)
g.plugins.append(p)
except:
logger.warning("Plugin '%s' failed to load from settings file." %
name)
if load('plugins', "_") == "_":
#lets load some default if we dont have presets
g.plugins.append(Scan_Path(g))
g.plugins.append(Vis_Polyline(g))
g.plugins.append(Vis_Circle(g))
# g.plugins.append(Vis_Light_Points(g))
#sort by exec order
g.plugins.sort(key=lambda p: p.order)
#init gui
for p in g.plugins:
if hasattr(p, 'init_gui'):
p.init_gui()
# gl_state settings
basic_gl_setup()
g.image_tex = create_named_texture((height, width, 3))
while not glfwWindowShouldClose(main_window):
update_fps()
#grab new frame
if g.play or g.new_seek:
try:
new_frame = cap.get_frame()
except EndofVideoFileError:
#end of video logic: pause at last frame.
g.play = False
if g.new_seek:
display_time = new_frame.timestamp
g.new_seek = False
frame = new_frame.copy()
#new positons and events we make a deepcopy just like the image is a copy.
current_pupil_positions = deepcopy(positions_by_frame[frame.index])
events = []
# allow each Plugin to do its work.
for p in g.plugins:
p.update(frame, current_pupil_positions, events)
#check if a plugin need to be destroyed
g.plugins = [p for p in g.plugins if p.alive]
# render camera image
glfwMakeContextCurrent(main_window)
make_coord_system_norm_based()
draw_named_texture(g.image_tex, frame.img)
make_coord_system_pixel_based(frame.img.shape)
# render visual feedback from loaded plugins
for p in g.plugins:
p.gl_display()
#present frames at appropriate speed
wait_time = frame.timestamp - display_time
display_time = frame.timestamp
try:
spent_time = time() - timestamp
sleep(wait_time - spent_time)
except:
pass
timestamp = time()
atb.draw()
glfwSwapBuffers(main_window)
glfwPollEvents()
plugin_save = []
for p in g.plugins:
try:
p_initializer = p.get_class_name(), p.get_init_dict()
plugin_save.append(p_initializer)
except AttributeError:
#not all plugins need to be savable, they will not have the init dict.
# any object without a get_init_dict method will throw this exception.
pass
# de-init all running plugins
for p in g.plugins:
p.alive = False
#reading p.alive actually runs plug-in cleanup
_ = p.alive
save('plugins', plugin_save)
save('window_size', bar.window_size.value)
session_settings.close()
cap.close()
bar.destroy()
glfwDestroyWindow(main_window)
glfwTerminate()
logger.debug("Process done")
def world(g_pool):
"""world
"""
# Callback functions
def on_resize(w, h):
atb.TwWindowSize(w, h);
adjust_gl_view(w,h)
def on_key(key, pressed):
if not atb.TwEventKeyboardGLFW(key,pressed):
if pressed:
if key == GLFW_KEY_ESC:
on_close()
def on_char(char, pressed):
if not atb.TwEventCharGLFW(char,pressed):
pass
def on_button(button, pressed):
if not atb.TwEventMouseButtonGLFW(button,pressed):
if pressed:
pos = glfwGetMousePos()
pos = normalize(pos,glfwGetWindowSize())
pos = denormalize(pos,(frame.img.shape[1],frame.img.shape[0]) ) # Position in img pixels
for p in g.plugins:
p.on_click(pos)
def on_pos(x, y):
if atb.TwMouseMotion(x,y):
pass
def on_scroll(pos):
if not atb.TwMouseWheel(pos):
pass
def on_close():
g_pool.quit.value = True
print "WORLD Process closing from window"
# load session persistent settings
session_settings = shelve.open('user_settings_world',protocol=2)
def load(var_name,default):
try:
return session_settings[var_name]
except:
return default
def save(var_name,var):
session_settings[var_name] = var
# gaze object
gaze = Temp()
gaze.map_coords = (0., 0.)
gaze.image_coords = (0., 0.)
# Initialize capture, check if it works
cap = autoCreateCapture(g_pool.world_src, g_pool.world_size,24)
if cap is None:
print "WORLD: Error could not create Capture"
return
frame = cap.get_frame()
if frame.img is None:
print "WORLD: Error could not get image"
return
height,width = frame.img.shape[:2]
# helpers called by the main atb bar
def update_fps():
old_time, bar.timestamp = bar.timestamp, time()
dt = bar.timestamp - old_time
if dt:
bar.fps.value += .05 * (1 / dt - bar.fps.value)
def set_window_size(mode,data):
height,width = frame.img.shape[:2]
ratio = (1,.75,.5,.25)[mode]
w,h = int(width*ratio),int(height*ratio)
glfwSetWindowSize(w,h)
data.value=mode # update the bar.value
def get_from_data(data):
"""
helper for atb getter and setter use
"""
return data.value
def open_calibration(selection,data):
# prepare destruction of old ref_detector.
if g.current_ref_detector:
g.current_ref_detector.alive = False
# remove old ref detector from list of plugins
g.plugins = [p for p in g.plugins if p.alive]
print "selected: ",reference_detectors.name_by_index[selection]
g.current_ref_detector = reference_detectors.detector_by_index[selection](global_calibrate=g_pool.calibrate,
shared_pos=g_pool.ref,
screen_marker_pos = g_pool.marker,
screen_marker_state = g_pool.marker_state,
atb_pos=bar.next_atb_pos)
g.plugins.append(g.current_ref_detector)
# save the value for atb bar
data.value=selection
def toggle_record_video():
if any([True for p in g.plugins if isinstance(p,recorder.Recorder)]):
for p in g.plugins:
if isinstance(p,recorder.Recorder):
p.alive = False
else:
# set up folder within recordings named by user input in atb
if not bar.rec_name.value:
bar.rec_name.value = recorder.get_auto_name()
recorder_instance = recorder.Recorder(bar.rec_name.value, bar.fps.value, frame.img.shape, g_pool.pos_record, g_pool.eye_tx)
g.plugins.append(recorder_instance)
def toggle_show_calib_result():
if any([True for p in g.plugins if isinstance(p,Show_Calibration)]):
for p in g.plugins:
if isinstance(p,Show_Calibration):
p.alive = False
else:
calib = Show_Calibration(frame.img.shape)
g.plugins.append(calib)
def show_calib_result():
# kill old if any
if any([True for p in g.plugins if isinstance(p,Show_Calibration)]):
for p in g.plugins:
if isinstance(p,Show_Calibration):
p.alive = False
g.plugins = [p for p in g.plugins if p.alive]
# make new
calib = Show_Calibration(frame.img.shape)
g.plugins.append(calib)
def hide_calib_result():
if any([True for p in g.plugins if isinstance(p,Show_Calibration)]):
for p in g.plugins:
if isinstance(p,Show_Calibration):
p.alive = False
# Initialize ant tweak bar - inherits from atb.Bar
atb.init()
bar = atb.Bar(name = "World", label="Controls",
help="Scene controls", color=(50, 50, 50), alpha=100,valueswidth=150,
text='light', position=(10, 10),refresh=.3, size=(300, 200))
bar.next_atb_pos = (10,220)
bar.fps = c_float(0.0)
bar.timestamp = time()
bar.calibration_type = c_int(load("calibration_type",0))
bar.show_calib_result = c_bool(0)
bar.record_video = c_bool(0)
bar.record_running = c_bool(0)
bar.play = g_pool.play
bar.window_size = c_int(load("window_size",0))
window_size_enum = atb.enum("Display Size",{"Full":0, "Medium":1,"Half":2,"Mini":3})
bar.calibrate_type_enum = atb.enum("Calibration Method",reference_detectors.index_by_name)
bar.rec_name = create_string_buffer(512)
bar.rec_name.value = recorder.get_auto_name()
# play and record can be tied together via pointers to the objects
# bar.play = bar.record_video
bar.add_var("fps", bar.fps, step=1., readonly=True)
bar.add_var("display size", vtype=window_size_enum,setter=set_window_size,getter=get_from_data,data=bar.window_size)
bar.add_var("calibration method",setter=open_calibration,getter=get_from_data,data=bar.calibration_type, vtype=bar.calibrate_type_enum,group="Calibration", help="Please choose your desired calibration method.")
bar.add_button("show calibration result",toggle_show_calib_result, group="Calibration", help="Click to show calibration result.")
bar.add_var("session name",bar.rec_name, group="Recording", help="creates folder Data_Name_XXX, where xxx is an increasing number")
bar.add_button("record", toggle_record_video, key="r", group="Recording", help="Start/Stop Recording")
bar.add_separator("Sep1")
bar.add_var("play video", bar.play, help="play a video in the Player window")
bar.add_var("exit", g_pool.quit)
# add uvc camera controls to a seperate ATB bar
cap.create_atb_bar(pos=(320,10))
# create container for globally scoped vars (within world)
g = Temp()
g.plugins = []
g.current_ref_detector = None
open_calibration(bar.calibration_type.value,bar.calibration_type)
# Initialize glfw
glfwInit()
height,width = frame.img.shape[:2]
glfwOpenWindow(width, height, 0, 0, 0, 8, 0, 0, GLFW_WINDOW)
glfwSetWindowTitle("World")
glfwSetWindowPos(0,0)
#set the last saved window size
set_window_size(bar.window_size.value,bar.window_size)
# Register callbacks
glfwSetWindowSizeCallback(on_resize)
glfwSetWindowCloseCallback(on_close)
glfwSetKeyCallback(on_key)
glfwSetCharCallback(on_char)
glfwSetMouseButtonCallback(on_button)
glfwSetMousePosCallback(on_pos)
glfwSetMouseWheelCallback(on_scroll)
# gl_state settings
import OpenGL.GL as gl
gl.glEnable(gl.GL_POINT_SMOOTH)
gl.glBlendFunc(gl.GL_SRC_ALPHA, gl.GL_ONE_MINUS_SRC_ALPHA)
gl.glEnable(gl.GL_BLEND)
del gl
# Event loop
while glfwGetWindowParam(GLFW_OPENED) and not g_pool.quit.value:
# Get input characters entered in player
if g_pool.player_input.value:
player_input = g_pool.player_input.value
g_pool.player_input.value = 0
on_char(player_input,True)
# Get an image from the grabber
frame = cap.get_frame()
update_fps()
for p in g.plugins:
p.update(frame)
g.plugins = [p for p in g.plugins if p.alive]
g_pool.player_refresh.set()
# render the screen
clear_gl_screen()
draw_gl_texture(frame.img)
# render visual feedback from loaded plugins
for p in g.plugins:
p.gl_display()
# update gaze point from shared variable pool and draw on screen. If both coords are 0: no pupil pos was detected.
if not g_pool.gaze[:] == [0.,0.]:
draw_gl_point_norm(g_pool.gaze[:],color=(1.,0.,0.,0.5))
atb.draw()
glfwSwapBuffers()
# end while running and clean-up
# de-init all running plugins
for p in g.plugins:
p.alive = False
g.plugins = [p for p in g.plugins if p.alive]
save('window_size',bar.window_size.value)
save('calibration_type',bar.calibration_type.value)
session_settings.close()
cap.close()
glfwCloseWindow()
glfwTerminate()
print "WORLD Process closed"
def world(g_pool, cap_src, cap_size):
"""world
Creates a window, gl context.
Grabs images from a capture.
Receives Pupil coordinates from g_pool.pupil_queue
Can run various plug-ins.
"""
# Callback functions
def on_resize(window, w, h):
active_window = glfwGetCurrentContext()
glfwMakeContextCurrent(window)
norm_size = normalize((w, h), glfwGetWindowSize(window))
fb_size = denormalize(norm_size, glfwGetFramebufferSize(window))
atb.TwWindowSize(*map(int, fb_size))
adjust_gl_view(w, h, window)
glfwMakeContextCurrent(active_window)
for p in g_pool.plugins:
p.on_window_resize(window, w, h)
def on_iconify(window, iconfied):
if not isinstance(cap, FakeCapture):
g_pool.update_textures.value = not iconfied
def on_key(window, key, scancode, action, mods):
if not atb.TwEventKeyboardGLFW(key, action):
if action == GLFW_PRESS:
if key == GLFW_KEY_ESCAPE:
on_close(window)
def on_char(window, char):
if not atb.TwEventCharGLFW(char, 1):
pass
def on_button(window, button, action, mods):
if not atb.TwEventMouseButtonGLFW(button, action):
pos = glfwGetCursorPos(window)
pos = normalize(pos, glfwGetWindowSize(world_window))
pos = denormalize(pos,
(frame.img.shape[1],
frame.img.shape[0])) # Position in img pixels
for p in g_pool.plugins:
p.on_click(pos, button, action)
def on_pos(window, x, y):
norm_pos = normalize((x, y), glfwGetWindowSize(window))
fb_x, fb_y = denormalize(norm_pos, glfwGetFramebufferSize(window))
if atb.TwMouseMotion(int(fb_x), int(fb_y)):
pass
def on_scroll(window, x, y):
if not atb.TwMouseWheel(int(x)):
pass
def on_close(window):
g_pool.quit.value = True
logger.info('Process closing from window')
# load session persistent settings
session_settings = Persistent_Dict(
os.path.join(g_pool.user_dir, 'user_settings_world'))
def load(var_name, default):
return session_settings.get(var_name, default)
def save(var_name, var):
session_settings[var_name] = var
# Initialize capture
cap = autoCreateCapture(cap_src, cap_size, 24, timebase=g_pool.timebase)
# Get an image from the grabber
try:
frame = cap.get_frame()
except CameraCaptureError:
logger.error("Could not retrieve image from capture")
cap.close()
return
height, width = frame.img.shape[:2]
# load last calibration data
try:
pt_cloud = np.load(os.path.join(g_pool.user_dir, 'cal_pt_cloud.npy'))
logger.debug("Using calibration found in %s" % g_pool.user_dir)
map_pupil = calibrate.get_map_from_cloud(pt_cloud, (width, height))
except:
logger.debug("No calibration found.")
def map_pupil(vector):
""" 1 to 1 mapping """
return vector
# any object we attach to the g_pool object *from now on* will only be visible to this process!
# vars should be declared here to make them visible to the code reader.
g_pool.plugins = []
g_pool.map_pupil = map_pupil
g_pool.update_textures = c_bool(1)
if isinstance(cap, FakeCapture):
g_pool.update_textures.value = False
g_pool.capture = cap
g_pool.rec_name = recorder.get_auto_name()
# helpers called by the main atb bar
def update_fps():
old_time, bar.timestamp = bar.timestamp, time()
dt = bar.timestamp - old_time
if dt:
bar.fps.value += .05 * (1. / dt - bar.fps.value)
def set_window_size(mode, data):
height, width = frame.img.shape[:2]
ratio = (1, .75, .5, .25)[mode]
w, h = int(width * ratio), int(height * ratio)
glfwSetWindowSize(world_window, w, h)
data.value = mode # update the bar.value
def get_from_data(data):
"""
helper for atb getter and setter use
"""
return data.value
def set_rec_name(val):
if not val.value:
g_pool.rec_name = recorder.get_auto_name()
else:
g_pool.rec_name = val.value
def get_rec_name():
return create_string_buffer(g_pool.rec_name, 512)
def open_calibration(selection, data):
# prepare destruction of current ref_detector... and remove it
for p in g_pool.plugins:
if isinstance(p, calibration_routines.detector_by_index):
p.alive = False
g_pool.plugins = [p for p in g_pool.plugins if p.alive]
new_ref_detector = calibration_routines.detector_by_index[selection](
g_pool, atb_pos=bar.next_atb_pos)
g_pool.plugins.append(new_ref_detector)
g_pool.plugins.sort(key=lambda p: p.order)
# save the value for atb bar
data.value = selection
def toggle_record_video():
for p in g_pool.plugins:
if isinstance(p, recorder.Recorder):
p.alive = False
return
new_plugin = recorder.Recorder(g_pool, g_pool.rec_name, bar.fps.value,
frame.img.shape, bar.record_eye.value,
g_pool.eye_tx, bar.audio.value)
g_pool.plugins.append(new_plugin)
g_pool.plugins.sort(key=lambda p: p.order)
def toggle_show_calib_result():
for p in g_pool.plugins:
if isinstance(p, Show_Calibration):
p.alive = False
return
new_plugin = Show_Calibration(g_pool, frame.img.shape)
g_pool.plugins.append(new_plugin)
g_pool.plugins.sort(key=lambda p: p.order)
def toggle_server():
for p in g_pool.plugins:
if isinstance(p, Pupil_Server):
p.alive = False
return
new_plugin = Pupil_Server(g_pool, (10, 300))
g_pool.plugins.append(new_plugin)
g_pool.plugins.sort(key=lambda p: p.order)
def toggle_remote():
for p in g_pool.plugins:
if isinstance(p, Pupil_Remote):
p.alive = False
return
new_plugin = Pupil_Remote(g_pool, (10, 360), on_char)
g_pool.plugins.append(new_plugin)
g_pool.plugins.sort(key=lambda p: p.order)
def toggle_ar():
for p in g_pool.plugins:
if isinstance(p, Marker_Detector):
p.alive = False
return
new_plugin = Marker_Detector(g_pool, (10, 400))
g_pool.plugins.append(new_plugin)
g_pool.plugins.sort(key=lambda p: p.order)
def reset_timebase():
#the last frame from worldcam will be t0
g_pool.timebase.value = g_pool.capure.get_now()
logger.info(
"New timebase set to %s all timestamps will count from here now." %
g_pool.timebase.value)
atb.init()
# add main controls ATB bar
bar = atb.Bar(name="World",
label="Controls",
help="Scene controls",
color=(50, 50, 50),
alpha=100,
valueswidth=150,
text='light',
position=(10, 10),
refresh=.3,
size=(300, 200))
bar.next_atb_pos = (10, 220)
bar.fps = c_float(0.0)
bar.timestamp = time()
bar.calibration_type = c_int(load("calibration_type", 0))
bar.record_eye = c_bool(load("record_eye", 0))
bar.audio = c_int(load("audio", -1))
bar.window_size = c_int(load("window_size", 0))
window_size_enum = atb.enum("Display Size", {
"Full": 0,
"Medium": 1,
"Half": 2,
"Mini": 3
})
calibrate_type_enum = atb.enum("Calibration Method",
calibration_routines.index_by_name)
audio_enum = atb.enum("Audio Input", dict(Audio_Input_List()))
bar.version = create_string_buffer(g_pool.version, 512)
bar.add_var(
"fps",
bar.fps,
step=1.,
readonly=True,
help=
"Refresh speed of this process. Especially during recording it should not drop below the camera set frame rate."
)
bar.add_var(
"display size",
vtype=window_size_enum,
setter=set_window_size,
getter=get_from_data,
data=bar.window_size,
help="Resize the world window. This has no effect on the actual image."
)
bar.add_var("calibration method",
setter=open_calibration,
getter=get_from_data,
data=bar.calibration_type,
vtype=calibrate_type_enum,
group="Calibration",
help="Please choose your desired calibration method.")
bar.add_button("show calibration result",
toggle_show_calib_result,
group="Calibration",
help="Click to show calibration result.")
bar.add_var("session name",
create_string_buffer(512),
getter=get_rec_name,
setter=set_rec_name,
group="Recording",
help="Give your recording session a custom name.")
bar.add_button("record",
toggle_record_video,
key="r",
group="Recording",
help="Start/Stop Recording")
bar.add_var("record eye",
bar.record_eye,
group="Recording",
help="check to save raw video of eye")
bar.add_var("record audio",
bar.audio,
vtype=audio_enum,
group="Recording",
help="Select from audio recording options.")
bar.add_button(
"start/stop marker tracking",
toggle_ar,
key="x",
help="find markers in scene to map gaze onto referace surfaces")
bar.add_button(
"start/stop server",
toggle_server,
key="s",
help=
"the server broadcasts pupil and gaze positions locally or via network"
)
bar.add_button("start/stop remote",
toggle_remote,
key="w",
help="remote allows seding commad to pupil via network")
bar.add_button(
"set timebase to now",
reset_timebase,
help="this button allows the timestamps to count from now on.",
key="t")
bar.add_var(
"update screen",
g_pool.update_textures,
help=
"if you dont need to see the camera image updated, you can turn this of to reduce CPU load."
)
bar.add_separator("Sep1")
bar.add_var("version", bar.version, readonly=True)
bar.add_var("exit", g_pool.quit)
# add uvc camera controls ATB bar
cap.create_atb_bar(pos=(320, 10))
# Initialize glfw
glfwInit()
world_window = glfwCreateWindow(width, height, "World", None, None)
glfwMakeContextCurrent(world_window)
# Register callbacks world_window
glfwSetWindowSizeCallback(world_window, on_resize)
glfwSetWindowCloseCallback(world_window, on_close)
glfwSetWindowIconifyCallback(world_window, on_iconify)
glfwSetKeyCallback(world_window, on_key)
glfwSetCharCallback(world_window, on_char)
glfwSetMouseButtonCallback(world_window, on_button)
glfwSetCursorPosCallback(world_window, on_pos)
glfwSetScrollCallback(world_window, on_scroll)
#set the last saved window size
set_window_size(bar.window_size.value, bar.window_size)
on_resize(world_window, *glfwGetWindowSize(world_window))
glfwSetWindowPos(world_window, 0, 0)
# gl_state settings
basic_gl_setup()
g_pool.image_tex = create_named_texture(frame.img)
# refresh speed settings
glfwSwapInterval(0)
#load calibration plugin
open_calibration(bar.calibration_type.value, bar.calibration_type)
#load gaze_display plugin
g_pool.plugins.append(Display_Recent_Gaze(g_pool))
# Event loop
while not g_pool.quit.value:
# Get an image from the grabber
try:
frame = cap.get_frame()
except CameraCaptureError:
logger.error("Capture from Camera Failed. Stopping.")
break
except EndofVideoFileError:
logger.warning("Video File is done. Stopping")
break
update_fps()
#a container that allows plugins to post and read events
events = []
#receive and map pupil positions
recent_pupil_positions = []
while not g_pool.pupil_queue.empty():
p = g_pool.pupil_queue.get()
if p['norm_pupil'] is None:
p['norm_gaze'] = None
else:
p['norm_gaze'] = g_pool.map_pupil(p['norm_pupil'])
recent_pupil_positions.append(p)
# allow each Plugin to do its work.
for p in g_pool.plugins:
p.update(frame, recent_pupil_positions, events)
#check if a plugin need to be destroyed
g_pool.plugins = [p for p in g_pool.plugins if p.alive]
# render camera image
glfwMakeContextCurrent(world_window)
make_coord_system_norm_based()
if g_pool.update_textures.value:
draw_named_texture(g_pool.image_tex, frame.img)
else:
draw_named_texture(g_pool.image_tex)
make_coord_system_pixel_based(frame.img.shape)
# render visual feedback from loaded plugins
for p in g_pool.plugins:
p.gl_display()
atb.draw()
glfwSwapBuffers(world_window)
glfwPollEvents()
# de-init all running plugins
for p in g_pool.plugins:
p.alive = False
#reading p.alive actually runs plug-in cleanup
_ = p.alive
save('window_size', bar.window_size.value)
save('calibration_type', bar.calibration_type.value)
save('record_eye', bar.record_eye.value)
save('audio', bar.audio.value)
session_settings.close()
cap.close()
atb.terminate()
glfwDestroyWindow(world_window)
glfwTerminate()
logger.debug("Process done")
def create_atb_bar(self,pos):
''' create advanced tweak bar with setting for Mr. Gaze '''
self.bar = atb.Bar(name = "Mr_Gaze_Detector", label="Mr. Gaze Controls",
help="Mr. Gaze Params", color=(50, 50, 50), alpha=100,
text='light', position=pos, refresh=.3, size=(200, 250))
# VIDEO section
self.bar.add_var("downsampling", vtype=c_long, setter=self.set_downsampling, getter=self.get_downsampling, min=1, max=5)
# PUPILDETECT section
self.bar.add_var("pupil detect", vtype=c_bool, setter=self.set_pupildetect_enabled, getter=self.get_pupildetect_enabled)
self.bar.add_var("specificity", vtype=c_long, setter=self.set_specificity, getter=self.get_specificity, min=0, max=100)
self.bar.add_var("scalefactor", vtype=c_float, setter=self.set_scalefactor, getter=self.get_scalefactor, min=0, max=2, step=0.01)
# PUPILSEG section
self.bar.pupilseg_method_enum = atb.enum("pupilseg M",{"otsu":0,
"manual":1})
self.bar.add_var("Pupilseg M", vtype=self.bar.pupilseg_method_enum, setter=self.set_pupilseg_method, getter=self.get_pupilseg_method, help="select pupil seg method")
self.bar.add_var("pupildiameterperc", vtype=c_float, setter=self.set_pupildiameterperc, getter=self.get_pupildiameterperc, min=0, max=100)
self.bar.add_var("glintdiameterperc", vtype=c_float, setter=self.set_glintdiameterperc, getter=self.get_glintdiameterperc, min=0, max=100)
self.bar.add_var("pupilthresholdperc", vtype=c_float, setter=self.set_pupilthresholdperc, getter=self.get_pupilthresholdperc, min=0, max=100)
# self.bar.add_var("pupilhigh", vtype=c_float, setter=self.set_pupilhigh, getter=self.get_pupilhigh, min=0, max=100)
# self.bar.add_var("glintlow", vtype=c_float, setter=self.set_glintlow, getter=self.get_glintlow, min=0, max=100)
# self.bar.add_var("sigma", vtype=c_float, setter=self.set_sigma, getter=self.get_sigma, min=0, max=5)
# PUPILFIT section -----------
method_dict = {0:"ROBUST_LSQ", 1:"LSQ", 2:"RANSAC", 3:"RANSAC_SUPPORT"}
rev_method_dict = {y:x for x,y in method_dict.iteritems()}
self.bar.pupilfit_method_enum = atb.enum("pupilfit M", rev_method_dict)
self.bar.add_var("Pupilfit M", vtype=self.bar.pupilfit_method_enum, setter=self.set_pupilfit_method, getter=self.get_pupilfit_method, help="select pupil fit method")
self.bar.add_var("maxiterations", vtype=c_long, setter=self.set_maxiterations, getter=self.get_maxiterations, min=0, max=25)
self.bar.add_var("maxrefinements", vtype=c_long, setter=self.set_maxrefinements, getter=self.get_maxrefinements, min=0, max=25)
self.bar.add_var("maxinlierperc", vtype=c_float, setter=self.set_maxinlierperc, getter=self.get_maxinlierperc, min=0, max=100)
# PREPROC section -------------
self.bar.add_var("perclow", vtype=c_float, setter=self.set_perclow, getter=self.get_perclow, min=0, max=100)
self.bar.add_var("perchigh", vtype=c_float, setter=self.set_perchigh, getter=self.get_perchigh, min=0, max=100)
# # ARTIFACTS section ------------------
# self.bar.add_var("MR clean", vtype=c_bool, setter=self.set_mrclean_enabled, getter=self.get_mrclean_enabled)
# self.bar.add_var("zthresh", vtype=c_float, setter=self.set_zthresh, getter=self.get_zthresh, min=0, max=25)
# method_dict = {0:"highpass", 1:"knownfixations"}
# rev_method_dict = {y:x for x,y in method_dict.iteritems()}
# self.bar.motioncorr_method_enum = atb.enum("MoCo Method", rev_method_dict)
# self.bar.add_var("Motioncorr M", vtype=self.bar.motioncorr_method_enum, setter=self.set_motioncorr_method, getter=self.get_motioncorr_method, help="select motion correction method")
# self.bar.add_var("mocokernel", vtype=c_long, setter=self.set_mocokernel, getter=self.get_mocokernel, min=0, max=250)
# OUTPUT section ------------
self.bar.add_var("graphics", vtype=c_bool, setter=self.set_graphics, getter=self.get_graphics)
self.bar.add_var("verbose", vtype=c_bool, setter=self.set_verbose, getter=self.get_verbose)