def __init__(self, g_pool, show_segmentation=True):
super().__init__(g_pool)
self.storage = model.Classified_Segment_Storage(plugin=self,
rec_dir=g_pool.rec_dir)
self.seek_controller = controller.Eye_Movement_Seek_Controller(
plugin=self,
storage=self.storage,
seek_to_timestamp=self.seek_to_timestamp)
self.offline_controller = controller.Eye_Movement_Offline_Controller(
plugin=self,
storage=self.storage,
on_status=self.on_task_status,
on_progress=self.on_task_progress,
on_exception=self.on_task_exception,
on_completed=self.on_task_completed,
)
self.menu_content = ui.Menu_Content(
plugin=self,
label_text=self.MENU_LABEL_TEXT,
show_segmentation=show_segmentation,
)
self.prev_segment_button = ui.Prev_Segment_Button(
on_click=self.seek_controller.jump_to_prev_segment)
self.next_segment_button = ui.Next_Segment_Button(
on_click=self.seek_controller.jump_to_next_segment)
self._gaze_changed_listener = Listener(plugin=self,
topic="gaze_positions",
rec_dir=g_pool.rec_dir)
self._gaze_changed_listener.add_observer(
method_name="on_data_changed",
observer=self.offline_controller.classify)
self._eye_movement_changed_announcer = Announcer(
plugin=self,
topic=EYE_MOVEMENT_ANNOUNCER_TOPIC,
rec_dir=g_pool.rec_dir)
def __init__(self,
g_pool,
polyline_style_init_dict={},
scan_path_init_dict={},
**kwargs):
super().__init__(g_pool)
self.polyline_style_controller = PolylineStyleController(
**polyline_style_init_dict)
self.scan_path_controller = ScanPathController(g_pool,
**scan_path_init_dict)
self.scan_path_controller.add_observer("on_update_ui",
self._update_scan_path_ui)
self._gaze_changed_listener = Listener(plugin=self,
topic="gaze_positions",
rec_dir=g_pool.rec_dir)
self._gaze_changed_listener.add_observer(
method_name="on_data_changed",
observer=self.scan_path_controller.on_gaze_data_changed,
)
class Vis_Polyline(Visualizer_Plugin_Base, Observable):
order = 0.9
uniqueness = "not_unique"
icon_chr = chr(0xE922)
icon_font = "pupil_icons"
def __init__(self,
g_pool,
polyline_style_init_dict={},
scan_path_init_dict={},
**kwargs):
super().__init__(g_pool)
self.polyline_style_controller = PolylineStyleController(
**polyline_style_init_dict)
self.scan_path_controller = ScanPathController(g_pool,
**scan_path_init_dict)
self.scan_path_controller.add_observer("on_update_ui",
self._update_scan_path_ui)
self._gaze_changed_listener = Listener(plugin=self,
topic="gaze_positions",
rec_dir=g_pool.rec_dir)
self._gaze_changed_listener.add_observer(
method_name="on_data_changed",
observer=self.scan_path_controller.on_gaze_data_changed,
)
def get_init_dict(self):
return {
"polyline_style_init_dict":
self.polyline_style_controller.get_init_dict(),
"scan_path_init_dict":
self.scan_path_controller.get_init_dict(),
}
def init_ui(self):
polyline_style_thickness_slider = ui.Slider(
"thickness",
self.polyline_style_controller,
min=self.polyline_style_controller.thickness_min,
max=self.polyline_style_controller.thickness_max,
step=self.polyline_style_controller.thickness_step,
label="Line thickness",
)
polyline_style_color_info_text = ui.Info_Text(
"Set RGB color component values.")
polyline_style_color_r_slider = ui.Slider(
"r",
self.polyline_style_controller,
min=self.polyline_style_controller.rgba_min,
max=self.polyline_style_controller.rgba_max,
step=self.polyline_style_controller.rgba_step,
label="Red",
)
polyline_style_color_g_slider = ui.Slider(
"g",
self.polyline_style_controller,
min=self.polyline_style_controller.rgba_min,
max=self.polyline_style_controller.rgba_max,
step=self.polyline_style_controller.rgba_step,
label="Green",
)
polyline_style_color_b_slider = ui.Slider(
"b",
self.polyline_style_controller,
min=self.polyline_style_controller.rgba_min,
max=self.polyline_style_controller.rgba_max,
step=self.polyline_style_controller.rgba_step,
label="Blue",
)
scan_path_timeframe_range = ui.Slider(
"timeframe",
self.scan_path_controller,
min=self.scan_path_controller.min_timeframe,
max=self.scan_path_controller.max_timeframe,
step=self.scan_path_controller.timeframe_step,
label="Duration",
)
scan_path_doc = ui.Info_Text(
"Duration of past gaze to include in polyline.")
scan_path_status = ui.Info_Text("")
polyline_style_color_menu = ui.Growing_Menu("Color")
polyline_style_color_menu.collapsed = True
polyline_style_color_menu.append(polyline_style_color_info_text)
polyline_style_color_menu.append(polyline_style_color_r_slider)
polyline_style_color_menu.append(polyline_style_color_g_slider)
polyline_style_color_menu.append(polyline_style_color_b_slider)
scan_path_menu = ui.Growing_Menu("Gaze History")
scan_path_menu.collapsed = False
scan_path_menu.append(scan_path_doc)
scan_path_menu.append(scan_path_timeframe_range)
scan_path_menu.append(scan_path_status)
self.add_menu()
self.menu.label = "Gaze Polyline"
self.menu.append(polyline_style_thickness_slider)
self.menu.append(polyline_style_color_menu)
self.menu.append(scan_path_menu)
self.scan_path_timeframe_range = scan_path_timeframe_range
self.scan_path_status = scan_path_status
self._update_scan_path_ui()
def deinit_ui(self):
self.remove_menu()
self.scan_path_timeframe_range = None
self.scan_path_status = None
def recent_events(self, events):
self.scan_path_controller.process()
frame = events.get("frame")
if not frame:
return
self._draw_polyline_path(frame, events)
# self._draw_scan_path_debug(frame, events)
def cleanup(self):
self.scan_path_controller.cleanup()
def _update_scan_path_ui(self):
if self.menu_icon:
self.menu_icon.indicator_stop = self.scan_path_controller.progress
if self.scan_path_status:
self.scan_path_status.text = self.scan_path_controller.status_string
def _polyline_points(self, image_size, base_gaze_data,
scan_path_gaze_data):
if scan_path_gaze_data is not None:
points_fields = ["norm_x", "norm_y"]
gaze_points = scan_path_gaze_data[points_fields]
gaze_points = np.array(
gaze_points.tolist(),
dtype=gaze_points.dtype[0]) # FIXME: This is a workaround
gaze_points = gaze_points.reshape((-1, len(points_fields)))
gaze_points = np_denormalize(gaze_points, image_size, flip_y=True)
return gaze_points.tolist()
else:
return [
denormalize(datum["norm_pos"], image_size, flip_y=True)
for datum in base_gaze_data
if datum["confidence"] >= self.g_pool.min_data_confidence
]
def _draw_polyline_path(self, frame, events):
pts = self._polyline_points(
image_size=frame.img.shape[:-1][::-1],
base_gaze_data=events.get("gaze", []),
scan_path_gaze_data=self.scan_path_controller.
scan_path_gaze_for_frame(frame),
)
if not pts:
return
pts = np.array([pts], dtype=np.int32)
cv2.polylines(
frame.img,
pts,
isClosed=False,
color=self.polyline_style_controller.cv2_bgra,
thickness=self.polyline_style_controller.thickness,
lineType=cv2.LINE_AA,
)
def _draw_scan_path_debug(self, frame, events):
from methods import denormalize
from player_methods import transparent_circle
gaze_data = self.scan_path_controller.scan_path_gaze_for_frame(frame)
if gaze_data is None:
return
points_to_draw_count = len(gaze_data)
image_size = frame.img.shape[:-1][::-1]
for idx, datum in enumerate(gaze_data):
point = (datum["norm_x"], datum["norm_y"])
point = denormalize(point, image_size, flip_y=True)
gray = float(idx) / points_to_draw_count
transparent_circle(frame.img,
point,
radius=20,
color=(gray, gray, gray, 0.9),
thickness=2)
class Offline_Eye_Movement_Detector(Observable, Eye_Movement_Detector_Base):
"""
Eye movement classification detector based on segmented linear regression.
Event identification is based on segmentation that simultaneously denoises the signal and determines event
boundaries. The full gaze position time-series is segmented into an approximately optimal piecewise linear
function in O(n) time. Gaze feature parameters for classification into fixations, saccades, smooth pursuits and post-saccadic oscillations
are derived from human labeling in a data-driven manner.
More details about this approach can be found here:
https://www.nature.com/articles/s41598-017-17983-x
The open source implementation can be found here:
https://gitlab.com/nslr/nslr-hmm
"""
MENU_LABEL_TEXT = "Eye Movement Detector"
def __init__(self, g_pool, show_segmentation=True):
super().__init__(g_pool)
self.storage = model.Classified_Segment_Storage(plugin=self,
rec_dir=g_pool.rec_dir)
self.add_observer("on_task_completed", self.storage.save_to_disk)
self.seek_controller = controller.Eye_Movement_Seek_Controller(
plugin=self,
storage=self.storage,
seek_to_timestamp=self.seek_to_timestamp)
self.offline_controller = controller.Eye_Movement_Offline_Controller(
plugin=self,
storage=self.storage,
on_started=self.on_task_started,
on_status=self.on_task_status,
on_progress=self.on_task_progress,
on_exception=self.on_task_exception,
on_completed=self.on_task_completed,
)
self.menu_content = ui.Menu_Content(
plugin=self,
label_text=self.MENU_LABEL_TEXT,
show_segmentation=show_segmentation,
)
self.prev_segment_button = ui.Prev_Segment_Button(
on_click=self.seek_controller.jump_to_prev_segment)
self.next_segment_button = ui.Next_Segment_Button(
on_click=self.seek_controller.jump_to_next_segment)
self._gaze_changed_listener = Listener(plugin=self,
topic="gaze_positions",
rec_dir=g_pool.rec_dir)
self._gaze_changed_listener.add_observer(
method_name="on_data_changed",
observer=self.offline_controller.classify)
self._eye_movement_changed_announcer = Announcer(
plugin=self,
topic=EYE_MOVEMENT_ANNOUNCER_TOPIC,
rec_dir=g_pool.rec_dir)
#
def trigger_recalculate(self):
self.notify_all({
"subject": Notification_Subject.SHOULD_RECALCULATE,
"delay": 0.5
})
def seek_to_timestamp(self, timestamp):
self.notify_all({
"subject": "seek_control.should_seek",
"timestamp": timestamp
})
def on_task_started(self):
self.menu_content.update_error_text("")
def on_task_progress(self, progress: float):
self.menu_content.update_progress(progress)
def on_task_status(self, status: str):
self.menu_content.update_status(status)
def on_task_exception(self, exception: Exception):
error_message = f"{exception}"
logger.error(error_message)
self.menu_content.update_error_text(error_message)
def on_task_completed(self):
self._eye_movement_changed_announcer.announce_new()
#
def init_ui(self):
self.add_menu()
self.menu_content.add_to_menu(self.menu)
self.prev_segment_button.add_to_quickbar(self.g_pool.quickbar)
self.next_segment_button.add_to_quickbar(self.g_pool.quickbar)
if len(self.storage):
status = "Loaded from cache"
self.menu_content.update_status(status)
else:
self.trigger_recalculate()
def deinit_ui(self):
self.remove_menu()
self.prev_segment_button.remove_from_quickbar(self.g_pool.quickbar)
self.next_segment_button.remove_from_quickbar(self.g_pool.quickbar)
def get_init_dict(self):
return {"show_segmentation": self.menu_content.show_segmentation}
def on_notify(self, notification):
if notification["subject"] in (
Notification_Subject.SHOULD_RECALCULATE,
Notification_Subject.MIN_DATA_CONFIDENCE_CHANGED,
):
self.offline_controller.classify()
elif notification["subject"] == "should_export":
self.export_eye_movement(notification["ts_window"],
notification["export_dir"])
def recent_events(self, events):
frame = events.get("frame")
if not frame:
return
visible_segments = self.storage.segments_in_frame(frame)
self.seek_controller.update_visible_segments(visible_segments)
self.menu_content.update_detail_text(
current_index=self.seek_controller.current_segment_index,
total_segment_count=self.seek_controller.total_segment_count,
current_segment=self.seek_controller.current_segment,
prev_segment=self.seek_controller.prev_segment,
next_segment=self.seek_controller.next_segment,
)
if self.menu_content.show_segmentation:
segment_renderer = ui.Segment_Overlay_Image_Renderer(
canvas_size=(frame.width, frame.height), image=frame.img)
for segment in visible_segments:
segment_renderer.draw(segment)
events[utils.EYE_MOVEMENT_EVENT_KEY] = visible_segments
def export_eye_movement(self, export_window, export_dir):
segments_in_section = self.storage.segments_in_timestamp_window(
export_window)
if segments_in_section:
by_segment_csv_exporter = controller.Eye_Movement_By_Segment_CSV_Exporter(
)
by_segment_csv_exporter.csv_export(segments_in_section,
export_dir=export_dir)
export_window_start, export_window_stop = export_window
ts_segment_class_pairs = ((gaze["timestamp"], seg.segment_class)
for seg in segments_in_section
for gaze in seg.segment_data
if export_window_start <=
gaze["timestamp"] <= export_window_stop)
by_gaze_csv_exporter = controller.Eye_Movement_By_Gaze_CSV_Exporter(
)
by_gaze_csv_exporter.csv_export(ts_segment_class_pairs,
export_dir=export_dir)
else:
logger.warning(
"The selected export range does not include eye movement detections"
)
def cleanup(self):
self.remove_observer("on_task_completed", self.storage.save_to_disk)