class Offline_Pupil_Detection(Pupil_Producer_Base):
"""docstring for Offline_Pupil_Detection"""
session_data_version = 4
session_data_name = "offline_pupil"
@classmethod
def is_available_within_context(cls, g_pool) -> bool:
if g_pool.app == "player":
recording = PupilRecording(rec_dir=g_pool.rec_dir)
meta_info = recording.meta_info
if (
meta_info.recording_software_name
== RecordingInfo.RECORDING_SOFTWARE_NAME_PUPIL_INVISIBLE
):
# Disable post-hoc pupil detector in Player if Pupil Invisible recording
return False
return super().is_available_within_context(g_pool)
@classmethod
def plugin_menu_label(cls) -> str:
return "Post-Hoc Pupil Detection"
@classmethod
def pupil_data_source_selection_order(cls) -> float:
return 2.0
def __init__(self, g_pool):
super().__init__(g_pool)
self._detection_paused = False
zmq_ctx = zmq.Context()
self.data_sub = zmq_tools.Msg_Receiver(
zmq_ctx,
g_pool.ipc_sub_url,
topics=("pupil", "notify.file_source"),
hwm=100_000,
)
self.data_dir = os.path.join(g_pool.rec_dir, "offline_data")
os.makedirs(self.data_dir, exist_ok=True)
try:
session_meta_data = fm.load_object(
os.path.join(self.data_dir, self.session_data_name + ".meta")
)
assert session_meta_data.get("version") == self.session_data_version
except (AssertionError, FileNotFoundError):
session_meta_data = {}
session_meta_data["detection_status"] = ["unknown", "unknown"]
self.detection_status = session_meta_data["detection_status"]
self._pupil_data_store = pm.PupilDataCollector()
pupil_data_from_cache = pm.PupilDataBisector.load_from_file(
self.data_dir, self.session_data_name
)
self.publish_existing(pupil_data_from_cache)
# Start offline pupil detection if not complete yet:
self.eye_video_loc = [None, None]
self.eye_frame_num = [0, 0]
self.eye_frame_idx = [-1, -1]
# start processes
for eye_id in range(2):
if self.detection_status[eye_id] != "complete":
self.start_eye_process(eye_id)
def start_eye_process(self, eye_id):
potential_locs = [
os.path.join(self.g_pool.rec_dir, "eye{}{}".format(eye_id, ext))
for ext in (".mjpeg", ".mp4", ".mkv")
]
existing_locs = [loc for loc in potential_locs if os.path.exists(loc)]
if not existing_locs:
logger.error("no eye video for eye '{}' found.".format(eye_id))
self.detection_status[eye_id] = "No eye video found."
return
rec, file_ = os.path.split(existing_locs[0])
set_name = os.path.splitext(file_)[0]
self.videoset = VideoSet(rec, set_name, fill_gaps=False)
self.videoset.load_or_build_lookup()
if self.videoset.is_empty():
logger.error(f"No videos for eye '{eye_id}' found.")
self.detection_status[eye_id] = "No eye video found."
return
video_loc = existing_locs[0]
n_valid_frames = np.count_nonzero(self.videoset.lookup.container_idx > -1)
self.eye_frame_num[eye_id] = n_valid_frames
self.eye_frame_idx = [-1, -1]
capure_settings = "File_Source", {"source_path": video_loc, "timing": None}
self.notify_all(
{
"subject": "eye_process.should_start",
"eye_id": eye_id,
"overwrite_cap_settings": capure_settings,
}
)
self.eye_video_loc[eye_id] = video_loc
self.detection_status[eye_id] = "Detecting..."
@property
def detection_progress(self) -> float:
if not sum(self.eye_frame_num):
return 0.0
progress_by_eye = [0.0, 0.0]
for eye_id in (0, 1):
total_frames = self.eye_frame_num[eye_id]
if total_frames > 0:
current_index = self.eye_frame_idx[eye_id]
progress = (current_index + 1) / total_frames
progress = max(0.0, min(progress, 1.0))
else:
progress = 1.0
progress_by_eye[eye_id] = progress
return min(progress_by_eye)
def stop_eye_process(self, eye_id):
self.notify_all({"subject": "eye_process.should_stop", "eye_id": eye_id})
self.eye_video_loc[eye_id] = None
def recent_events(self, events):
super().recent_events(events)
while self.data_sub.new_data:
topic = self.data_sub.recv_topic()
remaining_frames = self.data_sub.recv_remaining_frames()
if topic.startswith("pupil."):
# pupil data only has one remaining frame
payload_serialized = next(remaining_frames)
pupil_datum = fm.Serialized_Dict(msgpack_bytes=payload_serialized)
assert pm.PupilTopic.match(topic, eye_id=pupil_datum["id"])
timestamp = pupil_datum["timestamp"]
self._pupil_data_store.append(topic, pupil_datum, timestamp)
else:
payload = self.data_sub.deserialize_payload(*remaining_frames)
if payload["subject"] == "file_source.video_finished":
for eye_id in (0, 1):
if self.eye_video_loc[eye_id] == payload["source_path"]:
logger.debug("eye {} process complete".format(eye_id))
self.eye_frame_idx[eye_id] = self.eye_frame_num[eye_id]
self.detection_status[eye_id] = "complete"
self.stop_eye_process(eye_id)
break
if self.eye_video_loc == [None, None]:
data = self._pupil_data_store.as_pupil_data_bisector()
self.publish_new(pupil_data_bisector=data)
if payload["subject"] == "file_source.current_frame_index":
for eye_id in (0, 1):
if self.eye_video_loc[eye_id] == payload["source_path"]:
self.eye_frame_idx[eye_id] = payload["index"]
self.menu_icon.indicator_stop = self.detection_progress
def publish_existing(self, pupil_data_bisector):
self.g_pool.pupil_positions = pupil_data_bisector
self._pupil_changed_announcer.announce_existing()
def publish_new(self, pupil_data_bisector):
self.g_pool.pupil_positions = pupil_data_bisector
self._pupil_changed_announcer.announce_new()
logger.debug("pupil positions changed")
self.save_offline_data()
def on_notify(self, notification):
super().on_notify(notification)
if notification["subject"] == "eye_process.started":
pass
elif notification["subject"] == "eye_process.stopped":
self.eye_video_loc[notification["eye_id"]] = None
def cleanup(self):
self.stop_eye_process(0)
self.stop_eye_process(1)
# close sockets before context is terminated
self.data_sub = None
self.save_offline_data()
def save_offline_data(self):
self.g_pool.pupil_positions.save_to_file(self.data_dir, "offline_pupil")
session_data = {}
session_data["detection_status"] = self.detection_status
session_data["version"] = self.session_data_version
cache_path = os.path.join(self.data_dir, "offline_pupil.meta")
fm.save_object(session_data, cache_path)
logger.info("Cached detected pupil data to {}".format(cache_path))
def redetect(self):
self._pupil_data_store.clear()
self.g_pool.pupil_positions = self._pupil_data_store.as_pupil_data_bisector()
self._pupil_changed_announcer.announce_new()
self.detection_finished_flag = False
self.detection_paused = False
for eye_id in range(2):
if self.eye_video_loc[eye_id] is None:
self.start_eye_process(eye_id)
else:
self.notify_all(
{
"subject": "file_source.seek",
"frame_index": 0,
"source_path": self.eye_video_loc[eye_id],
}
)
def init_ui(self):
super().init_ui()
self.menu.append(ui.Info_Text("Detect pupil positions from eye videos."))
self.menu.append(ui.Switch("detection_paused", self, label="Pause detection"))
self.menu.append(ui.Button("Redetect", self.redetect))
self.menu.append(
ui.Text_Input(
"0",
label="eye0:",
getter=lambda: self.detection_status[0],
setter=lambda _: _,
)
)
self.menu.append(
ui.Text_Input(
"1",
label="eye1:",
getter=lambda: self.detection_status[1],
setter=lambda _: _,
)
)
progress_slider = ui.Slider(
"detection_progress",
label="Detection Progress",
getter=lambda: 100 * self.detection_progress,
setter=lambda _: _,
)
progress_slider.display_format = "%3.0f%%"
self.menu.append(progress_slider)
@property
def detection_paused(self):
return self._detection_paused
@detection_paused.setter
def detection_paused(self, should_pause):
self._detection_paused = should_pause
for eye_id in range(2):
if self.eye_video_loc[eye_id] is not None:
subject = "file_source." + (
"should_pause" if should_pause else "should_play"
)
self.notify_all(
{"subject": subject, "source_path": self.eye_video_loc[eye_id]}
)
class Offline_Pupil_Detection(Pupil_Producer_Base):
"""docstring for Offline_Pupil_Detection"""
session_data_version = 3
session_data_name = "offline_pupil"
def __init__(self, g_pool):
super().__init__(g_pool)
zmq_ctx = zmq.Context()
self.data_sub = zmq_tools.Msg_Receiver(
zmq_ctx,
g_pool.ipc_sub_url,
topics=("pupil", "notify.file_source.video_finished"),
hwm=100_000,
)
self.data_dir = os.path.join(g_pool.rec_dir, "offline_data")
os.makedirs(self.data_dir, exist_ok=True)
try:
session_meta_data = fm.load_object(
os.path.join(self.data_dir, self.session_data_name + ".meta"))
assert session_meta_data.get(
"version") == self.session_data_version
except (AssertionError, FileNotFoundError):
session_meta_data = {}
session_meta_data["detection_method"] = "3d"
session_meta_data["detection_status"] = ["unknown", "unknown"]
self.detection_method = session_meta_data["detection_method"]
self.detection_status = session_meta_data["detection_status"]
pupil = fm.load_pldata_file(self.data_dir, self.session_data_name)
ts_data_zip = zip(pupil.timestamps, pupil.data)
ts_topic_zip = zip(pupil.timestamps, pupil.topics)
self.pupil_positions = collections.OrderedDict(ts_data_zip)
self.id_topics = collections.OrderedDict(ts_topic_zip)
self.eye_video_loc = [None, None]
self.eye_frame_num = [0, 0]
for topic in self.id_topics.values():
eye_id = int(topic[-1])
self.eye_frame_num[eye_id] += 1
self.pause_switch = None
self.detection_paused = False
# start processes
for eye_id in range(2):
if self.detection_status[eye_id] != "complete":
self.start_eye_process(eye_id)
# either we did not start them or they failed to start (mono setup etc)
# either way we are done and can publish
if self.eye_video_loc == [None, None]:
self.correlate_publish()
def start_eye_process(self, eye_id):
potential_locs = [
os.path.join(self.g_pool.rec_dir, "eye{}{}".format(eye_id, ext))
for ext in (".mjpeg", ".mp4", ".mkv")
]
existing_locs = [loc for loc in potential_locs if os.path.exists(loc)]
if not existing_locs:
logger.error("no eye video for eye '{}' found.".format(eye_id))
self.detection_status[eye_id] = "No eye video found."
return
rec, file_ = os.path.split(existing_locs[0])
set_name = os.path.splitext(file_)[0]
self.videoset = VideoSet(rec, set_name, fill_gaps=False)
self.videoset.load_or_build_lookup()
if self.videoset.is_empty():
logger.error(f"No videos for eye '{eye_id}' found.")
self.detection_status[eye_id] = "No eye video found."
return
video_loc = existing_locs[0]
n_valid_frames = np.count_nonzero(
self.videoset.lookup.container_idx > -1)
self.eye_frame_num[eye_id] = n_valid_frames
capure_settings = "File_Source", {
"source_path": video_loc,
"timing": None
}
self.notify_all({
"subject": "eye_process.should_start",
"eye_id": eye_id,
"overwrite_cap_settings": capure_settings,
})
self.eye_video_loc[eye_id] = video_loc
self.detection_status[eye_id] = "Detecting..."
def stop_eye_process(self, eye_id):
self.notify_all({
"subject": "eye_process.should_stop",
"eye_id": eye_id
})
self.eye_video_loc[eye_id] = None
def recent_events(self, events):
super().recent_events(events)
while self.data_sub.new_data:
topic = self.data_sub.recv_topic()
remaining_frames = self.data_sub.recv_remaining_frames()
if topic.startswith("pupil."):
# pupil data only has one remaining frame
payload_serialized = next(remaining_frames)
pupil_datum = fm.Serialized_Dict(
msgpack_bytes=payload_serialized)
assert int(topic[-1]) == pupil_datum["id"]
self.pupil_positions[pupil_datum["timestamp"]] = pupil_datum
self.id_topics[pupil_datum["timestamp"]] = topic
else:
payload = self.data_sub.deserialize_payload(*remaining_frames)
if payload["subject"] == "file_source.video_finished":
for eyeid in (0, 1):
if self.eye_video_loc[eyeid] == payload["source_path"]:
logger.debug(
"eye {} process complete".format(eyeid))
self.detection_status[eyeid] = "complete"
self.stop_eye_process(eyeid)
break
if self.eye_video_loc == [None, None]:
self.correlate_publish()
total = sum(self.eye_frame_num)
self.menu_icon.indicator_stop = (len(self.pupil_positions) /
total if total else 0.0)
def correlate_publish(self):
self.g_pool.pupil_positions = pm.Bisector(
tuple(self.pupil_positions.values()),
tuple(self.pupil_positions.keys()))
self.g_pool.pupil_positions_by_id = self.create_pupil_positions_by_id(
self.pupil_positions, self.id_topics)
self._pupil_changed_announcer.announce_new()
logger.debug("pupil positions changed")
self.save_offline_data()
def on_notify(self, notification):
super().on_notify(notification)
if notification["subject"] == "eye_process.started":
self.set_detection_mapping_mode(self.detection_method)
elif notification["subject"] == "eye_process.stopped":
self.eye_video_loc[notification["eye_id"]] = None
def cleanup(self):
self.stop_eye_process(0)
self.stop_eye_process(1)
# close sockets before context is terminated
self.data_sub = None
self.save_offline_data()
def save_offline_data(self):
topic_data_ts = ((self.id_topics[ts], datum, ts)
for ts, datum in self.pupil_positions.items())
with fm.PLData_Writer(self.data_dir, "offline_pupil") as writer:
for topic, datum, timestamp in topic_data_ts:
writer.append_serialized(timestamp, topic, datum.serialized)
session_data = {}
session_data["detection_method"] = self.detection_method
session_data["detection_status"] = self.detection_status
session_data["version"] = self.session_data_version
cache_path = os.path.join(self.data_dir, "offline_pupil.meta")
fm.save_object(session_data, cache_path)
logger.info("Cached detected pupil data to {}".format(cache_path))
def redetect(self):
self.pupil_positions.clear(
) # delete previously detected pupil positions
self.id_topics.clear()
self.g_pool.pupil_positions = pm.Bisector([], [])
self.detection_finished_flag = False
self.detection_paused = False
for eye_id in range(2):
if self.eye_video_loc[eye_id] is None:
self.start_eye_process(eye_id)
else:
self.notify_all({
"subject": "file_source.seek",
"frame_index": 0,
"source_path": self.eye_video_loc[eye_id],
})
def set_detection_mapping_mode(self, new_mode):
n = {"subject": "set_detection_mapping_mode", "mode": new_mode}
self.notify_all(n)
self.redetect()
self.detection_method = new_mode
def init_ui(self):
super().init_ui()
self.menu.label = "Offline Pupil Detector"
self.menu.append(
ui.Info_Text(
"Detects pupil positions from the recording's eye videos."))
self.menu.append(
ui.Selector(
"detection_method",
self,
label="Detection Method",
selection=["2d", "3d"],
setter=self.set_detection_mapping_mode,
))
self.menu.append(
ui.Switch("detection_paused", self, label="Pause detection"))
self.menu.append(ui.Button("Redetect", self.redetect))
self.menu.append(
ui.Text_Input(
"0",
label="eye0:",
getter=lambda: self.detection_status[0],
setter=lambda _: _,
))
self.menu.append(
ui.Text_Input(
"1",
label="eye1:",
getter=lambda: self.detection_status[1],
setter=lambda _: _,
))
def detection_progress():
total = sum(self.eye_frame_num)
return 100 * len(self.pupil_positions) / total if total else 0.0
progress_slider = ui.Slider(
"detection_progress",
label="Detection Progress",
getter=detection_progress,
setter=lambda _: _,
)
progress_slider.display_format = "%3.0f%%"
self.menu.append(progress_slider)
@property
def detection_paused(self):
return self._detection_paused
@detection_paused.setter
def detection_paused(self, should_pause):
self._detection_paused = should_pause
for eye_id in range(2):
if self.eye_video_loc[eye_id] is not None:
subject = "file_source." + ("should_pause"
if should_pause else "should_play")
self.notify_all({
"subject": subject,
"source_path": self.eye_video_loc[eye_id]
})