def __init__(self, video_path):
self.source = File_Source(
SimpleNamespace(), source_path=video_path, timing=None, fill_gaps=True
)
if not self.source.initialised:
raise FileNotFoundError(video_path)
self.current_frame = self.source.get_frame()
def __init__(self, video_path):
try:
self.source = File_Source(_Empty(),
source_path=video_path,
timing=None)
except AssertionError as err:
raise FileNotFoundError(video_path) from err
self.current_frame = self.source.get_frame()
def initliaze_video(self, rec_dir, world_timestamps):
eye_loc = os.path.join(rec_dir, 'eye{}.*'.format(self.eyeid))
try:
self.source = File_Source(Empty(), source_path=glob(eye_loc)[0])
self.current_eye_frame = self.source.get_frame()
except (FileNotFoundError, IndexError, FileCaptureError):
logger.warning('Video for eye{} was not found or could not be opened.'.format(self.eyeid))
else:
self.eye_world_frame_map = correlate_eye_world(self.source.timestamps, world_timestamps)
if self.menu is not None:
self.menu.read_only = False
def _convert_video_file(
input_file,
output_file,
export_range,
world_timestamps,
process_frame,
timestamp_export_format,
):
yield "Export video", 0.0
input_source = File_Source(SimpleNamespace(), input_file, fill_gaps=True)
if not input_source.initialised:
yield "Exporting video failed", 0.0
return
# yield progress results two times per second
update_rate = int(input_source.frame_rate / 2)
export_start, export_stop = export_range # export_stop is exclusive
export_window = pm.exact_window(world_timestamps, (export_start, export_stop - 1))
(export_from_index, export_to_index) = pm.find_closest(
input_source.timestamps, export_window
)
# NOTE: Start time of the export recording will be synced with world recording
# export! This means that if the recording to export started later than the world
# video, the first frame of the exported recording will not be at timestamp 0 in
# the recording, but later. Some video players (e.g. VLC on windows) might display
# the video weirdly in this case, but we rather want syncronization between the
# exported video!
start_time = export_window[0]
writer = MPEG_Writer(output_file, start_time)
input_source.seek_to_frame(export_from_index)
next_update_idx = export_from_index + update_rate
while True:
try:
input_frame = input_source.get_frame()
except EndofVideoError:
break
if input_frame.index >= export_to_index:
break
output_img = process_frame(input_source, input_frame)
output_frame = input_frame
output_frame._img = output_img # it's ._img because .img has no setter
writer.write_video_frame(output_frame)
if input_source.get_frame_index() >= next_update_idx:
progress = (input_source.get_frame_index() - export_from_index) / (
export_to_index - export_from_index
)
yield "Exporting video", progress * 100.0
next_update_idx += update_rate
writer.close(timestamp_export_format)
input_source.cleanup()
yield "Exporting video completed", 100.0
class FrameFetcher:
__slots__ = ("source", "current_frame")
def __init__(self, video_path):
self.source = File_Source(SimpleNamespace(),
source_path=video_path,
timing=None,
fill_gaps=True)
if not self.source.initialised:
raise FileNotFoundError(video_path)
self.current_frame = self.source.get_frame()
def closest_frame_to_ts(self, ts):
closest_idx = pm.find_closest(self.source.timestamps, ts)
return self.frame_for_idx(closest_idx)
def frame_for_idx(self, requested_frame_idx):
if requested_frame_idx != self.current_frame.index:
if requested_frame_idx == self.source.get_frame_index() + 2:
# if we just need to seek by one frame,
# its faster to just read one and and throw it away.
self.source.get_frame()
if requested_frame_idx != self.source.get_frame_index() + 1:
self.source.seek_to_frame(int(requested_frame_idx))
try:
self.current_frame = self.source.get_frame()
except EndofVideoError:
logger.info("End of video {}.".format(self.source.source_path))
return self.current_frame
def _convert_video_file(
input_file,
output_file,
export_range,
world_timestamps,
process_frame,
timestamp_export_format,
):
yield "Export video", 0.0
input_source = File_Source(EmptyGPool(), input_file, fill_gaps=True)
if not input_source.initialised:
yield "Exporting video failed", 0.0
return
# yield progress results two times per second
update_rate = int(input_source.frame_rate / 2)
export_start, export_stop = export_range # export_stop is exclusive
export_window = pm.exact_window(world_timestamps,
(export_start, export_stop - 1))
(export_from_index,
export_to_index) = pm.find_closest(input_source.timestamps, export_window)
writer = AV_Writer(output_file,
fps=input_source.frame_rate,
audio_dir=None,
use_timestamps=True)
input_source.seek_to_frame(export_from_index)
next_update_idx = export_from_index + update_rate
while True:
try:
input_frame = input_source.get_frame()
except EndofVideoError:
break
if input_frame.index >= export_to_index:
break
output_img = process_frame(input_source, input_frame)
output_frame = input_frame
output_frame._img = output_img # it's ._img because .img has no setter
writer.write_video_frame(output_frame)
if input_source.get_frame_index() >= next_update_idx:
progress = (input_source.get_frame_index() - export_from_index) / (
export_to_index - export_from_index)
yield "Exporting video", progress * 100.0
next_update_idx += update_rate
writer.close(timestamp_export_format)
input_source.cleanup()
yield "Exporting video completed", 100.0
def __init__(self,g_pool,alpha=0.6,eye_scale_factor=.5,move_around=0,mirror={'0':False,'1':False}, flip={'0':False,'1':False},pos=[(640,10),(10,10)], show_ellipses=True):
super().__init__(g_pool)
self.order = .6
self.menu = None
# user controls
self.alpha = alpha #opacity level of eyes
self.eye_scale_factor = eye_scale_factor #scale
self.showeyes = 0,1 #modes: any text containg both means both eye is present, on 'only eye1' if only one eye recording
self.move_around = move_around #boolean whether allow to move clip around screen or not
self.video_size = [0,0] #video_size of recording (bc scaling)
self.show_ellipses = show_ellipses
#variables specific to each eye
self.eye_frames = []
self.eye_world_frame_map = []
self.eye_cap = []
self.mirror = mirror #do we horiz flip first eye
self.flip = flip #do we vert flip first eye
self.pos = [list(pos[0]),list(pos[1])] #positions of 2 eyes
self.drag_offset = [None,None]
# load eye videos and eye timestamps
if VersionFormat(self.g_pool.meta_info['Capture Software Version'][1:]) < VersionFormat('0.4'):
eye_video_path = os.path.join(g_pool.rec_dir,'eye.avi'),'None'
eye_timestamps_path = os.path.join(g_pool.rec_dir,'eye_timestamps.npy'),'None'
else:
eye_video_path = os.path.join(g_pool.rec_dir,'eye0.*'),os.path.join(g_pool.rec_dir,'eye1.*')
eye_timestamps_path = os.path.join(g_pool.rec_dir,'eye0_timestamps.npy'),os.path.join(g_pool.rec_dir,'eye1_timestamps.npy')
#try to load eye video and ts for each eye.
for video,ts in zip(eye_video_path,eye_timestamps_path):
try:
class empty(object):
pass
self.eye_cap.append(File_Source(empty(),source_path=glob(video)[0],timestamps=np.load(ts)))
except(IndexError,FileCaptureError):
pass
else:
self.eye_frames.append(self.eye_cap[-1].get_frame())
try:
eye_timestamps = list(np.load(ts))
except:
pass
else:
self.eye_world_frame_map.append(correlate_eye_world(eye_timestamps,g_pool.timestamps))
if len(self.eye_cap) == 2:
logger.debug("Loaded binocular eye video data.")
elif len(self.eye_cap) == 1:
logger.debug("Loaded monocular eye video data")
self.showeyes = (0,)
else:
logger.error("Could not load eye video.")
self.alive = False
return
def _convert_video_file(
input_file,
output_file,
export_range,
world_timestamps,
process_frame,
timestamp_export_format,
):
yield "Export video", 0.0
input_source = File_Source(SimpleNamespace(), input_file, fill_gaps=True)
if not input_source.initialised:
yield "Exporting video failed", 0.0
return
# yield progress results two times per second
update_rate = int(input_source.frame_rate / 2)
export_start, export_stop = export_range # export_stop is exclusive
export_window = pm.exact_window(world_timestamps, (export_start, export_stop - 1))
(export_from_index, export_to_index) = pm.find_closest(
input_source.timestamps, export_window
)
writer = AV_Writer(
output_file, fps=input_source.frame_rate, audio_dir=None, use_timestamps=True
)
input_source.seek_to_frame(export_from_index)
next_update_idx = export_from_index + update_rate
while True:
try:
input_frame = input_source.get_frame()
except EndofVideoError:
break
if input_frame.index >= export_to_index:
break
output_img = process_frame(input_source, input_frame)
output_frame = input_frame
output_frame._img = output_img # it's ._img because .img has no setter
writer.write_video_frame(output_frame)
if input_source.get_frame_index() >= next_update_idx:
progress = (input_source.get_frame_index() - export_from_index) / (
export_to_index - export_from_index
)
yield "Exporting video", progress * 100.0
next_update_idx += update_rate
writer.close(timestamp_export_format)
input_source.cleanup()
yield "Exporting video completed", 100.0
def __init__(self,g_pool,alpha=0.6,eye_scale_factor=.5,move_around=0,mirror={'0':False,'1':False}, flip={'0':False,'1':False},pos=[(640,10),(10,10)]):
super(Eye_Video_Overlay, self).__init__(g_pool)
self.order = .6
self.menu = None
# user controls
self.alpha = alpha #opacity level of eyes
self.eye_scale_factor = eye_scale_factor #scale
self.showeyes = 0,1 #modes: any text containg both means both eye is present, on 'only eye1' if only one eye recording
self.move_around = move_around #boolean whether allow to move clip around screen or not
self.video_size = [0,0] #video_size of recording (bc scaling)
#variables specific to each eye
self.eye_frames = []
self.eye_world_frame_map = []
self.eye_cap = []
self.mirror = mirror #do we horiz flip first eye
self.flip = flip #do we vert flip first eye
self.pos = [list(pos[0]),list(pos[1])] #positions of 2 eyes
self.drag_offset = [None,None]
# load eye videos and eye timestamps
if g_pool.rec_version < VersionFormat('0.4'):
eye_video_path = os.path.join(g_pool.rec_dir,'eye.avi'),'None'
eye_timestamps_path = os.path.join(g_pool.rec_dir,'eye_timestamps.npy'),'None'
else:
eye_video_path = os.path.join(g_pool.rec_dir,'eye0.*'),os.path.join(g_pool.rec_dir,'eye1.*')
eye_timestamps_path = os.path.join(g_pool.rec_dir,'eye0_timestamps.npy'),os.path.join(g_pool.rec_dir,'eye1_timestamps.npy')
#try to load eye video and ts for each eye.
for video,ts in zip(eye_video_path,eye_timestamps_path):
try:
self.eye_cap.append(File_Source(self.g_pool,source_path=glob(video)[0],timestamps=np.load(ts)))
except IndexError,FileCaptureError:
pass
else:
self.eye_frames.append(self.eye_cap[-1].get_frame())
try:
eye_timestamps = list(np.load(ts))
except:
pass
else:
self.eye_world_frame_map.append(correlate_eye_world(eye_timestamps,g_pool.timestamps))
def export_undistorted_h264(distorted_video_loc, target_video_loc,
export_range):
yield "Converting scene video", .1
capture = File_Source(Empty(), distorted_video_loc)
if not capture.initialised:
yield "Converting scene video failed", 0.
return
update_rate = 10
start_time = None
time_base = Fraction(1, 65535)
average_fps = int(
len(capture.timestamps) /
(capture.timestamps[-1] - capture.timestamps[0]))
target_container = av.open(target_video_loc, 'w')
video_stream = target_container.add_stream('mpeg4', 1 / time_base)
video_stream.bit_rate = 150e6
video_stream.bit_rate_tolerance = video_stream.bit_rate / 20
video_stream.thread_count = max(1, mp.cpu_count() - 1)
video_stream.width, video_stream.height = capture.frame_size
av_frame = av.VideoFrame(*capture.frame_size, 'bgr24')
av_frame.time_base = time_base
capture.seek_to_frame(export_range[0])
next_update_idx = export_range[0] + update_rate
while True:
try:
frame = capture.get_frame()
except EndofVideoError:
break
if frame.index > export_range[1]:
break
if start_time is None:
start_time = frame.timestamp
undistorted_img = capture.intrinsics.undistort(frame.img)
av_frame.planes[0].update(undistorted_img)
av_frame.pts = int((frame.timestamp - start_time) / time_base)
packet = video_stream.encode(av_frame)
if packet:
target_container.mux(packet)
if capture.current_frame_idx >= next_update_idx:
progress = ((capture.current_frame_idx - export_range[0]) /
(export_range[1] - export_range[0])) * .9 + .1
yield "Converting scene video", progress * 100.
next_update_idx += update_rate
while True: # flush encoder
packet = video_stream.encode()
if packet:
target_container.mux(packet)
else:
break
target_container.close()
capture.cleanup()
yield "Converting scene video completed", 1. * 100.
def _export_world_video(
rec_dir,
user_dir,
min_data_confidence,
start_frame,
end_frame,
plugin_initializers,
out_file_path,
pre_computed_eye_data,
):
"""
Simulates the generation for the world video and saves a certain time range as a video.
It simulates a whole g_pool such that all plugins run as normal.
"""
from glob import glob
from time import time
import file_methods as fm
import player_methods as pm
from av_writer import AV_Writer
# we are not importing manual gaze correction. In Player corrections have already been applied.
# in batch exporter this plugin makes little sense.
from fixation_detector import Offline_Fixation_Detector
# Plug-ins
from plugin import Plugin_List, import_runtime_plugins
from video_capture import EndofVideoError, File_Source
from vis_circle import Vis_Circle
from vis_cross import Vis_Cross
from vis_eye_video_overlay import Vis_Eye_Video_Overlay
from vis_light_points import Vis_Light_Points
from vis_polyline import Vis_Polyline
from vis_scan_path import Vis_Scan_Path
from vis_watermark import Vis_Watermark
PID = str(os.getpid())
logger = logging.getLogger(__name__ + " with pid: " + PID)
start_status = "Starting video export with pid: {}".format(PID)
logger.info(start_status)
yield start_status, 0
try:
vis_plugins = sorted(
[
Vis_Circle,
Vis_Cross,
Vis_Polyline,
Vis_Light_Points,
Vis_Watermark,
Vis_Scan_Path,
Vis_Eye_Video_Overlay,
],
key=lambda x: x.__name__,
)
analysis_plugins = [Offline_Fixation_Detector]
user_plugins = sorted(
import_runtime_plugins(os.path.join(user_dir, "plugins")),
key=lambda x: x.__name__,
)
available_plugins = vis_plugins + analysis_plugins + user_plugins
name_by_index = [p.__name__ for p in available_plugins]
plugin_by_name = dict(zip(name_by_index, available_plugins))
meta_info = pm.load_meta_info(rec_dir)
g_pool = GlobalContainer()
g_pool.app = "exporter"
g_pool.min_data_confidence = min_data_confidence
valid_ext = (".mp4", ".mkv", ".avi", ".h264", ".mjpeg", ".fake")
try:
video_path = next(f for f in glob(os.path.join(rec_dir, "world.*"))
if os.path.splitext(f)[1] in valid_ext)
except StopIteration:
raise FileNotFoundError("No Video world found")
cap = File_Source(g_pool,
source_path=video_path,
fill_gaps=True,
timing=None)
timestamps = cap.timestamps
file_name = os.path.basename(out_file_path)
dir_name = os.path.dirname(out_file_path)
out_file_path = os.path.expanduser(os.path.join(dir_name, file_name))
if os.path.isfile(out_file_path):
logger.warning("Video out file already exsists. I will overwrite!")
os.remove(out_file_path)
logger.debug("Saving Video to {}".format(out_file_path))
# Trim mark verification
# make sure the trim marks (start frame, end frame) make sense:
# We define them like python list slices, thus we can test them like such.
trimmed_timestamps = timestamps[start_frame:end_frame]
if len(trimmed_timestamps) == 0:
warn = "Start and end frames are set such that no video will be exported."
logger.warning(warn)
yield warn, 0.0
return
if start_frame is None:
start_frame = 0
# these two vars are shared with the launching process and give a job length and progress report.
frames_to_export = len(trimmed_timestamps)
current_frame = 0
exp_info = (
"Will export from frame {} to frame {}. This means I will export {} frames."
)
logger.debug(
exp_info.format(start_frame, start_frame + frames_to_export,
frames_to_export))
# setup of writer
writer = AV_Writer(out_file_path,
fps=cap.frame_rate,
audio_dir=rec_dir,
use_timestamps=True)
cap.seek_to_frame(start_frame)
start_time = time()
g_pool.plugin_by_name = plugin_by_name
g_pool.capture = cap
g_pool.rec_dir = rec_dir
g_pool.user_dir = user_dir
g_pool.meta_info = meta_info
g_pool.timestamps = timestamps
g_pool.delayed_notifications = {}
g_pool.notifications = []
for initializers in pre_computed_eye_data.values():
initializers["data"] = [
fm.Serialized_Dict(msgpack_bytes=serialized)
for serialized in initializers["data"]
]
g_pool.pupil_positions = pm.Bisector(**pre_computed_eye_data["pupil"])
g_pool.gaze_positions = pm.Bisector(**pre_computed_eye_data["gaze"])
g_pool.fixations = pm.Affiliator(**pre_computed_eye_data["fixations"])
# add plugins
g_pool.plugins = Plugin_List(g_pool, plugin_initializers)
while frames_to_export > current_frame:
try:
frame = cap.get_frame()
except EndofVideoError:
break
events = {"frame": frame}
# new positions and events
frame_window = pm.enclosing_window(g_pool.timestamps, frame.index)
events["gaze"] = g_pool.gaze_positions.by_ts_window(frame_window)
events["pupil"] = g_pool.pupil_positions.by_ts_window(frame_window)
# publish delayed notifications when their time has come.
for n in list(g_pool.delayed_notifications.values()):
if n["_notify_time_"] < time():
del n["_notify_time_"]
del g_pool.delayed_notifications[n["subject"]]
g_pool.notifications.append(n)
# notify each plugin if there are new notifications:
while g_pool.notifications:
n = g_pool.notifications.pop(0)
for p in g_pool.plugins:
p.on_notify(n)
# allow each Plugin to do its work.
for p in g_pool.plugins:
p.recent_events(events)
writer.write_video_frame(frame)
current_frame += 1
yield "Exporting with pid {}".format(PID), current_frame
writer.close(timestamp_export_format="all")
duration = time() - start_time
effective_fps = float(current_frame) / duration
result = "Export done: Exported {} frames to {}. This took {} seconds. Exporter ran at {} frames per second."
logger.info(
result.format(current_frame, out_file_path, duration,
effective_fps))
yield "Export done. This took {:.0f} seconds.".format(
duration), current_frame
except GeneratorExit:
logger.warning("Video export with pid {} was canceled.".format(
os.getpid()))
class Eye_Wrapper(object):
def __init__(self,
g_pool,
eyeid,
pos,
hdpi_fac=1.0,
hflip=False,
vflip=False):
super().__init__()
self.g_pool = g_pool
self.eyeid = eyeid
self.pos = pos
self.hflip = hflip
self.vflip = vflip
self.source = None
self.eye_world_frame_map = None
self.current_eye_frame = None
self.drag_offset = None
self.menu = None
self.hdpi_fac = hdpi_fac
def initliaze_video(self, rec_dir, world_timestamps):
eye_loc = os.path.join(rec_dir, "eye{}.*".format(self.eyeid))
try:
self.source = File_Source(SimpleNamespace(),
source_path=glob(eye_loc)[0],
timing=None)
self.current_eye_frame = self.source.get_frame()
except (FileNotFoundError, IndexError):
logger.warning(
"Video for eye{} was not found or could not be opened.".format(
self.eyeid))
else:
self.eye_world_frame_map = correlate_eye_world(
self.source.timestamps, world_timestamps)
if self.menu is not None:
self.menu.read_only = False
def add_eye_menu(self, parent):
self.menu = ui.Growing_Menu("Eye {}".format(self.eyeid))
parent.append(self.menu)
self.menu.append(ui.Switch("hflip", self, label="Horizontal flip"))
self.menu.append(ui.Switch("vflip", self, label="Vertical flip"))
self.menu.read_only = not self.initialized
def remove_eye_menu(self, parent):
parent.remove(self.menu)
self.menu = None
def deinitliaze_video(self):
self.source = None
self.eye_world_frame_map = None
self.current_eye_frame = None
if self.menu is not None:
self.menu.read_only = True
@property
def initialized(self):
return self.source is not None
@property
def config(self):
return {"pos": self.pos, "hflip": self.hflip, "vflip": self.vflip}
def visualize(self, frame, alpha, scale, show_ellipses, pupil_positions):
if not self.initialized:
return
requested_eye_frame_idx = self.eye_world_frame_map[frame.index]
# 1. do we need a new frame?
if requested_eye_frame_idx != self.current_eye_frame.index:
if requested_eye_frame_idx == self.source.get_frame_index() + 2:
# if we just need to seek by one frame, its faster to just read one and and throw it away.
self.source.get_frame()
if requested_eye_frame_idx != self.source.get_frame_index() + 1:
self.source.seek_to_frame(int(requested_eye_frame_idx))
try:
self.current_eye_frame = self.source.get_frame()
except EndofVideoError:
logger.info(
"Reached the end of the eye video for eye video {}.".
format(self.eyeid))
# 2. dragging image
if self.drag_offset is not None:
x, y = glfwGetCursorPos(glfwGetCurrentContext())
pos = x * self.hdpi_fac, y * self.hdpi_fac
pos = normalize(pos, self.g_pool.camera_render_size)
# Position in img pixels
pos = denormalize(pos, (frame.img.shape[1], frame.img.shape[0]))
self.pos = (
int(pos[0] + self.drag_offset[0]),
int(pos[1] + self.drag_offset[1]),
)
# 3. keep in image bounds, do this even when not dragging because the image video_sizes could change.
video_size = (
round(self.current_eye_frame.width * scale),
round(self.current_eye_frame.height * scale),
)
# frame.img.shape[0] is height, frame.img.shape[1] is width of screen
self.pos = (
min(frame.img.shape[1] - video_size[0], max(self.pos[0], 0)),
min(frame.img.shape[0] - video_size[1], max(self.pos[1], 0)),
)
# 4. flipping images, converting to greyscale
eyeimage = self.current_eye_frame.gray
eyeimage = cv2.cvtColor(eyeimage, cv2.COLOR_GRAY2BGR)
if show_ellipses:
try:
pp = next(
(pp for pp in pupil_positions if pp["id"] == self.eyeid
and pp["timestamp"] == self.current_eye_frame.timestamp))
except StopIteration:
pass
else:
draw_pupil_on_image(eyeimage, pp)
# flip and scale
eyeimage = cv2.resize(eyeimage, (0, 0), fx=scale, fy=scale)
if self.hflip:
eyeimage = np.fliplr(eyeimage)
if self.vflip:
eyeimage = np.flipud(eyeimage)
transparent_image_overlay(self.pos, eyeimage, frame.img, alpha)
def on_click(self, pos, button, action, hdpi_fac, eye_scale):
self.hdpi_fac = hdpi_fac
if not self.initialized:
return False # click event has not been consumed
video_size = (
round(self.current_eye_frame.width * eye_scale),
round(self.current_eye_frame.height * eye_scale),
)
if (self.pos[0] < pos[0] < self.pos[0] + video_size[0]
and self.pos[1] < pos[1] < self.pos[1] + video_size[1]):
self.drag_offset = self.pos[0] - pos[0], self.pos[1] - pos[1]
return True
else:
self.drag_offset = None
return False
def export(rec_dir,
user_dir,
min_data_confidence,
start_frame=None,
end_frame=None,
plugin_initializers=(),
out_file_path=None,
pre_computed={}):
logger = logging.getLogger(__name__ + ' with pid: ' + str(os.getpid()))
start_status = 'Starting video export with pid: {}'.format(os.getpid())
print(start_status)
yield start_status, 0
try:
update_recording_to_recent(rec_dir)
vis_plugins = sorted([
Vis_Circle, Vis_Cross, Vis_Polyline, Vis_Light_Points,
Vis_Watermark, Vis_Scan_Path, Vis_Eye_Video_Overlay
],
key=lambda x: x.__name__)
analysis_plugins = [Offline_Fixation_Detector]
user_plugins = sorted(import_runtime_plugins(
os.path.join(user_dir, 'plugins')),
key=lambda x: x.__name__)
available_plugins = vis_plugins + analysis_plugins + user_plugins
name_by_index = [p.__name__ for p in available_plugins]
plugin_by_name = dict(zip(name_by_index, available_plugins))
update_recording_to_recent(rec_dir)
video_path = [
f for f in glob(os.path.join(rec_dir, "world.*"))
if os.path.splitext(f)[-1] in ('.mp4', '.mkv', '.avi', '.mjpeg')
][0]
pupil_data_path = os.path.join(rec_dir, "pupil_data")
audio_path = os.path.join(rec_dir, "audio.mp4")
meta_info = load_meta_info(rec_dir)
g_pool = Global_Container()
g_pool.app = 'exporter'
g_pool.min_data_confidence = min_data_confidence
cap = File_Source(g_pool, video_path)
timestamps = cap.timestamps
# Out file path verification, we do this before but if one uses a separate tool, this will kick in.
if out_file_path is None:
out_file_path = os.path.join(rec_dir, "world_viz.mp4")
else:
file_name = os.path.basename(out_file_path)
dir_name = os.path.dirname(out_file_path)
if not dir_name:
dir_name = rec_dir
if not file_name:
file_name = 'world_viz.mp4'
out_file_path = os.path.expanduser(
os.path.join(dir_name, file_name))
if os.path.isfile(out_file_path):
logger.warning("Video out file already exsists. I will overwrite!")
os.remove(out_file_path)
logger.debug("Saving Video to {}".format(out_file_path))
# Trim mark verification
# make sure the trim marks (start frame, endframe) make sense:
# We define them like python list slices, thus we can test them like such.
trimmed_timestamps = timestamps[start_frame:end_frame]
if len(trimmed_timestamps) == 0:
warn = "Start and end frames are set such that no video will be exported."
logger.warning(warn)
yield warn, 0.
return
if start_frame is None:
start_frame = 0
# these two vars are shared with the lauching process and give a job length and progress report.
frames_to_export = len(trimmed_timestamps)
current_frame = 0
exp_info = "Will export from frame {} to frame {}. This means I will export {} frames."
logger.debug(
exp_info.format(start_frame, start_frame + frames_to_export,
frames_to_export))
# setup of writer
writer = AV_Writer(out_file_path,
fps=cap.frame_rate,
audio_loc=audio_path,
use_timestamps=True)
cap.seek_to_frame(start_frame)
start_time = time()
g_pool.plugin_by_name = plugin_by_name
g_pool.capture = cap
g_pool.rec_dir = rec_dir
g_pool.user_dir = user_dir
g_pool.meta_info = meta_info
g_pool.timestamps = timestamps
g_pool.delayed_notifications = {}
g_pool.notifications = []
# load pupil_positions, gaze_positions
pupil_data = pre_computed.get("pupil_data") or load_object(
pupil_data_path)
g_pool.pupil_data = pupil_data
g_pool.pupil_positions = pre_computed.get(
"pupil_positions") or pupil_data['pupil_positions']
g_pool.gaze_positions = pre_computed.get(
"gaze_positions") or pupil_data['gaze_positions']
g_pool.fixations = [] # populated by the fixation detector plugin
g_pool.pupil_positions_by_frame = correlate_data(
g_pool.pupil_positions, g_pool.timestamps)
g_pool.gaze_positions_by_frame = correlate_data(
g_pool.gaze_positions, g_pool.timestamps)
g_pool.fixations_by_frame = [
[] for x in g_pool.timestamps
] # populated by the fixation detector plugin
# add plugins
g_pool.plugins = Plugin_List(g_pool, plugin_initializers)
while frames_to_export > current_frame:
try:
frame = cap.get_frame()
except EndofVideoFileError:
break
events = {'frame': frame}
# new positons and events
events['gaze_positions'] = g_pool.gaze_positions_by_frame[
frame.index]
events['pupil_positions'] = g_pool.pupil_positions_by_frame[
frame.index]
# publish delayed notifiactions when their time has come.
for n in list(g_pool.delayed_notifications.values()):
if n['_notify_time_'] < time():
del n['_notify_time_']
del g_pool.delayed_notifications[n['subject']]
g_pool.notifications.append(n)
# notify each plugin if there are new notifactions:
while g_pool.notifications:
n = g_pool.notifications.pop(0)
for p in g_pool.plugins:
p.on_notify(n)
# allow each Plugin to do its work.
for p in g_pool.plugins:
p.recent_events(events)
writer.write_video_frame(frame)
current_frame += 1
yield 'Exporting', current_frame
writer.close()
writer = None
duration = time() - start_time
effective_fps = float(current_frame) / duration
result = "Export done: Exported {} frames to {}. This took {} seconds. Exporter ran at {} frames per second."
print(
result.format(current_frame, out_file_path, duration,
effective_fps))
yield 'Export done. This took {:.0f} seconds.'.format(
duration), current_frame
except GeneratorExit:
print('Video export with pid {} was canceled.'.format(os.getpid()))
except:
from time import sleep
import traceback
trace = traceback.format_exc()
print('Process Export (pid: {}) crashed with trace:\n{}'.format(
os.getpid(), trace))
sleep(1.0)
def session(rec_dir):
plugin_dir = os.path.join(user_dir, 'plugins')
if not os.path.isdir(plugin_dir):
os.mkdir(plugin_dir)
runtime_plugins = import_runtime_plugins(plugin_dir)
system_plugins = [Log_Display, Seek_Bar, Trim_Marks]
vis_plugins = sorted([Vis_Circle, Vis_Fixation, Vis_Polyline, Vis_Light_Points, Vis_Cross,
Vis_Watermark, Vis_Eye_Video_Overlay, Vis_Scan_Path], key=lambda x: x.__name__)
analysis_plugins = sorted([Gaze_Position_2D_Fixation_Detector, Pupil_Angle_3D_Fixation_Detector,
Manual_Gaze_Correction, Video_Export_Launcher, Offline_Surface_Tracker,
Raw_Data_Exporter, Batch_Exporter, Annotation_Player], key=lambda x: x.__name__)
other_plugins = sorted([Log_History, Marker_Auto_Trim_Marks], key=lambda x: x.__name__)
user_plugins = sorted(runtime_plugins, key=lambda x: x.__name__)
user_launchable_plugins = vis_plugins + analysis_plugins + other_plugins + user_plugins
available_plugins = system_plugins + user_launchable_plugins
name_by_index = [p.__name__ for p in available_plugins]
plugin_by_name = dict(zip(name_by_index, available_plugins))
# Callback functions
def on_resize(window, w, h):
if gl_utils.is_window_visible(window):
hdpi_factor = float(glfwGetFramebufferSize(window)[0] / glfwGetWindowSize(window)[0])
g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor
g_pool.gui.update_window(w, h)
g_pool.gui.collect_menus()
for g in g_pool.graphs:
g.scale = hdpi_factor
g.adjust_window_size(w, h)
gl_utils.adjust_gl_view(w, h)
for p in g_pool.plugins:
p.on_window_resize(window, w, h)
def on_key(window, key, scancode, action, mods):
g_pool.gui.update_key(key, scancode, action, mods)
def on_char(window, char):
g_pool.gui.update_char(char)
def on_button(window, button, action, mods):
g_pool.gui.update_button(button, action, mods)
pos = glfwGetCursorPos(window)
pos = normalize(pos, glfwGetWindowSize(window))
pos = denormalize(pos, (frame.img.shape[1], frame.img.shape[0])) # Position in img pixels
for p in g_pool.plugins:
p.on_click(pos, button, action)
def on_pos(window, x, y):
hdpi_factor = float(glfwGetFramebufferSize(window)[0]/glfwGetWindowSize(window)[0])
g_pool.gui.update_mouse(x*hdpi_factor, y*hdpi_factor)
def on_scroll(window, x, y):
g_pool.gui.update_scroll(x, y*scroll_factor)
def on_drop(window, count, paths):
for x in range(count):
new_rec_dir = paths[x].decode('utf-8')
if is_pupil_rec_dir(new_rec_dir):
logger.debug("Starting new session with '{}'".format(new_rec_dir))
global rec_dir
rec_dir = new_rec_dir
glfwSetWindowShouldClose(window, True)
else:
logger.error("'{}' is not a valid pupil recording".format(new_rec_dir))
tick = delta_t()
def get_dt():
return next(tick)
update_recording_to_recent(rec_dir)
video_path = [f for f in glob(os.path.join(rec_dir, "world.*")) if f[-3:] in ('mp4', 'mkv', 'avi')][0]
timestamps_path = os.path.join(rec_dir, "world_timestamps.npy")
pupil_data_path = os.path.join(rec_dir, "pupil_data")
meta_info = load_meta_info(rec_dir)
app_version = get_version(version_file)
# log info about Pupil Platform and Platform in player.log
logger.info('Application Version: {}'.format(app_version))
logger.info('System Info: {}'.format(get_system_info()))
timestamps = np.load(timestamps_path)
# create container for globally scoped vars
g_pool = Global_Container()
g_pool.app = 'player'
# Initialize capture
cap = File_Source(g_pool, video_path, timestamps=list(timestamps))
# load session persistent settings
session_settings = Persistent_Dict(os.path.join(user_dir, "user_settings"))
if VersionFormat(session_settings.get("version", '0.0')) < get_version(version_file):
logger.info("Session setting are from older version of this app. I will not use those.")
session_settings.clear()
width, height = session_settings.get('window_size', cap.frame_size)
window_pos = session_settings.get('window_position', window_position_default)
main_window = glfwCreateWindow(width, height, "Pupil Player: "+meta_info["Recording Name"]+" - "
+ rec_dir.split(os.path.sep)[-1], None, None)
glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
glfwMakeContextCurrent(main_window)
cygl.utils.init()
def set_scale(new_scale):
g_pool.gui_user_scale = new_scale
on_resize(main_window, *glfwGetFramebufferSize(main_window))
# load pupil_positions, gaze_positions
pupil_data = load_object(pupil_data_path)
pupil_list = pupil_data['pupil_positions']
gaze_list = pupil_data['gaze_positions']
g_pool.pupil_data = pupil_data
g_pool.binocular = meta_info.get('Eye Mode', 'monocular') == 'binocular'
g_pool.version = app_version
g_pool.capture = cap
g_pool.timestamps = timestamps
g_pool.play = False
g_pool.new_seek = True
g_pool.user_dir = user_dir
g_pool.rec_dir = rec_dir
g_pool.meta_info = meta_info
g_pool.min_data_confidence = session_settings.get('min_data_confidence', 0.6)
g_pool.pupil_positions_by_frame = correlate_data(pupil_list, g_pool.timestamps)
g_pool.gaze_positions_by_frame = correlate_data(gaze_list, g_pool.timestamps)
g_pool.fixations_by_frame = [[] for x in g_pool.timestamps] # populated by the fixation detector plugin
def next_frame(_):
try:
cap.seek_to_frame(cap.get_frame_index())
except(FileSeekError):
logger.warning("Could not seek to next frame.")
else:
g_pool.new_seek = True
def prev_frame(_):
try:
cap.seek_to_frame(cap.get_frame_index()-2)
except(FileSeekError):
logger.warning("Could not seek to previous frame.")
else:
g_pool.new_seek = True
def toggle_play(new_state):
if cap.get_frame_index() >= cap.get_frame_count()-5:
cap.seek_to_frame(1) # avoid pause set by hitting trimmark pause.
logger.warning("End of video - restart at beginning.")
g_pool.play = new_state
def set_data_confidence(new_confidence):
g_pool.min_data_confidence = new_confidence
notification = {'subject': 'min_data_confidence_changed'}
notification['_notify_time_'] = time()+.8
g_pool.delayed_notifications[notification['subject']] = notification
def open_plugin(plugin):
if plugin == "Select to load":
return
g_pool.plugins.add(plugin)
def purge_plugins():
for p in g_pool.plugins:
if p.__class__ in user_launchable_plugins:
p.alive = False
g_pool.plugins.clean()
def do_export(_):
export_range = slice(g_pool.trim_marks.in_mark, g_pool.trim_marks.out_mark)
export_dir = os.path.join(g_pool.rec_dir, 'exports', '{}-{}'.format(export_range.start, export_range.stop))
try:
os.makedirs(export_dir)
except OSError as e:
if e.errno != errno.EEXIST:
logger.error("Could not create export dir")
raise e
else:
overwrite_warning = "Previous export for range [{}-{}] already exsits - overwriting."
logger.warning(overwrite_warning.format(export_range.start, export_range.stop))
else:
logger.info('Created export dir at "{}"'.format(export_dir))
notification = {'subject': 'should_export', 'range': export_range, 'export_dir': export_dir}
g_pool.notifications.append(notification)
g_pool.gui = ui.UI()
g_pool.gui_user_scale = session_settings.get('gui_scale', 1.)
g_pool.main_menu = ui.Scrolling_Menu("Settings", pos=(-350, 20), size=(300, 500))
g_pool.main_menu.append(ui.Button('Reset window size',
lambda: glfwSetWindowSize(main_window, cap.frame_size[0], cap.frame_size[1])))
g_pool.main_menu.append(ui.Selector('gui_user_scale', g_pool, setter=set_scale, selection=[.8, .9, 1., 1.1, 1.2], label='Interface Size'))
g_pool.main_menu.append(ui.Info_Text('Player Version: {}'.format(g_pool.version)))
g_pool.main_menu.append(ui.Info_Text('Capture Version: {}'.format(meta_info['Capture Software Version'])))
g_pool.main_menu.append(ui.Info_Text('Data Format Version: {}'.format(meta_info['Data Format Version'])))
g_pool.main_menu.append(ui.Slider('min_data_confidence', g_pool, setter=set_data_confidence,
step=.05, min=0.0, max=1.0, label='Confidence threshold'))
selector_label = "Select to load"
vis_labels = [" " + p.__name__.replace('_', ' ') for p in vis_plugins]
analysis_labels = [" " + p.__name__.replace('_', ' ') for p in analysis_plugins]
other_labels = [" " + p.__name__.replace('_', ' ') for p in other_plugins]
user_labels = [" " + p.__name__.replace('_', ' ') for p in user_plugins]
plugins = ([selector_label, selector_label] + vis_plugins + [selector_label] + analysis_plugins + [selector_label]
+ other_plugins + [selector_label] + user_plugins)
labels = ([selector_label, "Visualization"] + vis_labels + ["Analysis"] + analysis_labels + ["Other"]
+ other_labels + ["User added"] + user_labels)
g_pool.main_menu.append(ui.Selector('Open plugin:',
selection=plugins,
labels=labels,
setter=open_plugin,
getter=lambda: selector_label))
g_pool.main_menu.append(ui.Button('Close all plugins', purge_plugins))
g_pool.quickbar = ui.Stretching_Menu('Quick Bar', (0, 100), (120, -100))
g_pool.play_button = ui.Thumb('play',
g_pool,
label=chr(0xf04b),
setter=toggle_play,
hotkey=GLFW_KEY_SPACE,
label_font='fontawesome',
label_offset_x=5,
label_offset_y=0,
label_offset_size=-24)
g_pool.play_button.on_color[:] = (0, 1., .0, .8)
g_pool.forward_button = ui.Thumb('forward',
label=chr(0xf04e),
getter=lambda: False,
setter=next_frame,
hotkey=GLFW_KEY_RIGHT,
label_font='fontawesome',
label_offset_x=5,
label_offset_y=0,
label_offset_size=-24)
g_pool.backward_button = ui.Thumb('backward',
label=chr(0xf04a),
getter=lambda: False,
setter=prev_frame,
hotkey=GLFW_KEY_LEFT,
label_font='fontawesome',
label_offset_x=-5,
label_offset_y=0,
label_offset_size=-24)
g_pool.export_button = ui.Thumb('export',
label=chr(0xf063),
getter=lambda: False,
setter=do_export,
hotkey='e',
label_font='fontawesome',
label_offset_x=0,
label_offset_y=2,
label_offset_size=-24)
g_pool.quickbar.extend([g_pool.play_button, g_pool.forward_button, g_pool.backward_button, g_pool.export_button])
g_pool.gui.append(g_pool.quickbar)
g_pool.gui.append(g_pool.main_menu)
# we always load these plugins
system_plugins = [('Trim_Marks', {}), ('Seek_Bar', {})]
default_plugins = [('Log_Display', {}), ('Vis_Scan_Path', {}), ('Vis_Polyline', {}), ('Vis_Circle', {}), ('Video_Export_Launcher', {})]
previous_plugins = session_settings.get('loaded_plugins', default_plugins)
g_pool.notifications = []
g_pool.delayed_notifications = {}
g_pool.plugins = Plugin_List(g_pool, plugin_by_name, system_plugins+previous_plugins)
# Register callbacks main_window
glfwSetFramebufferSizeCallback(main_window, on_resize)
glfwSetKeyCallback(main_window, on_key)
glfwSetCharCallback(main_window, on_char)
glfwSetMouseButtonCallback(main_window, on_button)
glfwSetCursorPosCallback(main_window, on_pos)
glfwSetScrollCallback(main_window, on_scroll)
glfwSetDropCallback(main_window, on_drop)
g_pool.gui.configuration = session_settings.get('ui_config', {})
# gl_state settings
gl_utils.basic_gl_setup()
g_pool.image_tex = Named_Texture()
# set up performace graphs:
pid = os.getpid()
ps = psutil.Process(pid)
ts = None
cpu_graph = graph.Bar_Graph()
cpu_graph.pos = (20, 110)
cpu_graph.update_fn = ps.cpu_percent
cpu_graph.update_rate = 5
cpu_graph.label = 'CPU %0.1f'
fps_graph = graph.Bar_Graph()
fps_graph.pos = (140, 110)
fps_graph.update_rate = 5
fps_graph.label = "%0.0f REC FPS"
pupil_graph = graph.Bar_Graph(max_val=1.0)
pupil_graph.pos = (260, 110)
pupil_graph.update_rate = 5
pupil_graph.label = "Confidence: %0.2f"
g_pool.graphs = [cpu_graph, fps_graph, pupil_graph]
# trigger on_resize
on_resize(main_window, *glfwGetFramebufferSize(main_window))
while not glfwWindowShouldClose(main_window):
# grab new frame
if g_pool.play or g_pool.new_seek:
g_pool.new_seek = False
try:
new_frame = cap.get_frame()
except EndofVideoFileError:
# end of video logic: pause at last frame.
g_pool.play = False
logger.warning("end of video")
update_graph = True
else:
update_graph = False
frame = new_frame.copy()
events = {}
events['frame'] = frame
# report time between now and the last loop interation
events['dt'] = get_dt()
# new positons we make a deepcopy just like the image is a copy.
events['gaze_positions'] = deepcopy(g_pool.gaze_positions_by_frame[frame.index])
events['pupil_positions'] = deepcopy(g_pool.pupil_positions_by_frame[frame.index])
if update_graph:
# update performace graphs
for p in events['pupil_positions']:
pupil_graph.add(p['confidence'])
t = new_frame.timestamp
if ts and ts != t:
dt, ts = t-ts, t
fps_graph.add(1./dt)
else:
ts = new_frame.timestamp
g_pool.play_button.status_text = str(frame.index)
# always update the CPU graph
cpu_graph.update()
# publish delayed notifiactions when their time has come.
for n in list(g_pool.delayed_notifications.values()):
if n['_notify_time_'] < time():
del n['_notify_time_']
del g_pool.delayed_notifications[n['subject']]
g_pool.notifications.append(n)
# notify each plugin if there are new notifactions:
while g_pool.notifications:
n = g_pool.notifications.pop(0)
for p in g_pool.plugins:
p.on_notify(n)
# allow each Plugin to do its work.
for p in g_pool.plugins:
p.recent_events(events)
# check if a plugin need to be destroyed
g_pool.plugins.clean()
# render camera image
glfwMakeContextCurrent(main_window)
gl_utils.make_coord_system_norm_based()
g_pool.image_tex.update_from_frame(frame)
g_pool.image_tex.draw()
gl_utils.make_coord_system_pixel_based(frame.img.shape)
# render visual feedback from loaded plugins
for p in g_pool.plugins:
p.gl_display()
fps_graph.draw()
cpu_graph.draw()
pupil_graph.draw()
g_pool.gui.update()
# present frames at appropriate speed
cap.wait(frame)
glfwSwapBuffers(main_window)
glfwPollEvents()
session_settings['loaded_plugins'] = g_pool.plugins.get_initializers()
session_settings['min_data_confidence'] = g_pool.min_data_confidence
session_settings['gui_scale'] = g_pool.gui_user_scale
session_settings['ui_config'] = g_pool.gui.configuration
session_settings['window_size'] = glfwGetWindowSize(main_window)
session_settings['window_position'] = glfwGetWindowPos(main_window)
session_settings['version'] = str(g_pool.version)
session_settings.close()
# de-init all running plugins
for p in g_pool.plugins:
p.alive = False
g_pool.plugins.clean()
cap.cleanup()
g_pool.gui.terminate()
glfwDestroyWindow(main_window)
def circle_detector(ipc_push_url, pair_url,
source_path, timestamps_path, batch_size=20):
# ipc setup
import zmq
import zmq_tools
zmq_ctx = zmq.Context()
process_pipe = zmq_tools.Msg_Pair_Client(zmq_ctx, pair_url)
# logging setup
import logging
logging.getLogger("OpenGL").setLevel(logging.ERROR)
logger = logging.getLogger()
logger.handlers = []
logger.setLevel(logging.INFO)
logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url))
# create logger for the context of this function
logger = logging.getLogger(__name__)
# imports
import cv2
import numpy as np
from time import sleep
from circle_detector import find_concetric_circles
from video_capture import File_Source, EndofVideoFileError
from methods import normalize
try:
src = File_Source(Empty(), source_path, np.load(timestamps_path), timed_playback=False)
frame = src.get_frame()
logger.info('Starting calibration marker detection...')
frame_count = src.get_frame_count()
queue = []
while True:
while process_pipe.new_data:
topic, n = process_pipe.recv()
if topic == 'terminate':
process_pipe.send(topic='exception', payload={"reason": "User terminated."})
logger.debug("Process terminated")
sleep(1.0)
return
progress = 100.*frame.index/frame_count
markers = find_concetric_circles(frame.gray, min_ring_count=3)
if len(markers) > 0:
detected = True
marker_pos = markers[0][0][0] # first marker innermost ellipse, pos
pos = normalize(marker_pos, (frame.width, frame.height), flip_y=True)
else:
detected = False
pos = None
if detected:
second_ellipse = markers[0][1]
col_slice = int(second_ellipse[0][0]-second_ellipse[1][0]/2), int(second_ellipse[0][0]+second_ellipse[1][0]/2)
row_slice = int(second_ellipse[0][1]-second_ellipse[1][1]/2), int(second_ellipse[0][1]+second_ellipse[1][1]/2)
marker_gray = frame.gray[slice(*row_slice), slice(*col_slice)]
avg = cv2.mean(marker_gray)[0]
center = marker_gray[int(second_ellipse[1][1])//2, int(second_ellipse[1][0])//2]
rel_shade = center-avg
ref = {}
ref["norm_pos"] = pos
ref["screen_pos"] = marker_pos
ref["timestamp"] = frame.timestamp
ref['index'] = frame.index
if rel_shade > 30:
ref['type'] = 'stop_marker'
else:
ref['type'] = 'calibration_marker'
queue.append((progress, ref))
else:
queue.append((progress, None))
if len(queue) > batch_size:
# dequeue batch
data = queue[:batch_size]
del queue[:batch_size]
process_pipe.send(topic='progress', payload={'data': data})
frame = src.get_frame()
except EndofVideoFileError:
process_pipe.send(topic='progress', payload={'data': queue})
process_pipe.send(topic='finished', payload={})
logger.debug("Process finished")
except:
import traceback
process_pipe.send(topic='exception', payload={'reason': traceback.format_exc()})
logger.debug("Process raised Exception")
sleep(1.0)
def export(should_terminate,
frames_to_export,
current_frame,
rec_dir,
user_dir,
min_data_confidence,
start_frame=None,
end_frame=None,
plugin_initializers=(),
out_file_path=None):
vis_plugins = sorted([
Vis_Circle, Vis_Cross, Vis_Polyline, Vis_Light_Points, Vis_Watermark,
Scan_Path
],
key=lambda x: x.__name__)
analysis_plugins = sorted([
Manual_Gaze_Correction, Eye_Video_Overlay,
Pupil_Angle_3D_Fixation_Detector, Gaze_Position_2D_Fixation_Detector
],
key=lambda x: x.__name__)
user_plugins = sorted(import_runtime_plugins(
os.path.join(user_dir, 'plugins')),
key=lambda x: x.__name__)
available_plugins = vis_plugins + analysis_plugins + user_plugins
name_by_index = [p.__name__ for p in available_plugins]
index_by_name = dict(zip(name_by_index, range(len(name_by_index))))
plugin_by_name = dict(zip(name_by_index, available_plugins))
logger = logging.getLogger(__name__ + ' with pid: ' + str(os.getpid()))
update_recording_to_recent(rec_dir)
video_path = [
f for f in glob(os.path.join(rec_dir, "world.*"))
if f[-3:] in ('mp4', 'mkv', 'avi')
][0]
timestamps_path = os.path.join(rec_dir, "world_timestamps.npy")
pupil_data_path = os.path.join(rec_dir, "pupil_data")
meta_info = load_meta_info(rec_dir)
rec_version = read_rec_version(meta_info)
g_pool = Global_Container()
g_pool.app = 'exporter'
g_pool.min_data_confidence = min_data_confidence
timestamps = np.load(timestamps_path)
cap = File_Source(g_pool, video_path, timestamps=timestamps)
#Out file path verification, we do this before but if one uses a seperate tool, this will kick in.
if out_file_path is None:
out_file_path = os.path.join(rec_dir, "world_viz.mp4")
else:
file_name = os.path.basename(out_file_path)
dir_name = os.path.dirname(out_file_path)
if not dir_name:
dir_name = rec_dir
if not file_name:
file_name = 'world_viz.mp4'
out_file_path = os.path.expanduser(os.path.join(dir_name, file_name))
if os.path.isfile(out_file_path):
logger.warning("Video out file already exsists. I will overwrite!")
os.remove(out_file_path)
logger.debug("Saving Video to %s" % out_file_path)
#Trim mark verification
#make sure the trim marks (start frame, endframe) make sense: We define them like python list slices,thus we can test them like such.
trimmed_timestamps = timestamps[start_frame:end_frame]
if len(trimmed_timestamps) == 0:
logger.warn(
"Start and end frames are set such that no video will be exported."
)
return False
if start_frame == None:
start_frame = 0
#these two vars are shared with the lauching process and give a job length and progress report.
frames_to_export.value = len(trimmed_timestamps)
current_frame.value = 0
logger.debug(
"Will export from frame %s to frame %s. This means I will export %s frames."
% (start_frame, start_frame + frames_to_export.value,
frames_to_export.value))
#setup of writer
writer = AV_Writer(out_file_path, fps=cap.frame_rate, use_timestamps=True)
cap.seek_to_frame(start_frame)
start_time = time()
g_pool.capture = cap
g_pool.rec_dir = rec_dir
g_pool.user_dir = user_dir
g_pool.rec_version = rec_version
g_pool.timestamps = timestamps
g_pool.delayed_notifications = {}
g_pool.notifications = []
# load pupil_positions, gaze_positions
pupil_data = load_object(pupil_data_path)
pupil_list = pupil_data['pupil_positions']
gaze_list = pupil_data['gaze_positions']
g_pool.pupil_positions_by_frame = correlate_data(pupil_list,
g_pool.timestamps)
g_pool.gaze_positions_by_frame = correlate_data(gaze_list,
g_pool.timestamps)
g_pool.fixations_by_frame = [[] for x in g_pool.timestamps
] #populated by the fixation detector plugin
#add plugins
g_pool.plugins = Plugin_List(g_pool, plugin_by_name, plugin_initializers)
while frames_to_export.value > current_frame.value:
if should_terminate.value:
logger.warning("User aborted export. Exported %s frames to %s." %
(current_frame.value, out_file_path))
#explicit release of VideoWriter
writer.close()
writer = None
return False
try:
frame = cap.get_frame_nowait()
except EndofVideoFileError:
break
events = {}
#new positons and events
events['gaze_positions'] = g_pool.gaze_positions_by_frame[frame.index]
events['pupil_positions'] = g_pool.pupil_positions_by_frame[
frame.index]
# publish delayed notifiactions when their time has come.
for n in g_pool.delayed_notifications.values():
if n['_notify_time_'] < time():
del n['_notify_time_']
del g_pool.delayed_notifications[n['subject']]
g_pool.notifications.append(n)
# notify each plugin if there are new notifactions:
while g_pool.notifications:
n = g_pool.notifications.pop(0)
for p in g_pool.plugins:
p.on_notify(n)
# allow each Plugin to do its work.
for p in g_pool.plugins:
p.update(frame, events)
writer.write_video_frame(frame)
current_frame.value += 1
writer.close()
writer = None
duration = time() - start_time
effective_fps = float(current_frame.value) / duration
logger.info(
"Export done: Exported %s frames to %s. This took %s seconds. Exporter ran at %s frames per second"
% (current_frame.value, out_file_path, duration, effective_fps))
return True
class Eye_Wrapper(object):
def __init__(self, g_pool, eyeid, pos, hdpi_fac=1., hflip=False, vflip=False):
super().__init__()
self.g_pool = g_pool
self.eyeid = eyeid
self.pos = pos
self.hflip = hflip
self.vflip = vflip
self.source = None
self.eye_world_frame_map = None
self.current_eye_frame = None
self.drag_offset = None
self.menu = None
self.hdpi_fac = hdpi_fac
def initliaze_video(self, rec_dir, world_timestamps):
eye_loc = os.path.join(rec_dir, 'eye{}.*'.format(self.eyeid))
try:
self.source = File_Source(Empty(), source_path=glob(eye_loc)[0])
self.current_eye_frame = self.source.get_frame()
except (FileNotFoundError, IndexError, FileCaptureError):
logger.warning('Video for eye{} was not found or could not be opened.'.format(self.eyeid))
else:
self.eye_world_frame_map = correlate_eye_world(self.source.timestamps, world_timestamps)
if self.menu is not None:
self.menu.read_only = False
def add_eye_menu(self, parent):
self.menu = ui.Growing_Menu('Eye {}'.format(self.eyeid))
parent.append(self.menu)
self.menu.append(ui.Switch('hflip', self, label='Horizontal flip'))
self.menu.append(ui.Switch('vflip', self, label='Vertical flip'))
self.menu.read_only = not self.initialized
def remove_eye_menu(self, parent):
parent.remove(self.menu)
self.menu = None
def deinitliaze_video(self):
self.source = None
self.eye_world_frame_map = None
self.current_eye_frame = None
if self.menu is not None:
self.menu.read_only = True
@property
def initialized(self):
return self.source is not None
@property
def config(self):
return {'pos': self.pos, 'hflip': self.hflip, 'vflip': self.vflip}
def visualize(self, frame, alpha, scale, show_ellipses, pupil_positions):
if not self.initialized:
return
requested_eye_frame_idx = self.eye_world_frame_map[frame.index]
# 1. do we need a new frame?
if requested_eye_frame_idx != self.current_eye_frame.index:
if requested_eye_frame_idx == self.source.get_frame_index() + 2:
# if we just need to seek by one frame, its faster to just read one and and throw it away.
self.source.get_frame()
if requested_eye_frame_idx != self.source.get_frame_index() + 1:
self.source.seek_to_frame(requested_eye_frame_idx)
try:
self.current_eye_frame = self.source.get_frame()
except EndofVideoError:
logger.info("Reached the end of the eye video for eye video {}.".format(self.eyeid))
# 2. dragging image
if self.drag_offset is not None:
x, y = glfwGetCursorPos(glfwGetCurrentContext())
pos = x * self.hdpi_fac, y * self.hdpi_fac
pos = normalize(pos, self.g_pool.camera_render_size)
# Position in img pixels
pos = denormalize(pos, (frame.img.shape[1], frame.img.shape[0]))
self.pos = int(pos[0] + self.drag_offset[0]), int(pos[1] + self.drag_offset[1])
# 3. keep in image bounds, do this even when not dragging because the image video_sizes could change.
video_size = round(self.current_eye_frame.width * scale), round(self.current_eye_frame.height * scale)
# frame.img.shape[0] is height, frame.img.shape[1] is width of screen
self.pos = (min(frame.img.shape[1] - video_size[0], max(self.pos[0], 0)),
min(frame.img.shape[0] - video_size[1], max(self.pos[1], 0)))
# 4. vflipping images, converting to greyscale
eyeimage = self.current_eye_frame.gray
eyeimage = cv2.cvtColor(eyeimage, cv2.COLOR_GRAY2BGR)
if show_ellipses:
try:
pp = next((pp for pp in pupil_positions if pp['id'] == self.eyeid and pp['timestamp'] == self.current_eye_frame.timestamp))
except StopIteration:
pass
else:
el = pp['ellipse']
conf = int(pp.get('model_confidence', pp.get('confidence', 0.1)) * 255)
el_points = getEllipsePts((el['center'], el["axes"], el['angle']))
cv2.polylines(eyeimage, [np.asarray(el_points,dtype='i')], True, (0, 0, 255, conf), thickness=1)
cv2.circle(eyeimage,(int(el['center'][0]),int(el['center'][1])), 5, (0, 0, 255, conf), thickness=-1)
#flip and scale
eyeimage = cv2.resize(eyeimage, (0, 0), fx=scale, fy=scale)
if self.hflip:
eyeimage = np.fliplr(eyeimage)
if self.vflip:
eyeimage = np.flipud(eyeimage)
transparent_image_overlay(self.pos, eyeimage, frame.img, alpha)
def on_click(self, pos, button, action, hdpi_fac, eye_scale):
self.hdpi_fac = hdpi_fac
if not self.initialized:
return False # click event has not been consumed
video_size = round(self.current_eye_frame.width * eye_scale), round(self.current_eye_frame.height * eye_scale)
if (self.pos[0] < pos[0] < self.pos[0] + video_size[0] and
self.pos[1] < pos[1] < self.pos[1] + video_size[1]):
self.drag_offset = self.pos[0] - pos[0], self.pos[1] - pos[1]
return True
else:
self.drag_offset = None
return False
def player(rec_dir, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir,
app_version, debug):
# general imports
from time import sleep
import logging
from glob import glob
from time import time, strftime, localtime
# networking
import zmq
import zmq_tools
import numpy as np
# zmq ipc setup
zmq_ctx = zmq.Context()
ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url)
notify_sub = zmq_tools.Msg_Receiver(zmq_ctx,
ipc_sub_url,
topics=("notify", ))
# log setup
logging.getLogger("OpenGL").setLevel(logging.ERROR)
logger = logging.getLogger()
logger.handlers = []
logger.setLevel(logging.NOTSET)
logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url))
# create logger for the context of this function
logger = logging.getLogger(__name__)
try:
from background_helper import IPC_Logging_Task_Proxy
IPC_Logging_Task_Proxy.push_url = ipc_push_url
from tasklib.background.patches import IPCLoggingPatch
IPCLoggingPatch.ipc_push_url = ipc_push_url
# imports
from file_methods import Persistent_Dict, next_export_sub_dir
# display
import glfw
# check versions for our own depedencies as they are fast-changing
from pyglui import __version__ as pyglui_version
from pyglui import ui, cygl
from pyglui.cygl.utils import Named_Texture, RGBA
import gl_utils
# capture
from video_capture import File_Source
# helpers/utils
from version_utils import VersionFormat
from methods import normalize, denormalize, delta_t, get_system_info
import player_methods as pm
from pupil_recording import PupilRecording
from csv_utils import write_key_value_file
# Plug-ins
from plugin import Plugin, Plugin_List, import_runtime_plugins
from plugin_manager import Plugin_Manager
from vis_circle import Vis_Circle
from vis_cross import Vis_Cross
from vis_polyline import Vis_Polyline
from vis_light_points import Vis_Light_Points
from vis_watermark import Vis_Watermark
from vis_fixation import Vis_Fixation
from seek_control import Seek_Control
from surface_tracker import Surface_Tracker_Offline
# from marker_auto_trim_marks import Marker_Auto_Trim_Marks
from fixation_detector import Offline_Fixation_Detector
from log_display import Log_Display
from annotations import Annotation_Player
from raw_data_exporter import Raw_Data_Exporter
from log_history import Log_History
from pupil_producers import Pupil_From_Recording, Offline_Pupil_Detection
from gaze_producer.gaze_from_recording import GazeFromRecording
from gaze_producer.gaze_from_offline_calibration import (
GazeFromOfflineCalibration, )
from system_graphs import System_Graphs
from system_timelines import System_Timelines
from blink_detection import Offline_Blink_Detection
from audio_playback import Audio_Playback
from video_export.plugins.imotions_exporter import iMotions_Exporter
from video_export.plugins.eye_video_exporter import Eye_Video_Exporter
from video_export.plugins.world_video_exporter import World_Video_Exporter
from head_pose_tracker.offline_head_pose_tracker import (
Offline_Head_Pose_Tracker, )
from video_capture import File_Source
from video_overlay.plugins import Video_Overlay, Eye_Overlay
from pupil_recording import (
assert_valid_recording_type,
InvalidRecordingException,
)
assert VersionFormat(pyglui_version) >= VersionFormat(
"1.27"), "pyglui out of date, please upgrade to newest version"
process_was_interrupted = False
def interrupt_handler(sig, frame):
import traceback
trace = traceback.format_stack(f=frame)
logger.debug(f"Caught signal {sig} in:\n" + "".join(trace))
nonlocal process_was_interrupted
process_was_interrupted = True
signal.signal(signal.SIGINT, interrupt_handler)
runtime_plugins = import_runtime_plugins(
os.path.join(user_dir, "plugins"))
system_plugins = [
Log_Display,
Seek_Control,
Plugin_Manager,
System_Graphs,
System_Timelines,
Audio_Playback,
]
user_plugins = [
Vis_Circle,
Vis_Fixation,
Vis_Polyline,
Vis_Light_Points,
Vis_Cross,
Vis_Watermark,
Eye_Overlay,
Video_Overlay,
Offline_Fixation_Detector,
Offline_Blink_Detection,
Surface_Tracker_Offline,
Raw_Data_Exporter,
Annotation_Player,
Log_History,
Pupil_From_Recording,
Offline_Pupil_Detection,
GazeFromRecording,
GazeFromOfflineCalibration,
World_Video_Exporter,
iMotions_Exporter,
Eye_Video_Exporter,
Offline_Head_Pose_Tracker,
] + runtime_plugins
plugins = system_plugins + user_plugins
# Callback functions
def on_resize(window, w, h):
nonlocal window_size
nonlocal hdpi_factor
if w == 0 or h == 0:
return
hdpi_factor = glfw.getHDPIFactor(window)
g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor
window_size = w, h
g_pool.camera_render_size = w - int(
icon_bar_width * g_pool.gui.scale), h
g_pool.gui.update_window(*window_size)
g_pool.gui.collect_menus()
for p in g_pool.plugins:
p.on_window_resize(window, *g_pool.camera_render_size)
def on_window_key(window, key, scancode, action, mods):
g_pool.gui.update_key(key, scancode, action, mods)
def on_window_char(window, char):
g_pool.gui.update_char(char)
def on_window_mouse_button(window, button, action, mods):
g_pool.gui.update_button(button, action, mods)
def on_pos(window, x, y):
x, y = x * hdpi_factor, y * hdpi_factor
g_pool.gui.update_mouse(x, y)
pos = x, y
pos = normalize(pos, g_pool.camera_render_size)
# Position in img pixels
pos = denormalize(pos, g_pool.capture.frame_size)
for p in g_pool.plugins:
p.on_pos(pos)
def on_scroll(window, x, y):
g_pool.gui.update_scroll(x, y * scroll_factor)
def on_drop(window, count, paths):
paths = [paths[x].decode("utf-8") for x in range(count)]
for path in paths:
try:
assert_valid_recording_type(path)
_restart_with_recording(path)
return
except InvalidRecordingException as err:
logger.debug(str(err))
for plugin in g_pool.plugins:
if plugin.on_drop(paths):
break
def _restart_with_recording(rec_dir):
logger.debug("Starting new session with '{}'".format(rec_dir))
ipc_pub.notify({
"subject": "player_drop_process.should_start",
"rec_dir": rec_dir
})
glfw.glfwSetWindowShouldClose(g_pool.main_window, True)
tick = delta_t()
def get_dt():
return next(tick)
recording = PupilRecording(rec_dir)
meta_info = recording.meta_info
# log info about Pupil Platform and Platform in player.log
logger.info("Application Version: {}".format(app_version))
logger.info("System Info: {}".format(get_system_info()))
logger.debug(f"Debug flag: {debug}")
icon_bar_width = 50
window_size = None
hdpi_factor = 1.0
# create container for globally scoped vars
g_pool = SimpleNamespace()
g_pool.app = "player"
g_pool.zmq_ctx = zmq_ctx
g_pool.ipc_pub = ipc_pub
g_pool.ipc_pub_url = ipc_pub_url
g_pool.ipc_sub_url = ipc_sub_url
g_pool.ipc_push_url = ipc_push_url
g_pool.plugin_by_name = {p.__name__: p for p in plugins}
g_pool.camera_render_size = None
video_path = recording.files().core().world().videos()[0].resolve()
File_Source(
g_pool,
timing="external",
source_path=video_path,
buffered_decoding=True,
fill_gaps=True,
)
# load session persistent settings
session_settings = Persistent_Dict(
os.path.join(user_dir, "user_settings_player"))
if VersionFormat(session_settings.get("version",
"0.0")) != app_version:
logger.info(
"Session setting are a different version of this app. I will not use those."
)
session_settings.clear()
width, height = g_pool.capture.frame_size
width += icon_bar_width
width, height = session_settings.get("window_size", (width, height))
window_pos = session_settings.get("window_position",
window_position_default)
window_name = f"Pupil Player: {meta_info.recording_name} - {rec_dir}"
glfw.glfwInit()
main_window = glfw.glfwCreateWindow(width, height, window_name, None,
None)
glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
glfw.glfwMakeContextCurrent(main_window)
cygl.utils.init()
g_pool.main_window = main_window
def set_scale(new_scale):
g_pool.gui_user_scale = new_scale
window_size = (
g_pool.camera_render_size[0] +
int(icon_bar_width * g_pool.gui_user_scale * hdpi_factor),
glfw.glfwGetFramebufferSize(main_window)[1],
)
logger.warning(icon_bar_width * g_pool.gui_user_scale *
hdpi_factor)
glfw.glfwSetWindowSize(main_window, *window_size)
g_pool.version = app_version
g_pool.timestamps = g_pool.capture.timestamps
g_pool.get_timestamp = lambda: 0.0
g_pool.user_dir = user_dir
g_pool.rec_dir = rec_dir
g_pool.meta_info = meta_info
g_pool.min_data_confidence = session_settings.get(
"min_data_confidence", MIN_DATA_CONFIDENCE_DEFAULT)
g_pool.min_calibration_confidence = session_settings.get(
"min_calibration_confidence", MIN_CALIBRATION_CONFIDENCE_DEFAULT)
# populated by producers
g_pool.pupil_positions = pm.PupilDataBisector()
g_pool.gaze_positions = pm.Bisector()
g_pool.fixations = pm.Affiliator()
g_pool.eye_movements = pm.Affiliator()
def set_data_confidence(new_confidence):
g_pool.min_data_confidence = new_confidence
notification = {"subject": "min_data_confidence_changed"}
notification["_notify_time_"] = time() + 0.8
g_pool.ipc_pub.notify(notification)
def do_export(_):
left_idx = g_pool.seek_control.trim_left
right_idx = g_pool.seek_control.trim_right
export_range = left_idx, right_idx + 1 # exclusive range.stop
export_ts_window = pm.exact_window(g_pool.timestamps,
(left_idx, right_idx))
export_dir = os.path.join(g_pool.rec_dir, "exports")
export_dir = next_export_sub_dir(export_dir)
os.makedirs(export_dir)
logger.info('Created export dir at "{}"'.format(export_dir))
export_info = {
"Player Software Version":
str(g_pool.version),
"Data Format Version":
meta_info.min_player_version,
"Export Date":
strftime("%d.%m.%Y", localtime()),
"Export Time":
strftime("%H:%M:%S", localtime()),
"Frame Index Range:":
g_pool.seek_control.get_frame_index_trim_range_string(),
"Relative Time Range":
g_pool.seek_control.get_rel_time_trim_range_string(),
"Absolute Time Range":
g_pool.seek_control.get_abs_time_trim_range_string(),
}
with open(os.path.join(export_dir, "export_info.csv"), "w") as csv:
write_key_value_file(csv, export_info)
notification = {
"subject": "should_export",
"range": export_range,
"ts_window": export_ts_window,
"export_dir": export_dir,
}
g_pool.ipc_pub.notify(notification)
def reset_restart():
logger.warning("Resetting all settings and restarting Player.")
glfw.glfwSetWindowShouldClose(main_window, True)
ipc_pub.notify({"subject": "clear_settings_process.should_start"})
ipc_pub.notify({
"subject": "player_process.should_start",
"rec_dir": rec_dir,
"delay": 2.0,
})
def toggle_general_settings(collapsed):
# this is the menu toggle logic.
# Only one menu can be open.
# If no menu is open the menubar should collapse.
g_pool.menubar.collapsed = collapsed
for m in g_pool.menubar.elements:
m.collapsed = True
general_settings.collapsed = collapsed
g_pool.gui = ui.UI()
g_pool.gui_user_scale = session_settings.get("gui_scale", 1.0)
g_pool.menubar = ui.Scrolling_Menu("Settings",
pos=(-500, 0),
size=(-icon_bar_width, 0),
header_pos="left")
g_pool.iconbar = ui.Scrolling_Menu("Icons",
pos=(-icon_bar_width, 0),
size=(0, 0),
header_pos="hidden")
g_pool.timelines = ui.Container((0, 0), (0, 0), (0, 0))
g_pool.timelines.horizontal_constraint = g_pool.menubar
g_pool.user_timelines = ui.Timeline_Menu("User Timelines",
pos=(0.0, -150.0),
size=(0.0, 0.0),
header_pos="headline")
g_pool.user_timelines.color = RGBA(a=0.0)
g_pool.user_timelines.collapsed = True
# add container that constaints itself to the seekbar height
vert_constr = ui.Container((0, 0), (0, -50.0), (0, 0))
vert_constr.append(g_pool.user_timelines)
g_pool.timelines.append(vert_constr)
def set_window_size():
f_width, f_height = g_pool.capture.frame_size
f_width += int(icon_bar_width * g_pool.gui.scale)
glfw.glfwSetWindowSize(main_window, f_width, f_height)
general_settings = ui.Growing_Menu("General", header_pos="headline")
general_settings.append(ui.Button("Reset window size",
set_window_size))
general_settings.append(
ui.Selector(
"gui_user_scale",
g_pool,
setter=set_scale,
selection=[0.8, 0.9, 1.0, 1.1, 1.2] +
list(np.arange(1.5, 5.1, 0.5)),
label="Interface Size",
))
general_settings.append(
ui.Info_Text(
f"Minimum Player Version: {meta_info.min_player_version}"))
general_settings.append(
ui.Info_Text(f"Player Version: {g_pool.version}"))
general_settings.append(
ui.Info_Text(
f"Recording Software: {meta_info.recording_software_name}"))
general_settings.append(
ui.Info_Text(
f"Recording Software Version: {meta_info.recording_software_version}"
))
general_settings.append(
ui.Info_Text(
"High level data, e.g. fixations, or visualizations only consider gaze data that has an equal or higher confidence than the minimum data confidence."
))
general_settings.append(
ui.Slider(
"min_data_confidence",
g_pool,
setter=set_data_confidence,
step=0.05,
min=0.0,
max=1.0,
label="Minimum data confidence",
))
general_settings.append(
ui.Button("Restart with default settings", reset_restart))
g_pool.menubar.append(general_settings)
icon = ui.Icon(
"collapsed",
general_settings,
label=chr(0xE8B8),
on_val=False,
off_val=True,
setter=toggle_general_settings,
label_font="pupil_icons",
)
icon.tooltip = "General Settings"
g_pool.iconbar.append(icon)
user_plugin_separator = ui.Separator()
user_plugin_separator.order = 0.35
g_pool.iconbar.append(user_plugin_separator)
g_pool.quickbar = ui.Stretching_Menu("Quick Bar", (0, 100),
(100, -100))
g_pool.export_button = ui.Thumb(
"export",
label=chr(0xE2C5),
getter=lambda: False,
setter=do_export,
hotkey="e",
label_font="pupil_icons",
)
g_pool.quickbar.extend([g_pool.export_button])
g_pool.gui.append(g_pool.menubar)
g_pool.gui.append(g_pool.timelines)
g_pool.gui.append(g_pool.iconbar)
g_pool.gui.append(g_pool.quickbar)
# we always load these plugins
default_plugins = [
("Plugin_Manager", {}),
("Seek_Control", {}),
("Log_Display", {}),
("Raw_Data_Exporter", {}),
("Vis_Polyline", {}),
("Vis_Circle", {}),
("System_Graphs", {}),
("System_Timelines", {}),
("World_Video_Exporter", {}),
("Pupil_From_Recording", {}),
("GazeFromRecording", {}),
("Audio_Playback", {}),
]
g_pool.plugins = Plugin_List(
g_pool, session_settings.get("loaded_plugins", default_plugins))
# Manually add g_pool.capture to the plugin list
g_pool.plugins._plugins.append(g_pool.capture)
g_pool.plugins._plugins.sort(key=lambda p: p.order)
g_pool.capture.init_ui()
general_settings.insert(
-1,
ui.Text_Input(
"rel_time_trim_section",
getter=g_pool.seek_control.get_rel_time_trim_range_string,
setter=g_pool.seek_control.set_rel_time_trim_range_string,
label="Relative time range to export",
),
)
general_settings.insert(
-1,
ui.Text_Input(
"frame_idx_trim_section",
getter=g_pool.seek_control.get_frame_index_trim_range_string,
setter=g_pool.seek_control.set_frame_index_trim_range_string,
label="Frame index range to export",
),
)
# Register callbacks main_window
glfw.glfwSetFramebufferSizeCallback(main_window, on_resize)
glfw.glfwSetKeyCallback(main_window, on_window_key)
glfw.glfwSetCharCallback(main_window, on_window_char)
glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button)
glfw.glfwSetCursorPosCallback(main_window, on_pos)
glfw.glfwSetScrollCallback(main_window, on_scroll)
glfw.glfwSetDropCallback(main_window, on_drop)
toggle_general_settings(True)
g_pool.gui.configuration = session_settings.get("ui_config", {})
# gl_state settings
gl_utils.basic_gl_setup()
g_pool.image_tex = Named_Texture()
# trigger on_resize
on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))
def handle_notifications(n):
subject = n["subject"]
if subject == "start_plugin":
g_pool.plugins.add(g_pool.plugin_by_name[n["name"]],
args=n.get("args", {}))
elif subject.startswith("meta.should_doc"):
ipc_pub.notify({
"subject": "meta.doc",
"actor": g_pool.app,
"doc": player.__doc__
})
for p in g_pool.plugins:
if (p.on_notify.__doc__
and p.__class__.on_notify != Plugin.on_notify):
ipc_pub.notify({
"subject": "meta.doc",
"actor": p.class_name,
"doc": p.on_notify.__doc__,
})
while (not glfw.glfwWindowShouldClose(main_window)
and not process_was_interrupted):
# fetch newest notifications
new_notifications = []
while notify_sub.new_data:
t, n = notify_sub.recv()
new_notifications.append(n)
# notify each plugin if there are new notifications:
for n in new_notifications:
handle_notifications(n)
for p in g_pool.plugins:
p.on_notify(n)
events = {}
# report time between now and the last loop interation
events["dt"] = get_dt()
# pupil and gaze positions are added by their respective producer plugins
events["pupil"] = []
events["gaze"] = []
# allow each Plugin to do its work.
for p in g_pool.plugins:
p.recent_events(events)
# check if a plugin need to be destroyed
g_pool.plugins.clean()
glfw.glfwMakeContextCurrent(main_window)
glfw.glfwPollEvents()
# render visual feedback from loaded plugins
if gl_utils.is_window_visible(main_window):
gl_utils.glViewport(0, 0, *g_pool.camera_render_size)
g_pool.capture.gl_display()
for p in g_pool.plugins:
p.gl_display()
gl_utils.glViewport(0, 0, *window_size)
try:
clipboard = glfw.glfwGetClipboardString(
main_window).decode()
except AttributeError: # clipbaord is None, might happen on startup
clipboard = ""
g_pool.gui.update_clipboard(clipboard)
user_input = g_pool.gui.update()
if user_input.clipboard and user_input.clipboard != clipboard:
# only write to clipboard if content changed
glfw.glfwSetClipboardString(main_window,
user_input.clipboard.encode())
for b in user_input.buttons:
button, action, mods = b
x, y = glfw.glfwGetCursorPos(main_window)
pos = x * hdpi_factor, y * hdpi_factor
pos = normalize(pos, g_pool.camera_render_size)
pos = denormalize(pos, g_pool.capture.frame_size)
for plugin in g_pool.plugins:
if plugin.on_click(pos, button, action):
break
for key, scancode, action, mods in user_input.keys:
for plugin in g_pool.plugins:
if plugin.on_key(key, scancode, action, mods):
break
for char_ in user_input.chars:
for plugin in g_pool.plugins:
if plugin.on_char(char_):
break
# present frames at appropriate speed
g_pool.seek_control.wait(events["frame"].timestamp)
glfw.glfwSwapBuffers(main_window)
session_settings["loaded_plugins"] = g_pool.plugins.get_initializers()
session_settings["min_data_confidence"] = g_pool.min_data_confidence
session_settings[
"min_calibration_confidence"] = g_pool.min_calibration_confidence
session_settings["gui_scale"] = g_pool.gui_user_scale
session_settings["ui_config"] = g_pool.gui.configuration
session_settings["window_position"] = glfw.glfwGetWindowPos(
main_window)
session_settings["version"] = str(g_pool.version)
session_window_size = glfw.glfwGetWindowSize(main_window)
if 0 not in session_window_size:
session_settings["window_size"] = session_window_size
session_settings.close()
# de-init all running plugins
for p in g_pool.plugins:
p.alive = False
g_pool.plugins.clean()
g_pool.gui.terminate()
glfw.glfwDestroyWindow(main_window)
except Exception:
import traceback
trace = traceback.format_exc()
logger.error("Process Player crashed with trace:\n{}".format(trace))
finally:
logger.info("Process shutting down.")
ipc_pub.notify({"subject": "player_process.stopped"})
sleep(1.0)
def player(rec_dir, ipc_pub_url, ipc_sub_url,
ipc_push_url, user_dir, app_version):
# general imports
import logging
import errno
from glob import glob
from copy import deepcopy
from time import time
# networking
import zmq
import zmq_tools
# zmq ipc setup
zmq_ctx = zmq.Context()
ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url)
notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=('notify',))
# log setup
logging.getLogger("OpenGL").setLevel(logging.ERROR)
logger = logging.getLogger()
logger.handlers = []
logger.setLevel(logging.INFO)
logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url))
# create logger for the context of this function
logger = logging.getLogger(__name__)
# imports
from file_methods import Persistent_Dict, load_object
import numpy as np
# display
import glfw
# check versions for our own depedencies as they are fast-changing
from pyglui import __version__ as pyglui_version
from pyglui import ui, graph, cygl
from pyglui.cygl.utils import Named_Texture
import gl_utils
# capture
from video_capture import File_Source, EndofVideoFileError, FileSeekError
# helpers/utils
from version_utils import VersionFormat
from methods import normalize, denormalize, delta_t, get_system_info
from player_methods import correlate_data, is_pupil_rec_dir, load_meta_info
# monitoring
import psutil
# Plug-ins
from plugin import Plugin, Plugin_List, import_runtime_plugins, Visualizer_Plugin_Base, Analysis_Plugin_Base, Producer_Plugin_Base
from vis_circle import Vis_Circle
from vis_cross import Vis_Cross
from vis_polyline import Vis_Polyline
from vis_light_points import Vis_Light_Points
from vis_watermark import Vis_Watermark
from vis_fixation import Vis_Fixation
from vis_scan_path import Vis_Scan_Path
from vis_eye_video_overlay import Vis_Eye_Video_Overlay
from seek_bar import Seek_Bar
from trim_marks import Trim_Marks
from video_export_launcher import Video_Export_Launcher
from offline_surface_tracker import Offline_Surface_Tracker
from marker_auto_trim_marks import Marker_Auto_Trim_Marks
from fixation_detector import Gaze_Position_2D_Fixation_Detector, Pupil_Angle_3D_Fixation_Detector
# from manual_gaze_correction import Manual_Gaze_Correction
from batch_exporter import Batch_Exporter
from log_display import Log_Display
from annotations import Annotation_Player
from raw_data_exporter import Raw_Data_Exporter
from log_history import Log_History
from pupil_producers import Pupil_From_Recording, Offline_Pupil_Detection
from gaze_producers import Gaze_From_Recording, Offline_Calibration
assert pyglui_version >= '1.5'
runtime_plugins = import_runtime_plugins(os.path.join(user_dir, 'plugins'))
system_plugins = [Log_Display, Seek_Bar, Trim_Marks]
user_launchable_plugins = [Vis_Circle, Vis_Fixation, Vis_Polyline, Vis_Light_Points, Vis_Cross, Vis_Watermark,
Vis_Eye_Video_Overlay, Vis_Scan_Path, Gaze_Position_2D_Fixation_Detector,
Pupil_Angle_3D_Fixation_Detector, Video_Export_Launcher,
Offline_Surface_Tracker, Raw_Data_Exporter, Batch_Exporter, Annotation_Player,
Log_History, Marker_Auto_Trim_Marks, Pupil_From_Recording, Offline_Pupil_Detection,
Gaze_From_Recording, Offline_Calibration] + runtime_plugins
available_plugins = system_plugins + user_launchable_plugins
name_by_index = [p.__name__ for p in available_plugins]
plugin_by_name = dict(zip(name_by_index, available_plugins))
# Callback functions
def on_resize(window, w, h):
if gl_utils.is_window_visible(window):
hdpi_factor = float(glfw.glfwGetFramebufferSize(window)[0] / glfw.glfwGetWindowSize(window)[0])
g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor
g_pool.gui.update_window(w, h)
g_pool.gui.collect_menus()
for g in g_pool.graphs:
g.scale = hdpi_factor
g.adjust_window_size(w, h)
gl_utils.adjust_gl_view(w, h)
for p in g_pool.plugins:
p.on_window_resize(window, w, h)
def on_key(window, key, scancode, action, mods):
g_pool.gui.update_key(key, scancode, action, mods)
def on_char(window, char):
g_pool.gui.update_char(char)
def on_button(window, button, action, mods):
g_pool.gui.update_button(button, action, mods)
def on_pos(window, x, y):
hdpi_factor = float(glfw.glfwGetFramebufferSize(window)[0]/glfw.glfwGetWindowSize(window)[0])
g_pool.gui.update_mouse(x*hdpi_factor, y*hdpi_factor)
def on_scroll(window, x, y):
g_pool.gui.update_scroll(x, y*scroll_factor)
def on_drop(window, count, paths):
for x in range(count):
new_rec_dir = paths[x].decode('utf-8')
if is_pupil_rec_dir(new_rec_dir):
logger.debug("Starting new session with '{}'".format(new_rec_dir))
ipc_pub.notify({"subject": "player_drop_process.should_start", "rec_dir": new_rec_dir})
glfw.glfwSetWindowShouldClose(window, True)
else:
logger.error("'{}' is not a valid pupil recording".format(new_rec_dir))
tick = delta_t()
def get_dt():
return next(tick)
video_path = [f for f in glob(os.path.join(rec_dir, "world.*"))
if os.path.splitext(f)[1] in ('.mp4', '.mkv', '.avi', '.h264', '.mjpeg')][0]
timestamps_path = os.path.join(rec_dir, "world_timestamps.npy")
pupil_data_path = os.path.join(rec_dir, "pupil_data")
meta_info = load_meta_info(rec_dir)
# log info about Pupil Platform and Platform in player.log
logger.info('Application Version: {}'.format(app_version))
logger.info('System Info: {}'.format(get_system_info()))
timestamps = np.load(timestamps_path)
# create container for globally scoped vars
g_pool = Global_Container()
g_pool.app = 'player'
g_pool.zmq_ctx = zmq_ctx
g_pool.ipc_pub = ipc_pub
g_pool.ipc_pub_url = ipc_pub_url
g_pool.ipc_sub_url = ipc_sub_url
g_pool.ipc_push_url = ipc_push_url
# Initialize capture
cap = File_Source(g_pool, video_path, timestamps=list(timestamps))
# load session persistent settings
session_settings = Persistent_Dict(os.path.join(user_dir, "user_settings"))
if VersionFormat(session_settings.get("version", '0.0')) != app_version:
logger.info("Session setting are a different version of this app. I will not use those.")
session_settings.clear()
width, height = session_settings.get('window_size', cap.frame_size)
window_pos = session_settings.get('window_position', window_position_default)
glfw.glfwInit()
main_window = glfw.glfwCreateWindow(width, height, "Pupil Player: "+meta_info["Recording Name"]+" - "
+ rec_dir.split(os.path.sep)[-1], None, None)
glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
glfw.glfwMakeContextCurrent(main_window)
cygl.utils.init()
def set_scale(new_scale):
g_pool.gui_user_scale = new_scale
on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))
# load pupil_positions, gaze_positions
g_pool.pupil_data = load_object(pupil_data_path)
g_pool.binocular = meta_info.get('Eye Mode', 'monocular') == 'binocular'
g_pool.version = app_version
g_pool.capture = cap
g_pool.timestamps = timestamps
g_pool.get_timestamp = lambda: 0.
g_pool.play = False
g_pool.new_seek = True
g_pool.user_dir = user_dir
g_pool.rec_dir = rec_dir
g_pool.meta_info = meta_info
g_pool.min_data_confidence = session_settings.get('min_data_confidence', 0.6)
g_pool.pupil_positions = []
g_pool.gaze_positions = []
g_pool.fixations = []
g_pool.notifications_by_frame = correlate_data(g_pool.pupil_data['notifications'], g_pool.timestamps)
g_pool.pupil_positions_by_frame = [[] for x in g_pool.timestamps] # populated by producer`
g_pool.gaze_positions_by_frame = [[] for x in g_pool.timestamps] # populated by producer
g_pool.fixations_by_frame = [[] for x in g_pool.timestamps] # populated by the fixation detector plugin
def next_frame(_):
try:
cap.seek_to_frame(cap.get_frame_index() + 1)
except(FileSeekError):
logger.warning("Could not seek to next frame.")
else:
g_pool.new_seek = True
def prev_frame(_):
try:
cap.seek_to_frame(cap.get_frame_index() - 1)
except(FileSeekError):
logger.warning("Could not seek to previous frame.")
else:
g_pool.new_seek = True
def toggle_play(new_state):
if cap.get_frame_index() >= cap.get_frame_count()-5:
cap.seek_to_frame(1) # avoid pause set by hitting trimmark pause.
logger.warning("End of video - restart at beginning.")
g_pool.play = new_state
def set_data_confidence(new_confidence):
g_pool.min_data_confidence = new_confidence
notification = {'subject': 'min_data_confidence_changed'}
notification['_notify_time_'] = time()+.8
g_pool.ipc_pub.notify(notification)
def open_plugin(plugin):
if plugin == "Select to load":
return
g_pool.plugins.add(plugin)
def purge_plugins():
for p in g_pool.plugins:
if p.__class__ in user_launchable_plugins:
p.alive = False
g_pool.plugins.clean()
def do_export(_):
export_range = g_pool.trim_marks.in_mark, g_pool.trim_marks.out_mark
export_dir = os.path.join(g_pool.rec_dir, 'exports', '{}-{}'.format(*export_range))
try:
os.makedirs(export_dir)
except OSError as e:
if e.errno != errno.EEXIST:
logger.error("Could not create export dir")
raise e
else:
overwrite_warning = "Previous export for range [{}-{}] already exsits - overwriting."
logger.warning(overwrite_warning.format(*export_range))
else:
logger.info('Created export dir at "{}"'.format(export_dir))
notification = {'subject': 'should_export', 'range': export_range, 'export_dir': export_dir}
g_pool.ipc_pub.notify(notification)
g_pool.gui = ui.UI()
g_pool.gui_user_scale = session_settings.get('gui_scale', 1.)
g_pool.main_menu = ui.Scrolling_Menu("Settings", pos=(-350, 20), size=(300, 560))
g_pool.main_menu.append(ui.Button('Reset window size',
lambda: glfw.glfwSetWindowSize(main_window, cap.frame_size[0], cap.frame_size[1])))
g_pool.main_menu.append(ui.Selector('gui_user_scale', g_pool, setter=set_scale, selection=[.8, .9, 1., 1.1, 1.2], label='Interface Size'))
g_pool.main_menu.append(ui.Info_Text('Player Version: {}'.format(g_pool.version)))
g_pool.main_menu.append(ui.Info_Text('Capture Version: {}'.format(meta_info['Capture Software Version'])))
g_pool.main_menu.append(ui.Info_Text('Data Format Version: {}'.format(meta_info['Data Format Version'])))
g_pool.main_menu.append(ui.Slider('min_data_confidence', g_pool, setter=set_data_confidence,
step=.05, min=0.0, max=1.0, label='Confidence threshold'))
g_pool.main_menu.append(ui.Info_Text('Open plugins'))
selector_label = "Select to load"
def append_selector(label, plugins):
plugins.sort(key=lambda p: p.__name__)
plugin_labels = [p.__name__.replace('_', ' ') for p in plugins]
g_pool.main_menu.append(ui.Selector(label,
selection=[selector_label] + plugins,
labels=[selector_label] + plugin_labels,
setter=open_plugin,
getter=lambda: selector_label))
base_plugins = [Visualizer_Plugin_Base, Analysis_Plugin_Base, Producer_Plugin_Base]
base_labels = ['Visualizer:', 'Analyser:', 'Data Source:']
launchable = user_launchable_plugins.copy()
for base_class, label in zip(base_plugins, base_labels):
member_plugins = []
for p in user_launchable_plugins:
if issubclass(p, base_class):
member_plugins.append(p)
launchable.remove(p)
append_selector(label, member_plugins)
# launchable only contains plugins that could not be assigned to any of the above categories
append_selector('Other', launchable)
g_pool.main_menu.append(ui.Button('Close all plugins', purge_plugins))
g_pool.quickbar = ui.Stretching_Menu('Quick Bar', (0, 100), (120, -100))
g_pool.play_button = ui.Thumb('play',
g_pool,
label=chr(0xf04b),
setter=toggle_play,
hotkey=glfw.GLFW_KEY_SPACE,
label_font='fontawesome',
label_offset_x=5,
label_offset_y=0,
label_offset_size=-24)
g_pool.play_button.on_color[:] = (0, 1., .0, .8)
g_pool.forward_button = ui.Thumb('forward',
label=chr(0xf04e),
getter=lambda: False,
setter=next_frame,
hotkey=glfw.GLFW_KEY_RIGHT,
label_font='fontawesome',
label_offset_x=5,
label_offset_y=0,
label_offset_size=-24)
g_pool.backward_button = ui.Thumb('backward',
label=chr(0xf04a),
getter=lambda: False,
setter=prev_frame,
hotkey=glfw.GLFW_KEY_LEFT,
label_font='fontawesome',
label_offset_x=-5,
label_offset_y=0,
label_offset_size=-24)
g_pool.export_button = ui.Thumb('export',
label=chr(0xf063),
getter=lambda: False,
setter=do_export,
hotkey='e',
label_font='fontawesome',
label_offset_x=0,
label_offset_y=2,
label_offset_size=-24)
g_pool.quickbar.extend([g_pool.play_button, g_pool.forward_button, g_pool.backward_button, g_pool.export_button])
g_pool.gui.append(g_pool.quickbar)
g_pool.gui.append(g_pool.main_menu)
# we always load these plugins
system_plugins = [('Trim_Marks', {}), ('Seek_Bar', {})]
default_plugins = [('Log_Display', {}), ('Vis_Scan_Path', {}), ('Vis_Polyline', {}),
('Vis_Circle', {}), ('Video_Export_Launcher', {}),
('Pupil_From_Recording', {}), ('Gaze_From_Recording', {})]
previous_plugins = session_settings.get('loaded_plugins', default_plugins)
g_pool.plugins = Plugin_List(g_pool, plugin_by_name, system_plugins+previous_plugins)
# Register callbacks main_window
glfw.glfwSetFramebufferSizeCallback(main_window, on_resize)
glfw.glfwSetKeyCallback(main_window, on_key)
glfw.glfwSetCharCallback(main_window, on_char)
glfw.glfwSetMouseButtonCallback(main_window, on_button)
glfw.glfwSetCursorPosCallback(main_window, on_pos)
glfw.glfwSetScrollCallback(main_window, on_scroll)
glfw.glfwSetDropCallback(main_window, on_drop)
g_pool.gui.configuration = session_settings.get('ui_config', {})
# gl_state settings
gl_utils.basic_gl_setup()
g_pool.image_tex = Named_Texture()
# set up performace graphs:
pid = os.getpid()
ps = psutil.Process(pid)
ts = None
cpu_graph = graph.Bar_Graph()
cpu_graph.pos = (20, 110)
cpu_graph.update_fn = ps.cpu_percent
cpu_graph.update_rate = 5
cpu_graph.label = 'CPU %0.1f'
fps_graph = graph.Bar_Graph()
fps_graph.pos = (140, 110)
fps_graph.update_rate = 5
fps_graph.label = "%0.0f REC FPS"
pupil_graph = graph.Bar_Graph(max_val=1.0)
pupil_graph.pos = (260, 110)
pupil_graph.update_rate = 5
pupil_graph.label = "Confidence: %0.2f"
g_pool.graphs = [cpu_graph, fps_graph, pupil_graph]
# trigger on_resize
on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))
def handle_notifications(n):
subject = n['subject']
if subject == 'start_plugin':
g_pool.plugins.add(
plugin_by_name[n['name']], args=n.get('args', {}))
elif subject.startswith('meta.should_doc'):
ipc_pub.notify({'subject': 'meta.doc',
'actor': g_pool.app,
'doc': player.__doc__})
for p in g_pool.plugins:
if (p.on_notify.__doc__
and p.__class__.on_notify != Plugin.on_notify):
ipc_pub.notify({'subject': 'meta.doc',
'actor': p.class_name,
'doc': p.on_notify.__doc__})
while not glfw.glfwWindowShouldClose(main_window):
# fetch newest notifications
new_notifications = []
while notify_sub.new_data:
t, n = notify_sub.recv()
new_notifications.append(n)
# notify each plugin if there are new notifications:
for n in new_notifications:
handle_notifications(n)
for p in g_pool.plugins:
p.on_notify(n)
# grab new frame
if g_pool.play or g_pool.new_seek:
g_pool.new_seek = False
try:
new_frame = cap.get_frame()
except EndofVideoFileError:
# end of video logic: pause at last frame.
g_pool.play = False
logger.warning("end of video")
update_graph = True
else:
update_graph = False
frame = new_frame.copy()
events = {}
events['frame'] = frame
# report time between now and the last loop interation
events['dt'] = get_dt()
# new positons we make a deepcopy just like the image is a copy.
events['gaze_positions'] = deepcopy(g_pool.gaze_positions_by_frame[frame.index])
events['pupil_positions'] = deepcopy(g_pool.pupil_positions_by_frame[frame.index])
if update_graph:
# update performace graphs
for p in events['pupil_positions']:
pupil_graph.add(p['confidence'])
t = new_frame.timestamp
if ts and ts != t:
dt, ts = t-ts, t
fps_graph.add(1./dt)
else:
ts = new_frame.timestamp
g_pool.play_button.status_text = str(frame.index)
# always update the CPU graph
cpu_graph.update()
# allow each Plugin to do its work.
for p in g_pool.plugins:
p.recent_events(events)
# check if a plugin need to be destroyed
g_pool.plugins.clean()
# render camera image
glfw.glfwMakeContextCurrent(main_window)
gl_utils.make_coord_system_norm_based()
g_pool.image_tex.update_from_frame(frame)
g_pool.image_tex.draw()
gl_utils.make_coord_system_pixel_based(frame.img.shape)
# render visual feedback from loaded plugins
for p in g_pool.plugins:
p.gl_display()
fps_graph.draw()
cpu_graph.draw()
pupil_graph.draw()
unused_buttons = g_pool.gui.update()
for b in unused_buttons:
button,action,mods = b
pos = glfw.glfwGetCursorPos(main_window)
pos = normalize(pos, glfw.glfwGetWindowSize(main_window))
pos = denormalize(pos, (frame.img.shape[1], frame.img.shape[0])) # Position in img pixels
for p in g_pool.plugins:
p.on_click(pos, button, action)
# present frames at appropriate speed
cap.wait(frame)
glfw.glfwSwapBuffers(main_window)
glfw.glfwPollEvents()
session_settings['loaded_plugins'] = g_pool.plugins.get_initializers()
session_settings['min_data_confidence'] = g_pool.min_data_confidence
session_settings['gui_scale'] = g_pool.gui_user_scale
session_settings['ui_config'] = g_pool.gui.configuration
session_settings['window_size'] = glfw.glfwGetWindowSize(main_window)
session_settings['window_position'] = glfw.glfwGetWindowPos(main_window)
session_settings['version'] = str(g_pool.version)
session_settings.close()
# de-init all running plugins
for p in g_pool.plugins:
p.alive = False
g_pool.plugins.clean()
cap.cleanup()
g_pool.gui.terminate()
glfw.glfwDestroyWindow(main_window)
logger.info("Process shutting down.")
ipc_pub.notify({'subject': 'player_process.stopped'})
def export_processed_h264(
world_timestamps,
unprocessed_video_loc,
target_video_loc,
export_range,
process_frame,
export_timestamps,
):
yield "Converting video", 0.1
capture = File_Source(Empty(), unprocessed_video_loc)
if not capture.initialised:
yield "Converting scene video failed", 0.0
return
export_window = pm.exact_window(world_timestamps, export_range)
(export_from_index,
export_to_index) = pm.find_closest(capture.timestamps, export_window)
update_rate = 10
start_time = None
time_base = Fraction(1, 65535)
target_container = av.open(target_video_loc, "w")
video_stream = target_container.add_stream("mpeg4", 1 / time_base)
video_stream.bit_rate = 150e6
video_stream.bit_rate_tolerance = video_stream.bit_rate / 20
video_stream.thread_count = max(1, mp.cpu_count() - 1)
video_stream.width, video_stream.height = capture.frame_size
av_frame = av.VideoFrame(*capture.frame_size, "bgr24")
av_frame.time_base = time_base
capture.seek_to_frame(export_from_index)
next_update_idx = export_from_index + update_rate
timestamps = []
while True:
try:
frame = capture.get_frame()
except EndofVideoError:
break
if frame.index > export_to_index:
break
if start_time is None:
start_time = frame.timestamp
undistorted_img = process_frame(capture, frame)
av_frame.planes[0].update(undistorted_img)
av_frame.pts = int((frame.timestamp - start_time) / time_base)
if export_timestamps:
timestamps.append(frame.timestamp)
packet = video_stream.encode(av_frame)
if packet:
target_container.mux(packet)
if capture.current_frame_idx >= next_update_idx:
progress = ((capture.current_frame_idx - export_from_index) /
(export_to_index - export_from_index)) * 0.9 + 0.1
yield "Converting video", progress * 100.0
next_update_idx += update_rate
while True: # flush encoder
packet = video_stream.encode()
if packet:
target_container.mux(packet)
else:
break
if export_timestamps:
write_timestamps(target_video_loc, timestamps)
target_container.close()
capture.cleanup()
yield "Converting video completed", 1.0 * 100.0
def player(rec_dir, ipc_pub_url, ipc_sub_url,
ipc_push_url, user_dir, app_version):
# general imports
import logging
import errno
from glob import glob
from copy import deepcopy
from time import time
# networking
import zmq
import zmq_tools
# zmq ipc setup
zmq_ctx = zmq.Context()
ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url)
notify_sub = zmq_tools.Msg_Receiver(zmq_ctx, ipc_sub_url, topics=('notify',))
# log setup
logging.getLogger("OpenGL").setLevel(logging.ERROR)
logger = logging.getLogger()
logger.handlers = []
logger.setLevel(logging.INFO)
logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url))
# create logger for the context of this function
logger = logging.getLogger(__name__)
# imports
from file_methods import Persistent_Dict, load_object
# display
import glfw
# check versions for our own depedencies as they are fast-changing
from pyglui import __version__ as pyglui_version
from pyglui import ui, graph, cygl
from pyglui.cygl.utils import Named_Texture
import gl_utils
# capture
from video_capture import File_Source, EndofVideoFileError, FileSeekError
# helpers/utils
from version_utils import VersionFormat
from methods import normalize, denormalize, delta_t, get_system_info
from player_methods import correlate_data, is_pupil_rec_dir, load_meta_info
# monitoring
import psutil
# Plug-ins
from plugin import Plugin, Plugin_List, import_runtime_plugins, Visualizer_Plugin_Base, Analysis_Plugin_Base, Producer_Plugin_Base
from vis_circle import Vis_Circle
from vis_cross import Vis_Cross
from vis_polyline import Vis_Polyline
from vis_light_points import Vis_Light_Points
from vis_watermark import Vis_Watermark
from vis_fixation import Vis_Fixation
from vis_scan_path import Vis_Scan_Path
from vis_eye_video_overlay import Vis_Eye_Video_Overlay
from seek_bar import Seek_Bar
from trim_marks import Trim_Marks
from video_export_launcher import Video_Export_Launcher
from offline_surface_tracker import Offline_Surface_Tracker
from marker_auto_trim_marks import Marker_Auto_Trim_Marks
from fixation_detector import Gaze_Position_2D_Fixation_Detector, Pupil_Angle_3D_Fixation_Detector
# from manual_gaze_correction import Manual_Gaze_Correction
from batch_exporter import Batch_Exporter
from log_display import Log_Display
from annotations import Annotation_Player
from raw_data_exporter import Raw_Data_Exporter
from log_history import Log_History
from pupil_producers import Pupil_From_Recording, Offline_Pupil_Detection
from gaze_producers import Gaze_From_Recording, Offline_Calibration
assert pyglui_version >= '1.7'
runtime_plugins = import_runtime_plugins(os.path.join(user_dir, 'plugins'))
system_plugins = [Log_Display, Seek_Bar, Trim_Marks]
user_launchable_plugins = [Vis_Circle, Vis_Fixation, Vis_Polyline, Vis_Light_Points, Vis_Cross, Vis_Watermark,
Vis_Eye_Video_Overlay, Vis_Scan_Path, Gaze_Position_2D_Fixation_Detector,
Pupil_Angle_3D_Fixation_Detector, Video_Export_Launcher,
Offline_Surface_Tracker, Raw_Data_Exporter, Batch_Exporter, Annotation_Player,
Log_History, Marker_Auto_Trim_Marks, Pupil_From_Recording, Offline_Pupil_Detection,
Gaze_From_Recording, Offline_Calibration] + runtime_plugins
available_plugins = system_plugins + user_launchable_plugins
name_by_index = [p.__name__ for p in available_plugins]
plugin_by_name = dict(zip(name_by_index, available_plugins))
# Callback functions
def on_resize(window, w, h):
if gl_utils.is_window_visible(window):
hdpi_factor = float(glfw.glfwGetFramebufferSize(window)[0] / glfw.glfwGetWindowSize(window)[0])
g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor
g_pool.gui.update_window(w, h)
g_pool.gui.collect_menus()
for g in g_pool.graphs:
g.scale = hdpi_factor
g.adjust_window_size(w, h)
gl_utils.adjust_gl_view(w, h)
for p in g_pool.plugins:
p.on_window_resize(window, w, h)
def on_window_key(window, key, scancode, action, mods):
g_pool.gui.update_key(key, scancode, action, mods)
def on_window_char(window, char):
g_pool.gui.update_char(char)
def on_window_mouse_button(window, button, action, mods):
g_pool.gui.update_button(button, action, mods)
def on_pos(window, x, y):
hdpi_factor = float(glfw.glfwGetFramebufferSize(window)[0]/glfw.glfwGetWindowSize(window)[0])
g_pool.gui.update_mouse(x*hdpi_factor, y*hdpi_factor)
def on_scroll(window, x, y):
g_pool.gui.update_scroll(x, y*scroll_factor)
def on_drop(window, count, paths):
for x in range(count):
new_rec_dir = paths[x].decode('utf-8')
if is_pupil_rec_dir(new_rec_dir):
logger.debug("Starting new session with '{}'".format(new_rec_dir))
ipc_pub.notify({"subject": "player_drop_process.should_start", "rec_dir": new_rec_dir})
glfw.glfwSetWindowShouldClose(window, True)
else:
logger.error("'{}' is not a valid pupil recording".format(new_rec_dir))
tick = delta_t()
def get_dt():
return next(tick)
video_path = [f for f in glob(os.path.join(rec_dir, "world.*"))
if os.path.splitext(f)[1] in ('.mp4', '.mkv', '.avi', '.h264', '.mjpeg')][0]
pupil_data_path = os.path.join(rec_dir, "pupil_data")
meta_info = load_meta_info(rec_dir)
# log info about Pupil Platform and Platform in player.log
logger.info('Application Version: {}'.format(app_version))
logger.info('System Info: {}'.format(get_system_info()))
# create container for globally scoped vars
g_pool = Global_Container()
g_pool.app = 'player'
g_pool.zmq_ctx = zmq_ctx
g_pool.ipc_pub = ipc_pub
g_pool.ipc_pub_url = ipc_pub_url
g_pool.ipc_sub_url = ipc_sub_url
g_pool.ipc_push_url = ipc_push_url
# Initialize capture
cap = File_Source(g_pool, video_path)
# load session persistent settings
session_settings = Persistent_Dict(os.path.join(user_dir, "user_settings"))
if VersionFormat(session_settings.get("version", '0.0')) != app_version:
logger.info("Session setting are a different version of this app. I will not use those.")
session_settings.clear()
width, height = session_settings.get('window_size', cap.frame_size)
window_pos = session_settings.get('window_position', window_position_default)
glfw.glfwInit()
main_window = glfw.glfwCreateWindow(width, height, "Pupil Player: "+meta_info["Recording Name"]+" - "
+ rec_dir.split(os.path.sep)[-1], None, None)
glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
glfw.glfwMakeContextCurrent(main_window)
cygl.utils.init()
def set_scale(new_scale):
g_pool.gui_user_scale = new_scale
on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))
# load pupil_positions, gaze_positions
g_pool.pupil_data = load_object(pupil_data_path)
g_pool.binocular = meta_info.get('Eye Mode', 'monocular') == 'binocular'
g_pool.version = app_version
g_pool.capture = cap
g_pool.timestamps = g_pool.capture.timestamps
g_pool.get_timestamp = lambda: 0.
g_pool.play = False
g_pool.new_seek = True
g_pool.user_dir = user_dir
g_pool.rec_dir = rec_dir
g_pool.meta_info = meta_info
g_pool.min_data_confidence = session_settings.get('min_data_confidence', 0.6)
g_pool.pupil_positions = []
g_pool.gaze_positions = []
g_pool.fixations = []
g_pool.notifications_by_frame = correlate_data(g_pool.pupil_data['notifications'], g_pool.timestamps)
g_pool.pupil_positions_by_frame = [[] for x in g_pool.timestamps] # populated by producer`
g_pool.gaze_positions_by_frame = [[] for x in g_pool.timestamps] # populated by producer
g_pool.fixations_by_frame = [[] for x in g_pool.timestamps] # populated by the fixation detector plugin
def next_frame(_):
try:
cap.seek_to_frame(cap.get_frame_index() + 1)
except(FileSeekError):
logger.warning("Could not seek to next frame.")
else:
g_pool.new_seek = True
def prev_frame(_):
try:
cap.seek_to_frame(cap.get_frame_index() - 1)
except(FileSeekError):
logger.warning("Could not seek to previous frame.")
else:
g_pool.new_seek = True
def toggle_play(new_state):
if cap.get_frame_index() >= cap.get_frame_count()-5:
cap.seek_to_frame(1) # avoid pause set by hitting trimmark pause.
logger.warning("End of video - restart at beginning.")
g_pool.play = new_state
def set_data_confidence(new_confidence):
g_pool.min_data_confidence = new_confidence
notification = {'subject': 'min_data_confidence_changed'}
notification['_notify_time_'] = time()+.8
g_pool.ipc_pub.notify(notification)
def open_plugin(plugin):
if plugin == "Select to load":
return
g_pool.plugins.add(plugin)
def purge_plugins():
for p in g_pool.plugins:
if p.__class__ in user_launchable_plugins:
p.alive = False
g_pool.plugins.clean()
def do_export(_):
export_range = g_pool.trim_marks.in_mark, g_pool.trim_marks.out_mark
export_dir = os.path.join(g_pool.rec_dir, 'exports', '{}-{}'.format(*export_range))
try:
os.makedirs(export_dir)
except OSError as e:
if e.errno != errno.EEXIST:
logger.error("Could not create export dir")
raise e
else:
overwrite_warning = "Previous export for range [{}-{}] already exsits - overwriting."
logger.warning(overwrite_warning.format(*export_range))
else:
logger.info('Created export dir at "{}"'.format(export_dir))
notification = {'subject': 'should_export', 'range': export_range, 'export_dir': export_dir}
g_pool.ipc_pub.notify(notification)
g_pool.gui = ui.UI()
g_pool.gui_user_scale = session_settings.get('gui_scale', 1.)
g_pool.main_menu = ui.Scrolling_Menu("Settings", pos=(-350, 20), size=(300, 560))
g_pool.main_menu.append(ui.Button('Reset window size',
lambda: glfw.glfwSetWindowSize(main_window, cap.frame_size[0], cap.frame_size[1])))
g_pool.main_menu.append(ui.Selector('gui_user_scale', g_pool, setter=set_scale, selection=[.8, .9, 1., 1.1, 1.2], label='Interface Size'))
g_pool.main_menu.append(ui.Info_Text('Player Version: {}'.format(g_pool.version)))
g_pool.main_menu.append(ui.Info_Text('Capture Version: {}'.format(meta_info['Capture Software Version'])))
g_pool.main_menu.append(ui.Info_Text('Data Format Version: {}'.format(meta_info['Data Format Version'])))
g_pool.main_menu.append(ui.Slider('min_data_confidence', g_pool, setter=set_data_confidence,
step=.05, min=0.0, max=1.0, label='Confidence threshold'))
g_pool.main_menu.append(ui.Info_Text('Open plugins'))
selector_label = "Select to load"
def append_selector(label, plugins):
plugins.sort(key=lambda p: p.__name__)
plugin_labels = [p.__name__.replace('_', ' ') for p in plugins]
g_pool.main_menu.append(ui.Selector(label,
selection=[selector_label] + plugins,
labels=[selector_label] + plugin_labels,
setter=open_plugin,
getter=lambda: selector_label))
base_plugins = [Visualizer_Plugin_Base, Analysis_Plugin_Base, Producer_Plugin_Base]
base_labels = ['Visualizer:', 'Analyser:', 'Data Source:']
launchable = user_launchable_plugins.copy()
for base_class, label in zip(base_plugins, base_labels):
member_plugins = []
for p in user_launchable_plugins:
if issubclass(p, base_class):
member_plugins.append(p)
launchable.remove(p)
append_selector(label, member_plugins)
# launchable only contains plugins that could not be assigned to any of the above categories
append_selector('Other', launchable)
g_pool.main_menu.append(ui.Button('Close all plugins', purge_plugins))
g_pool.quickbar = ui.Stretching_Menu('Quick Bar', (0, 100), (120, -100))
g_pool.play_button = ui.Thumb('play',
g_pool,
label=chr(0xf04b),
setter=toggle_play,
hotkey=glfw.GLFW_KEY_SPACE,
label_font='fontawesome',
label_offset_x=5,
label_offset_y=0,
label_offset_size=-24)
g_pool.play_button.on_color[:] = (0, 1., .0, .8)
g_pool.forward_button = ui.Thumb('forward',
label=chr(0xf04e),
getter=lambda: False,
setter=next_frame,
hotkey=glfw.GLFW_KEY_RIGHT,
label_font='fontawesome',
label_offset_x=5,
label_offset_y=0,
label_offset_size=-24)
g_pool.backward_button = ui.Thumb('backward',
label=chr(0xf04a),
getter=lambda: False,
setter=prev_frame,
hotkey=glfw.GLFW_KEY_LEFT,
label_font='fontawesome',
label_offset_x=-5,
label_offset_y=0,
label_offset_size=-24)
g_pool.export_button = ui.Thumb('export',
label=chr(0xf063),
getter=lambda: False,
setter=do_export,
hotkey='e',
label_font='fontawesome',
label_offset_x=0,
label_offset_y=2,
label_offset_size=-24)
g_pool.quickbar.extend([g_pool.play_button, g_pool.forward_button, g_pool.backward_button, g_pool.export_button])
g_pool.gui.append(g_pool.quickbar)
g_pool.gui.append(g_pool.main_menu)
# we always load these plugins
system_plugins = [('Trim_Marks', {}), ('Seek_Bar', {})]
default_plugins = [('Log_Display', {}), ('Vis_Scan_Path', {}), ('Vis_Polyline', {}),
('Vis_Circle', {}), ('Video_Export_Launcher', {}),
('Pupil_From_Recording', {}), ('Gaze_From_Recording', {})]
previous_plugins = session_settings.get('loaded_plugins', default_plugins)
g_pool.plugins = Plugin_List(g_pool, plugin_by_name, system_plugins+previous_plugins)
# Register callbacks main_window
glfw.glfwSetFramebufferSizeCallback(main_window, on_resize)
glfw.glfwSetKeyCallback(main_window, on_window_key)
glfw.glfwSetCharCallback(main_window, on_window_char)
glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button)
glfw.glfwSetCursorPosCallback(main_window, on_pos)
glfw.glfwSetScrollCallback(main_window, on_scroll)
glfw.glfwSetDropCallback(main_window, on_drop)
g_pool.gui.configuration = session_settings.get('ui_config', {})
# gl_state settings
gl_utils.basic_gl_setup()
g_pool.image_tex = Named_Texture()
# set up performace graphs:
pid = os.getpid()
ps = psutil.Process(pid)
ts = None
cpu_graph = graph.Bar_Graph()
cpu_graph.pos = (20, 110)
cpu_graph.update_fn = ps.cpu_percent
cpu_graph.update_rate = 5
cpu_graph.label = 'CPU %0.1f'
fps_graph = graph.Bar_Graph()
fps_graph.pos = (140, 110)
fps_graph.update_rate = 5
fps_graph.label = "%0.0f REC FPS"
pupil_graph = graph.Bar_Graph(max_val=1.0)
pupil_graph.pos = (260, 110)
pupil_graph.update_rate = 5
pupil_graph.label = "Confidence: %0.2f"
g_pool.graphs = [cpu_graph, fps_graph, pupil_graph]
# trigger on_resize
on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))
def handle_notifications(n):
subject = n['subject']
if subject == 'start_plugin':
g_pool.plugins.add(
plugin_by_name[n['name']], args=n.get('args', {}))
elif subject.startswith('meta.should_doc'):
ipc_pub.notify({'subject': 'meta.doc',
'actor': g_pool.app,
'doc': player.__doc__})
for p in g_pool.plugins:
if (p.on_notify.__doc__
and p.__class__.on_notify != Plugin.on_notify):
ipc_pub.notify({'subject': 'meta.doc',
'actor': p.class_name,
'doc': p.on_notify.__doc__})
while not glfw.glfwWindowShouldClose(main_window):
# fetch newest notifications
new_notifications = []
while notify_sub.new_data:
t, n = notify_sub.recv()
new_notifications.append(n)
# notify each plugin if there are new notifications:
for n in new_notifications:
handle_notifications(n)
for p in g_pool.plugins:
p.on_notify(n)
# grab new frame
if g_pool.play or g_pool.new_seek:
g_pool.new_seek = False
try:
new_frame = cap.get_frame()
except EndofVideoFileError:
# end of video logic: pause at last frame.
g_pool.play = False
logger.warning("end of video")
update_graph = True
else:
update_graph = False
frame = new_frame.copy()
events = {}
events['frame'] = frame
# report time between now and the last loop interation
events['dt'] = get_dt()
# new positons we make a deepcopy just like the image is a copy.
events['gaze_positions'] = deepcopy(g_pool.gaze_positions_by_frame[frame.index])
events['pupil_positions'] = deepcopy(g_pool.pupil_positions_by_frame[frame.index])
if update_graph:
# update performace graphs
for p in events['pupil_positions']:
pupil_graph.add(p['confidence'])
t = new_frame.timestamp
if ts and ts != t:
dt, ts = t-ts, t
fps_graph.add(1./dt)
else:
ts = new_frame.timestamp
g_pool.play_button.status_text = str(frame.index)
# always update the CPU graph
cpu_graph.update()
# allow each Plugin to do its work.
for p in g_pool.plugins:
p.recent_events(events)
# check if a plugin need to be destroyed
g_pool.plugins.clean()
# render camera image
glfw.glfwMakeContextCurrent(main_window)
gl_utils.make_coord_system_norm_based()
g_pool.image_tex.update_from_ndarray(frame.bgr)
g_pool.image_tex.draw()
gl_utils.make_coord_system_pixel_based(frame.img.shape)
# render visual feedback from loaded plugins
for p in g_pool.plugins:
p.gl_display()
fps_graph.draw()
cpu_graph.draw()
pupil_graph.draw()
unused_elements = g_pool.gui.update()
for b in unused_elements.buttons:
button, action, mods = b
pos = glfw.glfwGetCursorPos(main_window)
pos = normalize(pos, glfw.glfwGetWindowSize(main_window))
pos = denormalize(pos, (frame.img.shape[1], frame.img.shape[0])) # Position in img pixels
for p in g_pool.plugins:
p.on_click(pos, button, action)
for key, scancode, action, mods in unused_elements.keys:
for p in g_pool.plugins:
p.on_key(key, scancode, action, mods)
for char_ in unused_elements.chars:
for p in g_pool.plugins:
p.on_char(char_)
# present frames at appropriate speed
cap.wait(frame)
glfw.glfwSwapBuffers(main_window)
glfw.glfwPollEvents()
session_settings['loaded_plugins'] = g_pool.plugins.get_initializers()
session_settings['min_data_confidence'] = g_pool.min_data_confidence
session_settings['gui_scale'] = g_pool.gui_user_scale
session_settings['ui_config'] = g_pool.gui.configuration
session_settings['window_size'] = glfw.glfwGetWindowSize(main_window)
session_settings['window_position'] = glfw.glfwGetWindowPos(main_window)
session_settings['version'] = str(g_pool.version)
session_settings.close()
# de-init all running plugins
for p in g_pool.plugins:
p.alive = False
g_pool.plugins.clean()
cap.cleanup()
g_pool.gui.terminate()
glfw.glfwDestroyWindow(main_window)
logger.info("Process shutting down.")
ipc_pub.notify({'subject': 'player_process.stopped'})
def circle_detector(ipc_push_url, pair_url, source_path, batch_size=20):
# ipc setup
import zmq
import zmq_tools
zmq_ctx = zmq.Context()
process_pipe = zmq_tools.Msg_Pair_Client(zmq_ctx, pair_url)
# logging setup
import logging
logging.getLogger("OpenGL").setLevel(logging.ERROR)
logger = logging.getLogger()
logger.handlers = []
logger.setLevel(logging.INFO)
logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url))
# create logger for the context of this function
logger = logging.getLogger(__name__)
# imports
import cv2
from time import sleep
from circle_detector import find_concetric_circles
from video_capture import File_Source, EndofVideoFileError
from methods import normalize
try:
src = File_Source(Empty(), source_path, timed_playback=False)
frame = src.get_frame()
logger.info('Starting calibration marker detection...')
frame_count = src.get_frame_count()
queue = []
while True:
while process_pipe.new_data:
topic, n = process_pipe.recv()
if topic == 'terminate':
process_pipe.send(topic='exception',
payload={"reason": "User terminated."})
logger.debug("Process terminated")
sleep(1.0)
return
progress = 100. * frame.index / frame_count
markers = find_concetric_circles(frame.gray, min_ring_count=3)
if len(markers) > 0:
detected = True
marker_pos = markers[0][0][
0] # first marker innermost ellipse, pos
pos = normalize(marker_pos, (frame.width, frame.height),
flip_y=True)
else:
detected = False
pos = None
if detected:
second_ellipse = markers[0][1]
col_slice = int(second_ellipse[0][0] - second_ellipse[1][0] /
2), int(second_ellipse[0][0] +
second_ellipse[1][0] / 2)
row_slice = int(second_ellipse[0][1] - second_ellipse[1][1] /
2), int(second_ellipse[0][1] +
second_ellipse[1][1] / 2)
marker_gray = frame.gray[slice(*row_slice), slice(*col_slice)]
avg = cv2.mean(marker_gray)[0]
center = marker_gray[int(second_ellipse[1][1]) // 2,
int(second_ellipse[1][0]) // 2]
rel_shade = center - avg
ref = {}
ref["norm_pos"] = pos
ref["screen_pos"] = marker_pos
ref["timestamp"] = frame.timestamp
ref['index'] = frame.index
if rel_shade > 30:
ref['type'] = 'stop_marker'
else:
ref['type'] = 'calibration_marker'
queue.append((progress, ref))
else:
queue.append((progress, None))
if len(queue) > batch_size:
# dequeue batch
data = queue[:batch_size]
del queue[:batch_size]
process_pipe.send(topic='progress', payload={'data': data})
frame = src.get_frame()
except EndofVideoFileError:
process_pipe.send(topic='progress', payload={'data': queue})
process_pipe.send(topic='finished', payload={})
logger.debug("Process finished")
except:
import traceback
process_pipe.send(topic='exception',
payload={'reason': traceback.format_exc()})
logger.debug("Process raised Exception")
sleep(1.0)
def player(rec_dir, ipc_pub_url, ipc_sub_url, ipc_push_url, user_dir,
app_version):
# general imports
from time import sleep
import logging
import errno
from glob import glob
from time import time
# networking
import zmq
import zmq_tools
import numpy as np
# zmq ipc setup
zmq_ctx = zmq.Context()
ipc_pub = zmq_tools.Msg_Dispatcher(zmq_ctx, ipc_push_url)
notify_sub = zmq_tools.Msg_Receiver(zmq_ctx,
ipc_sub_url,
topics=('notify', ))
# log setup
logging.getLogger("OpenGL").setLevel(logging.ERROR)
logger = logging.getLogger()
logger.handlers = []
logger.setLevel(logging.INFO)
logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url))
# create logger for the context of this function
logger = logging.getLogger(__name__)
try:
# imports
from file_methods import Persistent_Dict, load_object
# display
import glfw
# check versions for our own depedencies as they are fast-changing
from pyglui import __version__ as pyglui_version
from pyglui import ui, cygl
from pyglui.cygl.utils import Named_Texture, RGBA
import gl_utils
# capture
from video_capture import File_Source, EndofVideoFileError
# helpers/utils
from version_utils import VersionFormat
from methods import normalize, denormalize, delta_t, get_system_info
from player_methods import correlate_data, is_pupil_rec_dir, load_meta_info
# Plug-ins
from plugin import Plugin, Plugin_List, import_runtime_plugins
from plugin_manager import Plugin_Manager
from vis_circle import Vis_Circle
from vis_cross import Vis_Cross
from vis_polyline import Vis_Polyline
from vis_light_points import Vis_Light_Points
from vis_watermark import Vis_Watermark
from vis_fixation import Vis_Fixation
from vis_scan_path import Vis_Scan_Path
from vis_eye_video_overlay import Vis_Eye_Video_Overlay
from seek_control import Seek_Control
from video_export_launcher import Video_Export_Launcher
from offline_surface_tracker import Offline_Surface_Tracker
# from marker_auto_trim_marks import Marker_Auto_Trim_Marks
from fixation_detector import Offline_Fixation_Detector
from batch_exporter import Batch_Exporter, Batch_Export
from log_display import Log_Display
from annotations import Annotation_Player
from raw_data_exporter import Raw_Data_Exporter
from log_history import Log_History
from pupil_producers import Pupil_From_Recording, Offline_Pupil_Detection
from gaze_producers import Gaze_From_Recording, Offline_Calibration
from system_graphs import System_Graphs
from system_timelines import System_Timelines
from blink_detection import Offline_Blink_Detection
assert VersionFormat(pyglui_version) >= VersionFormat(
'1.17'), 'pyglui out of date, please upgrade to newest version'
runtime_plugins = import_runtime_plugins(
os.path.join(user_dir, 'plugins'))
system_plugins = [
Log_Display, Seek_Control, Plugin_Manager, System_Graphs,
Batch_Export, System_Timelines
]
user_plugins = [
Vis_Circle, Vis_Fixation, Vis_Polyline, Vis_Light_Points,
Vis_Cross, Vis_Watermark, Vis_Eye_Video_Overlay, Vis_Scan_Path,
Offline_Fixation_Detector, Offline_Blink_Detection, Batch_Exporter,
Video_Export_Launcher, Offline_Surface_Tracker, Raw_Data_Exporter,
Annotation_Player, Log_History, Pupil_From_Recording,
Offline_Pupil_Detection, Gaze_From_Recording, Offline_Calibration
] + runtime_plugins
plugins = system_plugins + user_plugins
# Callback functions
def on_resize(window, w, h):
nonlocal window_size
nonlocal hdpi_factor
hdpi_factor = float(
glfw.glfwGetFramebufferSize(window)[0] /
glfw.glfwGetWindowSize(window)[0])
g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor
window_size = w, h
g_pool.camera_render_size = w - int(
icon_bar_width * g_pool.gui.scale), h
g_pool.gui.update_window(*window_size)
g_pool.gui.collect_menus()
for p in g_pool.plugins:
p.on_window_resize(window, *g_pool.camera_render_size)
def on_window_key(window, key, scancode, action, mods):
g_pool.gui.update_key(key, scancode, action, mods)
def on_window_char(window, char):
g_pool.gui.update_char(char)
def on_window_mouse_button(window, button, action, mods):
g_pool.gui.update_button(button, action, mods)
def on_pos(window, x, y):
x, y = x * hdpi_factor, y * hdpi_factor
g_pool.gui.update_mouse(x, y)
pos = x, y
pos = normalize(pos, g_pool.camera_render_size)
# Position in img pixels
pos = denormalize(pos, g_pool.capture.frame_size)
for p in g_pool.plugins:
p.on_pos(pos)
def on_scroll(window, x, y):
g_pool.gui.update_scroll(x, y * scroll_factor)
def on_drop(window, count, paths):
for x in range(count):
new_rec_dir = paths[x].decode('utf-8')
if is_pupil_rec_dir(new_rec_dir):
logger.debug(
"Starting new session with '{}'".format(new_rec_dir))
ipc_pub.notify({
"subject": "player_drop_process.should_start",
"rec_dir": new_rec_dir
})
glfw.glfwSetWindowShouldClose(window, True)
else:
logger.error("'{}' is not a valid pupil recording".format(
new_rec_dir))
tick = delta_t()
def get_dt():
return next(tick)
video_path = [
f for f in glob(os.path.join(rec_dir, "world.*"))
if os.path.splitext(f)[1] in ('.mp4', '.mkv', '.avi', '.h264',
'.mjpeg')
][0]
pupil_data_path = os.path.join(rec_dir, "pupil_data")
meta_info = load_meta_info(rec_dir)
# log info about Pupil Platform and Platform in player.log
logger.info('Application Version: {}'.format(app_version))
logger.info('System Info: {}'.format(get_system_info()))
icon_bar_width = 50
window_size = None
hdpi_factor = 1.0
# create container for globally scoped vars
g_pool = Global_Container()
g_pool.app = 'player'
g_pool.zmq_ctx = zmq_ctx
g_pool.ipc_pub = ipc_pub
g_pool.ipc_pub_url = ipc_pub_url
g_pool.ipc_sub_url = ipc_sub_url
g_pool.ipc_push_url = ipc_push_url
g_pool.plugin_by_name = {p.__name__: p for p in plugins}
g_pool.camera_render_size = None
# sets itself to g_pool.capture
File_Source(g_pool, video_path)
# load session persistent settings
session_settings = Persistent_Dict(
os.path.join(user_dir, "user_settings_player"))
if VersionFormat(session_settings.get("version",
'0.0')) != app_version:
logger.info(
"Session setting are a different version of this app. I will not use those."
)
session_settings.clear()
g_pool.capture.playback_speed = session_settings.get(
'playback_speed', 1.)
width, height = session_settings.get('window_size',
g_pool.capture.frame_size)
window_pos = session_settings.get('window_position',
window_position_default)
glfw.glfwInit()
main_window = glfw.glfwCreateWindow(
width, height, "Pupil Player: " + meta_info["Recording Name"] +
" - " + rec_dir.split(os.path.sep)[-1], None, None)
glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
glfw.glfwMakeContextCurrent(main_window)
cygl.utils.init()
g_pool.main_window = main_window
def set_scale(new_scale):
g_pool.gui_user_scale = new_scale
window_size = (
g_pool.camera_render_size[0] +
int(icon_bar_width * g_pool.gui_user_scale * hdpi_factor),
glfw.glfwGetFramebufferSize(main_window)[1])
logger.warning(icon_bar_width * g_pool.gui_user_scale *
hdpi_factor)
glfw.glfwSetWindowSize(main_window, *window_size)
# load pupil_positions, gaze_positions
g_pool.pupil_data = load_object(pupil_data_path)
g_pool.binocular = meta_info.get('Eye Mode',
'monocular') == 'binocular'
g_pool.version = app_version
g_pool.timestamps = g_pool.capture.timestamps
g_pool.get_timestamp = lambda: 0.
g_pool.new_seek = True
g_pool.user_dir = user_dir
g_pool.rec_dir = rec_dir
g_pool.meta_info = meta_info
g_pool.min_data_confidence = session_settings.get(
'min_data_confidence', 0.6)
g_pool.pupil_positions = []
g_pool.gaze_positions = []
g_pool.fixations = []
g_pool.notifications_by_frame = correlate_data(
g_pool.pupil_data['notifications'], g_pool.timestamps)
g_pool.pupil_positions_by_frame = [[] for x in g_pool.timestamps
] # populated by producer`
g_pool.gaze_positions_by_frame = [[] for x in g_pool.timestamps
] # populated by producer
g_pool.fixations_by_frame = [
[] for x in g_pool.timestamps
] # populated by the fixation detector plugin
def set_data_confidence(new_confidence):
g_pool.min_data_confidence = new_confidence
notification = {'subject': 'min_data_confidence_changed'}
notification['_notify_time_'] = time() + .8
g_pool.ipc_pub.notify(notification)
def open_plugin(plugin):
if plugin == "Select to load":
return
g_pool.plugins.add(plugin)
def purge_plugins():
for p in g_pool.plugins:
if p.__class__ in user_plugins:
p.alive = False
g_pool.plugins.clean()
def do_export(_):
export_range = g_pool.seek_control.trim_left, g_pool.seek_control.trim_right
export_dir = os.path.join(g_pool.rec_dir, 'exports',
'{}-{}'.format(*export_range))
try:
os.makedirs(export_dir)
except OSError as e:
if e.errno != errno.EEXIST:
logger.error("Could not create export dir")
raise e
else:
overwrite_warning = "Previous export for range [{}-{}] already exists - overwriting."
logger.warning(overwrite_warning.format(*export_range))
else:
logger.info('Created export dir at "{}"'.format(export_dir))
notification = {
'subject': 'should_export',
'range': export_range,
'export_dir': export_dir
}
g_pool.ipc_pub.notify(notification)
def reset_restart():
logger.warning("Resetting all settings and restarting Player.")
glfw.glfwSetWindowShouldClose(main_window, True)
ipc_pub.notify({'subject': 'clear_settings_process.should_start'})
ipc_pub.notify({
'subject': 'player_process.should_start',
'rec_dir': rec_dir,
'delay': 2.
})
def toggle_general_settings(collapsed):
# this is the menu toggle logic.
# Only one menu can be open.
# If no menu is open the menubar should collapse.
g_pool.menubar.collapsed = collapsed
for m in g_pool.menubar.elements:
m.collapsed = True
general_settings.collapsed = collapsed
g_pool.gui = ui.UI()
g_pool.gui_user_scale = session_settings.get('gui_scale', 1.)
g_pool.menubar = ui.Scrolling_Menu("Settings",
pos=(-500, 0),
size=(-icon_bar_width, 0),
header_pos='left')
g_pool.iconbar = ui.Scrolling_Menu("Icons",
pos=(-icon_bar_width, 0),
size=(0, 0),
header_pos='hidden')
g_pool.timelines = ui.Container((0, 0), (0, 0), (0, 0))
g_pool.timelines.horizontal_constraint = g_pool.menubar
g_pool.user_timelines = ui.Timeline_Menu('User Timelines',
pos=(0., -150.),
size=(0., 0.),
header_pos='headline')
g_pool.user_timelines.color = RGBA(a=0.)
g_pool.user_timelines.collapsed = True
# add container that constaints itself to the seekbar height
vert_constr = ui.Container((0, 0), (0, -50.), (0, 0))
vert_constr.append(g_pool.user_timelines)
g_pool.timelines.append(vert_constr)
general_settings = ui.Growing_Menu('General', header_pos='headline')
general_settings.append(
ui.Button(
'Reset window size', lambda: glfw.glfwSetWindowSize(
main_window, g_pool.capture.frame_size[0], g_pool.capture.
frame_size[1])))
general_settings.append(
ui.Selector('gui_user_scale',
g_pool,
setter=set_scale,
selection=[.8, .9, 1., 1.1, 1.2] +
list(np.arange(1.5, 5.1, .5)),
label='Interface Size'))
general_settings.append(
ui.Info_Text('Player Version: {}'.format(g_pool.version)))
general_settings.append(
ui.Info_Text('Capture Version: {}'.format(
meta_info['Capture Software Version'])))
general_settings.append(
ui.Info_Text('Data Format Version: {}'.format(
meta_info['Data Format Version'])))
general_settings.append(
ui.Slider('min_data_confidence',
g_pool,
setter=set_data_confidence,
step=.05,
min=0.0,
max=1.0,
label='Confidence threshold'))
general_settings.append(
ui.Button('Restart with default settings', reset_restart))
g_pool.menubar.append(general_settings)
icon = ui.Icon('collapsed',
general_settings,
label=chr(0xe8b8),
on_val=False,
off_val=True,
setter=toggle_general_settings,
label_font='pupil_icons')
icon.tooltip = 'General Settings'
g_pool.iconbar.append(icon)
user_plugin_separator = ui.Separator()
user_plugin_separator.order = 0.35
g_pool.iconbar.append(user_plugin_separator)
g_pool.quickbar = ui.Stretching_Menu('Quick Bar', (0, 100),
(100, -100))
g_pool.export_button = ui.Thumb('export',
label=chr(0xe2c5),
getter=lambda: False,
setter=do_export,
hotkey='e',
label_font='pupil_icons')
g_pool.quickbar.extend([g_pool.export_button])
g_pool.gui.append(g_pool.menubar)
g_pool.gui.append(g_pool.timelines)
g_pool.gui.append(g_pool.iconbar)
g_pool.gui.append(g_pool.quickbar)
# we always load these plugins
default_plugins = [('Plugin_Manager', {}), ('Seek_Control', {}),
('Log_Display', {}), ('Raw_Data_Exporter', {}),
('Vis_Polyline', {}), ('Vis_Circle', {}),
('System_Graphs', {}), ('System_Timelines', {}),
('Video_Export_Launcher', {}),
('Pupil_From_Recording', {}),
('Gaze_From_Recording', {})]
g_pool.plugins = Plugin_List(
g_pool, session_settings.get('loaded_plugins', default_plugins))
# Register callbacks main_window
glfw.glfwSetFramebufferSizeCallback(main_window, on_resize)
glfw.glfwSetKeyCallback(main_window, on_window_key)
glfw.glfwSetCharCallback(main_window, on_window_char)
glfw.glfwSetMouseButtonCallback(main_window, on_window_mouse_button)
glfw.glfwSetCursorPosCallback(main_window, on_pos)
glfw.glfwSetScrollCallback(main_window, on_scroll)
glfw.glfwSetDropCallback(main_window, on_drop)
toggle_general_settings(True)
g_pool.gui.configuration = session_settings.get('ui_config', {})
# gl_state settings
gl_utils.basic_gl_setup()
g_pool.image_tex = Named_Texture()
# trigger on_resize
on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window))
def handle_notifications(n):
subject = n['subject']
if subject == 'start_plugin':
g_pool.plugins.add(g_pool.plugin_by_name[n['name']],
args=n.get('args', {}))
elif subject.startswith('meta.should_doc'):
ipc_pub.notify({
'subject': 'meta.doc',
'actor': g_pool.app,
'doc': player.__doc__
})
for p in g_pool.plugins:
if (p.on_notify.__doc__
and p.__class__.on_notify != Plugin.on_notify):
ipc_pub.notify({
'subject': 'meta.doc',
'actor': p.class_name,
'doc': p.on_notify.__doc__
})
while not glfw.glfwWindowShouldClose(main_window):
# fetch newest notifications
new_notifications = []
while notify_sub.new_data:
t, n = notify_sub.recv()
new_notifications.append(n)
# notify each plugin if there are new notifications:
for n in new_notifications:
handle_notifications(n)
for p in g_pool.plugins:
p.on_notify(n)
# grab new frame
if g_pool.capture.play or g_pool.new_seek:
g_pool.new_seek = False
try:
new_frame = g_pool.capture.get_frame()
except EndofVideoFileError:
# end of video logic: pause at last frame.
g_pool.capture.play = False
logger.warning("end of video")
frame = new_frame.copy()
events = {}
events['frame'] = frame
# report time between now and the last loop interation
events['dt'] = get_dt()
# pupil and gaze positions are added by their respective producer plugins
events['pupil_positions'] = []
events['gaze_positions'] = []
# allow each Plugin to do its work.
for p in g_pool.plugins:
p.recent_events(events)
# check if a plugin need to be destroyed
g_pool.plugins.clean()
glfw.glfwMakeContextCurrent(main_window)
# render visual feedback from loaded plugins
if gl_utils.is_window_visible(main_window):
gl_utils.glViewport(0, 0, *g_pool.camera_render_size)
g_pool.capture._recent_frame = frame
g_pool.capture.gl_display()
for p in g_pool.plugins:
p.gl_display()
gl_utils.glViewport(0, 0, *window_size)
try:
clipboard = glfw.glfwGetClipboardString(
main_window).decode()
except AttributeError: # clipbaord is None, might happen on startup
clipboard = ''
g_pool.gui.update_clipboard(clipboard)
user_input = g_pool.gui.update()
if user_input.clipboard and user_input.clipboard != clipboard:
# only write to clipboard if content changed
glfw.glfwSetClipboardString(main_window,
user_input.clipboard.encode())
for b in user_input.buttons:
button, action, mods = b
x, y = glfw.glfwGetCursorPos(main_window)
pos = x * hdpi_factor, y * hdpi_factor
pos = normalize(pos, g_pool.camera_render_size)
pos = denormalize(pos, g_pool.capture.frame_size)
for p in g_pool.plugins:
p.on_click(pos, button, action)
for key, scancode, action, mods in user_input.keys:
for p in g_pool.plugins:
p.on_key(key, scancode, action, mods)
for char_ in user_input.chars:
for p in g_pool.plugins:
p.on_char(char_)
glfw.glfwSwapBuffers(main_window)
# present frames at appropriate speed
g_pool.capture.wait(frame)
glfw.glfwPollEvents()
session_settings['playback_speed'] = g_pool.capture.playback_speed
session_settings['loaded_plugins'] = g_pool.plugins.get_initializers()
session_settings['min_data_confidence'] = g_pool.min_data_confidence
session_settings['gui_scale'] = g_pool.gui_user_scale
session_settings['ui_config'] = g_pool.gui.configuration
session_settings['window_size'] = glfw.glfwGetWindowSize(main_window)
session_settings['window_position'] = glfw.glfwGetWindowPos(
main_window)
session_settings['version'] = str(g_pool.version)
session_settings.close()
# de-init all running plugins
for p in g_pool.plugins:
p.alive = False
g_pool.plugins.clean()
g_pool.capture.cleanup()
g_pool.gui.terminate()
glfw.glfwDestroyWindow(main_window)
except:
import traceback
trace = traceback.format_exc()
logger.error('Process Player crashed with trace:\n{}'.format(trace))
finally:
logger.info("Process shutting down.")
ipc_pub.notify({'subject': 'player_process.stopped'})
sleep(1.0)
def fill_cache(visited_list, video_file_path, timestamps, q, seek_idx, run,
min_marker_perimeter):
'''
this function is part of marker_detector it is run as a seperate process.
it must be kept in a seperate file for namespace sanatisation
'''
import os
import logging
logger = logging.getLogger(__name__ + ' with pid: ' + str(os.getpid()))
logger.debug('Started cacher process for Marker Detector')
import cv2
from video_capture import File_Source, EndofVideoFileError, FileSeekError
from square_marker_detect import detect_markers_robust
aperture = 9
markers = []
cap = File_Source(Global_Container(),
video_file_path,
timestamps=timestamps)
def next_unvisited_idx(frame_idx):
try:
visited = visited_list[frame_idx]
except IndexError:
visited = True # trigger search
if not visited:
next_unvisited = frame_idx
else:
# find next unvisited site in the future
try:
next_unvisited = visited_list.index(False, frame_idx)
except ValueError:
# any thing in the past?
try:
next_unvisited = visited_list.index(False, 0, frame_idx)
except ValueError:
#no unvisited sites left. Done!
logger.debug("Caching completed.")
next_unvisited = None
return next_unvisited
def handle_frame(next):
if next != cap.get_frame_index():
#we need to seek:
logger.debug("Seeking to Frame %s" % next)
try:
cap.seek_to_frame(next)
except FileSeekError:
#could not seek to requested position
logger.warning("Could not evaluate frame: %s." % next)
visited_list[next] = True # this frame is now visited.
q.put((next,
[])) # we cannot look at the frame, report no detection
return
#seeking invalidates prev markers for the detector
markers[:] = []
try:
frame = cap.get_frame_nowait()
except EndofVideoFileError:
logger.debug("Video File's last frame(s) not accesible")
#could not read frame
logger.warning("Could not evaluate frame: %s." % next)
visited_list[next] = True # this frame is now visited.
q.put(
(next, [])) # we cannot look at the frame, report no detection
return
markers[:] = detect_markers_robust(
frame.gray,
grid_size=5,
prev_markers=markers,
min_marker_perimeter=min_marker_perimeter,
aperture=aperture,
visualize=0,
true_detect_every_frame=1)
visited_list[frame.index] = True
q.put(
(frame.index, markers[:])
) #object passed will only be pickeled when collected from other process! need to make a copy ot avoid overwrite!!!
while run.value:
next = cap.get_frame_index()
if seek_idx.value != -1:
next = seek_idx.value
seek_idx.value = -1
logger.debug("User required seek. Marker caching at Frame: %s" %
next)
#check the visited list
next = next_unvisited_idx(next)
if next == None:
#we are done here:
break
else:
handle_frame(next)
logger.debug("Closing Cacher Process")
cap.cleanup()
q.close()
run.value = False
return
def fill_cache(visited_list, video_file_path, q, seek_idx, run,
min_marker_perimeter, invert_image):
"""
this function is part of marker_detector it is run as a seperate process.
it must be kept in a seperate file for namespace sanatisation
"""
import os
import logging
logger = logging.getLogger(__name__ + " with pid: " + str(os.getpid()))
logger.debug("Started cacher process for Marker Detector")
from video_capture import File_Source, EndofVideoError, FileSeekError
from square_marker_detect import detect_markers_robust
aperture = 9
markers = []
cap = File_Source(SimpleNamespace(),
source_path=video_file_path,
timing=None,
fill_gaps=True)
def next_unvisited_idx(frame_idx):
try:
visited = visited_list[frame_idx]
except IndexError:
visited = True # trigger search
if not visited:
next_unvisited = frame_idx
else:
# find next unvisited site in the future
try:
next_unvisited = visited_list.index(False, frame_idx)
except ValueError:
# any thing in the past?
try:
next_unvisited = visited_list.index(False, 0, frame_idx)
except ValueError:
# no unvisited sites left. Done!
logger.debug("Caching completed.")
next_unvisited = None
return next_unvisited
def handle_frame(next_frame):
if next_frame != cap.get_frame_index() + 1:
# we need to seek:
logger.debug("Seeking to Frame {}".format(next_frame))
try:
cap.seek_to_frame(next_frame)
except FileSeekError:
# could not seek to requested position
logger.warning(
"Could not evaluate frame: {}.".format(next_frame))
visited_list[next_frame] = True # this frame is now visited.
q.put((next_frame,
[])) # we cannot look at the frame, report no detection
return
# seeking invalidates prev markers for the detector
markers[:] = []
try:
frame = cap.get_frame()
except EndofVideoError:
logger.debug("Video File's last frame(s) not accesible")
# could not read frame
logger.warning("Could not evaluate frame: {}.".format(next_frame))
visited_list[next_frame] = True # this frame is now visited.
q.put((next_frame,
[])) # we cannot look at the frame, report no detection
return
markers[:] = detect_markers_robust(
frame.gray,
grid_size=5,
prev_markers=markers,
min_marker_perimeter=min_marker_perimeter,
aperture=aperture,
visualize=0,
true_detect_every_frame=1,
invert_image=invert_image,
)
visited_list[frame.index] = True
q.put(
(frame.index, markers[:])
) # object passed will only be pickeled when collected from other process! need to make a copy ot avoid overwrite!!!
while run.value:
next_frame = cap.get_frame_index()
if seek_idx.value != -1:
next_frame = seek_idx.value
seek_idx.value = -1
logger.debug(
"User required seek. Marker caching at Frame: {}".format(
next_frame))
# Delete the # to unfill the gap
# target_entry = cap.videoset.lookup[next_frame]
# if target_entry.container_idx == -1:
# continue
# check the visited list
next_frame = next_unvisited_idx(next_frame)
if next_frame is None:
# we are done here:
break
else:
handle_frame(next_frame)
logger.debug("Closing Cacher Process")
cap.cleanup()
q.close()
run.value = False
return
def fill_cache(visited_list,video_file_path,timestamps,q,seek_idx,run,min_marker_perimeter):
'''
this function is part of marker_detector it is run as a seperate process.
it must be kept in a seperate file for namespace sanatisation
'''
import os
#import logging
#logger = logging.getLogger(__name__+' with pid: '+str(os.getpid()) )
#logger.debug('Started cacher process for Marker Detector')
import cv2
from video_capture import File_Source, EndofVideoFileError,FileSeekError
from screen_detector_methods import detect_screens
#aperture = 9
markers = []
cap = File_Source(Global_Container(),video_file_path,timestamps=timestamps)
def next_unvisited_idx(frame_idx):
try:
visited = visited_list[frame_idx]
except IndexError:
visited = True # trigger search
if not visited:
next_unvisited = frame_idx
else:
# find next unvisited site in the future
try:
next_unvisited = visited_list.index(False,frame_idx)
except ValueError:
# any thing in the past?
try:
next_unvisited = visited_list.index(False,0,frame_idx)
except ValueError:
#no unvisited sites left. Done!
#logger.debug("Caching completed.")
next_unvisited = None
return next_unvisited
def handle_frame(next):
if next != cap.get_frame_index():
#we need to seek:
#logger.debug("Seeking to Frame %s" %next)
try:
cap.seek_to_frame(next)
except FileSeekError:
#could not seek to requested position
#logger.warning("Could not evaluate frame: %s."%next)
visited_list[next] = True # this frame is now visited.
q.put((next,[])) # we cannot look at the frame, report no detection
return
#seeking invalidates prev markers for the detector
markers[:] = []
try:
frame = cap.get_frame_nowait()
except EndofVideoFileError:
#logger.debug("Video File's last frame(s) not accesible")
#could not read frame
#logger.warning("Could not evaluate frame: %s."%next)
visited_list[next] = True # this frame is now visited.
q.put((next,[])) # we cannot look at the frame, report no detection
return
markers[:] = detect_screens(frame.gray)
visited_list[frame.index] = True
q.put((frame.index,markers[:])) #object passed will only be pickeled when collected from other process! need to make a copy ot avoid overwrite!!!
while run.value:
next = cap.get_frame_index()
if seek_idx.value != -1:
next = seek_idx.value
seek_idx.value = -1
#logger.debug("User required seek. Marker caching at Frame: %s"%next)
#check the visited list
next = next_unvisited_idx(next)
if next == None:
#we are done here:
break
else:
handle_frame(next)
#logger.debug("Closing Cacher Process")
cap.cleanup()
q.close()
run.value = False
return
def update_cache_hack(self):
from video_capture import File_Source, EndofVideoError, FileSeekError
from screen_detector_methods import detect_screens
def put_in_cache(frame_idx, detected_screen):
print(frame_idx)
visited_list[frame_idx] = True
self.cache.update(frame_idx, detected_screen)
for s in self.surfaces:
s.update_cache(self.cache,
min_marker_perimeter=self.min_marker_perimeter,
min_id_confidence=self.min_id_confidence,
idx=frame_idx)
def next_unvisited_idx(frame_idx):
try:
visited = visited_list[frame_idx]
except IndexError:
visited = True # trigger search
if not visited:
next_unvisited = frame_idx
else:
# find next unvisited site in the future
try:
next_unvisited = visited_list.index(False, frame_idx)
except ValueError:
# any thing in the past?
try:
next_unvisited = visited_list.index(
False, 0, frame_idx)
except ValueError:
#no unvisited sites left. Done!
#logger.debug("Caching completed.")
next_unvisited = None
return next_unvisited
def handle_frame(next_frame):
if next_frame != cap.get_frame_index():
#we need to seek:
#logger.debug("Seeking to Frame %s" %next_frame)
try:
cap.seek_to_frame(next_frame)
except FileSeekError:
put_in_cache(
next_frame,
[]) # we cannot look at the frame, report no detection
return
#seeking invalidates prev markers for the detector
# markers[:] = []
try:
frame = cap.get_frame()
except EndofVideoError:
put_in_cache(
next_frame,
[]) # we cannot look at the frame, report no detection
return
put_in_cache(frame.index, detect_screens(frame.gray))
self.cacher_seek_idx = 0
visited_list = [False for x in self.cache]
# markers = []
cap = File_Source(Global_Container(), self.g_pool.capture.source_path)
for _ in self.cache:
next_frame = cap.get_frame_index()
if next_frame is None or next_frame >= len(self.cache):
#we are done here:
break
else:
handle_frame(next_frame)
def circle_detector(ipc_push_url, pair_url, source_path, batch_size=20):
# ipc setup
import zmq
import zmq_tools
zmq_ctx = zmq.Context()
process_pipe = zmq_tools.Msg_Pair_Client(zmq_ctx, pair_url)
# logging setup
import logging
logging.getLogger("OpenGL").setLevel(logging.ERROR)
logger = logging.getLogger()
logger.handlers = []
logger.setLevel(logging.INFO)
logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url))
# create logger for the context of this function
logger = logging.getLogger(__name__)
# imports
from time import sleep
from video_capture import File_Source, EndofVideoFileError
from circle_detector import CircleTracker
try:
src = File_Source(Empty(), source_path, timed_playback=False)
frame = src.get_frame()
logger.info('Starting calibration marker detection...')
frame_count = src.get_frame_count()
queue = []
circle_tracker = CircleTracker()
while True:
while process_pipe.new_data:
topic, n = process_pipe.recv()
if topic == 'terminate':
process_pipe.send(topic='exception',
payload={"reason": "User terminated."})
logger.debug("Process terminated")
sleep(1.0)
return
progress = 100. * frame.index / frame_count
markers = [
m for m in circle_tracker.update(frame.gray)
if m['marker_type'] == 'Ref'
]
if len(markers):
ref = {
"norm_pos": markers[0]['norm_pos'],
"screen_pos": markers[0]['img_pos'],
"timestamp": frame.timestamp,
'index': frame.index
}
queue.append((progress, ref))
else:
queue.append((progress, None))
if len(queue) > batch_size:
# dequeue batch
data = queue[:batch_size]
del queue[:batch_size]
process_pipe.send(topic='progress', payload={'data': data})
frame = src.get_frame()
except EndofVideoFileError:
process_pipe.send(topic='progress', payload={'data': queue})
process_pipe.send(topic='finished', payload={})
logger.debug("Process finished")
except Exception:
import traceback
process_pipe.send(topic='exception',
payload={'reason': traceback.format_exc()})
logger.debug("Process raised Exception")
sleep(1.0)
def export(should_terminate, frames_to_export, current_frame, rec_dir, user_dir, min_data_confidence,
start_frame=None, end_frame=None, plugin_initializers=(), out_file_path=None,pre_computed={}):
vis_plugins = sorted([Vis_Circle,Vis_Cross,Vis_Polyline,Vis_Light_Points,
Vis_Watermark,Vis_Scan_Path,Vis_Eye_Video_Overlay], key=lambda x: x.__name__)
analysis_plugins = sorted([ Pupil_Angle_3D_Fixation_Detector,
Gaze_Position_2D_Fixation_Detector], key=lambda x: x.__name__)
user_plugins = sorted(import_runtime_plugins(os.path.join(user_dir, 'plugins')), key=lambda x: x.__name__)
available_plugins = vis_plugins + analysis_plugins + user_plugins
name_by_index = [p.__name__ for p in available_plugins]
plugin_by_name = dict(zip(name_by_index, available_plugins))
logger = logging.getLogger(__name__+' with pid: '+str(os.getpid()))
update_recording_to_recent(rec_dir)
video_path = [f for f in glob(os.path.join(rec_dir, "world.*")) if f[-3:] in ('mp4', 'mkv', 'avi')][0]
timestamps_path = os.path.join(rec_dir, "world_timestamps.npy")
pupil_data_path = os.path.join(rec_dir, "pupil_data")
audio_path = os.path.join(rec_dir, "audio.mp4")
meta_info = load_meta_info(rec_dir)
g_pool = Global_Container()
g_pool.app = 'exporter'
g_pool.min_data_confidence = min_data_confidence
timestamps = np.load(timestamps_path)
cap = File_Source(g_pool, video_path, timestamps=timestamps)
# Out file path verification, we do this before but if one uses a seperate tool, this will kick in.
if out_file_path is None:
out_file_path = os.path.join(rec_dir, "world_viz.mp4")
else:
file_name = os.path.basename(out_file_path)
dir_name = os.path.dirname(out_file_path)
if not dir_name:
dir_name = rec_dir
if not file_name:
file_name = 'world_viz.mp4'
out_file_path = os.path.expanduser(os.path.join(dir_name, file_name))
if os.path.isfile(out_file_path):
logger.warning("Video out file already exsists. I will overwrite!")
os.remove(out_file_path)
logger.debug("Saving Video to {}".format(out_file_path))
# Trim mark verification
# make sure the trim marks (start frame, endframe) make sense:
# We define them like python list slices, thus we can test them like such.
trimmed_timestamps = timestamps[start_frame:end_frame]
if len(trimmed_timestamps) == 0:
logger.warn("Start and end frames are set such that no video will be exported.")
return False
if start_frame is None:
start_frame = 0
# these two vars are shared with the lauching process and give a job length and progress report.
frames_to_export.value = len(trimmed_timestamps)
current_frame.value = 0
exp_info = "Will export from frame {} to frame {}. This means I will export {} frames."
logger.debug(exp_info.format(start_frame, start_frame + frames_to_export.value, frames_to_export.value))
# setup of writer
writer = AV_Writer(out_file_path, fps=cap.frame_rate, audio_loc=audio_path, use_timestamps=True)
cap.seek_to_frame(start_frame)
start_time = time()
g_pool.capture = cap
g_pool.rec_dir = rec_dir
g_pool.user_dir = user_dir
g_pool.meta_info = meta_info
g_pool.timestamps = timestamps
g_pool.delayed_notifications = {}
g_pool.notifications = []
# load pupil_positions, gaze_positions
pupil_data = pre_computed.get("pupil_data") or load_object(pupil_data_path)
g_pool.pupil_data = pupil_data
g_pool.pupil_positions = pre_computed.get("pupil_positions") or pupil_data['pupil_positions']
g_pool.gaze_positions = pre_computed.get("gaze_positions") or pupil_data['gaze_positions']
g_pool.fixations = [] # populated by the fixation detector plugin
g_pool.pupil_positions_by_frame = correlate_data(g_pool.pupil_positions,g_pool.timestamps)
g_pool.gaze_positions_by_frame = correlate_data(g_pool.gaze_positions,g_pool.timestamps)
g_pool.fixations_by_frame = [[] for x in g_pool.timestamps] # populated by the fixation detector plugin
# add plugins
g_pool.plugins = Plugin_List(g_pool, plugin_by_name, plugin_initializers)
while frames_to_export.value > current_frame.value:
if should_terminate.value:
logger.warning("User aborted export. Exported {} frames to {}.".format(current_frame.value, out_file_path))
# explicit release of VideoWriter
writer.close()
writer = None
return False
try:
frame = cap.get_frame()
except EndofVideoFileError:
break
events = {'frame':frame}
# new positons and events
events['gaze_positions'] = g_pool.gaze_positions_by_frame[frame.index]
events['pupil_positions'] = g_pool.pupil_positions_by_frame[frame.index]
# publish delayed notifiactions when their time has come.
for n in list(g_pool.delayed_notifications.values()):
if n['_notify_time_'] < time():
del n['_notify_time_']
del g_pool.delayed_notifications[n['subject']]
g_pool.notifications.append(n)
# notify each plugin if there are new notifactions:
while g_pool.notifications:
n = g_pool.notifications.pop(0)
for p in g_pool.plugins:
p.on_notify(n)
# allow each Plugin to do its work.
for p in g_pool.plugins:
p.recent_events(events)
writer.write_video_frame(frame)
current_frame.value += 1
writer.close()
writer = None
duration = time()-start_time
effective_fps = float(current_frame.value)/duration
result = "Export done: Exported {} frames to {}. This took {} seconds. Exporter ran at {} frames per second."
logger.info(result.format(current_frame.value, out_file_path, duration, effective_fps))
return True
def _export_world_video(
rec_dir,
user_dir,
min_data_confidence,
start_frame,
end_frame,
plugin_initializers,
out_file_path,
pre_computed_eye_data,
):
"""
Simulates the generation for the world video and saves a certain time range as a video.
It simulates a whole g_pool such that all plugins run as normal.
"""
from glob import glob
from time import time
import file_methods as fm
import player_methods as pm
from av_writer import MPEG_Audio_Writer
# We are not importing manual gaze correction. In Player corrections have already
# been applied.
from fixation_detector import Offline_Fixation_Detector
# Plug-ins
from plugin import Plugin_List, import_runtime_plugins
from video_capture import EndofVideoError, File_Source
from video_overlay.plugins import Video_Overlay, Eye_Overlay
from vis_circle import Vis_Circle
from vis_cross import Vis_Cross
from vis_light_points import Vis_Light_Points
from vis_polyline import Vis_Polyline
from vis_watermark import Vis_Watermark
PID = str(os.getpid())
logger = logging.getLogger(f"{__name__} with pid: {PID}")
start_status = f"Starting video export with pid: {PID}"
logger.info(start_status)
yield start_status, 0
try:
vis_plugins = sorted(
[
Vis_Circle,
Vis_Cross,
Vis_Polyline,
Vis_Light_Points,
Vis_Watermark,
Eye_Overlay,
Video_Overlay,
],
key=lambda x: x.__name__,
)
analysis_plugins = [Offline_Fixation_Detector]
user_plugins = sorted(
import_runtime_plugins(os.path.join(user_dir, "plugins")),
key=lambda x: x.__name__,
)
available_plugins = vis_plugins + analysis_plugins + user_plugins
name_by_index = [p.__name__ for p in available_plugins]
plugin_by_name = dict(zip(name_by_index, available_plugins))
recording = PupilRecording(rec_dir)
meta_info = recording.meta_info
g_pool = GlobalContainer()
g_pool.app = "exporter"
g_pool.process = "exporter"
g_pool.min_data_confidence = min_data_confidence
videos = recording.files().core().world().videos()
if not videos:
raise FileNotFoundError("No world video found")
source_path = videos[0].resolve()
cap = File_Source(g_pool,
source_path=source_path,
fill_gaps=True,
timing=None)
if not cap.initialised:
warn = "Trying to export zero-duration world video."
logger.warning(warn)
yield warn, 0.0
return
timestamps = cap.timestamps
file_name = os.path.basename(out_file_path)
dir_name = os.path.dirname(out_file_path)
out_file_path = os.path.expanduser(os.path.join(dir_name, file_name))
if os.path.isfile(out_file_path):
logger.warning("Video out file already exsists. I will overwrite!")
os.remove(out_file_path)
logger.debug("Saving Video to {}".format(out_file_path))
# Trim mark verification
# make sure the trim marks (start frame, end frame) make sense:
# We define them like python list slices, thus we can test them like such.
trimmed_timestamps = timestamps[start_frame:end_frame]
if len(trimmed_timestamps) == 0:
warn = "Start and end frames are set such that no video will be exported."
logger.warning(warn)
yield warn, 0.0
return
if start_frame is None:
start_frame = 0
# these two vars are shared with the launching process and
# give a job length and progress report.
frames_to_export = len(trimmed_timestamps)
current_frame = 0
logger.debug(
f"Will export from frame {start_frame} to frame "
f"{start_frame + frames_to_export}. This means I will export "
f"{frames_to_export} frames.")
cap.seek_to_frame(start_frame)
start_time = time()
g_pool.plugin_by_name = plugin_by_name
g_pool.capture = cap
g_pool.rec_dir = rec_dir
g_pool.user_dir = user_dir
g_pool.meta_info = meta_info
g_pool.timestamps = timestamps
g_pool.delayed_notifications = {}
g_pool.notifications = []
for initializers in pre_computed_eye_data.values():
initializers["data"] = [
fm.Serialized_Dict(msgpack_bytes=serialized)
for serialized in initializers["data"]
]
g_pool.pupil_positions = pm.PupilDataBisector.from_init_dict(
pre_computed_eye_data["pupil"])
g_pool.gaze_positions = pm.Bisector(**pre_computed_eye_data["gaze"])
g_pool.fixations = pm.Affiliator(**pre_computed_eye_data["fixations"])
# add plugins
g_pool.plugins = Plugin_List(g_pool, plugin_initializers)
try:
# setup of writer
writer = MPEG_Audio_Writer(
out_file_path,
start_time_synced=trimmed_timestamps[0],
audio_dir=rec_dir,
)
while frames_to_export > current_frame:
try:
frame = cap.get_frame()
except EndofVideoError:
break
events = {"frame": frame}
# new positions and events
frame_window = pm.enclosing_window(g_pool.timestamps,
frame.index)
events["gaze"] = g_pool.gaze_positions.by_ts_window(
frame_window)
events["pupil"] = g_pool.pupil_positions.by_ts_window(
frame_window)
# publish delayed notifications when their time has come.
for n in list(g_pool.delayed_notifications.values()):
if n["_notify_time_"] < time():
del n["_notify_time_"]
del g_pool.delayed_notifications[n["subject"]]
g_pool.notifications.append(n)
# notify each plugin if there are new notifications:
while g_pool.notifications:
n = g_pool.notifications.pop(0)
for p in g_pool.plugins:
p.on_notify(n)
# allow each Plugin to do its work.
for p in g_pool.plugins:
p.recent_events(events)
writer.write_video_frame(frame)
current_frame += 1
yield "Exporting with pid {}".format(PID), current_frame
except GeneratorExit:
logger.warning(f"Video export with pid {PID} was canceled.")
writer.close(timestamp_export_format=None,
closed_suffix=".canceled")
return
writer.close(timestamp_export_format="all")
duration = time() - start_time
effective_fps = float(current_frame) / duration
logger.info(
f"Export done: Exported {current_frame} frames to {out_file_path}. "
f"This took {duration} seconds. "
f"Exporter ran at {effective_fps} frames per second.")
yield "Export done. This took {:.0f} seconds.".format(
duration), current_frame
except GeneratorExit:
logger.warning(f"Video export with pid {PID} was canceled.")
def circle_detector(ipc_push_url, pair_url, source_path, batch_size=20):
# ipc setup
import zmq
import zmq_tools
zmq_ctx = zmq.Context()
process_pipe = zmq_tools.Msg_Pair_Client(zmq_ctx, pair_url)
# logging setup
import logging
logging.getLogger("OpenGL").setLevel(logging.ERROR)
logger = logging.getLogger()
logger.handlers = []
logger.setLevel(logging.INFO)
logger.addHandler(zmq_tools.ZMQ_handler(zmq_ctx, ipc_push_url))
# create logger for the context of this function
logger = logging.getLogger(__name__)
# imports
from time import sleep
from video_capture import File_Source, EndofVideoError
from circle_detector import CircleTracker
try:
# TODO: we need fill_gaps=True for correct frame indices to paint the circle
# markers on the world stream. But actually we don't want to process gap frames
# with the marker detector. We should make an option to only receive non-gap
# frames, but with gap-like indices?
src = File_Source(SimpleNamespace(),
source_path,
timing=None,
fill_gaps=True)
frame = src.get_frame()
logger.info("Starting calibration marker detection...")
frame_count = src.get_frame_count()
queue = []
circle_tracker = CircleTracker()
while True:
while process_pipe.new_data:
topic, n = process_pipe.recv()
if topic == "terminate":
process_pipe.send({
"topic": "exception",
"reason": "User terminated."
})
logger.debug("Process terminated")
sleep(1.0)
return
progress = 100.0 * frame.index / frame_count
markers = [
m for m in circle_tracker.update(frame.gray)
if m["marker_type"] == "Ref"
]
if len(markers):
ref = {
"norm_pos": markers[0]["norm_pos"],
"screen_pos": markers[0]["img_pos"],
"timestamp": frame.timestamp,
"index_range":
tuple(range(frame.index - 5, frame.index + 5)),
"index": frame.index,
}
queue.append((progress, ref))
else:
queue.append((progress, None))
if len(queue) > batch_size:
# dequeue batch
data = queue[:batch_size]
del queue[:batch_size]
process_pipe.send({"topic": "progress", "data": data})
frame = src.get_frame()
except EndofVideoError:
process_pipe.send({"topic": "progress", "data": queue})
process_pipe.send({"topic": "finished"})
logger.debug("Process finished")
except Exception:
import traceback
process_pipe.send({
"topic": "exception",
"reason": traceback.format_exc()
})
logger.debug("Process raised Exception")
sleep(1.0)
def session(rec_dir):
system_plugins = [Log_Display, Seek_Bar, Trim_Marks]
vis_plugins = sorted([Vis_Circle, Vis_Fixation, Vis_Polyline, Vis_Light_Points, Vis_Cross,
Vis_Watermark, Vis_Eye_Video_Overlay, Vis_Scan_Path], key=lambda x: x.__name__)
analysis_plugins = sorted([Gaze_Position_2D_Fixation_Detector, Pupil_Angle_3D_Fixation_Detector,
Manual_Gaze_Correction, Video_Export_Launcher, Offline_Surface_Tracker,
Raw_Data_Exporter, Batch_Exporter, Annotation_Player], key=lambda x: x.__name__)
other_plugins = sorted([Show_Calibration, Log_History, Marker_Auto_Trim_Marks], key=lambda x: x.__name__)
user_plugins = sorted(import_runtime_plugins(os.path.join(user_dir, 'plugins')), key=lambda x: x.__name__)
user_launchable_plugins = vis_plugins + analysis_plugins + other_plugins + user_plugins
available_plugins = system_plugins + user_launchable_plugins
name_by_index = [p.__name__ for p in available_plugins]
plugin_by_name = dict(zip(name_by_index, available_plugins))
# Callback functions
def on_resize(window, w, h):
g_pool.gui.update_window(w, h)
g_pool.gui.collect_menus()
graph.adjust_size(w, h)
gl_utils.adjust_gl_view(w, h)
for p in g_pool.plugins:
p.on_window_resize(window, w, h)
def on_key(window, key, scancode, action, mods):
g_pool.gui.update_key(key, scancode, action, mods)
def on_char(window, char):
g_pool.gui.update_char(char)
def on_button(window, button, action, mods):
g_pool.gui.update_button(button, action, mods)
pos = glfwGetCursorPos(window)
pos = normalize(pos, glfwGetWindowSize(window))
pos = denormalize(pos, (frame.img.shape[1], frame.img.shape[0])) # Position in img pixels
for p in g_pool.plugins:
p.on_click(pos, button, action)
def on_pos(window, x, y):
hdpi_factor = float(glfwGetFramebufferSize(window)[0]/glfwGetWindowSize(window)[0])
g_pool.gui.update_mouse(x*hdpi_factor, y*hdpi_factor)
def on_scroll(window, x, y):
g_pool.gui.update_scroll(x, y*scroll_factor)
def on_drop(window, count, paths):
for x in range(count):
new_rec_dir = paths[x].decode('utf-8')
if is_pupil_rec_dir(new_rec_dir):
logger.debug("Starting new session with '{}'".format(new_rec_dir))
global rec_dir
rec_dir = new_rec_dir
glfwSetWindowShouldClose(window, True)
else:
logger.error("'{}' is not a valid pupil recording".format(new_rec_dir))
tick = delta_t()
def get_dt():
return next(tick)
update_recording_to_recent(rec_dir)
video_path = [f for f in glob(os.path.join(rec_dir, "world.*")) if f[-3:] in ('mp4', 'mkv', 'avi')][0]
timestamps_path = os.path.join(rec_dir, "world_timestamps.npy")
pupil_data_path = os.path.join(rec_dir, "pupil_data")
meta_info = load_meta_info(rec_dir)
rec_version = read_rec_version(meta_info)
app_version = get_version(version_file)
# log info about Pupil Platform and Platform in player.log
logger.info('Application Version: {}'.format(app_version))
logger.info('System Info: {}'.format(get_system_info()))
timestamps = np.load(timestamps_path)
# create container for globally scoped vars
g_pool = Global_Container()
g_pool.app = 'player'
# Initialize capture
cap = File_Source(g_pool, video_path, timestamps=list(timestamps))
# load session persistent settings
session_settings = Persistent_Dict(os.path.join(user_dir, "user_settings"))
if session_settings.get("version", VersionFormat('0.0')) < get_version(version_file):
logger.info("Session setting are from older version of this app. I will not use those.")
session_settings.clear()
width, height = session_settings.get('window_size', cap.frame_size)
window_pos = session_settings.get('window_position', window_position_default)
main_window = glfwCreateWindow(width, height, "Pupil Player: "+meta_info["Recording Name"]+" - "
+ rec_dir.split(os.path.sep)[-1], None, None)
glfwSetWindowPos(main_window, window_pos[0], window_pos[1])
glfwMakeContextCurrent(main_window)
cygl.utils.init()
# load pupil_positions, gaze_positions
pupil_data = load_object(pupil_data_path)
pupil_list = pupil_data['pupil_positions']
gaze_list = pupil_data['gaze_positions']
g_pool.binocular = meta_info.get('Eye Mode', 'monocular') == 'binocular'
g_pool.version = app_version
g_pool.capture = cap
g_pool.timestamps = timestamps
g_pool.play = False
g_pool.new_seek = True
g_pool.user_dir = user_dir
g_pool.rec_dir = rec_dir
g_pool.rec_version = rec_version
g_pool.meta_info = meta_info
g_pool.min_data_confidence = session_settings.get('min_data_confidence', 0.6)
g_pool.pupil_positions_by_frame = correlate_data(pupil_list, g_pool.timestamps)
g_pool.gaze_positions_by_frame = correlate_data(gaze_list, g_pool.timestamps)
g_pool.fixations_by_frame = [[] for x in g_pool.timestamps] # populated by the fixation detector plugin
def next_frame(_):
try:
cap.seek_to_frame(cap.get_frame_index())
except(FileSeekError):
logger.warning("Could not seek to next frame.")
else:
g_pool.new_seek = True
def prev_frame(_):
try:
cap.seek_to_frame(cap.get_frame_index()-2)
except(FileSeekError):
logger.warning("Could not seek to previous frame.")
else:
g_pool.new_seek = True
def toggle_play(new_state):
if cap.get_frame_index() >= cap.get_frame_count()-5:
cap.seek_to_frame(1) # avoid pause set by hitting trimmark pause.
logger.warning("End of video - restart at beginning.")
g_pool.play = new_state
def set_scale(new_scale):
g_pool.gui.scale = new_scale
g_pool.gui.collect_menus()
def set_data_confidence(new_confidence):
g_pool.min_data_confidence = new_confidence
notification = {'subject': 'min_data_confidence_changed'}
notification['_notify_time_'] = time()+.8
g_pool.delayed_notifications[notification['subject']] = notification
def open_plugin(plugin):
if plugin == "Select to load":
return
g_pool.plugins.add(plugin)
def purge_plugins():
for p in g_pool.plugins:
if p.__class__ in user_launchable_plugins:
p.alive = False
g_pool.plugins.clean()
def do_export(_):
export_range = slice(g_pool.trim_marks.in_mark, g_pool.trim_marks.out_mark)
export_dir = os.path.join(g_pool.rec_dir, 'exports', '{}-{}'.format(export_range.start, export_range.stop))
try:
os.makedirs(export_dir)
except OSError as e:
if e.errno != errno.EEXIST:
logger.error("Could not create export dir")
raise e
else:
overwrite_warning = "Previous export for range [{}-{}] already exsits - overwriting."
logger.warning(overwrite_warning.format(export_range.start, export_range.stop))
else:
logger.info('Created export dir at "{}"'.format(export_dir))
notification = {'subject': 'should_export', 'range': export_range, 'export_dir': export_dir}
g_pool.notifications.append(notification)
g_pool.gui = ui.UI()
g_pool.gui.scale = session_settings.get('gui_scale', 1)
g_pool.main_menu = ui.Scrolling_Menu("Settings", pos=(-350, 20), size=(300, 500))
g_pool.main_menu.append(ui.Button("Close Pupil Player", lambda: glfwSetWindowShouldClose(main_window, True)))
g_pool.main_menu.append(ui.Slider('scale', g_pool.gui, setter=set_scale, step=.05, min=0.75, max=2.5, label='Interface Size'))
g_pool.main_menu.append(ui.Info_Text('Player Version: {}'.format(g_pool.version)))
g_pool.main_menu.append(ui.Info_Text('Recording Version: {}'.format(rec_version)))
g_pool.main_menu.append(ui.Slider('min_data_confidence', g_pool, setter=set_data_confidence,
step=.05, min=0.0, max=1.0, label='Confidence threshold'))
selector_label = "Select to load"
vis_labels = [" " + p.__name__.replace('_', ' ') for p in vis_plugins]
analysis_labels = [" " + p.__name__.replace('_', ' ') for p in analysis_plugins]
other_labels = [" " + p.__name__.replace('_', ' ') for p in other_plugins]
user_labels = [" " + p.__name__.replace('_', ' ') for p in user_plugins]
plugins = ([selector_label, selector_label] + vis_plugins + [selector_label] + analysis_plugins + [selector_label]
+ other_plugins + [selector_label] + user_plugins)
labels = ([selector_label, "Visualization"] + vis_labels + ["Analysis"] + analysis_labels + ["Other"]
+ other_labels + ["User added"] + user_labels)
g_pool.main_menu.append(ui.Selector('Open plugin:',
selection=plugins,
labels=labels,
setter=open_plugin,
getter=lambda: selector_label))
g_pool.main_menu.append(ui.Button('Close all plugins', purge_plugins))
g_pool.main_menu.append(ui.Button('Reset window size',
lambda: glfwSetWindowSize(main_window, cap.frame_size[0], cap.frame_size[1])))
g_pool.quickbar = ui.Stretching_Menu('Quick Bar', (0, 100), (120, -100))
g_pool.play_button = ui.Thumb('play',
g_pool,
label=chr(0xf04b),
setter=toggle_play,
hotkey=GLFW_KEY_SPACE,
label_font='fontawesome',
label_offset_x=5,
label_offset_y=0,
label_offset_size=-24)
g_pool.play_button.on_color[:] = (0, 1., .0, .8)
g_pool.forward_button = ui.Thumb('forward',
label=chr(0xf04e),
getter=lambda: False,
setter=next_frame,
hotkey=GLFW_KEY_RIGHT,
label_font='fontawesome',
label_offset_x=5,
label_offset_y=0,
label_offset_size=-24)
g_pool.backward_button = ui.Thumb('backward',
label=chr(0xf04a),
getter=lambda: False,
setter=prev_frame,
hotkey=GLFW_KEY_LEFT,
label_font='fontawesome',
label_offset_x=-5,
label_offset_y=0,
label_offset_size=-24)
g_pool.export_button = ui.Thumb('export',
label=chr(0xf063),
getter=lambda: False,
setter=do_export,
hotkey='e',
label_font='fontawesome',
label_offset_x=0,
label_offset_y=2,
label_offset_size=-24)
g_pool.quickbar.extend([g_pool.play_button, g_pool.forward_button, g_pool.backward_button, g_pool.export_button])
g_pool.gui.append(g_pool.quickbar)
g_pool.gui.append(g_pool.main_menu)
# we always load these plugins
system_plugins = [('Trim_Marks', {}), ('Seek_Bar', {})]
default_plugins = [('Log_Display', {}), ('Vis_Scan_Path', {}), ('Vis_Polyline', {}), ('Vis_Circle', {}), ('Video_Export_Launcher', {})]
previous_plugins = session_settings.get('loaded_plugins', default_plugins)
g_pool.notifications = []
g_pool.delayed_notifications = {}
g_pool.plugins = Plugin_List(g_pool, plugin_by_name, system_plugins+previous_plugins)
# Register callbacks main_window
glfwSetFramebufferSizeCallback(main_window, on_resize)
glfwSetKeyCallback(main_window, on_key)
glfwSetCharCallback(main_window, on_char)
glfwSetMouseButtonCallback(main_window, on_button)
glfwSetCursorPosCallback(main_window, on_pos)
glfwSetScrollCallback(main_window, on_scroll)
glfwSetDropCallback(main_window, on_drop)
# trigger on_resize
on_resize(main_window, *glfwGetFramebufferSize(main_window))
g_pool.gui.configuration = session_settings.get('ui_config', {})
# gl_state settings
gl_utils.basic_gl_setup()
g_pool.image_tex = Named_Texture()
# set up performace graphs:
pid = os.getpid()
ps = psutil.Process(pid)
ts = None
cpu_graph = graph.Bar_Graph()
cpu_graph.pos = (20, 110)
cpu_graph.update_fn = ps.cpu_percent
cpu_graph.update_rate = 5
cpu_graph.label = 'CPU %0.1f'
fps_graph = graph.Bar_Graph()
fps_graph.pos = (140, 110)
fps_graph.update_rate = 5
fps_graph.label = "%0.0f REC FPS"
pupil_graph = graph.Bar_Graph(max_val=1.0)
pupil_graph.pos = (260, 110)
pupil_graph.update_rate = 5
pupil_graph.label = "Confidence: %0.2f"
while not glfwWindowShouldClose(main_window):
# grab new frame
if g_pool.play or g_pool.new_seek:
g_pool.new_seek = False
try:
new_frame = cap.get_frame_nowait()
except EndofVideoFileError:
# end of video logic: pause at last frame.
g_pool.play = False
logger.warning("end of video")
update_graph = True
else:
update_graph = False
frame = new_frame.copy()
events = {}
# report time between now and the last loop interation
events['dt'] = get_dt()
# new positons we make a deepcopy just like the image is a copy.
events['gaze_positions'] = deepcopy(g_pool.gaze_positions_by_frame[frame.index])
events['pupil_positions'] = deepcopy(g_pool.pupil_positions_by_frame[frame.index])
if update_graph:
# update performace graphs
for p in events['pupil_positions']:
pupil_graph.add(p['confidence'])
t = new_frame.timestamp
if ts and ts != t:
dt, ts = t-ts, t
fps_graph.add(1./dt)
else:
ts = new_frame.timestamp
g_pool.play_button.status_text = str(frame.index)
# always update the CPU graph
cpu_graph.update()
# publish delayed notifiactions when their time has come.
for n in list(g_pool.delayed_notifications.values()):
if n['_notify_time_'] < time():
del n['_notify_time_']
del g_pool.delayed_notifications[n['subject']]
g_pool.notifications.append(n)
# notify each plugin if there are new notifactions:
while g_pool.notifications:
n = g_pool.notifications.pop(0)
for p in g_pool.plugins:
p.on_notify(n)
# allow each Plugin to do its work.
for p in g_pool.plugins:
p.update(frame, events)
# check if a plugin need to be destroyed
g_pool.plugins.clean()
# render camera image
glfwMakeContextCurrent(main_window)
gl_utils.make_coord_system_norm_based()
g_pool.image_tex.update_from_frame(frame)
g_pool.image_tex.draw()
gl_utils.make_coord_system_pixel_based(frame.img.shape)
# render visual feedback from loaded plugins
for p in g_pool.plugins:
p.gl_display()
graph.push_view()
fps_graph.draw()
cpu_graph.draw()
pupil_graph.draw()
graph.pop_view()
g_pool.gui.update()
# present frames at appropriate speed
cap.wait(frame)
glfwSwapBuffers(main_window)
glfwPollEvents()
session_settings['loaded_plugins'] = g_pool.plugins.get_initializers()
session_settings['min_data_confidence'] = g_pool.min_data_confidence
session_settings['gui_scale'] = g_pool.gui.scale
session_settings['ui_config'] = g_pool.gui.configuration
session_settings['window_size'] = glfwGetWindowSize(main_window)
session_settings['window_position'] = glfwGetWindowPos(main_window)
session_settings['version'] = g_pool.version
session_settings.close()
# de-init all running plugins
for p in g_pool.plugins:
p.alive = False
g_pool.plugins.clean()
cap.cleanup()
g_pool.gui.terminate()
glfwDestroyWindow(main_window)
def _export_world_video(
rec_dir,
user_dir,
min_data_confidence,
start_frame,
end_frame,
plugin_initializers,
out_file_path,
pre_computed_eye_data,
):
"""
Simulates the generation for the world video and saves a certain time range as a video.
It simulates a whole g_pool such that all plugins run as normal.
"""
from glob import glob
from time import time
import file_methods as fm
import player_methods as pm
from av_writer import AV_Writer
# we are not importing manual gaze correction. In Player corrections have already been applied.
# in batch exporter this plugin makes little sense.
from fixation_detector import Offline_Fixation_Detector
from eye_movement import Offline_Eye_Movement_Detector
# Plug-ins
from plugin import Plugin_List, import_runtime_plugins
from video_capture import EndofVideoError, File_Source
from video_overlay.plugins import Video_Overlay, Eye_Overlay
from vis_circle import Vis_Circle
from vis_cross import Vis_Cross
from vis_light_points import Vis_Light_Points
from vis_polyline import Vis_Polyline
from vis_scan_path import Vis_Scan_Path
from vis_watermark import Vis_Watermark
PID = str(os.getpid())
logger = logging.getLogger(__name__ + " with pid: " + PID)
start_status = "Starting video export with pid: {}".format(PID)
logger.info(start_status)
yield start_status, 0
try:
vis_plugins = sorted(
[
Vis_Circle,
Vis_Cross,
Vis_Polyline,
Vis_Light_Points,
Vis_Watermark,
Vis_Scan_Path,
Eye_Overlay,
Video_Overlay,
],
key=lambda x: x.__name__,
)
analysis_plugins = [Offline_Fixation_Detector, Offline_Eye_Movement_Detector]
user_plugins = sorted(
import_runtime_plugins(os.path.join(user_dir, "plugins")),
key=lambda x: x.__name__,
)
available_plugins = vis_plugins + analysis_plugins + user_plugins
name_by_index = [p.__name__ for p in available_plugins]
plugin_by_name = dict(zip(name_by_index, available_plugins))
meta_info = pm.load_meta_info(rec_dir)
g_pool = GlobalContainer()
g_pool.app = "exporter"
g_pool.min_data_confidence = min_data_confidence
valid_ext = (".mp4", ".mkv", ".avi", ".h264", ".mjpeg", ".fake")
try:
video_path = next(
f
for f in glob(os.path.join(rec_dir, "world.*"))
if os.path.splitext(f)[1] in valid_ext
)
except StopIteration:
raise FileNotFoundError("No Video world found")
cap = File_Source(g_pool, source_path=video_path, fill_gaps=True, timing=None)
timestamps = cap.timestamps
file_name = os.path.basename(out_file_path)
dir_name = os.path.dirname(out_file_path)
out_file_path = os.path.expanduser(os.path.join(dir_name, file_name))
if os.path.isfile(out_file_path):
logger.warning("Video out file already exsists. I will overwrite!")
os.remove(out_file_path)
logger.debug("Saving Video to {}".format(out_file_path))
# Trim mark verification
# make sure the trim marks (start frame, end frame) make sense:
# We define them like python list slices, thus we can test them like such.
trimmed_timestamps = timestamps[start_frame:end_frame]
if len(trimmed_timestamps) == 0:
warn = "Start and end frames are set such that no video will be exported."
logger.warning(warn)
yield warn, 0.0
return
if start_frame is None:
start_frame = 0
# these two vars are shared with the launching process and give a job length and progress report.
frames_to_export = len(trimmed_timestamps)
current_frame = 0
exp_info = (
"Will export from frame {} to frame {}. This means I will export {} frames."
)
logger.debug(
exp_info.format(
start_frame, start_frame + frames_to_export, frames_to_export
)
)
# setup of writer
writer = AV_Writer(
out_file_path, fps=cap.frame_rate, audio_dir=rec_dir, use_timestamps=True
)
cap.seek_to_frame(start_frame)
start_time = time()
g_pool.plugin_by_name = plugin_by_name
g_pool.capture = cap
g_pool.rec_dir = rec_dir
g_pool.user_dir = user_dir
g_pool.meta_info = meta_info
g_pool.timestamps = timestamps
g_pool.delayed_notifications = {}
g_pool.notifications = []
for initializers in pre_computed_eye_data.values():
initializers["data"] = [
fm.Serialized_Dict(msgpack_bytes=serialized)
for serialized in initializers["data"]
]
g_pool.pupil_positions = pm.Bisector(**pre_computed_eye_data["pupil"])
g_pool.pupil_positions_by_id = (
pm.Bisector(**pre_computed_eye_data["pupil_by_id_0"]),
pm.Bisector(**pre_computed_eye_data["pupil_by_id_1"]),
)
g_pool.gaze_positions = pm.Bisector(**pre_computed_eye_data["gaze"])
g_pool.fixations = pm.Affiliator(**pre_computed_eye_data["fixations"])
# add plugins
g_pool.plugins = Plugin_List(g_pool, plugin_initializers)
while frames_to_export > current_frame:
try:
frame = cap.get_frame()
except EndofVideoError:
break
events = {"frame": frame}
# new positions and events
frame_window = pm.enclosing_window(g_pool.timestamps, frame.index)
events["gaze"] = g_pool.gaze_positions.by_ts_window(frame_window)
events["pupil"] = g_pool.pupil_positions.by_ts_window(frame_window)
# publish delayed notifications when their time has come.
for n in list(g_pool.delayed_notifications.values()):
if n["_notify_time_"] < time():
del n["_notify_time_"]
del g_pool.delayed_notifications[n["subject"]]
g_pool.notifications.append(n)
# notify each plugin if there are new notifications:
while g_pool.notifications:
n = g_pool.notifications.pop(0)
for p in g_pool.plugins:
p.on_notify(n)
# allow each Plugin to do its work.
for p in g_pool.plugins:
p.recent_events(events)
writer.write_video_frame(frame)
current_frame += 1
yield "Exporting with pid {}".format(PID), current_frame
writer.close(timestamp_export_format="all")
duration = time() - start_time
effective_fps = float(current_frame) / duration
result = "Export done: Exported {} frames to {}. This took {} seconds. Exporter ran at {} frames per second."
logger.info(
result.format(current_frame, out_file_path, duration, effective_fps)
)
yield "Export done. This took {:.0f} seconds.".format(duration), current_frame
except GeneratorExit:
logger.warning("Video export with pid {} was canceled.".format(os.getpid()))