def _clone_task(clone_func: Callable[[], None], errors: Queue) -> None:
try:
clone_func()
except Exception as e:
exc_buffer = io.StringIO()
traceback.print_exc(file=exc_buffer)
errors.put_nowait(exc_buffer.getvalue())
raise e
def main():
timePipe, sigPipe = Pipe()
q = Queue()
clock = Process(target=tickTock, args=(timePipe, ))
testSignal = Process(target=signal, args=(q, sigPipe, [1]))
testSignal.start()
while True:
print(q.get())
def init_marker_cacher(self):
forking_enable(0) #for MacOs only
from marker_detector_cacher import fill_cache
visited_list = [False if x == False else True for x in self.cache]
video_file_path = os.path.join(self.g_pool.rec_dir,'world.avi')
self.cache_queue = Queue()
self.cacher_seek_idx = Value(c_int,0)
self.cacher_run = Value(c_bool,True)
self.cacher = Process(target=fill_cache, args=(visited_list,video_file_path,self.cache_queue,self.cacher_seek_idx,self.cacher_run))
self.cacher.start()
def test_set_pdeathsig(self):
success = "done"
q = Queue()
p = Process(target=parent_task, args=(q, success))
p.start()
child_proc = psutil.Process(q.get(timeout=3))
try:
p.terminate()
assert q.get(timeout=3) == success
finally:
child_proc.terminate()
def init_marker_cacher(self):
forking_enable(0) #for MacOs only
from marker_detector_cacher import fill_cache
visited_list = [False if x == False else True for x in self.cache]
video_file_path = self.g_pool.capture.src
timestamps = self.g_pool.capture.timestamps
self.cache_queue = Queue()
self.cacher_seek_idx = Value('i',0)
self.cacher_run = Value(c_bool,True)
self.cacher = Process(target=fill_cache, args=(visited_list,video_file_path,timestamps,self.cache_queue,self.cacher_seek_idx,self.cacher_run,self.min_marker_perimeter_cacher))
self.cacher.start()
def __init__(self, fun, args, postprocess, job):
""" Build multiprocessing queues and start worker. """
super(LongCalculation, self).__init__(job, "Cancel", 0, 0)
self.setModal(True)
self.input = Queue()
self.output = Queue()
self.input.put((fun, args, postprocess))
self.proc = Process(target=worker, args=(self.input, self.output))
self.proc.start()
self.timer = QTimer()
self.timer.timeout.connect(self.update)
self.timer.start(10)
class LongCalculation(QProgressDialog):
"""
Multiprocessing based worker for mesh and eigenvalue calculations.
This is necessary to make sure GUI is not blocked while mesh is built,
or when eigenvalue calculations are performed.
Transformations do not need as much time, unless there is one implemented
without numpy vectorized coordinate calculations.
"""
res = None
def __init__(self, fun, args, postprocess, job):
""" Build multiprocessing queues and start worker. """
super(LongCalculation, self).__init__(job, "Cancel", 0, 0)
self.setModal(True)
self.input = Queue()
self.output = Queue()
self.input.put((fun, args, postprocess))
self.proc = Process(target=worker, args=(self.input, self.output))
self.proc.start()
self.timer = QTimer()
self.timer.timeout.connect(self.update)
self.timer.start(10)
def update(self):
""" Check if worker is done, and close dialog. """
try:
out = self.output.get(block=False)
if isinstance(out, basestring):
self.setLabelText(out)
return
if out is None:
self.done(0)
return
self.res = out
self.timer.stop()
self.proc.join()
del self.proc
self.done(1)
except:
pass
def cleanUp(self):
""" Kill the running processes if cancelled/failed. """
if self.proc:
while self.proc.is_alive():
self.proc.terminate()
del self.proc
self.timer.stop()
def main():
q = Queue()
x = [1, 2, 3, 4, 5]
pl = []
for i in x:
pl.append(Process(target=f, args=(
q,
i,
)))
for p in pl:
p.start()
p.join()
for i in x:
print(q.get())
def run_in_childprocess(target, codec=None, *args, **kwargs):
assert codec is None or len(codec) == 2, codec
queue = Queue()
p = Process(target=_wrapper, args=(target, codec, queue, args, kwargs))
p.start()
e, r = queue.get()
p.join()
if e:
raise e
if codec:
r = codec[1](r)
return r
def clone_with_timeout(src: str, dest: str, clone_func: Callable[[], None],
timeout: float) -> None:
"""Clone a repository with timeout.
Args:
src: clone source
dest: clone destination
clone_func: callable that does the actual cloning
timeout: timeout in seconds
"""
errors: Queue = Queue()
process = Process(target=_clone_task, args=(clone_func, errors))
process.start()
process.join(timeout)
if process.is_alive():
process.terminate()
# Give it literally a second (in successive steps of 0.1 second),
# then kill it.
# Can't use `process.join(1)` here, billiard appears to be bugged
# https://github.com/celery/billiard/issues/270
killed = False
for _ in range(10):
time.sleep(0.1)
if not process.is_alive():
break
else:
killed = True
os.kill(process.pid, signal.SIGKILL)
raise CloneTimeout(src, timeout, killed)
if not errors.empty():
raise CloneFailure(src, dest, errors.get())
def main():
# To assign camera by name: put string(s) in list
# Parse command line arguments
parser = argparse.ArgumentParser(description='GUI for gaze tracking and pupillometry')
parser.add_argument('-eye', dest='eye_file', type=str, help="Work with existing video recording, instead of live feed", default='')
parser.add_argument('-world', dest='world_file', type=str, help="Work with existing video recording, instead of live feed", default='')
args = parser.parse_args()
# to use a pre-recorded video.
# Use a string to specify the path to your video file as demonstrated below
if args.eye_file == '':
eye_src = ["UI154xLE-M", "USB Camera-B4.09.24.1", "FaceTime Camera (Built-in)", "Microsoft", "6000","Integrated Camera"]
# to assign cameras directly, using integers as demonstrated below
# eye_src = 1
else:
# print "Using provide file: %s" % args.filename
eye_src = args.eye_file
if args.world_file == '':
world_src = ["Logitech Camera","(046d:081d)","C510","B525", "C525","C615","C920","C930e"]
# to assign cameras directly, using integers as demonstrated below
# world_src = 0
else:
world_src = args.world_file
# Camera video size in pixels (width,height)
eye_size = (260,216) #(1280,1024)
world_size = (640,480)
# on MacOS we will not use os.fork, elsewhere this does nothing.
forking_enable(0)
# Create and initialize IPC
g_pool = Temp()
g_pool.pupil_queue = Queue()
g_pool.eye_rx, g_pool.eye_tx = Pipe(False)
g_pool.quit = RawValue(c_bool,0)
# this value will be substracted form the capture timestamp
g_pool.timebase = RawValue(c_double,0)
# make some constants avaiable
g_pool.user_dir = user_dir
g_pool.rec_dir = rec_dir
g_pool.version = version
g_pool.app = 'capture'
# set up subprocesses
p_eye = Process(target=eye, args=(g_pool,eye_src,eye_size))
# Spawn subprocess:
p_eye.start()
if platform.system() == 'Linux':
# We need to give the camera driver some time before requesting another camera.
sleep(0.5)
world(g_pool,world_src,world_size)
# Exit / clean-up
p_eye.join()
def main():
# To assign camera by name: put string(s) in list
eye_cam_names = [
"USB 2.0 Camera", "Microsoft", "6000", "Integrated Camera",
"HD USB Camera"
]
world_src = [
"Logitech Camera", "(046d:081d)", "C510", "B525", "C525", "C615",
"C920", "C930e"
]
eye_src = (eye_cam_names,
0), (eye_cam_names, 1
) #first match for eye0 and second match for eye1
# to assign cameras directly, using integers as demonstrated below
# eye_src = 4 , 5 #second arg will be ignored for monocular eye trackers
# world_src = 1
# to use a pre-recorded video.
# Use a string to specify the path to your video file as demonstrated below
# eye_src = '/Users/mkassner/Downloads/000/eye0.mkv' , '/Users/mkassner/Downloads/eye.avi'
# world_src = "/Users/mkassner/Downloads/000/world.mkv"
# Camera video size in pixels (width,height)
eye_size = (640, 480)
world_size = (1280, 720)
# on MacOS we will not use os.fork, elsewhere this does nothing.
forking_enable(0)
#g_pool holds variables. Only if added here they are shared across processes.
g_pool = Global_Container()
# Create and initialize IPC
g_pool.pupil_queue = Queue()
g_pool.quit = Value(c_bool, 0)
g_pool.timebase = Value(c_double, 0)
g_pool.eye_tx = []
# make some constants avaiable
g_pool.user_dir = user_dir
g_pool.version = get_version(version_file)
g_pool.app = 'capture'
g_pool.binocular = binocular
p_eye = []
for eye_id in range(1 + 1 * binocular):
rx, tx = Pipe(False)
p_eye += [
Process(target=eye,
args=(g_pool, eye_src[eye_id], eye_size, rx, eye_id))
]
g_pool.eye_tx += [tx]
p_eye[-1].start()
world(g_pool, world_src, world_size)
# Exit / clean-up
for p in p_eye:
p.join()
def init_marker_cacher(self):
forking_enable(0) #for MacOs only
from marker_detector_cacher import fill_cache
visited_list = [False if x == False else True for x in self.cache]
video_file_path = self.g_pool.capture.src
self.cache_queue = Queue()
self.cacher_seek_idx = Value('i',0)
self.cacher_run = Value(c_bool,True)
self.cacher = Process(target=fill_cache, args=(visited_list,video_file_path,self.cache_queue,self.cacher_seek_idx,self.cacher_run,self.min_marker_perimeter))
self.cacher.start()
def main():
# To assign camera by name: put string(s) in list
eye_src = ["Microsoft", "6000", "Integrated Camera"]
world_src = [
"Logitech Camera", "(046d:081d)", "C510", "B525", "C525", "C615",
"C920", "C930e"
]
# to assign cameras directly, using integers as demonstrated below
# eye_src = 1
# world_src = 0
# to use a pre-recorded video.
# Use a string to specify the path to your video file as demonstrated below
# eye_src = '/Users/mkassner/Pupil/datasets/p1-left/frames/test.avi'
# world_src = "/Users/mkassner/Desktop/2014_01_21/000/world.avi"
# Camera video size in pixels (width,height)
eye_size = (640, 360)
world_size = (1280, 720)
# on MacOS we will not use os.fork, elsewhere this does nothing.
forking_enable(0)
# Create and initialize IPC
g_pool = Temp()
g_pool.pupil_queue = Queue()
g_pool.eye_rx, g_pool.eye_tx = Pipe(False)
g_pool.quit = RawValue(c_bool, 0)
# this value will be substracted form the capture timestamp
g_pool.timebase = RawValue(c_double, 0)
# make some constants avaiable
g_pool.user_dir = user_dir
g_pool.rec_dir = rec_dir
g_pool.version = version
g_pool.app = 'capture'
# set up subprocesses
p_eye = Process(target=eye, args=(g_pool, eye_src, eye_size))
# Spawn subprocess:
p_eye.start()
if platform.system() == 'Linux':
# We need to give the camera driver some time before requesting another camera.
sleep(0.5)
world(g_pool, world_src, world_size)
# Exit / clean-up
p_eye.join()
def main():
#IPC
pupil_queue = Queue()
timebase = Value(c_double, 0)
cmd_world_end, cmd_launcher_end = Pipe()
com0 = Pipe(True)
eyes_are_alive = Value(c_bool, 0), Value(c_bool, 0)
com1 = Pipe(True)
com_world_ends = com0[0], com1[0]
com_eye_ends = com0[1], com1[1]
p_world = Process(target=world,
args=(pupil_queue, timebase, cmd_world_end,
com_world_ends, eyes_are_alive, user_dir,
app_version, video_sources['world']))
p_world.start()
while True:
#block and listen for commands from world process.
cmd = cmd_launcher_end.recv()
if cmd == "Exit":
break
else:
eye_id = cmd
p_eye = Process(target=eye,
args=(pupil_queue, timebase, com_eye_ends[eye_id],
eyes_are_alive[eye_id], user_dir,
app_version, eye_id,
video_sources['eye%s' % eye_id]))
p_eye.start()
for p in active_children():
p.join()
logger.debug('Laucher exit')
class Offline_Marker_Detector(Plugin):
"""
Special version of marker detector for use with videofile source.
It uses a seperate process to search all frames in the world.avi file for markers.
- self.cache is a list containing marker positions for each frame.
- self.surfaces[i].cache is a list containing surface positions for each frame
Both caches are build up over time. The marker cache is also session persistent.
See marker_tracker.py for more info on this marker tracker.
"""
def __init__(self,
g_pool,
gui_settings={
'pos': (220, 200),
'size': (300, 300),
'iconified': False
}):
super(Offline_Marker_Detector, self).__init__()
self.g_pool = g_pool
self.gui_settings = gui_settings
self.order = .2
# all markers that are detected in the most recent frame
self.markers = []
# all registered surfaces
if g_pool.app == 'capture':
raise Exception('For Player only.')
#in player we load from the rec_dir: but we have a couple options:
self.surface_definitions = Persistent_Dict(
os.path.join(g_pool.rec_dir, 'surface_definitions'))
if self.load('offline_square_marker_surfaces', []) != []:
logger.debug(
"Found ref surfaces defined or copied in previous session.")
self.surfaces = [
Offline_Reference_Surface(
self.g_pool,
saved_definition=d,
gaze_positions_by_frame=self.g_pool.positions_by_frame)
for d in self.load('offline_square_marker_surfaces', [])
if isinstance(d, dict)
]
elif self.load('realtime_square_marker_surfaces', []) != []:
logger.debug(
"Did not find ref surfaces def created or used by the user in player from earlier session. Loading surfaces defined during capture."
)
self.surfaces = [
Offline_Reference_Surface(
self.g_pool,
saved_definition=d,
gaze_positions_by_frame=self.g_pool.positions_by_frame)
for d in self.load('realtime_square_marker_surfaces', [])
if isinstance(d, dict)
]
else:
logger.debug("No surface defs found. Please define using GUI.")
self.surfaces = []
# edit surfaces
self.surface_edit_mode = c_bool(0)
self.edit_surfaces = []
#detector vars
self.robust_detection = c_bool(1)
self.aperture = c_int(11)
self.min_marker_perimeter = 80
#check if marker cache is available from last session
self.persistent_cache = Persistent_Dict(
os.path.join(g_pool.rec_dir, 'square_marker_cache'))
self.cache = Cache_List(
self.persistent_cache.get('marker_cache',
[False for _ in g_pool.timestamps]))
logger.debug(
"Loaded marker cache %s / %s frames had been searched before" %
(len(self.cache) - self.cache.count(False), len(self.cache)))
self.init_marker_cacher()
#debug vars
self.draw_markers = c_bool(0)
self.show_surface_idx = c_int(0)
self.recent_pupil_positions = []
self.img_shape = None
self.img = None
def init_gui(self):
import atb
pos = self.gui_settings['pos']
atb_label = "Marker Detector"
self._bar = atb.Bar(name=self.__class__.__name__ + str(id(self)),
label=atb_label,
help="circle",
color=(50, 150, 50),
alpha=50,
text='light',
position=pos,
refresh=.1,
size=self.gui_settings['size'])
self._bar.iconified = self.gui_settings['iconified']
self.update_bar_markers()
#set up bar display padding
self.on_window_resize(glfwGetCurrentContext(),
*glfwGetWindowSize(glfwGetCurrentContext()))
def unset_alive(self):
self.alive = False
def load(self, var_name, default):
return self.surface_definitions.get(var_name, default)
def save(self, var_name, var):
self.surface_definitions[var_name] = var
def on_window_resize(self, window, w, h):
self.win_size = w, h
def on_click(self, pos, button, action):
if self.surface_edit_mode.value:
if self.edit_surfaces:
if action == GLFW_RELEASE:
self.edit_surfaces = []
# no surfaces verts in edit mode, lets see if the curser is close to one:
else:
if action == GLFW_PRESS:
surf_verts = ((0., 0.), (1., 0.), (1., 1.), (0., 1.))
x, y = pos
for s in self.surfaces:
if s.detected:
for (vx, vy), i in zip(
s.ref_surface_to_img(np.array(surf_verts)),
range(4)):
vx, vy = denormalize(
(vx, vy),
(self.img_shape[1], self.img_shape[0]),
flip_y=True)
if sqrt((x - vx)**2 +
(y - vy)**2) < 15: #img pixels
self.edit_surfaces.append((s, i))
def advance(self):
pass
def add_surface(self):
self.surfaces.append(
Offline_Reference_Surface(
self.g_pool,
gaze_positions_by_frame=self.g_pool.positions_by_frame))
self.update_bar_markers()
def remove_surface(self, i):
self.surfaces[i].cleanup()
del self.surfaces[i]
self.update_bar_markers()
def update_bar_markers(self):
self._bar.clear()
self._bar.add_button('close', self.unset_alive)
self._bar.add_var("draw markers", self.draw_markers)
self._bar.add_button(" add surface ", self.add_surface, key='a')
self._bar.add_var(" edit mode ", self.surface_edit_mode)
for s, i in zip(self.surfaces, range(len(self.surfaces)))[::-1]:
self._bar.add_var("%s_name" % i,
create_string_buffer(512),
getter=s.atb_get_name,
setter=s.atb_set_name,
group=str(i),
label='name')
self._bar.add_var("%s_markers" % i,
create_string_buffer(512),
getter=s.atb_marker_status,
group=str(i),
label='found/registered markers')
self._bar.add_var(
"%s_x_scale" % i,
vtype=c_float,
getter=s.atb_get_scale_x,
min=1,
setter=s.atb_set_scale_x,
group=str(i),
label='real width',
help=
'this scale factor is used to adjust the coordinate space for your needs (think photo pixels or mm or whatever)'
)
self._bar.add_var(
"%s_y_scale" % i,
vtype=c_float,
getter=s.atb_get_scale_y,
min=1,
setter=s.atb_set_scale_y,
group=str(i),
label='real height',
help=
'defining x and y scale factor you atumatically set the correct aspect ratio.'
)
self._bar.add_var("%s_window" % i,
setter=s.toggle_window,
getter=s.window_open,
group=str(i),
label='open in window')
self._bar.add_button("%s_hm" % i,
s.generate_heatmap,
label='generate_heatmap',
group=str(i))
self._bar.add_button("%s_export" % i,
self.save_surface_positions_to_file,
data=i,
label='export surface data',
group=str(i))
self._bar.add_button("%s_remove" % i,
self.remove_surface,
data=i,
label='remove',
group=str(i))
def update(self, frame, recent_pupil_positions, events):
self.img = frame.img
self.img_shape = frame.img.shape
self.update_marker_cache()
self.markers = self.cache[frame.index]
if self.markers == False:
# locate markers because precacher has not anayzed this frame yet. Most likely a seek event
self.markers = []
self.seek_marker_cacher(
frame.index
) # tell precacher that it better have every thing from here analyzed
# locate surfaces
for s in self.surfaces:
if not s.locate_from_cache(frame.index):
s.locate(self.markers)
if s.detected:
events.append({
'type': 'marker_ref_surface',
'name': s.name,
'uid': s.uid,
'm_to_screen': s.m_to_screen,
'm_from_screen': s.m_from_screen,
'timestamp': frame.timestamp,
'gaze_on_srf': s.gaze_on_srf
})
if self.draw_markers.value:
draw_markers(frame.img, self.markers)
# edit surfaces by user
if self.surface_edit_mode:
window = glfwGetCurrentContext()
pos = glfwGetCursorPos(window)
pos = normalize(pos, glfwGetWindowSize(window))
pos = denormalize(pos,
(frame.img.shape[1],
frame.img.shape[0])) # Position in img pixels
for s, v_idx in self.edit_surfaces:
if s.detected:
pos = normalize(pos,
(self.img_shape[1], self.img_shape[0]),
flip_y=True)
new_pos = s.img_to_ref_surface(np.array(pos))
s.move_vertex(v_idx, new_pos)
s.cache = None
else:
# update srf with no or invald cache:
for s in self.surfaces:
if s.cache == None:
s.init_cache(self.cache)
#allow surfaces to open/close windows
for s in self.surfaces:
if s.window_should_close:
s.close_window()
if s.window_should_open:
s.open_window()
def init_marker_cacher(self):
forking_enable(0) #for MacOs only
from marker_detector_cacher import fill_cache
visited_list = [False if x == False else True for x in self.cache]
video_file_path = os.path.join(self.g_pool.rec_dir, 'world.avi')
self.cache_queue = Queue()
self.cacher_seek_idx = Value(c_int, 0)
self.cacher_run = Value(c_bool, True)
self.cacher = Process(target=fill_cache,
args=(visited_list, video_file_path,
self.cache_queue, self.cacher_seek_idx,
self.cacher_run))
self.cacher.start()
def update_marker_cache(self):
while not self.cache_queue.empty():
idx, c_m = self.cache_queue.get()
self.cache.update(idx, c_m)
for s in self.surfaces:
s.update_cache(self.cache, idx=idx)
def seek_marker_cacher(self, idx):
self.cacher_seek_idx.value = idx
def close_marker_cacher(self):
self.update_marker_cache()
self.cacher_run.value = False
self.cacher.join()
def gl_display(self):
"""
Display marker and surface info inside world screen
"""
self.gl_display_cache_bars()
for m in self.markers:
hat = np.array(
[[[0, 0], [0, 1], [.5, 1.3], [1, 1], [1, 0], [0, 0]]],
dtype=np.float32)
hat = cv2.perspectiveTransform(hat, m_marker_to_screen(m))
draw_gl_polyline(hat.reshape((6, 2)), (0.1, 1., 1., .5))
for s in self.surfaces:
s.gl_draw_frame()
s.gl_display_in_window(self.g_pool.image_tex)
if self.surface_edit_mode.value:
for s in self.surfaces:
s.gl_draw_corners()
def gl_display_cache_bars(self):
"""
"""
padding = 20.
# Lines for areas that have be cached
cached_ranges = []
for r in self.cache.visited_ranges: # [[0,1],[3,4]]
cached_ranges += (r[0], 0), (r[1], 0) #[(0,0),(1,0),(3,0),(4,0)]
# Lines where surfaces have been found in video
cached_surfaces = []
for s in self.surfaces:
found_at = []
if s.cache is not None:
for r in s.cache.positive_ranges: # [[0,1],[3,4]]
found_at += (r[0], 0), (r[1], 0
) #[(0,0),(1,0),(3,0),(4,0)]
cached_surfaces.append(found_at)
glMatrixMode(GL_PROJECTION)
glPushMatrix()
glLoadIdentity()
width, height = self.win_size
h_pad = padding * (self.cache.length - 2) / float(width)
v_pad = padding * 1. / (height - 2)
gluOrtho2D(
-h_pad, (self.cache.length - 1) + h_pad, -v_pad, 1 + v_pad
) # ranging from 0 to cache_len-1 (horizontal) and 0 to 1 (vertical)
glMatrixMode(GL_MODELVIEW)
glPushMatrix()
glLoadIdentity()
color = (8., .6, .2, 8.)
draw_gl_polyline(cached_ranges, color=color, type='Lines', thickness=4)
color = (0., .7, .3, 8.)
for s in cached_surfaces:
glTranslatef(0, .02, 0)
draw_gl_polyline(s, color=color, type='Lines', thickness=2)
glMatrixMode(GL_PROJECTION)
glPopMatrix()
glMatrixMode(GL_MODELVIEW)
glPopMatrix()
def save_surface_positions_to_file(self, i):
s = self.surfaces[i]
in_mark = self.g_pool.trim_marks.in_mark
out_mark = self.g_pool.trim_marks.out_mark
if s.cache == None:
logger.warning(
"The surface is not cached. Please wait for the cacher to collect data."
)
return
srf_dir = os.path.join(
self.g_pool.rec_dir,
'surface_data' + '_' + s.name.replace('/', '') + '_' + s.uid)
logger.info("exporting surface gaze data to %s" % srf_dir)
if os.path.isdir(srf_dir):
logger.info(
"Will overwrite previous export for this referece surface")
else:
try:
os.mkdir(srf_dir)
except:
logger.warning("Could name make export dir %s" % srf_dir)
return
#save surface_positions as pickle file
save_object(s.cache.to_list(), os.path.join(srf_dir, 'srf_positons'))
#save surface_positions as csv
with open(os.path.join(srf_dir, 'srf_positons.csv'), 'wb') as csvfile:
csw_writer = csv.writer(csvfile,
delimiter='\t',
quotechar='|',
quoting=csv.QUOTE_MINIMAL)
csw_writer.writerow(('frame_idx', 'timestamp', 'm_to_screen',
'm_from_screen', 'detected_markers'))
for idx, ts, ref_srf_data in zip(
range(len(self.g_pool.timestamps)), self.g_pool.timestamps,
s.cache):
if in_mark <= idx <= out_mark:
if ref_srf_data is not None and ref_srf_data is not False:
csw_writer.writerow(
(idx, ts, ref_srf_data['m_to_screen'],
ref_srf_data['m_from_screen'],
ref_srf_data['detected_markers']))
#save gaze on srf as csv.
with open(os.path.join(srf_dir, 'gaze_positions_on_surface.csv'),
'wb') as csvfile:
csw_writer = csv.writer(csvfile,
delimiter='\t',
quotechar='|',
quoting=csv.QUOTE_MINIMAL)
csw_writer.writerow(
('world_frame_idx', 'world_timestamp', 'eye_timestamp',
'x_norm', 'y_norm', 'x_scaled', 'y_scaled', 'on_srf'))
for idx, ts, ref_srf_data in zip(
range(len(self.g_pool.timestamps)), self.g_pool.timestamps,
s.cache):
if in_mark <= idx <= out_mark:
if ref_srf_data is not None and ref_srf_data is not False:
for gp in ref_srf_data['gaze_on_srf']:
gp_x, gp_y = gp['norm_gaze_on_srf']
on_srf = (0 <= gp_x <= 1) and (0 <= gp_y <= 1)
csw_writer.writerow(
(idx, ts, gp['timestamp'], gp_x, gp_y,
gp_x * s.scale_factor[0],
gp_x * s.scale_factor[1], on_srf))
logger.info(
"Saved surface positon data and gaze on surface data for '%s' with uid:'%s'"
% (s.name, s.uid))
if s.heatmap is not None:
logger.info("Saved Heatmap as .png file.")
cv2.imwrite(os.path.join(srf_dir, 'heatmap.png'), s.heatmap)
if s.detected and self.img is not None:
#let save out the current surface image found in video
#here we get the verts of the surface quad in norm_coords
mapped_space_one = np.array(((0, 0), (1, 0), (1, 1), (0, 1)),
dtype=np.float32).reshape(-1, 1, 2)
screen_space = cv2.perspectiveTransform(mapped_space_one,
s.m_to_screen).reshape(
-1, 2)
#now we convert to image pixel coods
screen_space[:, 1] = 1 - screen_space[:, 1]
screen_space[:, 1] *= self.img.shape[0]
screen_space[:, 0] *= self.img.shape[1]
s_0, s_1 = s.scale_factor
#no we need to flip vertically again by setting the mapped_space verts accordingly.
mapped_space_scaled = np.array(
((0, s_1), (s_0, s_1), (s_0, 0), (0, 0)), dtype=np.float32)
M = cv2.getPerspectiveTransform(screen_space, mapped_space_scaled)
#here we do the actual perspactive transform of the image.
srf_in_video = cv2.warpPerspective(
self.img, M, (int(s.scale_factor[0]), int(s.scale_factor[1])))
cv2.imwrite(os.path.join(srf_dir, 'surface.png'), srf_in_video)
logger.info("Saved current image as .png file.")
def get_init_dict(self):
d = {}
if hasattr(self, '_bar'):
gui_settings = {
'pos': self._bar.position,
'size': self._bar.size,
'iconified': self._bar.iconified
}
d['gui_settings'] = gui_settings
return d
def cleanup(self):
""" called when the plugin gets terminated.
This happends either voluntary or forced.
if you have an atb bar or glfw window destroy it here.
"""
self.save("offline_square_marker_surfaces",
[rs.save_to_dict() for rs in self.surfaces if rs.defined])
self.close_marker_cacher()
self.persistent_cache["marker_cache"] = self.cache.to_list()
self.persistent_cache.close()
self.surface_definitions.close()
for s in self.surfaces:
s.close_window()
self._bar.destroy()
from billiard import Process, Queue
def f(q):
q.put([42, None, 'hello'])
if __name__ == '__main__':
q = Queue()
p = Process(target=f, args=(q, ))
p.start()
print(q.get()) # prints "[42, None, 'hello']"
p.join()
def main():
# To assign camera by name: put string(s) in list
world_src = [
"Pupil Cam1 ID2", "Logitech Camera", "(046d:081d)", "C510", "B525",
"C525", "C615", "C920", "C930e"
]
eye0 = [
"Pupil Cam1 ID0", "HD-6000", "Integrated Camera", "HD USB Camera",
"USB 2.0 Camera"
]
eye1 = ["Pupil Cam1 ID1", "HD-6000", "Integrated Camera"]
eye_src = eye0, eye1
# to assign cameras directly, using integers as demonstrated below
# eye_src = 1 , 1 #second arg will be ignored for monocular eye trackers
# world_src = 0
# to use a pre-recorded video.
# Use a string to specify the path to your video file as demonstrated below
# eye_src = '/Users/mkassner/Downloads/eye0.mkv' , '/Users/mkassner/Downloads/eye.avi'
# world_src = "/Users/mkassner/Downloads/000/world.mkv"
# Default camera video size in pixels (width,height)
eye_size = (640, 480)
world_size = (1280, 720)
# on MacOS we will not use os.fork, elsewhere this does nothing.
forking_enable(0)
#g_pool holds variables. Only if added here they are shared across processes.
g_pool = Global_Container()
# Create and initialize IPC
g_pool.pupil_queue = Queue()
g_pool.quit = Value(c_bool, 0)
g_pool.timebase = Value(c_double, 0)
g_pool.eye_tx = []
# make some constants avaiable
g_pool.user_dir = user_dir
g_pool.version = get_version(version_file)
g_pool.app = 'capture'
g_pool.binocular = binocular
p_eye = []
for eye_id in range(1 + 1 * binocular):
eye_end, world_end = Pipe(True)
p_eye += [
Process(target=eye,
args=(g_pool, eye_src[eye_id], eye_size, eye_end, eye_id))
]
p_eye[-1].start()
#wait for ready message from eye to sequentialize startup
logger.debug(world_end.recv())
g_pool.eye_tx += [world_end]
world(g_pool, world_src, world_size)
# Exit / clean-up
for p in p_eye:
p.join()
class Offline_Surface_Tracker(Surface_Tracker):
"""
Special version of surface tracker for use with videofile source.
It uses a seperate process to search all frames in the world video file for markers.
- self.cache is a list containing marker positions for each frame.
- self.surfaces[i].cache is a list containing surface positions for each frame
Both caches are build up over time. The marker cache is also session persistent.
See marker_tracker.py for more info on this marker tracker.
"""
def __init__(self,g_pool,mode="Show Markers and Surfaces",min_marker_perimeter = 100):
super(Offline_Surface_Tracker, self).__init__(g_pool,mode,min_marker_perimeter)
self.order = .2
if g_pool.app == 'capture':
raise Exception('For Player only.')
self.marker_cache_version = 1
self.min_marker_perimeter_cacher = 20 #find even super small markers. The surface locater will filter using min_marker_perimeter
#check if marker cache is available from last session
self.persistent_cache = Persistent_Dict(os.path.join(g_pool.rec_dir,'square_marker_cache'))
version = self.persistent_cache.get('version',0)
cache = self.persistent_cache.get('marker_cache',None)
if cache is None:
self.cache = Cache_List([False for _ in g_pool.timestamps])
self.persistent_cache['version'] = self.marker_cache_version
elif version != self.marker_cache_version:
self.persistent_cache['version'] = self.marker_cache_version
self.cache = Cache_List([False for _ in g_pool.timestamps])
logger.debug("Marker cache version missmatch. Rebuilding marker cache.")
else:
self.cache = Cache_List(cache)
logger.debug("Loaded marker cache %s / %s frames had been searched before"%(len(self.cache)-self.cache.count(False),len(self.cache)) )
self.init_marker_cacher()
for s in self.surfaces:
s.init_cache(self.cache,self.camera_calibration,self.min_marker_perimeter)
self.recalculate()
def load_surface_definitions_from_file(self):
self.surface_definitions = Persistent_Dict(os.path.join(self.g_pool.rec_dir,'surface_definitions'))
if self.surface_definitions.get('offline_square_marker_surfaces',[]) != []:
logger.debug("Found ref surfaces defined or copied in previous session.")
self.surfaces = [Offline_Reference_Surface(self.g_pool,saved_definition=d) for d in self.surface_definitions.get('offline_square_marker_surfaces',[]) if isinstance(d,dict)]
elif self.surface_definitions.get('realtime_square_marker_surfaces',[]) != []:
logger.debug("Did not find ref surfaces def created or used by the user in player from earlier session. Loading surfaces defined during capture.")
self.surfaces = [Offline_Reference_Surface(self.g_pool,saved_definition=d) for d in self.surface_definitions.get('realtime_square_marker_surfaces',[]) if isinstance(d,dict)]
else:
logger.debug("No surface defs found. Please define using GUI.")
self.surfaces = []
def init_gui(self):
self.menu = ui.Scrolling_Menu('Offline Surface Tracker')
self.g_pool.gui.append(self.menu)
self.add_button = ui.Thumb('add_surface',setter=lambda x: self.add_surface(),getter=lambda:False,label='Add Surface',hotkey='a')
self.g_pool.quickbar.append(self.add_button)
self.update_gui_markers()
self.on_window_resize(glfwGetCurrentContext(),*glfwGetWindowSize(glfwGetCurrentContext()))
def deinit_gui(self):
if self.menu:
self.g_pool.gui.remove(self.menu)
self.menu= None
if self.add_button:
self.g_pool.quickbar.remove(self.add_button)
self.add_button = None
def update_gui_markers(self):
def close():
self.alive=False
def set_min_marker_perimeter(val):
self.min_marker_perimeter = val
self.notify_all_delayed({'subject':'min_marker_perimeter_changed'},delay=1)
self.menu.elements[:] = []
self.menu.append(ui.Button('Close',close))
self.menu.append(ui.Slider('min_marker_perimeter',self,min=20,max=500,step=1,setter=set_min_marker_perimeter))
self.menu.append(ui.Info_Text('The offline surface tracker will look for markers in the entire video. By default it uses surfaces defined in capture. You can change and add more surfaces here.'))
self.menu.append(ui.Info_Text("Press the export button or type 'e' to start the export."))
self.menu.append(ui.Selector('mode',self,label='Mode',selection=["Show Markers and Surfaces","Show marker IDs","Show Heatmaps","Show Metrics"] ))
self.menu.append(ui.Info_Text('To see heatmap or surface metrics visualizations, click (re)-calculate gaze distributions. Set "X size" and "Y size" for each surface to see heatmap visualizations.'))
self.menu.append(ui.Button("(Re)-calculate gaze distributions", self.recalculate))
self.menu.append(ui.Button("Add surface", lambda:self.add_surface()))
for s in self.surfaces:
idx = self.surfaces.index(s)
s_menu = ui.Growing_Menu("Surface %s"%idx)
s_menu.collapsed=True
s_menu.append(ui.Text_Input('name',s))
s_menu.append(ui.Text_Input('x',s.real_world_size,label='X size'))
s_menu.append(ui.Text_Input('y',s.real_world_size,label='Y size'))
s_menu.append(ui.Button('Open Debug Window',s.open_close_window))
#closure to encapsulate idx
def make_remove_s(i):
return lambda: self.remove_surface(i)
remove_s = make_remove_s(idx)
s_menu.append(ui.Button('remove',remove_s))
self.menu.append(s_menu)
def on_notify(self,notification):
if notification['subject'] == 'gaze_positions_changed':
logger.info('Gaze postions changed. Recalculating.')
self.recalculate()
elif notification['subject'] == 'surfaces_changed':
logger.info('Surfaces changed. Recalculating.')
self.recalculate()
elif notification['subject'] == 'min_marker_perimeter_changed':
logger.info('Min marper perimeter adjusted. Re-detecting surfaces.')
self.invalidate_surface_caches()
elif notification['subject'] is "should_export":
self.save_surface_statsics_to_file(notification['range'],notification['export_dir'])
def on_window_resize(self,window,w,h):
self.win_size = w,h
def add_surface(self):
self.surfaces.append(Offline_Reference_Surface(self.g_pool))
self.update_gui_markers()
def recalculate(self):
in_mark = self.g_pool.trim_marks.in_mark
out_mark = self.g_pool.trim_marks.out_mark
section = slice(in_mark,out_mark)
# calc heatmaps
for s in self.surfaces:
if s.defined:
s.generate_heatmap(section)
# calc distirbution accross all surfaces.
results = []
for s in self.surfaces:
gaze_on_srf = s.gaze_on_srf_in_section(section)
results.append(len(gaze_on_srf))
self.metrics_gazecount = len(gaze_on_srf)
if results == []:
logger.warning("No surfaces defined.")
return
max_res = max(results)
results = np.array(results,dtype=np.float32)
if not max_res:
logger.warning("No gaze on any surface for this section!")
else:
results *= 255./max_res
results = np.uint8(results)
results_c_maps = cv2.applyColorMap(results, cv2.COLORMAP_JET)
for s,c_map in zip(self.surfaces,results_c_maps):
heatmap = np.ones((1,1,4),dtype=np.uint8)*125
heatmap[:,:,:3] = c_map
s.metrics_texture = Named_Texture()
s.metrics_texture.update_from_ndarray(heatmap)
def invalidate_surface_caches(self):
for s in self.surfaces:
s.cache = None
def update(self,frame,events):
self.img_shape = frame.img.shape
self.update_marker_cache()
# self.markers = [m for m in self.cache[frame.index] if m['perimeter'>=self.min_marker_perimeter]
self.markers = self.cache[frame.index]
if self.markers == False:
self.markers = []
self.seek_marker_cacher(frame.index) # tell precacher that it better have every thing from here on analyzed
events['surfaces'] = []
# locate surfaces
for s in self.surfaces:
if not s.locate_from_cache(frame.index):
s.locate(self.markers,self.camera_calibration,self.min_marker_perimeter)
if s.detected:
events['surfaces'].append({'name':s.name,'uid':s.uid,'m_to_screen':s.m_to_screen,'m_from_screen':s.m_from_screen, 'timestamp':frame.timestamp})
if self.mode == "Show marker IDs":
draw_markers(frame.img,self.markers)
elif self.mode == "Show Markers and Surfaces":
# edit surfaces by user
if self.edit_surf_verts:
window = glfwGetCurrentContext()
pos = glfwGetCursorPos(window)
pos = normalize(pos,glfwGetWindowSize(window),flip_y=True)
for s,v_idx in self.edit_surf_verts:
if s.detected:
new_pos = s.img_to_ref_surface(np.array(pos))
s.move_vertex(v_idx,new_pos)
else:
# update srf with no or invald cache:
for s in self.surfaces:
if s.cache == None and s not in [s for s,i in self.edit_surf_verts]:
s.init_cache(self.cache,self.camera_calibration,self.min_marker_perimeter)
self.notify_all_delayed({'subject':'surfaces_changed'})
#map recent gaze onto detected surfaces used for pupil server
for s in self.surfaces:
if s.detected:
s.gaze_on_srf = []
for p in events.get('gaze_positions',[]):
gp_on_s = tuple(s.img_to_ref_surface(np.array(p['norm_pos'])))
p['realtime gaze on ' + s.name] = gp_on_s
s.gaze_on_srf.append(gp_on_s)
#allow surfaces to open/close windows
for s in self.surfaces:
if s.window_should_close:
s.close_window()
if s.window_should_open:
s.open_window()
def init_marker_cacher(self):
forking_enable(0) #for MacOs only
from marker_detector_cacher import fill_cache
visited_list = [False if x == False else True for x in self.cache]
video_file_path = self.g_pool.capture.src
timestamps = self.g_pool.capture.timestamps
self.cache_queue = Queue()
self.cacher_seek_idx = Value('i',0)
self.cacher_run = Value(c_bool,True)
self.cacher = Process(target=fill_cache, args=(visited_list,video_file_path,timestamps,self.cache_queue,self.cacher_seek_idx,self.cacher_run,self.min_marker_perimeter_cacher))
self.cacher.start()
def update_marker_cache(self):
while not self.cache_queue.empty():
idx,c_m = self.cache_queue.get()
self.cache.update(idx,c_m)
for s in self.surfaces:
s.update_cache(self.cache,camera_calibration=self.camera_calibration,min_marker_perimeter=self.min_marker_perimeter,idx=idx)
if self.cacher_run.value == False:
self.recalculate()
def seek_marker_cacher(self,idx):
self.cacher_seek_idx.value = idx
def close_marker_cacher(self):
self.update_marker_cache()
self.cacher_run.value = False
self.cacher.join()
def gl_display(self):
"""
Display marker and surface info inside world screen
"""
self.gl_display_cache_bars()
super(Offline_Surface_Tracker,self).gl_display()
if self.mode == "Show Heatmaps":
for s in self.surfaces:
s.gl_display_heatmap()
if self.mode == "Show Metrics":
#todo: draw a backdrop to represent the gaze that is not on any surface
for s in self.surfaces:
#draw a quad on surface with false color of value.
s.gl_display_metrics()
def gl_display_cache_bars(self):
"""
"""
padding = 20.
# Lines for areas that have been cached
cached_ranges = []
for r in self.cache.visited_ranges: # [[0,1],[3,4]]
cached_ranges += (r[0],0),(r[1],0) #[(0,0),(1,0),(3,0),(4,0)]
# Lines where surfaces have been found in video
cached_surfaces = []
for s in self.surfaces:
found_at = []
if s.cache is not None:
for r in s.cache.positive_ranges: # [[0,1],[3,4]]
found_at += (r[0],0),(r[1],0) #[(0,0),(1,0),(3,0),(4,0)]
cached_surfaces.append(found_at)
glMatrixMode(GL_PROJECTION)
glPushMatrix()
glLoadIdentity()
width,height = self.win_size
h_pad = padding * (self.cache.length-2)/float(width)
v_pad = padding* 1./(height-2)
glOrtho(-h_pad, (self.cache.length-1)+h_pad, -v_pad, 1+v_pad,-1,1) # ranging from 0 to cache_len-1 (horizontal) and 0 to 1 (vertical)
glMatrixMode(GL_MODELVIEW)
glPushMatrix()
glLoadIdentity()
color = RGBA(.8,.6,.2,.8)
draw_polyline(cached_ranges,color=color,line_type=GL_LINES,thickness=4)
color = RGBA(0,.7,.3,.8)
for s in cached_surfaces:
glTranslatef(0,.02,0)
draw_polyline(s,color=color,line_type=GL_LINES,thickness=2)
glMatrixMode(GL_PROJECTION)
glPopMatrix()
glMatrixMode(GL_MODELVIEW)
glPopMatrix()
def save_surface_statsics_to_file(self,export_range,export_dir):
"""
between in and out mark
report: gaze distribution:
- total gazepoints
- gaze points on surface x
- gaze points not on any surface
report: surface visisbility
- total frames
- surface x visible framecount
surface events:
frame_no, ts, surface "name", "id" enter/exit
for each surface:
fixations_on_name.csv
gaze_on_name_id.csv
positions_of_name_id.csv
"""
metrics_dir = os.path.join(export_dir,'surfaces')
section = export_range
in_mark = export_range.start
out_mark = export_range.stop
logger.info("exporting metrics to %s"%metrics_dir)
if os.path.isdir(metrics_dir):
logger.info("Will overwrite previous export for this section")
else:
try:
os.mkdir(metrics_dir)
except:
logger.warning("Could not make metrics dir %s"%metrics_dir)
return
with open(os.path.join(metrics_dir,'surface_visibility.csv'),'wb') as csvfile:
csv_writer = csv.writer(csvfile, delimiter='\t',quotechar='|', quoting=csv.QUOTE_MINIMAL)
# surface visibility report
frame_count = len(self.g_pool.timestamps[section])
csv_writer.writerow(('frame_count',frame_count))
csv_writer.writerow((''))
csv_writer.writerow(('surface_name','visible_frame_count'))
for s in self.surfaces:
if s.cache == None:
logger.warning("The surface is not cached. Please wait for the cacher to collect data.")
return
visible_count = s.visible_count_in_section(section)
csv_writer.writerow( (s.name, visible_count) )
logger.info("Created 'surface_visibility.csv' file")
with open(os.path.join(metrics_dir,'surface_gaze_distribution.csv'),'wb') as csvfile:
csv_writer = csv.writer(csvfile, delimiter='\t',quotechar='|', quoting=csv.QUOTE_MINIMAL)
# gaze distribution report
gaze_in_section = list(chain(*self.g_pool.gaze_positions_by_frame[section]))
not_on_any_srf = set([gp['timestamp'] for gp in gaze_in_section])
csv_writer.writerow(('total_gaze_point_count',len(gaze_in_section)))
csv_writer.writerow((''))
csv_writer.writerow(('surface_name','gaze_count'))
for s in self.surfaces:
gaze_on_srf = s.gaze_on_srf_in_section(section)
gaze_on_srf = set([gp['base']['timestamp'] for gp in gaze_on_srf])
not_on_any_srf -= gaze_on_srf
csv_writer.writerow( (s.name, len(gaze_on_srf)) )
csv_writer.writerow(('not_on_any_surface', len(not_on_any_srf) ) )
logger.info("Created 'surface_gaze_distribution.csv' file")
with open(os.path.join(metrics_dir,'surface_events.csv'),'wb') as csvfile:
csv_writer = csv.writer(csvfile, delimiter='\t',quotechar='|', quoting=csv.QUOTE_MINIMAL)
# surface events report
csv_writer.writerow(('frame_number','timestamp','surface_name','surface_uid','event_type'))
events = []
for s in self.surfaces:
for enter_frame_id,exit_frame_id in s.cache.positive_ranges:
events.append({'frame_id':enter_frame_id,'srf_name':s.name,'srf_uid':s.uid,'event':'enter'})
events.append({'frame_id':exit_frame_id,'srf_name':s.name,'srf_uid':s.uid,'event':'exit'})
events.sort(key=lambda x: x['frame_id'])
for e in events:
csv_writer.writerow( ( e['frame_id'],self.g_pool.timestamps[e['frame_id']],e['srf_name'],e['srf_uid'],e['event'] ) )
logger.info("Created 'surface_events.csv' file")
for s in self.surfaces:
# per surface names:
surface_name = '_'+s.name.replace('/','')+'_'+s.uid
# save surface_positions as pickle file
save_object(s.cache.to_list(),os.path.join(metrics_dir,'srf_positions'+surface_name))
#save surface_positions as csv
with open(os.path.join(metrics_dir,'srf_positons'+surface_name+'.csv'),'wb') as csvfile:
csv_writer =csv.writer(csvfile, delimiter='\t',quotechar='|', quoting=csv.QUOTE_MINIMAL)
csv_writer.writerow(('frame_idx','timestamp','m_to_screen','m_from_screen','detected_markers'))
for idx,ts,ref_srf_data in zip(range(len(self.g_pool.timestamps)),self.g_pool.timestamps,s.cache):
if in_mark <= idx <= out_mark:
if ref_srf_data is not None and ref_srf_data is not False:
csv_writer.writerow( (idx,ts,ref_srf_data['m_to_screen'],ref_srf_data['m_from_screen'],ref_srf_data['detected_markers']) )
# save gaze on srf as csv.
with open(os.path.join(metrics_dir,'gaze_positions_on_surface'+surface_name+'.csv'),'wb') as csvfile:
csv_writer = csv.writer(csvfile, delimiter='\t',quotechar='|', quoting=csv.QUOTE_MINIMAL)
csv_writer.writerow(('world_timestamp','world_frame_idx','gaze_timestamp','x_norm','y_norm','x_scaled','y_scaled','on_srf'))
for idx,ts,ref_srf_data in zip(range(len(self.g_pool.timestamps)),self.g_pool.timestamps,s.cache):
if in_mark <= idx <= out_mark:
if ref_srf_data is not None and ref_srf_data is not False:
for gp in s.gaze_on_srf_by_frame_idx(idx,ref_srf_data['m_from_screen']):
csv_writer.writerow( (ts,idx,gp['base']['timestamp'],gp['norm_pos'][0],gp['norm_pos'][1],gp['norm_pos'][0]*s.real_world_size['x'],gp['norm_pos'][1]*s.real_world_size['y'],gp['on_srf']) )
# save fixation on srf as csv.
with open(os.path.join(metrics_dir,'fixations_on_surface'+surface_name+'.csv'),'wb') as csvfile:
csv_writer = csv.writer(csvfile, delimiter='\t',quotechar='|', quoting=csv.QUOTE_MINIMAL)
csv_writer.writerow(('id','start_timestamp','duration','start_frame','end_frame','norm_pos_x','norm_pos_y','x_scaled','y_scaled','on_srf'))
fixations_on_surface = []
for idx,ref_srf_data in zip(range(len(self.g_pool.timestamps)),s.cache):
if in_mark <= idx <= out_mark:
if ref_srf_data is not None and ref_srf_data is not False:
for f in s.fixations_on_srf_by_frame_idx(idx,ref_srf_data['m_from_screen']):
fixations_on_surface.append(f)
removed_dublicates = dict([(f['base']['id'],f) for f in fixations_on_surface]).values()
for f_on_s in removed_dublicates:
f = f_on_s['base']
f_x,f_y = f_on_s['norm_pos']
f_on_srf = f_on_s['on_srf']
csv_writer.writerow( (f['id'],f['timestamp'],f['duration'],f['start_frame_index'],f['end_frame_index'],f_x,f_y,f_x*s.real_world_size['x'],f_y*s.real_world_size['y'],f_on_srf) )
logger.info("Saved surface positon gaze and fixation data for '%s' with uid:'%s'"%(s.name,s.uid))
if s.heatmap is not None:
logger.info("Saved Heatmap as .png file.")
cv2.imwrite(os.path.join(metrics_dir,'heatmap'+surface_name+'.png'),s.heatmap)
logger.info("Done exporting reference surface data.")
# if s.detected and self.img is not None:
# #let save out the current surface image found in video
# #here we get the verts of the surface quad in norm_coords
# mapped_space_one = np.array(((0,0),(1,0),(1,1),(0,1)),dtype=np.float32).reshape(-1,1,2)
# screen_space = cv2.perspectiveTransform(mapped_space_one,s.m_to_screen).reshape(-1,2)
# #now we convert to image pixel coods
# screen_space[:,1] = 1-screen_space[:,1]
# screen_space[:,1] *= self.img.shape[0]
# screen_space[:,0] *= self.img.shape[1]
# s_0,s_1 = s.real_world_size
# #no we need to flip vertically again by setting the mapped_space verts accordingly.
# mapped_space_scaled = np.array(((0,s_1),(s_0,s_1),(s_0,0),(0,0)),dtype=np.float32)
# M = cv2.getPerspectiveTransform(screen_space,mapped_space_scaled)
# #here we do the actual perspactive transform of the image.
# srf_in_video = cv2.warpPerspective(self.img,M, (int(s.real_world_size['x']),int(s.real_world_size['y'])) )
# cv2.imwrite(os.path.join(metrics_dir,'surface'+surface_name+'.png'),srf_in_video)
# logger.info("Saved current image as .png file.")
# else:
# logger.info("'%s' is not currently visible. Seek to appropriate frame and repeat this command."%s.name)
def cleanup(self):
""" called when the plugin gets terminated.
This happens either voluntarily or forced.
if you have a GUI or glfw window destroy it here.
"""
self.surface_definitions["offline_square_marker_surfaces"] = [rs.save_to_dict() for rs in self.surfaces if rs.defined]
self.surface_definitions.close()
self.close_marker_cacher()
self.persistent_cache["marker_cache"] = self.cache.to_list()
self.persistent_cache.close()
for s in self.surfaces:
s.close_window()
self.deinit_gui()
pages.append(new_url[:-5] + "_" + str(n) + ".html")
return pages
def scra_list_page(pages):
ret = list()
for page_url in pages:
pq = PQ(url=page_url)
ret.extend(
re.findall(
r"(?P<ip>\d+\.\d+\.\d+\.\d+)\:(?P<port>\d+)@(?P<pro>\w+)#",
pq.text()))
return ret
queue = Queue()
all = list()
def test_proxy():
while 1:
try:
p = queue.get_nowait()
except:
break
proxies = {
p[2].lower(): '%s:%s' % (p[0], p[1]),
}
try:
begin = time.time()
code = requests.get("http://www.google.com.hk/",
class Offline_Marker_Detector(Plugin):
"""
Special version of marker detector for use with videofile source.
It uses a seperate process to search all frames in the world.avi file for markers.
- self.cache is a list containing marker positions for each frame.
- self.surfaces[i].cache is a list containing surface positions for each frame
Both caches are build up over time. The marker cache is also session persistent.
See marker_tracker.py for more info on this marker tracker.
"""
def __init__(self,g_pool,gui_settings={'pos':(220,200),'size':(300,300),'iconified':False}):
super(Offline_Marker_Detector, self).__init__()
self.g_pool = g_pool
self.gui_settings = gui_settings
self.order = .2
# all markers that are detected in the most recent frame
self.markers = []
# all registered surfaces
if g_pool.app == 'capture':
raise Exception('For Player only.')
#in player we load from the rec_dir: but we have a couple options:
self.surface_definitions = Persistent_Dict(os.path.join(g_pool.rec_dir,'surface_definitions'))
if self.load('offline_square_marker_surfaces',[]) != []:
logger.debug("Found ref surfaces defined or copied in previous session.")
self.surfaces = [Offline_Reference_Surface(self.g_pool,saved_definition=d,gaze_positions_by_frame=self.g_pool.positions_by_frame) for d in self.load('offline_square_marker_surfaces',[]) if isinstance(d,dict)]
elif self.load('realtime_square_marker_surfaces',[]) != []:
logger.debug("Did not find ref surfaces def created or used by the user in player from earlier session. Loading surfaces defined during capture.")
self.surfaces = [Offline_Reference_Surface(self.g_pool,saved_definition=d,gaze_positions_by_frame=self.g_pool.positions_by_frame) for d in self.load('realtime_square_marker_surfaces',[]) if isinstance(d,dict)]
else:
logger.debug("No surface defs found. Please define using GUI.")
self.surfaces = []
# ui mode settings
self.mode = c_int(0)
# edit surfaces
self.edit_surfaces = []
#detector vars
self.robust_detection = c_bool(1)
self.aperture = c_int(11)
self.min_marker_perimeter = 80
#check if marker cache is available from last session
self.persistent_cache = Persistent_Dict(os.path.join(g_pool.rec_dir,'square_marker_cache'))
self.cache = Cache_List(self.persistent_cache.get('marker_cache',[False for _ in g_pool.timestamps]))
logger.debug("Loaded marker cache %s / %s frames had been searched before"%(len(self.cache)-self.cache.count(False),len(self.cache)) )
self.init_marker_cacher()
#debug vars
self.show_surface_idx = c_int(0)
self.recent_pupil_positions = []
self.img_shape = None
self.img = None
def init_gui(self):
import atb
pos = self.gui_settings['pos']
atb_label = "Marker Detector"
self._bar = atb.Bar(name =self.__class__.__name__+str(id(self)), label=atb_label,
help="circle", color=(50, 150, 50), alpha=50,
text='light', position=pos,refresh=.1, size=self.gui_settings['size'])
self._bar.iconified = self.gui_settings['iconified']
self.update_bar_markers()
#set up bar display padding
self.on_window_resize(glfwGetCurrentContext(),*glfwGetWindowSize(glfwGetCurrentContext()))
def unset_alive(self):
self.alive = False
def load(self, var_name, default):
return self.surface_definitions.get(var_name,default)
def save(self, var_name, var):
self.surface_definitions[var_name] = var
def on_window_resize(self,window,w,h):
self.win_size = w,h
def on_click(self,pos,button,action):
if self.mode.value == 1:
if self.edit_surfaces:
if action == GLFW_RELEASE:
self.edit_surfaces = []
# no surfaces verts in edit mode, lets see if the curser is close to one:
else:
if action == GLFW_PRESS:
surf_verts = ((0.,0.),(1.,0.),(1.,1.),(0.,1.))
x,y = pos
for s in self.surfaces:
if s.detected:
for (vx,vy),i in zip(s.ref_surface_to_img(np.array(surf_verts)),range(4)):
vx,vy = denormalize((vx,vy),(self.img_shape[1],self.img_shape[0]),flip_y=True)
if sqrt((x-vx)**2 + (y-vy)**2) <15: #img pixels
self.edit_surfaces.append((s,i))
def advance(self):
pass
def add_surface(self):
self.surfaces.append(Offline_Reference_Surface(self.g_pool,gaze_positions_by_frame=self.g_pool.positions_by_frame))
self.update_bar_markers()
def remove_surface(self,i):
self.surfaces[i].cleanup()
del self.surfaces[i]
self.update_bar_markers()
def update_bar_markers(self):
self._bar.clear()
self._bar.add_button('close',self.unset_alive)
self._bar.add_button(" add surface ", self.add_surface, key='a')
# when cache is updated, when surface is edited, when trimmarks are changed.
# dropdown menue: markers and surface, surface edit mode, heatmaps, metrics
self._bar.mode_enum = atb.enum("Mode",{"Show Markers and Frames":0,"Show Marker Id's":4, "Surface edit mode":1,"Show Heatmaps":2,"Show Metrics":3})
self._bar.add_var("Mode",self.mode,vtype=self._bar.mode_enum)
self._bar.add_button(" (re)-calculate gaze distributions ", self.recalculate)
self._bar.add_button(" Export Gaze and Surface Data ", self.save_surface_statsics_to_file)
for s,i in zip(self.surfaces,range(len(self.surfaces)))[::-1]:
self._bar.add_var("%s_name"%i,create_string_buffer(512),getter=s.atb_get_name,setter=s.atb_set_name,group=str(i),label='name')
self._bar.add_var("%s_markers"%i,create_string_buffer(512), getter=s.atb_marker_status,group=str(i),label='found/registered markers' )
self._bar.add_var("%s_x_scale"%i,vtype=c_float, getter=s.atb_get_scale_x, min=1,setter=s.atb_set_scale_x,group=str(i),label='real width', help='this scale factor is used to adjust the coordinate space for your needs (think photo pixels or mm or whatever)' )
self._bar.add_var("%s_y_scale"%i,vtype=c_float, getter=s.atb_get_scale_y,min=1,setter=s.atb_set_scale_y,group=str(i),label='real height',help='defining x and y scale factor you atumatically set the correct aspect ratio.' )
self._bar.add_var("%s_window"%i,setter=s.toggle_window,getter=s.window_open,group=str(i),label='open in window')
# self._bar.add_button("%s_hm"%i, s.generate_heatmap, label='generate_heatmap',group=str(i))
# self._bar.add_button("%s_export"%i, self.save_surface_positions_to_file,data=i, label='export surface data',group=str(i))
self._bar.add_button("%s_remove"%i, self.remove_surface,data=i,label='remove',group=str(i))
def recalculate(self):
in_mark = self.g_pool.trim_marks.in_mark
out_mark = self.g_pool.trim_marks.out_mark
section = slice(in_mark,out_mark)
# calc heatmaps
for s in self.surfaces:
if s.defined:
s.generate_heatmap(section)
# calc metrics:
gaze_in_section = list(chain(*self.g_pool.positions_by_frame[section]))
results = []
for s in self.surfaces:
gaze_on_srf = s.gaze_on_srf_in_section(section)
results.append(len(gaze_on_srf))
self.metrics_gazecount = len(gaze_on_srf)
max_res = max(results)
results = np.array(results,dtype=np.float32)
if not max_res:
logger.warning("No gaze on any surface for this section!")
else:
results *= 255./max_res
results = np.uint8(results)
results_c_maps = cv2.applyColorMap(results, cv2.COLORMAP_JET)
for s,c_map in zip(self.surfaces,results_c_maps):
heatmap = np.ones((1,1,4),dtype=np.uint8)*125
heatmap[:,:,:3] = c_map
s.metrics_texture = create_named_texture(heatmap)
def update(self,frame,recent_pupil_positions,events):
self.img = frame.img
self.img_shape = frame.img.shape
self.update_marker_cache()
self.markers = self.cache[frame.index]
if self.markers == False:
self.markers = []
self.seek_marker_cacher(frame.index) # tell precacher that it better have every thing from here on analyzed
# locate surfaces
for s in self.surfaces:
if not s.locate_from_cache(frame.index):
s.locate(self.markers)
if s.detected:
events.append({'type':'marker_ref_surface','name':s.name,'uid':s.uid,'m_to_screen':s.m_to_screen,'m_from_screen':s.m_from_screen, 'timestamp':frame.timestamp,'gaze_on_srf':s.gaze_on_srf})
if self.mode.value == 4:
draw_markers(frame.img,self.markers)
# edit surfaces by user
if self.mode.value == 1:
window = glfwGetCurrentContext()
pos = glfwGetCursorPos(window)
pos = normalize(pos,glfwGetWindowSize(window))
pos = denormalize(pos,(frame.img.shape[1],frame.img.shape[0]) ) # Position in img pixels
for s,v_idx in self.edit_surfaces:
if s.detected:
pos = normalize(pos,(self.img_shape[1],self.img_shape[0]),flip_y=True)
new_pos = s.img_to_ref_surface(np.array(pos))
s.move_vertex(v_idx,new_pos)
s.cache = None
self.heatmap = None
else:
# update srf with no or invald cache:
for s in self.surfaces:
if s.cache == None:
s.init_cache(self.cache)
#allow surfaces to open/close windows
for s in self.surfaces:
if s.window_should_close:
s.close_window()
if s.window_should_open:
s.open_window()
def init_marker_cacher(self):
forking_enable(0) #for MacOs only
from marker_detector_cacher import fill_cache
visited_list = [False if x == False else True for x in self.cache]
video_file_path = os.path.join(self.g_pool.rec_dir,'world.avi')
self.cache_queue = Queue()
self.cacher_seek_idx = Value(c_int,0)
self.cacher_run = Value(c_bool,True)
self.cacher = Process(target=fill_cache, args=(visited_list,video_file_path,self.cache_queue,self.cacher_seek_idx,self.cacher_run))
self.cacher.start()
def update_marker_cache(self):
while not self.cache_queue.empty():
idx,c_m = self.cache_queue.get()
self.cache.update(idx,c_m)
for s in self.surfaces:
s.update_cache(self.cache,idx=idx)
def seek_marker_cacher(self,idx):
self.cacher_seek_idx.value = idx
def close_marker_cacher(self):
self.update_marker_cache()
self.cacher_run.value = False
self.cacher.join()
def gl_display(self):
"""
Display marker and surface info inside world screen
"""
self.gl_display_cache_bars()
for s in self.surfaces:
s.gl_display_in_window(self.g_pool.image_tex)
if self.mode.value in (0,1):
for m in self.markers:
hat = np.array([[[0,0],[0,1],[1,1],[1,0],[0,0]]],dtype=np.float32)
hat = cv2.perspectiveTransform(hat,m_marker_to_screen(m))
draw_gl_polyline(hat.reshape((5,2)),(0.1,1.,1.,.3),type='Polygon')
draw_gl_polyline(hat.reshape((5,2)),(0.1,1.,1.,.6))
for s in self.surfaces:
s.gl_draw_frame()
if self.mode.value == 1:
for s in self.surfaces:
s.gl_draw_corners()
if self.mode.value == 2:
for s in self.surfaces:
s.gl_display_heatmap()
if self.mode.value == 3:
#draw a backdrop to represent the gaze that is not on any surface
for s in self.surfaces:
#draw a quad on surface with false color of value.
s.gl_display_metrics()
def gl_display_cache_bars(self):
"""
"""
padding = 20.
# Lines for areas that have been cached
cached_ranges = []
for r in self.cache.visited_ranges: # [[0,1],[3,4]]
cached_ranges += (r[0],0),(r[1],0) #[(0,0),(1,0),(3,0),(4,0)]
# Lines where surfaces have been found in video
cached_surfaces = []
for s in self.surfaces:
found_at = []
if s.cache is not None:
for r in s.cache.positive_ranges: # [[0,1],[3,4]]
found_at += (r[0],0),(r[1],0) #[(0,0),(1,0),(3,0),(4,0)]
cached_surfaces.append(found_at)
glMatrixMode(GL_PROJECTION)
glPushMatrix()
glLoadIdentity()
width,height = self.win_size
h_pad = padding * (self.cache.length-2)/float(width)
v_pad = padding* 1./(height-2)
gluOrtho2D(-h_pad, (self.cache.length-1)+h_pad, -v_pad, 1+v_pad) # ranging from 0 to cache_len-1 (horizontal) and 0 to 1 (vertical)
glMatrixMode(GL_MODELVIEW)
glPushMatrix()
glLoadIdentity()
color = (8.,.6,.2,8.)
draw_gl_polyline(cached_ranges,color=color,type='Lines',thickness=4)
color = (0.,.7,.3,8.)
for s in cached_surfaces:
glTranslatef(0,.02,0)
draw_gl_polyline(s,color=color,type='Lines',thickness=2)
glMatrixMode(GL_PROJECTION)
glPopMatrix()
glMatrixMode(GL_MODELVIEW)
glPopMatrix()
def save_surface_statsics_to_file(self):
in_mark = self.g_pool.trim_marks.in_mark
out_mark = self.g_pool.trim_marks.out_mark
"""
between in and out mark
report: gaze distribution:
- total gazepoints
- gaze points on surface x
- gaze points not on any surface
report: surface visisbility
- total frames
- surface x visible framecount
surface events:
frame_no, ts, surface "name", "id" enter/exit
for each surface:
gaze_on_name_id.csv
positions_of_name_id.csv
"""
section = slice(in_mark,out_mark)
metrics_dir = os.path.join(self.g_pool.rec_dir,"metrics_%s-%s"%(in_mark,out_mark))
logger.info("exporting metrics to %s"%metrics_dir)
if os.path.isdir(metrics_dir):
logger.info("Will overwrite previous export for this section")
else:
try:
os.mkdir(metrics_dir)
except:
logger.warning("Could not make metrics dir %s"%metrics_dir)
return
with open(os.path.join(metrics_dir,'surface_visibility.csv'),'wb') as csvfile:
csv_writer = csv.writer(csvfile, delimiter='\t',quotechar='|', quoting=csv.QUOTE_MINIMAL)
# surface visibility report
frame_count = len(self.g_pool.timestamps[section])
csv_writer.writerow(('frame_count',frame_count))
csv_writer.writerow((''))
csv_writer.writerow(('surface_name','visible_frame_count'))
for s in self.surfaces:
if s.cache == None:
logger.warning("The surface is not cached. Please wait for the cacher to collect data.")
return
visible_count = s.visible_count_in_section(section)
csv_writer.writerow( (s.name, visible_count) )
logger.info("Created 'surface_visibility.csv' file")
with open(os.path.join(metrics_dir,'surface_gaze_distribution.csv'),'wb') as csvfile:
csv_writer = csv.writer(csvfile, delimiter='\t',quotechar='|', quoting=csv.QUOTE_MINIMAL)
# gaze distribution report
gaze_in_section = list(chain(*self.g_pool.positions_by_frame[section]))
not_on_any_srf = set([gp['timestamp'] for gp in gaze_in_section])
csv_writer.writerow(('total_gaze_point_count',len(gaze_in_section)))
csv_writer.writerow((''))
csv_writer.writerow(('surface_name','gaze_count'))
for s in self.surfaces:
gaze_on_srf = s.gaze_on_srf_in_section(section)
gaze_on_srf = set([gp["timestamp"] for gp in gaze_on_srf])
not_on_any_srf -= gaze_on_srf
csv_writer.writerow( (s.name, len(gaze_on_srf)) )
csv_writer.writerow(('not_on_any_surface', len(not_on_any_srf) ) )
logger.info("Created 'surface_gaze_distribution.csv' file")
with open(os.path.join(metrics_dir,'surface_events.csv'),'wb') as csvfile:
csv_writer = csv.writer(csvfile, delimiter='\t',quotechar='|', quoting=csv.QUOTE_MINIMAL)
# surface events report
csv_writer.writerow(('frame_number','timestamp','surface_name','surface_uid','event_type'))
events = []
for s in self.surfaces:
for enter_frame_id,exit_frame_id in s.cache.positive_ranges:
events.append({'frame_id':enter_frame_id,'srf_name':s.name,'srf_uid':s.uid,'event':'enter'})
events.append({'frame_id':exit_frame_id,'srf_name':s.name,'srf_uid':s.uid,'event':'exit'})
events.sort(key=lambda x: x['frame_id'])
for e in events:
csv_writer.writerow( ( e['frame_id'],self.g_pool.timestamps[e['frame_id']],e['srf_name'],e['srf_uid'],e['event'] ) )
logger.info("Created 'surface_events.csv' file")
for s in self.surfaces:
# per surface names:
surface_name = '_'+s.name.replace('/','')+'_'+s.uid
# save surface_positions as pickle file
save_object(s.cache.to_list(),os.path.join(metrics_dir,'srf_positions'+surface_name))
#save surface_positions as csv
with open(os.path.join(metrics_dir,'srf_positons'+surface_name+'.csv'),'wb') as csvfile:
csv_writer =csv.writer(csvfile, delimiter='\t',quotechar='|', quoting=csv.QUOTE_MINIMAL)
csv_writer.writerow(('frame_idx','timestamp','m_to_screen','m_from_screen','detected_markers'))
for idx,ts,ref_srf_data in zip(range(len(self.g_pool.timestamps)),self.g_pool.timestamps,s.cache):
if in_mark <= idx <= out_mark:
if ref_srf_data is not None and ref_srf_data is not False:
csv_writer.writerow( (idx,ts,ref_srf_data['m_to_screen'],ref_srf_data['m_from_screen'],ref_srf_data['detected_markers']) )
# save gaze on srf as csv.
with open(os.path.join(metrics_dir,'gaze_positions_on_surface'+surface_name+'.csv'),'wb') as csvfile:
csv_writer = csv.writer(csvfile, delimiter='\t',quotechar='|', quoting=csv.QUOTE_MINIMAL)
csv_writer.writerow(('world_frame_idx','world_timestamp','eye_timestamp','x_norm','y_norm','x_scaled','y_scaled','on_srf'))
for idx,ts,ref_srf_data in zip(range(len(self.g_pool.timestamps)),self.g_pool.timestamps,s.cache):
if in_mark <= idx <= out_mark:
if ref_srf_data is not None and ref_srf_data is not False:
for gp in ref_srf_data['gaze_on_srf']:
gp_x,gp_y = gp['norm_gaze_on_srf']
on_srf = (0 <= gp_x <= 1) and (0 <= gp_y <= 1)
csv_writer.writerow( (idx,ts,gp['timestamp'],gp_x,gp_y,gp_x*s.scale_factor[0],gp_x*s.scale_factor[1],on_srf) )
logger.info("Saved surface positon data and gaze on surface data for '%s' with uid:'%s'"%(s.name,s.uid))
if s.heatmap is not None:
logger.info("Saved Heatmap as .png file.")
cv2.imwrite(os.path.join(metrics_dir,'heatmap'+surface_name+'.png'),s.heatmap)
# if s.detected and self.img is not None:
# #let save out the current surface image found in video
# #here we get the verts of the surface quad in norm_coords
# mapped_space_one = np.array(((0,0),(1,0),(1,1),(0,1)),dtype=np.float32).reshape(-1,1,2)
# screen_space = cv2.perspectiveTransform(mapped_space_one,s.m_to_screen).reshape(-1,2)
# #now we convert to image pixel coods
# screen_space[:,1] = 1-screen_space[:,1]
# screen_space[:,1] *= self.img.shape[0]
# screen_space[:,0] *= self.img.shape[1]
# s_0,s_1 = s.scale_factor
# #no we need to flip vertically again by setting the mapped_space verts accordingly.
# mapped_space_scaled = np.array(((0,s_1),(s_0,s_1),(s_0,0),(0,0)),dtype=np.float32)
# M = cv2.getPerspectiveTransform(screen_space,mapped_space_scaled)
# #here we do the actual perspactive transform of the image.
# srf_in_video = cv2.warpPerspective(self.img,M, (int(s.scale_factor[0]),int(s.scale_factor[1])) )
# cv2.imwrite(os.path.join(metrics_dir,'surface'+surface_name+'.png'),srf_in_video)
# logger.info("Saved current image as .png file.")
# else:
# logger.info("'%s' is not currently visible. Seek to appropriate frame and repeat this command."%s.name)
def get_init_dict(self):
d = {}
if hasattr(self,'_bar'):
gui_settings = {'pos':self._bar.position,'size':self._bar.size,'iconified':self._bar.iconified}
d['gui_settings'] = gui_settings
return d
def cleanup(self):
""" called when the plugin gets terminated.
This happends either voluntary or forced.
if you have an atb bar or glfw window destroy it here.
"""
self.save("offline_square_marker_surfaces",[rs.save_to_dict() for rs in self.surfaces if rs.defined])
self.close_marker_cacher()
self.persistent_cache["marker_cache"] = self.cache.to_list()
self.persistent_cache.close()
self.surface_definitions.close()
for s in self.surfaces:
s.close_window()
self._bar.destroy()
class Offline_Marker_Detector(Plugin):
"""
Special version of marker detector for use with videofile source.
It uses a seperate process to search all frames in the world.avi file for markers.
- self.cache is a list containing marker positions for each frame.
- self.surfaces[i].cache is a list containing surface positions for each frame
Both caches are build up over time. The marker cache is also session persistent.
See marker_tracker.py for more info on this marker tracker.
"""
def __init__(self, g_pool, mode="Show Markers and Frames"):
super(Offline_Marker_Detector, self).__init__(g_pool)
self.order = .2
# all markers that are detected in the most recent frame
self.markers = []
# all registered surfaces
if g_pool.app == 'capture':
raise Exception('For Player only.')
#in player we load from the rec_dir: but we have a couple options:
self.surface_definitions = Persistent_Dict(
os.path.join(g_pool.rec_dir, 'surface_definitions'))
if self.surface_definitions.get('offline_square_marker_surfaces',
[]) != []:
logger.debug(
"Found ref surfaces defined or copied in previous session.")
self.surfaces = [
Offline_Reference_Surface(self.g_pool, saved_definition=d)
for d in self.surface_definitions.get(
'offline_square_marker_surfaces', [])
if isinstance(d, dict)
]
elif self.surface_definitions.get('realtime_square_marker_surfaces',
[]) != []:
logger.debug(
"Did not find ref surfaces def created or used by the user in player from earlier session. Loading surfaces defined during capture."
)
self.surfaces = [
Offline_Reference_Surface(self.g_pool, saved_definition=d)
for d in self.surface_definitions.get(
'realtime_square_marker_surfaces', [])
if isinstance(d, dict)
]
else:
logger.debug("No surface defs found. Please define using GUI.")
self.surfaces = []
# ui mode settings
self.mode = mode
# edit surfaces
self.edit_surfaces = []
#check if marker cache is available from last session
self.persistent_cache = Persistent_Dict(
os.path.join(g_pool.rec_dir, 'square_marker_cache'))
self.cache = Cache_List(
self.persistent_cache.get('marker_cache',
[False for _ in g_pool.timestamps]))
logger.debug(
"Loaded marker cache %s / %s frames had been searched before" %
(len(self.cache) - self.cache.count(False), len(self.cache)))
self.init_marker_cacher()
#debug vars
self.show_surface_idx = c_int(0)
self.img_shape = None
self.img = None
def init_gui(self):
self.menu = ui.Scrolling_Menu('Offline Marker Tracker')
self.g_pool.gui.append(self.menu)
self.add_button = ui.Thumb('add_surface',
setter=self.add_surface,
getter=lambda: False,
label='Add Surface',
hotkey='a')
self.g_pool.quickbar.append(self.add_button)
self.update_gui_markers()
self.on_window_resize(glfwGetCurrentContext(),
*glfwGetWindowSize(glfwGetCurrentContext()))
def deinit_gui(self):
if self.menu:
self.g_pool.gui.remove(self.menu)
self.menu = None
if self.add_button:
self.g_pool.quickbar.remove(self.add_button)
self.add_button = None
def update_gui_markers(self):
pass
self.menu.elements[:] = []
self.menu.append(
ui.Info_Text(
'The offline marker tracker will look for markers in the entire video. By default it uses surfaces defined in capture. You can change and add more surfaces here.'
))
self.menu.append(ui.Button('Close', self.close))
self.menu.append(
ui.Selector('mode',
self,
label='Mode',
selection=[
"Show Markers and Frames", "Show marker IDs",
"Surface edit mode", "Show Heatmaps",
"Show Metrics"
]))
self.menu.append(
ui.Info_Text(
'To see heatmap or surface metrics visualizations, click (re)-calculate gaze distributions. Set "X size" and "Y size" for each surface to see heatmap visualizations.'
))
self.menu.append(
ui.Button("(Re)-calculate gaze distributions", self.recalculate))
self.menu.append(
ui.Button("Export gaze and surface data",
self.save_surface_statsics_to_file))
self.menu.append(
ui.Button("Add surface", lambda: self.add_surface('_')))
for s in self.surfaces:
idx = self.surfaces.index(s)
s_menu = ui.Growing_Menu("Surface %s" % idx)
s_menu.collapsed = True
s_menu.append(ui.Text_Input('name', s))
s_menu.append(ui.Text_Input('x', s.real_world_size,
label='X size'))
s_menu.append(ui.Text_Input('y', s.real_world_size,
label='Y size'))
s_menu.append(ui.Button('Open Debug Window', s.open_close_window))
#closure to encapsulate idx
def make_remove_s(i):
return lambda: self.remove_surface(i)
remove_s = make_remove_s(idx)
s_menu.append(ui.Button('remove', remove_s))
self.menu.append(s_menu)
def close(self):
self.alive = False
def on_window_resize(self, window, w, h):
self.win_size = w, h
def on_click(self, pos, button, action):
if self.mode == "Surface edit mode":
if self.edit_surfaces:
if action == GLFW_RELEASE:
self.edit_surfaces = []
# no surfaces verts in edit mode, lets see if the curser is close to one:
else:
if action == GLFW_PRESS:
surf_verts = ((0., 0.), (1., 0.), (1., 1.), (0., 1.))
x, y = pos
for s in self.surfaces:
if s.detected and s.defined:
for (vx, vy), i in zip(
s.ref_surface_to_img(np.array(surf_verts)),
range(4)):
vx, vy = denormalize(
(vx, vy),
(self.img_shape[1], self.img_shape[0]),
flip_y=True)
if sqrt((x - vx)**2 +
(y - vy)**2) < 15: #img pixels
self.edit_surfaces.append((s, i))
def advance(self):
pass
def add_surface(self, _):
self.surfaces.append(Offline_Reference_Surface(self.g_pool))
self.update_gui_markers()
def remove_surface(self, i):
self.surfaces[i].cleanup()
del self.surfaces[i]
self.update_gui_markers()
def recalculate(self):
in_mark = self.g_pool.trim_marks.in_mark
out_mark = self.g_pool.trim_marks.out_mark
section = slice(in_mark, out_mark)
# calc heatmaps
for s in self.surfaces:
if s.defined:
s.generate_heatmap(section)
# calc distirbution accross all surfaces.
results = []
for s in self.surfaces:
gaze_on_srf = s.gaze_on_srf_in_section(section)
results.append(len(gaze_on_srf))
self.metrics_gazecount = len(gaze_on_srf)
if results == []:
logger.warning("No surfaces defined.")
return
max_res = max(results)
results = np.array(results, dtype=np.float32)
if not max_res:
logger.warning("No gaze on any surface for this section!")
else:
results *= 255. / max_res
results = np.uint8(results)
results_c_maps = cv2.applyColorMap(results, cv2.COLORMAP_JET)
for s, c_map in zip(self.surfaces, results_c_maps):
heatmap = np.ones((1, 1, 4), dtype=np.uint8) * 125
heatmap[:, :, :3] = c_map
s.metrics_texture = create_named_texture(heatmap.shape)
update_named_texture(s.metrics_texture, heatmap)
def update(self, frame, events):
self.img = frame.img
self.img_shape = frame.img.shape
self.update_marker_cache()
self.markers = self.cache[frame.index]
if self.markers == False:
self.markers = []
self.seek_marker_cacher(
frame.index
) # tell precacher that it better have every thing from here on analyzed
# locate surfaces
for s in self.surfaces:
if not s.locate_from_cache(frame.index):
s.locate(self.markers)
if s.detected:
pass
# events.append({'type':'marker_ref_surface','name':s.name,'uid':s.uid,'m_to_screen':s.m_to_screen,'m_from_screen':s.m_from_screen, 'timestamp':frame.timestamp,'gaze_on_srf':s.gaze_on_srf})
if self.mode == "Show marker IDs":
draw_markers(frame.img, self.markers)
# edit surfaces by user
if self.mode == "Surface edit mode":
window = glfwGetCurrentContext()
pos = glfwGetCursorPos(window)
pos = normalize(pos, glfwGetWindowSize(window), flip_y=True)
for s, v_idx in self.edit_surfaces:
if s.detected:
new_pos = s.img_to_ref_surface(np.array(pos))
s.move_vertex(v_idx, new_pos)
s.cache = None
self.heatmap = None
else:
# update srf with no or invald cache:
for s in self.surfaces:
if s.cache == None:
s.init_cache(self.cache)
#allow surfaces to open/close windows
for s in self.surfaces:
if s.window_should_close:
s.close_window()
if s.window_should_open:
s.open_window()
def init_marker_cacher(self):
forking_enable(0) #for MacOs only
from marker_detector_cacher import fill_cache
visited_list = [False if x == False else True for x in self.cache]
video_file_path = os.path.join(self.g_pool.rec_dir, 'world.mkv')
if not os.path.isfile(video_file_path):
video_file_path = os.path.join(self.g_pool.rec_dir, 'world.avi')
self.cache_queue = Queue()
self.cacher_seek_idx = Value('i', 0)
self.cacher_run = Value(c_bool, True)
self.cacher = Process(target=fill_cache,
args=(visited_list, video_file_path,
self.cache_queue, self.cacher_seek_idx,
self.cacher_run))
self.cacher.start()
def update_marker_cache(self):
while not self.cache_queue.empty():
idx, c_m = self.cache_queue.get()
self.cache.update(idx, c_m)
for s in self.surfaces:
s.update_cache(self.cache, idx=idx)
def seek_marker_cacher(self, idx):
self.cacher_seek_idx.value = idx
def close_marker_cacher(self):
self.update_marker_cache()
self.cacher_run.value = False
self.cacher.join()
def gl_display(self):
"""
Display marker and surface info inside world screen
"""
self.gl_display_cache_bars()
for s in self.surfaces:
s.gl_display_in_window(self.g_pool.image_tex)
if self.mode == "Show Markers and Frames":
for m in self.markers:
hat = np.array([[[0, 0], [0, 1], [1, 1], [1, 0], [0, 0]]],
dtype=np.float32)
hat = cv2.perspectiveTransform(hat, m_marker_to_screen(m))
draw_polyline(hat.reshape((5, 2)),
color=RGBA(0.1, 1., 1., .3),
line_type=GL_POLYGON)
draw_polyline(hat.reshape((5, 2)), color=RGBA(0.1, 1., 1., .6))
for s in self.surfaces:
s.gl_draw_frame(self.img_shape)
if self.mode == "Surface edit mode":
for s in self.surfaces:
s.gl_draw_frame(self.img_shape)
s.gl_draw_corners()
if self.mode == "Show Heatmaps":
for s in self.surfaces:
s.gl_display_heatmap()
if self.mode == "Show Metrics":
#todo: draw a backdrop to represent the gaze that is not on any surface
for s in self.surfaces:
#draw a quad on surface with false color of value.
s.gl_display_metrics()
def gl_display_cache_bars(self):
"""
"""
padding = 20.
# Lines for areas that have been cached
cached_ranges = []
for r in self.cache.visited_ranges: # [[0,1],[3,4]]
cached_ranges += (r[0], 0), (r[1], 0) #[(0,0),(1,0),(3,0),(4,0)]
# Lines where surfaces have been found in video
cached_surfaces = []
for s in self.surfaces:
found_at = []
if s.cache is not None:
for r in s.cache.positive_ranges: # [[0,1],[3,4]]
found_at += (r[0], 0), (r[1], 0
) #[(0,0),(1,0),(3,0),(4,0)]
cached_surfaces.append(found_at)
glMatrixMode(GL_PROJECTION)
glPushMatrix()
glLoadIdentity()
width, height = self.win_size
h_pad = padding * (self.cache.length - 2) / float(width)
v_pad = padding * 1. / (height - 2)
glOrtho(
-h_pad, (self.cache.length - 1) + h_pad, -v_pad, 1 + v_pad, -1, 1
) # ranging from 0 to cache_len-1 (horizontal) and 0 to 1 (vertical)
glMatrixMode(GL_MODELVIEW)
glPushMatrix()
glLoadIdentity()
color = RGBA(8., .6, .2, 8.)
draw_polyline(cached_ranges,
color=color,
line_type=GL_LINES,
thickness=4)
color = RGBA(0., .7, .3, 8.)
for s in cached_surfaces:
glTranslatef(0, .02, 0)
draw_polyline(s, color=color, line_type=GL_LINES, thickness=2)
glMatrixMode(GL_PROJECTION)
glPopMatrix()
glMatrixMode(GL_MODELVIEW)
glPopMatrix()
def save_surface_statsics_to_file(self):
in_mark = self.g_pool.trim_marks.in_mark
out_mark = self.g_pool.trim_marks.out_mark
"""
between in and out mark
report: gaze distribution:
- total gazepoints
- gaze points on surface x
- gaze points not on any surface
report: surface visisbility
- total frames
- surface x visible framecount
surface events:
frame_no, ts, surface "name", "id" enter/exit
for each surface:
fixations_on_name.csv
gaze_on_name_id.csv
positions_of_name_id.csv
"""
section = slice(in_mark, out_mark)
metrics_dir = os.path.join(self.g_pool.rec_dir,
"metrics_%s-%s" % (in_mark, out_mark))
logger.info("exporting metrics to %s" % metrics_dir)
if os.path.isdir(metrics_dir):
logger.info("Will overwrite previous export for this section")
else:
try:
os.mkdir(metrics_dir)
except:
logger.warning("Could not make metrics dir %s" % metrics_dir)
return
with open(os.path.join(metrics_dir, 'surface_visibility.csv'),
'wb') as csvfile:
csv_writer = csv.writer(csvfile,
delimiter='\t',
quotechar='|',
quoting=csv.QUOTE_MINIMAL)
# surface visibility report
frame_count = len(self.g_pool.timestamps[section])
csv_writer.writerow(('frame_count', frame_count))
csv_writer.writerow((''))
csv_writer.writerow(('surface_name', 'visible_frame_count'))
for s in self.surfaces:
if s.cache == None:
logger.warning(
"The surface is not cached. Please wait for the cacher to collect data."
)
return
visible_count = s.visible_count_in_section(section)
csv_writer.writerow((s.name, visible_count))
logger.info("Created 'surface_visibility.csv' file")
with open(os.path.join(metrics_dir, 'surface_gaze_distribution.csv'),
'wb') as csvfile:
csv_writer = csv.writer(csvfile,
delimiter='\t',
quotechar='|',
quoting=csv.QUOTE_MINIMAL)
# gaze distribution report
gaze_in_section = list(
chain(*self.g_pool.gaze_positions_by_frame[section]))
not_on_any_srf = set([gp['timestamp'] for gp in gaze_in_section])
csv_writer.writerow(
('total_gaze_point_count', len(gaze_in_section)))
csv_writer.writerow((''))
csv_writer.writerow(('surface_name', 'gaze_count'))
for s in self.surfaces:
gaze_on_srf = s.gaze_on_srf_in_section(section)
gaze_on_srf = set(
[gp['base']["timestamp"] for gp in gaze_on_srf])
not_on_any_srf -= gaze_on_srf
csv_writer.writerow((s.name, len(gaze_on_srf)))
csv_writer.writerow(('not_on_any_surface', len(not_on_any_srf)))
logger.info("Created 'surface_gaze_distribution.csv' file")
with open(os.path.join(metrics_dir, 'surface_events.csv'),
'wb') as csvfile:
csv_writer = csv.writer(csvfile,
delimiter='\t',
quotechar='|',
quoting=csv.QUOTE_MINIMAL)
# surface events report
csv_writer.writerow(('frame_number', 'timestamp', 'surface_name',
'surface_uid', 'event_type'))
events = []
for s in self.surfaces:
for enter_frame_id, exit_frame_id in s.cache.positive_ranges:
events.append({
'frame_id': enter_frame_id,
'srf_name': s.name,
'srf_uid': s.uid,
'event': 'enter'
})
events.append({
'frame_id': exit_frame_id,
'srf_name': s.name,
'srf_uid': s.uid,
'event': 'exit'
})
events.sort(key=lambda x: x['frame_id'])
for e in events:
csv_writer.writerow(
(e['frame_id'], self.g_pool.timestamps[e['frame_id']],
e['srf_name'], e['srf_uid'], e['event']))
logger.info("Created 'surface_events.csv' file")
for s in self.surfaces:
# per surface names:
surface_name = '_' + s.name.replace('/', '') + '_' + s.uid
# save surface_positions as pickle file
save_object(
s.cache.to_list(),
os.path.join(metrics_dir, 'srf_positions' + surface_name))
#save surface_positions as csv
with open(
os.path.join(metrics_dir,
'srf_positons' + surface_name + '.csv'),
'wb') as csvfile:
csv_writer = csv.writer(csvfile,
delimiter='\t',
quotechar='|',
quoting=csv.QUOTE_MINIMAL)
csv_writer.writerow(('frame_idx', 'timestamp', 'm_to_screen',
'm_from_screen', 'detected_markers'))
for idx, ts, ref_srf_data in zip(
range(len(self.g_pool.timestamps)),
self.g_pool.timestamps, s.cache):
if in_mark <= idx <= out_mark:
if ref_srf_data is not None and ref_srf_data is not False:
csv_writer.writerow(
(idx, ts, ref_srf_data['m_to_screen'],
ref_srf_data['m_from_screen'],
ref_srf_data['detected_markers']))
# save gaze on srf as csv.
with open(
os.path.join(
metrics_dir,
'gaze_positions_on_surface' + surface_name + '.csv'),
'wb') as csvfile:
csv_writer = csv.writer(csvfile,
delimiter='\t',
quotechar='|',
quoting=csv.QUOTE_MINIMAL)
csv_writer.writerow(
('world_timestamp', 'world_frame_idx', 'gaze_timestamp',
'x_norm', 'y_norm', 'x_scaled', 'y_scaled', 'on_srf'))
for idx, ts, ref_srf_data in zip(
range(len(self.g_pool.timestamps)),
self.g_pool.timestamps, s.cache):
if in_mark <= idx <= out_mark:
if ref_srf_data is not None and ref_srf_data is not False:
for gp in s.gaze_on_srf_by_frame_idx(
idx, ref_srf_data['m_from_screen']):
csv_writer.writerow(
(ts, idx, gp['base']['timestamp'],
gp['norm_pos'][0], gp['norm_pos'][1],
gp['norm_pos'][0] *
s.real_world_size['x'],
gp['norm_pos'][1] *
s.real_world_size['y'], gp['on_srf']))
# # save fixation on srf as csv.
with open(
os.path.join(
metrics_dir,
'fixations_on_surface' + surface_name + '.csv'),
'wb') as csvfile:
csv_writer = csv.writer(csvfile,
delimiter='\t',
quotechar='|',
quoting=csv.QUOTE_MINIMAL)
csv_writer.writerow(
('id', 'start_timestamp', 'duration', 'start_frame',
'end_frame', 'norm_pos_x', 'norm_pos_y', 'x_scaled',
'y_scaled', 'on_srf'))
fixations_on_surface = []
for idx, ref_srf_data in zip(
range(len(self.g_pool.timestamps)), s.cache):
if in_mark <= idx <= out_mark:
if ref_srf_data is not None and ref_srf_data is not False:
for f in s.fixations_on_srf_by_frame_idx(
idx, ref_srf_data['m_from_screen']):
fixations_on_surface.append(f)
removed_dublicates = dict([
(f['base']['id'], f) for f in fixations_on_surface
]).values()
for f_on_s in removed_dublicates:
f = f_on_s['base']
f_x, f_y = f_on_s['norm_pos']
f_on_srf = f_on_s['on_srf']
csv_writer.writerow(
(f['id'], f['timestamp'], f['duration'],
f['start_frame_index'], f['end_frame_index'], f_x,
f_y, f_x * s.real_world_size['x'],
f_y * s.real_world_size['y'], f_on_srf))
logger.info(
"Saved surface positon gaze and fixation data for '%s' with uid:'%s'"
% (s.name, s.uid))
if s.heatmap is not None:
logger.info("Saved Heatmap as .png file.")
cv2.imwrite(
os.path.join(metrics_dir,
'heatmap' + surface_name + '.png'), s.heatmap)
logger.info("Done exporting reference surface data.")
# if s.detected and self.img is not None:
# #let save out the current surface image found in video
# #here we get the verts of the surface quad in norm_coords
# mapped_space_one = np.array(((0,0),(1,0),(1,1),(0,1)),dtype=np.float32).reshape(-1,1,2)
# screen_space = cv2.perspectiveTransform(mapped_space_one,s.m_to_screen).reshape(-1,2)
# #now we convert to image pixel coods
# screen_space[:,1] = 1-screen_space[:,1]
# screen_space[:,1] *= self.img.shape[0]
# screen_space[:,0] *= self.img.shape[1]
# s_0,s_1 = s.real_world_size
# #no we need to flip vertically again by setting the mapped_space verts accordingly.
# mapped_space_scaled = np.array(((0,s_1),(s_0,s_1),(s_0,0),(0,0)),dtype=np.float32)
# M = cv2.getPerspectiveTransform(screen_space,mapped_space_scaled)
# #here we do the actual perspactive transform of the image.
# srf_in_video = cv2.warpPerspective(self.img,M, (int(s.real_world_size['x']),int(s.real_world_size['y'])) )
# cv2.imwrite(os.path.join(metrics_dir,'surface'+surface_name+'.png'),srf_in_video)
# logger.info("Saved current image as .png file.")
# else:
# logger.info("'%s' is not currently visible. Seek to appropriate frame and repeat this command."%s.name)
def get_init_dict(self):
return {'mode': self.mode}
def cleanup(self):
""" called when the plugin gets terminated.
This happens either voluntarily or forced.
if you have a GUI or glfw window destroy it here.
"""
self.surface_definitions["offline_square_marker_surfaces"] = [
rs.save_to_dict() for rs in self.surfaces if rs.defined
]
self.surface_definitions.close()
self.close_marker_cacher()
self.persistent_cache["marker_cache"] = self.cache.to_list()
self.persistent_cache.close()
for s in self.surfaces:
s.close_window()
self.deinit_gui()
class Offline_Marker_Detector(Plugin):
"""
Special version of marker detector for use with videofile source.
It uses a seperate process to search all frames in the world.avi file for markers.
- self.cache is a list containing marker positions for each frame.
- self.surfaces[i].cache is a list containing surface positions for each frame
Both caches are build up over time. The marker cache is also session persistent.
See marker_tracker.py for more info on this marker tracker.
"""
def __init__(self,g_pool,menu_conf={'pos':(300,200),'size':(300,300),'collapsed':False},mode="Show Markers and Frames"):
super(Offline_Marker_Detector, self).__init__(g_pool)
self.menu_conf = menu_conf
self.order = .2
# all markers that are detected in the most recent frame
self.markers = []
# all registered surfaces
if g_pool.app == 'capture':
raise Exception('For Player only.')
#in player we load from the rec_dir: but we have a couple options:
self.surface_definitions = Persistent_Dict(os.path.join(g_pool.rec_dir,'surface_definitions'))
if self.surface_definitions.get('offline_square_marker_surfaces',[]) != []:
logger.debug("Found ref surfaces defined or copied in previous session.")
self.surfaces = [Offline_Reference_Surface(self.g_pool,saved_definition=d) for d in self.surface_definitions.get('offline_square_marker_surfaces',[]) if isinstance(d,dict)]
elif self.surface_definitions.get('realtime_square_marker_surfaces',[]) != []:
logger.debug("Did not find ref surfaces def created or used by the user in player from earlier session. Loading surfaces defined during capture.")
self.surfaces = [Offline_Reference_Surface(self.g_pool,saved_definition=d) for d in self.surface_definitions.get('realtime_square_marker_surfaces',[]) if isinstance(d,dict)]
else:
logger.debug("No surface defs found. Please define using GUI.")
self.surfaces = []
# ui mode settings
self.mode = mode
# edit surfaces
self.edit_surfaces = []
#check if marker cache is available from last session
self.persistent_cache = Persistent_Dict(os.path.join(g_pool.rec_dir,'square_marker_cache'))
self.cache = Cache_List(self.persistent_cache.get('marker_cache',[False for _ in g_pool.timestamps]))
logger.debug("Loaded marker cache %s / %s frames had been searched before"%(len(self.cache)-self.cache.count(False),len(self.cache)) )
self.init_marker_cacher()
#debug vars
self.show_surface_idx = c_int(0)
self.img_shape = None
self.img = None
def init_gui(self):
self.menu = ui.Scrolling_Menu('Offline Marker Tracker')
self.menu.configuration = self.menu_conf
self.g_pool.gui.append(self.menu)
self.add_button = ui.Thumb('add_surface',setter=self.add_surface,getter=lambda:False,label='Add Surface',hotkey='a')
self.g_pool.quickbar.append(self.add_button)
self.update_gui_markers()
self.on_window_resize(glfwGetCurrentContext(),*glfwGetWindowSize(glfwGetCurrentContext()))
def deinit_gui(self):
if self.menu:
self.g_pool.gui.remove(self.menu)
self.menu_conf= self.menu.configuration
self.menu= None
if self.add_button:
self.g_pool.quickbar.remove(self.add_button)
self.add_button = None
def update_gui_markers(self):
pass
# self._bar.clear()
self.menu.elements[:] = []
self.menu.append(ui.Info_Text('The offline marker tracker will look for markers in the entire video. By default it uses surfaces defined in capture. You can change and add more surfaces here.'))
self.menu.append(ui.Button('Close',self.close))
self.menu.append(ui.Selector('mode',self,label='Mode',selection=["Show Markers and Frames","Show marker IDs", "Surface edit mode","Show Heatmaps","Show Metrics"] ))
self.menu.append(ui.Info_Text('To see heatmap or surface metrics visualizations, click (re)-calculate gaze distributions. Set "X size" and "Y size" for each surface to see heatmap visualizations.'))
self.menu.append(ui.Button("(Re)-calculate gaze distributions", self.recalculate))
self.menu.append(ui.Button("Export gaze and surface data", self.save_surface_statsics_to_file))
self.menu.append(ui.Button("Add surface", lambda:self.add_surface('_')))
for s in self.surfaces:
idx = self.surfaces.index(s)
s_menu = ui.Growing_Menu("Surface %s"%idx)
s_menu.collapsed=True
s_menu.append(ui.Text_Input('name',s))
# self._bar.add_var("%s_markers"%i,create_string_buffer(512), getter=s.atb_marker_status,group=str(i),label='found/registered markers' )
s_menu.append(ui.Text_Input('x',s.real_world_size,label='X size'))
s_menu.append(ui.Text_Input('y',s.real_world_size,label='Y size'))
s_menu.append(ui.Button('Open Debug Window',s.open_close_window))
#closure to encapsulate idx
def make_remove_s(i):
return lambda: self.remove_surface(i)
remove_s = make_remove_s(idx)
s_menu.append(ui.Button('remove',remove_s))
self.menu.append(s_menu)
def close(self):
self.alive = False
def on_window_resize(self,window,w,h):
self.win_size = w,h
def on_click(self,pos,button,action):
if self.mode=="Surface edit mode":
if self.edit_surfaces:
if action == GLFW_RELEASE:
self.edit_surfaces = []
# no surfaces verts in edit mode, lets see if the curser is close to one:
else:
if action == GLFW_PRESS:
surf_verts = ((0.,0.),(1.,0.),(1.,1.),(0.,1.))
x,y = pos
for s in self.surfaces:
if s.detected and s.defined:
for (vx,vy),i in zip(s.ref_surface_to_img(np.array(surf_verts)),range(4)):
vx,vy = denormalize((vx,vy),(self.img_shape[1],self.img_shape[0]),flip_y=True)
if sqrt((x-vx)**2 + (y-vy)**2) <15: #img pixels
self.edit_surfaces.append((s,i))
def advance(self):
pass
def add_surface(self,_):
self.surfaces.append(Offline_Reference_Surface(self.g_pool))
self.update_gui_markers()
def remove_surface(self,i):
self.surfaces[i].cleanup()
del self.surfaces[i]
self.update_gui_markers()
def recalculate(self):
in_mark = self.g_pool.trim_marks.in_mark
out_mark = self.g_pool.trim_marks.out_mark
section = slice(in_mark,out_mark)
# calc heatmaps
for s in self.surfaces:
if s.defined:
s.generate_heatmap(section)
# calc metrics:
results = []
for s in self.surfaces:
gaze_on_srf = s.gaze_on_srf_in_section(section)
results.append(len(gaze_on_srf))
self.metrics_gazecount = len(gaze_on_srf)
if results == []:
logger.warning("No surfaces defined.")
return
max_res = max(results)
results = np.array(results,dtype=np.float32)
if not max_res:
logger.warning("No gaze on any surface for this section!")
else:
results *= 255./max_res
results = np.uint8(results)
results_c_maps = cv2.applyColorMap(results, cv2.COLORMAP_JET)
for s,c_map in zip(self.surfaces,results_c_maps):
heatmap = np.ones((1,1,4),dtype=np.uint8)*125
heatmap[:,:,:3] = c_map
s.metrics_texture = create_named_texture(heatmap.shape)
update_named_texture(s.metrics_texture,heatmap)
def update(self,frame,events):
self.img = frame.img
self.img_shape = frame.img.shape
self.update_marker_cache()
self.markers = self.cache[frame.index]
if self.markers == False:
self.markers = []
self.seek_marker_cacher(frame.index) # tell precacher that it better have every thing from here on analyzed
# locate surfaces
for s in self.surfaces:
if not s.locate_from_cache(frame.index):
s.locate(self.markers)
if s.detected:
pass
# events.append({'type':'marker_ref_surface','name':s.name,'uid':s.uid,'m_to_screen':s.m_to_screen,'m_from_screen':s.m_from_screen, 'timestamp':frame.timestamp,'gaze_on_srf':s.gaze_on_srf})
if self.mode == "Show marker IDs":
draw_markers(frame.img,self.markers)
# edit surfaces by user
if self.mode == "Surface edit mode":
window = glfwGetCurrentContext()
pos = glfwGetCursorPos(window)
pos = normalize(pos,glfwGetWindowSize(window),flip_y=True)
for s,v_idx in self.edit_surfaces:
if s.detected:
new_pos = s.img_to_ref_surface(np.array(pos))
s.move_vertex(v_idx,new_pos)
s.cache = None
self.heatmap = None
else:
# update srf with no or invald cache:
for s in self.surfaces:
if s.cache == None:
s.init_cache(self.cache)
#allow surfaces to open/close windows
for s in self.surfaces:
if s.window_should_close:
s.close_window()
if s.window_should_open:
s.open_window()
def init_marker_cacher(self):
forking_enable(0) #for MacOs only
from marker_detector_cacher import fill_cache
visited_list = [False if x == False else True for x in self.cache]
video_file_path = os.path.join(self.g_pool.rec_dir,'world.mkv')
if not os.path.isfile(video_file_path):
video_file_path = os.path.join(self.g_pool.rec_dir,'world.avi')
self.cache_queue = Queue()
self.cacher_seek_idx = Value('i',0)
self.cacher_run = Value(c_bool,True)
self.cacher = Process(target=fill_cache, args=(visited_list,video_file_path,self.cache_queue,self.cacher_seek_idx,self.cacher_run))
self.cacher.start()
def update_marker_cache(self):
while not self.cache_queue.empty():
idx,c_m = self.cache_queue.get()
self.cache.update(idx,c_m)
for s in self.surfaces:
s.update_cache(self.cache,idx=idx)
def seek_marker_cacher(self,idx):
self.cacher_seek_idx.value = idx
def close_marker_cacher(self):
self.update_marker_cache()
self.cacher_run.value = False
self.cacher.join()
def gl_display(self):
"""
Display marker and surface info inside world screen
"""
self.gl_display_cache_bars()
for s in self.surfaces:
s.gl_display_in_window(self.g_pool.image_tex)
if self.mode == "Show Markers and Frames":
for m in self.markers:
hat = np.array([[[0,0],[0,1],[1,1],[1,0],[0,0]]],dtype=np.float32)
hat = cv2.perspectiveTransform(hat,m_marker_to_screen(m))
draw_polyline(hat.reshape((5,2)),color=RGBA(0.1,1.,1.,.3),line_type=GL_POLYGON)
draw_polyline(hat.reshape((5,2)),color=RGBA(0.1,1.,1.,.6))
for s in self.surfaces:
s.gl_draw_frame(self.img_shape)
if self.mode == "Surface edit mode":
for s in self.surfaces:
s.gl_draw_frame(self.img_shape)
s.gl_draw_corners()
if self.mode == "Show Heatmaps":
for s in self.surfaces:
s.gl_display_heatmap()
if self.mode == "Show Metrics":
#todo: draw a backdrop to represent the gaze that is not on any surface
for s in self.surfaces:
#draw a quad on surface with false color of value.
s.gl_display_metrics()
def gl_display_cache_bars(self):
"""
"""
padding = 20.
# Lines for areas that have been cached
cached_ranges = []
for r in self.cache.visited_ranges: # [[0,1],[3,4]]
cached_ranges += (r[0],0),(r[1],0) #[(0,0),(1,0),(3,0),(4,0)]
# Lines where surfaces have been found in video
cached_surfaces = []
for s in self.surfaces:
found_at = []
if s.cache is not None:
for r in s.cache.positive_ranges: # [[0,1],[3,4]]
found_at += (r[0],0),(r[1],0) #[(0,0),(1,0),(3,0),(4,0)]
cached_surfaces.append(found_at)
glMatrixMode(GL_PROJECTION)
glPushMatrix()
glLoadIdentity()
width,height = self.win_size
h_pad = padding * (self.cache.length-2)/float(width)
v_pad = padding* 1./(height-2)
glOrtho(-h_pad, (self.cache.length-1)+h_pad, -v_pad, 1+v_pad,-1,1) # ranging from 0 to cache_len-1 (horizontal) and 0 to 1 (vertical)
glMatrixMode(GL_MODELVIEW)
glPushMatrix()
glLoadIdentity()
color = RGBA(8.,.6,.2,8.)
draw_polyline(cached_ranges,color=color,line_type=GL_LINES,thickness=4)
color = RGBA(0.,.7,.3,8.)
for s in cached_surfaces:
glTranslatef(0,.02,0)
draw_polyline(s,color=color,line_type=GL_LINES,thickness=2)
glMatrixMode(GL_PROJECTION)
glPopMatrix()
glMatrixMode(GL_MODELVIEW)
glPopMatrix()
def save_surface_statsics_to_file(self):
in_mark = self.g_pool.trim_marks.in_mark
out_mark = self.g_pool.trim_marks.out_mark
"""
between in and out mark
report: gaze distribution:
- total gazepoints
- gaze points on surface x
- gaze points not on any surface
report: surface visisbility
- total frames
- surface x visible framecount
surface events:
frame_no, ts, surface "name", "id" enter/exit
for each surface:
gaze_on_name_id.csv
positions_of_name_id.csv
"""
section = slice(in_mark,out_mark)
metrics_dir = os.path.join(self.g_pool.rec_dir,"metrics_%s-%s"%(in_mark,out_mark))
logger.info("exporting metrics to %s"%metrics_dir)
if os.path.isdir(metrics_dir):
logger.info("Will overwrite previous export for this section")
else:
try:
os.mkdir(metrics_dir)
except:
logger.warning("Could not make metrics dir %s"%metrics_dir)
return
with open(os.path.join(metrics_dir,'surface_visibility.csv'),'wb') as csvfile:
csv_writer = csv.writer(csvfile, delimiter='\t',quotechar='|', quoting=csv.QUOTE_MINIMAL)
# surface visibility report
frame_count = len(self.g_pool.timestamps[section])
csv_writer.writerow(('frame_count',frame_count))
csv_writer.writerow((''))
csv_writer.writerow(('surface_name','visible_frame_count'))
for s in self.surfaces:
if s.cache == None:
logger.warning("The surface is not cached. Please wait for the cacher to collect data.")
return
visible_count = s.visible_count_in_section(section)
csv_writer.writerow( (s.name, visible_count) )
logger.info("Created 'surface_visibility.csv' file")
with open(os.path.join(metrics_dir,'surface_gaze_distribution.csv'),'wb') as csvfile:
csv_writer = csv.writer(csvfile, delimiter='\t',quotechar='|', quoting=csv.QUOTE_MINIMAL)
# gaze distribution report
gaze_in_section = list(chain(*self.g_pool.gaze_positions_by_frame[section]))
not_on_any_srf = set([gp['timestamp'] for gp in gaze_in_section])
csv_writer.writerow(('total_gaze_point_count',len(gaze_in_section)))
csv_writer.writerow((''))
csv_writer.writerow(('surface_name','gaze_count'))
for s in self.surfaces:
gaze_on_srf = s.gaze_on_srf_in_section(section)
gaze_on_srf = set([gp["timestamp"] for gp in gaze_on_srf])
not_on_any_srf -= gaze_on_srf
csv_writer.writerow( (s.name, len(gaze_on_srf)) )
csv_writer.writerow(('not_on_any_surface', len(not_on_any_srf) ) )
logger.info("Created 'surface_gaze_distribution.csv' file")
with open(os.path.join(metrics_dir,'surface_events.csv'),'wb') as csvfile:
csv_writer = csv.writer(csvfile, delimiter='\t',quotechar='|', quoting=csv.QUOTE_MINIMAL)
# surface events report
csv_writer.writerow(('frame_number','timestamp','surface_name','surface_uid','event_type'))
events = []
for s in self.surfaces:
for enter_frame_id,exit_frame_id in s.cache.positive_ranges:
events.append({'frame_id':enter_frame_id,'srf_name':s.name,'srf_uid':s.uid,'event':'enter'})
events.append({'frame_id':exit_frame_id,'srf_name':s.name,'srf_uid':s.uid,'event':'exit'})
events.sort(key=lambda x: x['frame_id'])
for e in events:
csv_writer.writerow( ( e['frame_id'],self.g_pool.timestamps[e['frame_id']],e['srf_name'],e['srf_uid'],e['event'] ) )
logger.info("Created 'surface_events.csv' file")
for s in self.surfaces:
# per surface names:
surface_name = '_'+s.name.replace('/','')+'_'+s.uid
# save surface_positions as pickle file
save_object(s.cache.to_list(),os.path.join(metrics_dir,'srf_positions'+surface_name))
#save surface_positions as csv
with open(os.path.join(metrics_dir,'srf_positons'+surface_name+'.csv'),'wb') as csvfile:
csv_writer =csv.writer(csvfile, delimiter='\t',quotechar='|', quoting=csv.QUOTE_MINIMAL)
csv_writer.writerow(('frame_idx','timestamp','m_to_screen','m_from_screen','detected_markers'))
for idx,ts,ref_srf_data in zip(range(len(self.g_pool.timestamps)),self.g_pool.timestamps,s.cache):
if in_mark <= idx <= out_mark:
if ref_srf_data is not None and ref_srf_data is not False:
csv_writer.writerow( (idx,ts,ref_srf_data['m_to_screen'],ref_srf_data['m_from_screen'],ref_srf_data['detected_markers']) )
# save gaze on srf as csv.
with open(os.path.join(metrics_dir,'gaze_positions_on_surface'+surface_name+'.csv'),'wb') as csvfile:
csv_writer = csv.writer(csvfile, delimiter='\t',quotechar='|', quoting=csv.QUOTE_MINIMAL)
csv_writer.writerow(('world_frame_idx','world_timestamp','eye_timestamp','x_norm','y_norm','x_scaled','y_scaled','on_srf'))
for idx,ts,ref_srf_data in zip(range(len(self.g_pool.timestamps)),self.g_pool.timestamps,s.cache):
if in_mark <= idx <= out_mark:
if ref_srf_data is not None and ref_srf_data is not False:
for gp in s.gaze_on_srf_by_frame_idx(idx,ref_srf_data['m_from_screen']):
gp_x,gp_y = gp['norm_pos']
on_srf = (0 <= gp_x <= 1) and (0 <= gp_y <= 1)
csv_writer.writerow( (idx,ts,gp['timestamp'],gp_x,gp_y,gp_x*s.real_world_size['x'],gp_x*s.real_world_size['y'],on_srf) )
logger.info("Saved surface positon data and gaze on surface data for '%s' with uid:'%s'"%(s.name,s.uid))
if s.heatmap is not None:
logger.info("Saved Heatmap as .png file.")
cv2.imwrite(os.path.join(metrics_dir,'heatmap'+surface_name+'.png'),s.heatmap)
# if s.detected and self.img is not None:
# #let save out the current surface image found in video
# #here we get the verts of the surface quad in norm_coords
# mapped_space_one = np.array(((0,0),(1,0),(1,1),(0,1)),dtype=np.float32).reshape(-1,1,2)
# screen_space = cv2.perspectiveTransform(mapped_space_one,s.m_to_screen).reshape(-1,2)
# #now we convert to image pixel coods
# screen_space[:,1] = 1-screen_space[:,1]
# screen_space[:,1] *= self.img.shape[0]
# screen_space[:,0] *= self.img.shape[1]
# s_0,s_1 = s.real_world_size
# #no we need to flip vertically again by setting the mapped_space verts accordingly.
# mapped_space_scaled = np.array(((0,s_1),(s_0,s_1),(s_0,0),(0,0)),dtype=np.float32)
# M = cv2.getPerspectiveTransform(screen_space,mapped_space_scaled)
# #here we do the actual perspactive transform of the image.
# srf_in_video = cv2.warpPerspective(self.img,M, (int(s.real_world_size['x']),int(s.real_world_size['y'])) )
# cv2.imwrite(os.path.join(metrics_dir,'surface'+surface_name+'.png'),srf_in_video)
# logger.info("Saved current image as .png file.")
# else:
# logger.info("'%s' is not currently visible. Seek to appropriate frame and repeat this command."%s.name)
def get_init_dict(self):
if self.menu:
d = {'menu_conf':self.menu.configuration,'mode':self.mode}
else:
d = {'menu_conf':self.menu_conf,'mode':self.mode}
return d
def cleanup(self):
""" called when the plugin gets terminated.
This happens either voluntarily or forced.
if you have a GUI or glfw window destroy it here.
"""
self.surface_definitions["offline_square_marker_surfaces"] = [rs.save_to_dict() for rs in self.surfaces if rs.defined]
self.surface_definitions.close()
self.close_marker_cacher()
self.persistent_cache["marker_cache"] = self.cache.to_list()
self.persistent_cache.close()
for s in self.surfaces:
s.close_window()
self.deinit_gui()
def output_monitor_queue(out):
q = Queue()
p = Process(target=enqueue_output, args=(out, q))
p.start()
return q, p
class Offline_Screen_Tracker(Offline_Surface_Tracker,Screen_Tracker):
"""
Special version of screen tracker for use with videofile source.
It uses a seperate process to search all frames in the world.avi file for markers.
- self.cache is a list containing marker positions for each frame.
- self.surfaces[i].cache is a list containing surface positions for each frame
Both caches are build up over time. The marker cache is also session persistent.
See marker_tracker.py for more info on this marker tracker.
"""
def __init__(self,g_pool,mode="Show Markers and Surfaces", min_marker_perimeter = 100,robust_detection=True, matrix=None):
#self.g_pool = g_pool
Trim_Marks_Extended_Exist = False
for p in g_pool.plugins:
if p.class_name == 'Trim_Marks_Extended':
Trim_Marks_Extended_Exist = True
break
if not Trim_Marks_Extended_Exist:
from trim_marks_patch import Trim_Marks_Extended
g_pool.plugins.add(Trim_Marks_Extended)
del Trim_Marks_Extended
# heatmap
self.matrix = matrix
self.heatmap_blur = True
self.heatmap_blur_gradation = 0.12
self.heatmap_colormap = "viridis"
self.gaze_correction_block_size = '1000'
self.gaze_correction_min_confidence = 0.98
self.gaze_correction_k = 2
self.heatmap_use_kdata = False
super(Offline_Screen_Tracker, self).__init__(g_pool,mode,min_marker_perimeter,robust_detection)
def load_surface_definitions_from_file(self):
self.surface_definitions = Persistent_Dict(os.path.join(self.g_pool.rec_dir,'surface_definitions'))
if self.surface_definitions.get('offline_square_marker_surfaces',[]) != []:
logger.debug("Found ref surfaces defined or copied in previous session.")
self.surfaces = [Offline_Reference_Surface_Extended(self.g_pool,saved_definition=d) for d in self.surface_definitions.get('offline_square_marker_surfaces',[]) if isinstance(d,dict)]
elif self.surface_definitions.get('realtime_square_marker_surfaces',[]) != []:
logger.debug("Did not find ref surfaces def created or used by the user in player from earlier session. Loading surfaces defined during capture.")
self.surfaces = [Offline_Reference_Surface_Extended(self.g_pool,saved_definition=d) for d in self.surface_definitions.get('realtime_square_marker_surfaces',[]) if isinstance(d,dict)]
else:
logger.debug("No surface defs found. Please define using GUI.")
self.surfaces = []
def init_gui(self):
self.menu = ui.Scrolling_Menu('Offline Screen Tracker')
self.g_pool.gui.append(self.menu)
self.update_gui_markers()
self.on_window_resize(glfwGetCurrentContext(),*glfwGetWindowSize(glfwGetCurrentContext()))
def init_marker_cacher(self):
forking_enable(0) #for MacOs only
from screen_detector_cacher import fill_cache
visited_list = [False if x == False else True for x in self.cache]
video_file_path = self.g_pool.capture.source_path
timestamps = self.g_pool.capture.timestamps
self.cache_queue = Queue()
self.cacher_seek_idx = Value('i',0)
self.cacher_run = Value(c_bool,True)
self.cacher = Process(target=fill_cache, args=(visited_list,video_file_path,timestamps,self.cache_queue,self.cacher_seek_idx,self.cacher_run,self.min_marker_perimeter_cacher))
self.cacher.start()
def update_marker_cache(self):
while not self.cache_queue.empty():
idx,c_m = self.cache_queue.get()
self.cache.update(idx,c_m)
for s in self.surfaces:
s.update_cache(self.cache,camera_calibration=self.camera_calibration,min_marker_perimeter=self.min_marker_perimeter,min_id_confidence=self.min_id_confidence,idx=idx)
# if self.cacher_run.value == False:
# self.recalculate()
# function TMatrixForm.GetMatrix(AMonitor: integer): TStmMatrix;
# var
# i,j,
# LRowCount,LColCount,SHeight,SWidth,SYGap,SXGap,SLeft,STop: integer;
# begin
# SHeight := 150;
# SWidth := 150;
# SYGap := 100;
# SXGap := 100;
# SLeft := 0;
# STop := 0;
# LRowCount := 3;
# LColCount := 3;
# SetLength(Result, LRowCount,LColCount);
# for i := Low(Result) to High(Result) do
# begin
# SLeft := ((SWidth+SXGap)*i)+(Screen.Monitors[AMonitor].Width div 2)
# -(((SWidth+SXGap)*LColCount) div 2)+((SXGap) div 2);
# for j:= Low(Result[i]) to High(Result[i]) do
# begin
# STop := ((SHeight+SYGap)*j)+(Screen.Monitors[AMonitor].Height div 2)
# -(((SHeight+SYGap)*LRowCount) div 2)+(SYGap div 2);
# Result[i][j].Left := SLeft;
# Result[i][j].Top := STop;
# Result[i][j].Width := SWidth;
# Result[i][j].Height := SHeight;
# end;
# end;
# end;
def matrix_segmentation(self):
if not self.mode == 'Show Markers and Surfaces':
logger.error('Please, select the "Show Markers and Surfaces" option at the Mode Selector.')
return
screen_width = 1280
screen_height = -768
def move_srf_to_stm(s,p):
"""
######### #########
# 0 . 1 # # lt.rt #
# . . # # . . #
# 3 . 2 # # lb.rb #
# uv #### #########
#########
# 3 . 2 #
# . . #
# 0 . 1 #
# sv ####
"""
sw = 150./screen_width
sh = 150./screen_height
before = s.markers.values()[0].uv_coords
#before = np.array(((0,0),(1,0),(1,1),(0,1)),dtype=np.float32)
after = before.copy()
after[0] = p
after[1] = p + np.array([sw,0])
after[2] = p + np.array([sw,sh])
after[3] = p + np.array([0,sh])
transform = cv2.getPerspectiveTransform(after,before)
for m in s.markers.values():
m.uv_coords = cv2.perspectiveTransform(m.uv_coords,transform)
n = 3
namei = 0
for i in xrange(0,n):
for j in xrange(0,n):
namei += 1
sname = 'S'+str(namei)
for s in self.surfaces:
if s.name == sname:
move_srf_to_stm(s, self.matrix[i][j])
for s in self.surfaces:
s.invalidate()
self.update_gui_markers()
def add_matrix_surfaces(self):
if not self.mode == 'Show Markers and Surfaces':
logger.error('Please, select the "Show Markers and Surfaces" option at the Mode Selector.')
return
screen_width = 1280
screen_height = -768
def midpoint(v1, v2):
return np.array([(v1[0]+v2[0])/2,(v1[1]+v2[1])/2])
def get_m(s, n=3):
def get_coord(index,midxy, y=False):
if y:
rws = screen_height # must flip
else:
rws = screen_width
return ((250./rws)*index)+midxy-(((250./rws)*n)/2)+((100./rws)/2)
rwsx = screen_width
rwsy = screen_height
lt = s.left_top
rt = s.right_top
lb = s.left_bottom
rb = s.right_bottom
m = [[[] for _ in xrange(0,n)] for _ in xrange(0,n)]
for j in xrange(0,n):
xt = get_coord(j,midpoint(lt,rt)[0])
# yt = get_coord(j,midpoint(lt,rt)[1])
# xb = get_coord(j,midpoint(lb,rb)[0])
# yb = get_coord(j,midpoint(lb,rb)[1])
for i in xrange(0,n):
yt = get_coord(i,midpoint(lt,rb)[1],True)
# yt = get_coord(i,midpoint([xt,yt],[xb,yb])[1],True)
m[i][j] = np.array([xt, yt])
return m
def create_surface(name):
self.surfaces.append(Offline_Reference_Surface_Extended(self.g_pool))
self.surfaces[-1].name = name
self.surfaces[-1].real_world_size['x'] = 150
self.surfaces[-1].real_world_size['y'] = 150
# self.surfaces[-1].markers = markers
n = 3
for s in self.surfaces:
if s.name == 'Screen':
self.matrix = get_m(s,n)
# markers = s.markers
for i in xrange(0,n*n):
create_surface('S'+str(i+1))
for s in self.surfaces:
s.invalidate()
self.update_gui_markers()
def screen_segmentation(self):
"""
no standards here, uv_coords ordering differing from the surface vertice one.
0 . 1
. .
3 . 2
uv
3 . 2
. .
0 . 1
sv
"""
if not self.mode == 'Show Markers and Surfaces':
logger.error('Please, select the "Show Markers and Surfaces" option at the Mode Selector.')
return
correcly_named = [False, False]
for s in self.surfaces:
if s.name == 'Left':
correcly_named[0] = (s.name == 'Left')
if s.name == 'Right':
correcly_named[1] = (s.name == 'Right')
if not (correcly_named[0] and correcly_named[1]):
logger.error('Please, create two identical surfaces and name them as "Left" and "Right".')
return
for s in self.surfaces:
s.real_world_size['x'] = s.real_world_size['x']/2.
lt = s.left_top
rt = s.right_top
lb = s.left_bottom
rb = s.right_bottom
midtop = np.array([(lt[0]+rt[0])/2,(lt[1]+rt[1])/2])
midbottom = np.array([(lb[0]+rb[0])/2,(lb[1]+rb[1])/2])
if s.name == 'Left':
s.right_top = midtop
s.right_bottom = midbottom
if s.name == 'Right':
s.left_top = midtop
s.left_bottom = midbottom
self.update_gui_markers()
def raise_bug(self):
raise 's'
def update_gui_markers(self):
def close():
self.alive = False
def set_min_marker_perimeter(val):
self.min_marker_perimeter = val
self.notify_all_delayed({'subject':'min_marker_perimeter_changed'},delay=1)
self.menu.elements[:] = []
self.menu.append(ui.Button('Close',close))
self.menu.append(ui.Slider('min_marker_perimeter',self,min=20,max=500,step=1,setter=set_min_marker_perimeter))
self.menu.append(ui.Info_Text('The offline screen tracker will look for a screen for each frame of the video. By default it uses surfaces defined in capture. You can change and add more surfaces here.'))
self.menu.append(ui.Selector('mode',self,setter=self.set_mode,label='Mode',selection=["Show Markers and Surfaces","Show marker IDs","Show Heatmaps","Show Gaze Cloud", "Show Kmeans Correction","Show Mean Correction","Show Metrics"] ))
if self.mode == 'Show Markers and Surfaces':
self.menu.append(ui.Info_Text('To split the screen in two (left,right) surfaces 1) add two surfaces; 2) name them as "Left" and "Right"; 3) press Left Right segmentation'))
self.menu.append(ui.Button("Left Right segmentation",self.screen_segmentation))
self.menu.append(ui.Button("Matrix segmentation", self.matrix_segmentation))
self.menu.append(ui.Button("Add M surfaces", self.add_matrix_surfaces))
self.menu.append(ui.Button("bug", self.raise_bug))
if self.mode == 'Show Kmeans Correction':
self.menu.append(ui.Info_Text('Gaze Correction requires a non segmented screen. It requires k equally distributed stimuli on the screen.'))
self.menu.append(ui.Text_Input('gaze_correction_block_size',self,label='Block Size'))
self.menu.append(ui.Slider('gaze_correction_min_confidence',self,min=0.0,step=0.01,max=1.0,label='Minimun gaze confidence'))
self.menu.append(ui.Slider('gaze_correction_k',self,min=1,step=1,max=24,label='K clusters'))
if self.mode == 'Show Gaze Cloud':
self.menu.append(ui.Slider('gaze_correction_min_confidence',self,min=0.0,step=0.01,max=1.0,label='Minimun gaze confidence'))
self.menu.append(ui.Slider('gaze_correction_k',self,min=1,step=1,max=24,label='K clusters'))
if self.mode == 'Show Heatmaps':
self.menu.append(ui.Info_Text('Heatmap Settings'))
self.menu.append(ui.Switch('heatmap_blur',self,label='Blur'))
self.menu.append(ui.Slider('heatmap_blur_gradation',self,min=0.01,step=0.01,max=1.0,label='Blur Gradation'))
self.menu.append(ui.Selector('heatmap_colormap',self,label='Color Map',selection=['magma', 'inferno', 'plasma', 'viridis', 'jet']))
self.menu.append(ui.Switch('heatmap_use_kdata',self,label='Use K Data'))
self.menu.append(ui.Info_Text('Select a section. To see heatmap, surface metrics, gaze cloud or gaze correction visualizations, click (re)-calculate gaze distributions. Set "X size" and "Y size" for each surface to see heatmap visualizations.'))
self.menu.append(ui.Button("(Re)-calculate gaze distributions",self.recalculate))
self.menu.append(ui.Info_Text('To use data from all sections to generate visualizations click the next button instead.'))
self.menu.append(ui.Button("(Re)-calculate",self.recalculate_all_sections))
self.menu.append(ui.Button("Add screen surface",lambda:self.add_surface('_')))
self.menu.append(ui.Info_Text('Export gaze metrics. We recalculate metrics for each section when exporting all sections. Press the recalculate button before export the current selected section.'))
self.menu.append(ui.Info_Text("Press the export button or type 'e' to start the export for the current section."))
self.menu.append(ui.Button("Export all sections", self.export_all_sections))
self.menu.append(ui.Info_Text('Requires segmentation plugin.'))
self.menu.append(ui.Button("Export all distances", self.export_all_distances))
self.menu.append(ui.Button("Precision Report", self.precision_report))
self.menu.append(ui.Button("Slice 1.5 - precision", self.export_all_precision))
for s in self.surfaces:
idx = self.surfaces.index(s)
s_menu = ui.Growing_Menu("Surface %s"%idx)
s_menu.collapsed=True
s_menu.append(ui.Text_Input('name',s))
s_menu.append(ui.Text_Input('x',s.real_world_size,label='X size'))
s_menu.append(ui.Text_Input('y',s.real_world_size,label='Y size'))
s_menu.append(ui.Button('Open Debug Window',s.open_close_window))
#closure to encapsulate idx
def make_remove_s(i):
return lambda: self.remove_surface(i)
remove_s = make_remove_s(idx)
s_menu.append(ui.Button('remove',remove_s))
self.menu.append(s_menu)
def set_mode(self, value):
self.mode = value
self.update_gui_markers()
def add_surface(self,_):
self.surfaces.append(Offline_Reference_Surface_Extended(self.g_pool))
self.surfaces[0].name = 'Screen'
self.surfaces[0].real_world_size['x'] = 1280
self.surfaces[0].real_world_size['y'] = 768
# self.surfaces[0].name = 'Left'
# self.surfaces[0].real_world_size['x'] = 1280
# self.surfaces[0].real_world_size['y'] = 768
# self.surfaces[1].name = 'Right'
# self.surfaces[1].real_world_size['x'] = 1280
# self.surfaces[1].real_world_size['y'] = 768
self.update_gui_markers()
# def update(self,frame,events):
# super(Offline_Screen_Tracker, self).update(frame, events)
# # locate surfaces
# for s in self.surfaces:
# if not s.locate_from_cache(frame.index):
# s.locate(self.markers)
# # if s.detected:
# # pass
# # # events.append({'type':'marker_ref_surface','name':s.name,'uid':s.uid,'m_to_screen':s.m_to_screen,'m_from_screen':s.m_from_screen, 'timestamp':frame.timestamp,'gaze_on_srf':s.gaze_on_srf})
def recalculate(self):
pass
# def recalculate(self):
# #super(Offline_Screen_Tracker, self).recalculate()
# # calc heatmaps
# in_mark = self.g_pool.trim_marks.in_mark
# out_mark = self.g_pool.trim_marks.out_mark
# section = slice(in_mark,out_mark)
# for s in self.surfaces:
# if s.defined:
# s.heatmap_blur = self.heatmap_blur
# s.heatmap_blur_gradation = self.heatmap_blur_gradation
# s.heatmap_colormap = self.heatmap_colormap
# s.heatmap_use_kdata = self.heatmap_use_kdata
# s.gaze_correction_block_size = self.gaze_correction_block_size
# s.gaze_correction_min_confidence = self.gaze_correction_min_confidence
# s.gaze_correction_k = self.gaze_correction_k
# s.generate_gaze_cloud(section)
# s.generate_gaze_correction(section)
# s.generate_mean_correction(section)
# s.generate_heatmap(section)
# # calc distirbution accross all surfaces.
# results = []
# for s in self.surfaces:
# gaze_on_srf = s.gaze_on_srf_in_section(section)
# results.append(len(gaze_on_srf))
# self.metrics_gazecount = len(gaze_on_srf)
# if results == []:
# logger.warning("No surfaces defined.")
# return
# max_res = max(results)
# results = np.array(results,dtype=np.float32)
# if not max_res:
# logger.warning("No gaze on any surface for this section!")
# else:
# results *= 255./max_res
# results = np.uint8(results)
# results_c_maps = cv2.applyColorMap(results, cv2.COLORMAP_JET)
# for s,c_map in zip(self.surfaces,results_c_maps):
# heatmap = np.ones((1,1,4),dtype=np.uint8)*125
# heatmap[:,:,:3] = c_map
# s.metrics_texture = Named_Texture()
# s.metrics_texture.update_from_ndarray(heatmap)
def recalculate_all_sections(self):
"""
treats all sections as one
should not be used to overlaid sections
"""
# for now, it requires trim_marks_patch.py
sections_alive = False
for p in self.g_pool.plugins:
if p.class_name == 'Trim_Marks_Extended':
sections_alive = True
if sections_alive:
sections = self.g_pool.trim_marks.sections
for s in self.surfaces:
if s.defined:
# assign user defined variables
s.heatmap_blur = self.heatmap_blur
s.heatmap_blur_gradation = self.heatmap_blur_gradation
s.heatmap_use_kdata = self.heatmap_use_kdata
s.heatmap_colormap = self.heatmap_colormap
s.gaze_correction_block_size = self.gaze_correction_block_size
s.gaze_correction_min_confidence = self.gaze_correction_min_confidence
s.gaze_correction_k = self.gaze_correction_k
# generate visualizations
s.generate_gaze_cloud(sections, True)
s.generate_gaze_correction(sections, True)
s.generate_heatmap(sections, True)
s.generate_mean_correction(sections, True)
logger.info("Recalculate visualizations done.")
else:
logger.error("Trim_Marks_Extended not found. Have you opened it?")
def gl_display(self):
"""
Display marker and surface info inside world screen
"""
super(Offline_Screen_Tracker, self).gl_display()
if self.mode == "Show Gaze Cloud":
for s in self.surfaces:
s.gl_display_gaze_cloud()
if self.mode == "Show Kmeans Correction":
for s in self.surfaces:
s.gl_display_gaze_correction()
if self.mode == "Show Heatmap Correction":
for s in self.surfaces:
s.gl_display_mean_correction()
if self.mode == "Show Mean Correction":
for s in self.surfaces:
s.gl_display_mean_correction()
def precision_report(self, custom_tag=None):
sections_alive = False
if self.g_pool.trim_marks.class_name == 'Trim_Marks_Extended':
sections_alive = True
segmentation = None
for p in self.g_pool.plugins:
if p.class_name == 'Segmentation':
if p.alive:
segmentation = p
break
if (segmentation is not None) and sections_alive:
export_path = os.path.join(self.g_pool.rec_dir,'exports')
save_path = os.path.join(export_path,"precision_report")
if os.path.isdir(save_path):
logger.info("Overwriting data on precision_report")
else:
try:
os.mkdir(save_path)
except:
logger.warning("Could not make dir %s"%save_path)
return
angles,x1,y1 = segmentation.scapp_report['Angle'],segmentation.scapp_report['X1'],segmentation.scapp_report['Y1']
unique_distances = sorted(set(zip(angles,x1,y1)))
unique_responses = sorted(set(segmentation.scapp_report['ExpcResp']))
segmentation.filter_by_expresp = True
segmentation.filter_by_distance = True
segmentation.filter_by_angle = False
segmentation.mode = 'in out pairs'
filtered_gaze = []
metadata=[]
for unique_distance in unique_distances:
segmentation.distance = str(unique_distance)
for unique_response in unique_responses:
(s1, s2, s3) = unique_distance
metadata.append("r_%s_distance_%s-%s-%s"%(unique_response, s1, s2, s3))
segmentation.expected_response = str(unique_response)
segmentation.clean_add_trim()
sections = self.g_pool.trim_marks.sections
gaze_no_confidence = 0
no_surface = 0
all_gaze = []
for s in self.surfaces:
if s.defined:
for sec in sections:
in_mark = sec[0]
out_mark = sec[1]
sec = slice(in_mark,out_mark)
for frame_idx,c_e in enumerate(s.cache[sec]):
if c_e:
frame_idx+=sec.start
for i, gp in enumerate(s.gaze_on_srf_by_frame_idx(frame_idx,c_e['m_from_screen'])):
if gp['base']['confidence'] >= self.gaze_correction_min_confidence:
all_gaze.append({'frame':frame_idx,'i':i,'norm_pos':gp['norm_pos'],'metatag':'%s-%s-%s-%s'%(unique_response, s1, s2, s3)})
else:
gaze_no_confidence += 1
else:
no_surface += 1
if not all_gaze:
logger.error("No Gaze points found.")
metadata.append("No gaze points found.")
return
else:
gaze_count = len(all_gaze)
metadata.append('Found %s frames with no screen/surface.'%no_surface)
metadata.append("Found %s gaze points."%gaze_count)
metadata.append("Removed '{0}' with confidence < '{1}'".format(gaze_no_confidence, self.gaze_correction_min_confidence))
filtered_gaze.append(all_gaze)
if custom_tag:
np.save(os.path.join(save_path,'data_ordered_by_metatag'+custom_tag),filtered_gaze)
else:
np.save(os.path.join(save_path,'data_ordered_by_metatag'),filtered_gaze)
#np.savetxt(os.path.join(save_path,'metadata.txt'),metadata)
segmentation.clean_custom_events()
for unique_distance in unique_distances:
segmentation.distance = str(unique_distance)
for unique_response in unique_responses:
segmentation.expected_response = str(unique_response)
segmentation.add_filtered_events()
segmentation.auto_trim()
filtered_gaze = []
mean_at_zero_cluster = []
norm_gaze = []
for s in self.surfaces:
if s.defined:
for sec in self.g_pool.trim_marks.sections:
section_gaze = []
in_mark = sec[0]
out_mark = sec[1]
sec = slice(in_mark,out_mark)
for frame_idx,c_e in enumerate(s.cache[sec]):
if c_e:
frame_idx+=sec.start
for i, gp in enumerate(s.gaze_on_srf_by_frame_idx(frame_idx,c_e['m_from_screen'])):
trial = segmentation.trial_from_timestamp(gp['base']['timestamp'])
if gp['base']['confidence'] >= self.gaze_correction_min_confidence:
section_gaze.append({'frame':frame_idx,'i':i,'norm_pos':gp['norm_pos'],'trial':trial})
filtered_gaze.append(section_gaze)
if custom_tag:
np.save(os.path.join(save_path,'data_ordered_by_trial'+custom_tag),filtered_gaze)
else:
np.save(os.path.join(save_path,'data_ordered_by_trial'),filtered_gaze)
else:
logger.error("Please, open the segmentation plugin.")
def export_all_precision(self):
segmentation = None
for p in self.g_pool.plugins:
if p.class_name == 'Segmentation':
if p.alive:
segmentation = p
break
segmentation.onset = 0.0
segmentation.offset = 1.5
for status in range(16):
tag = '_%s-%s'%(segmentation.onset,segmentation.offset)
tag = tag.replace('.','-')
logger.info(str(status)+tag)
self.precision_report(tag)
segmentation.onset += 0.1
segmentation.offset -= 0.1
logger.info('end')
def export_all_distances(self):
segmentation = None
for p in self.g_pool.plugins:
if p.class_name == 'Segmentation':
if p.alive:
segmentation = p
break
if segmentation is not None:
angles,x1,y1 = segmentation.scapp_report['Angle'],segmentation.scapp_report['X1'],segmentation.scapp_report['Y1']
unique_items = sorted(set(zip(angles,x1,y1)))
for unique_distance in unique_items:
segmentation.distance = str(unique_distance)
segmentation.clean_add_trim()
in_mark = self.g_pool.trim_marks.in_mark
out_mark = self.g_pool.trim_marks.out_mark
# generate visualizations and data
self.recalculate_all_sections()
export_path = os.path.join(self.g_pool.rec_dir,'exports')
save_path = os.path.join(export_path,"distance_%s-%s-%s"%unique_distance)
if os.path.isdir(save_path):
logger.info("Overwriting data on distance %s-%s-%s"%unique_distance)
else:
try:
os.mkdir(save_path)
except:
logger.warning("Could not make dir %s"%save_path)
return
for s in self.surfaces:
surface_name = '_'+s.name.replace('/','')+'_'+s.uid
if s.heatmap is not None:
logger.info("Saved Heatmap as .png file.")
cv2.imwrite(os.path.join(save_path,'heatmap'+surface_name+'.png'),s.heatmap)
if s.gaze_cloud is not None:
logger.info("Saved Gaze Cloud as .png file.")
cv2.imwrite(os.path.join(save_path,'gaze_cloud'+surface_name+'.png'),s.gaze_cloud)
if s.gaze_correction is not None:
logger.info("Saved Gaze Correction as .png file.")
cv2.imwrite(os.path.join(save_path,'gaze_correction'+surface_name+'.png'),s.gaze_correction)
# export a surface image from the center of the first section for visualization purposes only
self.export_section_image(save_path, s, in_mark, out_mark, os.path.join(save_path,'surface'+surface_name+'.png'))
# if s.gaze_correction_mean is not None:
# logger.info("Saved Gaze Correction Mean as .png file.")
# cv2.imwrite(os.path.join(save_path,'gaze_correction_mean'+surface_name+'.png'),s.gaze_correction_mean)
np.save(os.path.join(save_path,'source_data'),s.output_data)
def export_section_image(self,save_path,s,in_mark,out_mark,surface_path):
# lets save out the current surface image found in video
seek_pos = in_mark + ((out_mark - in_mark)/2)
self.g_pool.capture.seek_to_frame(seek_pos)
new_frame = self.g_pool.capture.get_frame()
frame = new_frame.copy()
self.update(frame, {})
if s.detected and frame.img is not None:
#here we get the verts of the surface quad in norm_coords
mapped_space_one = np.array(((0,0),(1,0),(1,1),(0,1)),dtype=np.float32).reshape(-1,1,2)
screen_space = cv2.perspectiveTransform(mapped_space_one,s.m_to_screen).reshape(-1,2)
#now we convert to image pixel coords
screen_space[:,1] = 1-screen_space[:,1]
screen_space[:,1] *= frame.img.shape[0]
screen_space[:,0] *= frame.img.shape[1]
s_0,s_1 = s.real_world_size['x'], s.real_world_size['y']
#now we need to flip vertically again by setting the mapped_space verts accordingly.
mapped_space_scaled = np.array(((0,s_1),(s_0,s_1),(s_0,0),(0,0)),dtype=np.float32)
M = cv2.getPerspectiveTransform(screen_space,mapped_space_scaled)
#here we do the actual perspactive transform of the image.
srf_in_video = cv2.warpPerspective(frame.img,M, (int(s.real_world_size['x']),int(s.real_world_size['y'])) )
cv2.imwrite(surface_path,srf_in_video)
logger.info("Saved: '%s'"%surface_path)
else:
logger.info("'%s' is not currently visible. Seek to appropriate frame and repeat this command."%s.name)
def export_all_sections(self):
for section in self.g_pool.trim_marks.sections:
self.g_pool.trim_marks.focus = self.g_pool.trim_marks.sections.index(section)
in_mark = self.g_pool.trim_marks.in_mark
out_mark = self.g_pool.trim_marks.out_mark
export_path = export_path = os.path.join(self.g_pool.rec_dir,'exports')
if os.path.isdir(export_path):
logger.info("Will overwrite export_path")
else:
try:
os.mkdir(export_path)
except:
logger.warning("Could not make metrics_dir %s"%export_path)
return
metrics_dir = os.path.join(export_path,"%s-%s"%(in_mark,out_mark))
if os.path.isdir(metrics_dir):
logger.info("Will overwrite metrics_dir")
else:
try:
os.mkdir(metrics_dir)
except:
logger.warning("Could not make metrics_dir %s"%metrics_dir)
return
self.recalculate()
self.save_surface_statsics_to_file(slice(in_mark,out_mark), metrics_dir)
surface_dir = os.path.join(metrics_dir,'surfaces')
for s in self.surfaces:
surface_name = '_'+s.name.replace('/','')+'_'+s.uid
if s.heatmap is not None:
logger.info("Saved Heatmap as .png file.")
cv2.imwrite(os.path.join(surface_dir,'heatmap'+surface_name+'.png'),s.heatmap)
if s.gaze_cloud is not None:
logger.info("Saved Gaze Cloud as .png file.")
cv2.imwrite(os.path.join(surface_dir,'gaze_cloud'+surface_name+'.png'),s.gaze_cloud)
if s.gaze_correction is not None:
logger.info("Saved Gaze Correction as .png file.")
cv2.imwrite(os.path.join(surface_dir,'gaze_correction'+surface_name+'.png'),s.gaze_correction)
surface_path = os.path.join(surface_dir,'surface'+surface_name+'.png')
# export a surface image from the center of the section for visualization purposes only
self.export_section_image(surface_dir, s, in_mark, out_mark, surface_path)
# lets create alternative versions of the surfaces *.pngs
src1 = cv2.imread(surface_path)
for g in s.output_data['gaze']:
cv2.circle(src1, (int(g[0]),int(g[1])), 3, (0, 0, 0), 0)
for c in s.output_data['kmeans']:
cv2.circle(src1, (int(c[0]),int(c[1])), 5, (0, 0, 255), -1)
cv2.imwrite(os.path.join(surface_dir,'surface-gaze_cloud'+surface_name+'.png'),src1)
np.savetxt(os.path.join(surface_dir,'surface-gaze_cloud'+surface_name+'.txt'), s.output_data['gaze'])
#src2 = cv2.imread(os.path.join(surface_dir,'heatmap'+surface_name+'.png'))
#dst = cv2.addWeighted(src1, .9, src2, .1, 0.0);
#cv2.imwrite(os.path.join(surface_dir,'surface-heatmap'+surface_name+'.png'),dst)
self.g_pool.capture.seek_to_frame(in_mark)
logger.info("Done exporting reference surface data.")
def export_raw_data(self):
"""
.surface_gaze_positions - gaze_timestamp, surface_norm_x, surface_norm_y
"""
sections_alive = False
if self.g_pool.trim_marks.class_name == 'Trim_Marks_Extended':
sections_alive = True
segmentation = None
for p in self.g_pool.plugins:
if p.class_name == 'Segmentation':
if p.alive:
segmentation = p
break
def get_init_dict(self):
return {'mode':self.mode,
'matrix':self.matrix}
def on_notify(self,notification):
if notification['subject'] == 'gaze_positions_changed':
logger.info('Gaze positions changed. Please, recalculate.')
#self.recalculate()
if notification['subject'] == 'gaze_positions_changed':
logger.info('Gaze postions changed. Please, recalculate..')
#self.recalculate()
elif notification['subject'] == 'surfaces_changed':
logger.info('Surfaces changed. Please, recalculate..')
#self.recalculate()
elif notification['subject'] == 'min_marker_perimeter_changed':
logger.info('min_marker.. not implemented')
#logger.info('Min marper perimeter adjusted. Re-detecting surfaces.')
#self.invalidate_surface_caches()
elif notification['subject'] is "should_export":
logger.info('should_export.. not implemented')
