def extract_data_subjects(
subjects,
condition,
type=0
): #Extract the data for n number de subjects and condition (silently or aloudly)
# both lists of subjects and conditions must be string,
# type = 0 returns the sentence data (raw),
# type = 1 returns the eye and rt (times),
# type = 2 returns eye data (fixations and saccades) only for the sentences (without wait times),
# type = 3 returns eye data (fixations and saccades) for all images (with wait times)
usr_sent = {}
usr_time = {}
cond = {}
times = {}
usr_eye_data = {}
eye_data = {}
#Windows
#dir= 'C:\Users\JARS\Dropbox'
#LINUX
#dir= '/home/jars/Dropbox'
#PC LAB
dir = 'C:\Users\CG\Dropbox'
for i in subjects:
for h in condition:
filename_eye = os.path.join(dir, 'data_exp', str(i), str(h),
'raw_eye.txt')
filename_rt = os.path.join(dir, 'data_exp', str(i), str(h),
'rt.txt')
st, x, y = import_eyedata(filename_eye)
st_rt, rt = import_rtdata(filename_rt)
raw = extracting_eye(st, x, y, st_rt, rt)
cond[h] = raw.copy()
if type == 1: #extracting the time data (eye and rt)
times[h] = {}
times[h]['time_eye'] = []
times[h]['time_wait'] = []
times[h]['time_key'] = []
for k in range(0, len(rt) - 1):
w = (st_rt[k + 1] - st_rt[k])
z = ((raw[k + 1]['st'][-1] - raw[k + 1]['st'][0]))
if rt[k] == 0 and rt[k + 1] == 1:
times[h]['time_wait'].append(
str(np.round(w, 2)).replace('.', ','))
else:
times[h]['time_key'].append(
str(np.round(w, 2)).replace('.', ','))
times[h]['time_eye'].append(
str(np.round(z, 2)).replace('.', ','))
ult = (st[-1] - st_rt[-1])
times[h]['time_wait'].append(
str(np.round(ult, 2)).replace('.', ','))
elif type == 2: #extracting eye data for each sentence (80) Without wait
eye_data[h] = {}
eye_data[h]['reading'] = []
for j in range(0, len(rt) - 1):
if rt[j] == 0 and rt[j + 1] == 1:
pass
else:
eye_data[h][j + 1] = {}
eye_data[h][j + 1]['fixations'] = []
eye_data[h][j + 1]['saccades'] = []
eye_data[h][j + 1]['dur_fix'] = []
eye_data[h][j + 1]['dur_sacc'] = []
eye_data[h][j + 1]['amplitude'] = []
st = np.array(raw[j + 1]['st'])
x = np.array(raw[j + 1]['x'])
y = np.array(raw[j + 1]['y'])
Sfix, Efix = fixation_detection(x, y, st)
Ssac, Esac, ampl = saccade_detection(x, y, st)
count_fix = 0
#Center wrong fixation detection
if len(Efix) is not 0:
#If the three first saccades are in the center (due to the recall time)
if (Efix[0][3] <= 1355 and Efix[0][3] >= 1130
) and (Efix[0][4] <= 760
and Efix[0][4] >= 596): #center area
Efix = Efix[1:]
Esac = Esac[1:]
Sfix = Sfix[1:]
Ssac = Ssac[1:]
ampl = ampl[1:]
else:
#print(["Wrong data A", j+1])
eye_data[h]['reading'].append(0)
continue
if len(Efix) is not 0:
#If the three first saccades are in the center (due to the recall time)
if (Efix[0][3] <= 1355 and Efix[0][3] >= 1130
) and (Efix[0][4] <= 760
and Efix[0][4] >= 596): #center area
Efix = Efix[1:]
Esac = Esac[1:]
Sfix = Sfix[1:]
Ssac = Ssac[1:]
ampl = ampl[1:]
else:
#print(["Wrong data B", j+1])
eye_data[h]['reading'].append(0)
continue
if len(Efix) is not 0:
#If the three first saccades are in the center (due to the recall time)
if (Efix[0][3] <= 1355 and Efix[0][3] >= 1130
) and (Efix[0][4] <= 760
and Efix[0][4] >= 596): #center area
Efix = Efix[1:]
Esac = Esac[1:]
Sfix = Sfix[1:]
Ssac = Ssac[1:]
ampl = ampl[1:]
else:
#print(["Wrong data C", j+1])
eye_data[h]['reading'].append(0)
continue
if len(Esac) is not 0:
#Double reading and final word saccade detection
for ind in range(len(Esac)):
if (Esac[ind][3] >= 954 and Esac[ind][3] <=
1964) and (Esac[ind][4] >= 730
and Esac[ind][4] <= 850
): #final word area
if (Esac[ind][5] >= 18 and Esac[ind][5] <=
918) and (Esac[ind][6] >= 600
and Esac[ind][6] <= 720):
if (
ind >= 3
): #Saccades index: After the third saccade
ix = len(Esac) - ind # difference
if ix <= 6:
#print(["Eliminating saccades: ", j + 1, Esac[ind][3:]], len(Esac), ind)
Esac = Esac[:-ix]
Ssac = Ssac[:-ix]
ampl = ampl[:-ix]
if ix > 4: # Holding the last fixation
Efix = Efix[:-ix + 2]
Sfix = Sfix[:-ix + 2]
break
else:
eye_data[h]['reading'].append(j +
1)
if len(Efix) is not 0:
for ind in range(len(Efix)):
if (ind <= 3):
if (Efix[ind][3] <= 2200
and Efix[ind][3] >= 400
) and (
Efix[ind][4] <= 850
and Efix[ind][4] >= 730
): # Innitial fixation at final word
if (
Sfix[ind] >
np.mean(Sfix) - np.std(Sfix)
): # is it an aware fixation (longer than the mean of the fixations minus the standart deviation)
# print(["Fixation final word: ", j+1, Sfix[ind], np.mean(Sfix), ind])
#eye_data[h]['reading'].append(2)
break
else:
if (
ind >= 1
): # were multiple short fixations?
# print(["Fixation final word double: ", j + 1, Sfix[ind], np.mean(Sfix), ind])
#eye_data[h]['reading'].append(2)
break
else: # Elimiting wrong fixations.
# print(["Wrong final word fixation: ", j+1, Sfix[ind], np.mean(Sfix), ind])#final word wrong (due the last fixation in the previous sentence)
Efix = Efix[1:]
Esac = Esac[1:]
Sfix = Sfix[1:]
Ssac = Ssac[1:]
ampl = ampl[1:]
#eye_data[h]['reading'].append(4)
eye_data[h][j + 1]['fixations'] = Efix
eye_data[h][j + 1]['saccades'] = Esac
eye_data[h][j + 1]['dur_fix'] = Sfix
eye_data[h][j + 1]['dur_sacc'] = Ssac
eye_data[h][j + 1]['amplitude'] = ampl
elif type == 3: #extracting eye data for all images (sentences + wait time)
eye_data[h] = {}
for k in raw:
eye_data[h][k] = {}
eye_data[h][k]['fixations'] = []
eye_data[h][k]['saccades'] = []
st = np.array(raw[k]['st'])
x = np.array(raw[k]['x'])
y = np.array(raw[k]['y'])
Sfix, Efix = fixation_detection(x, y, st)
Ssac, Esac, ampl = saccade_detection(x, y, st)
eye_data[h][k]['fixations'] = Efix
eye_data[h][k]['saccades'] = Esac
usr_eye_data[i] = eye_data.copy()
eye_data.clear()
usr_time[i] = times.copy()
times.clear()
usr_sent[i] = cond.copy()
cond.clear()
if type == 0:
return usr_sent
elif type == 1:
return usr_time
elif type == 2 or type == 3:
return usr_eye_data
def read_eyetribe(filename, start, stop=None, missing=0.0, debug=False):
"""Returns a list with dicts for every trial. A trial dict contains the
following keys:
x - numpy array of x positions
y - numpy array of y positions
size - numpy array of pupil size
time - numpy array of timestamps, t=0 at trialstart
trackertime- numpy array of timestamps, according to the tracker
events - dict with the following keys:
Sfix - list of lists, each containing [starttime]
Ssac - EMPTY! list of lists, each containing [starttime]
Sblk - list of lists, each containing [starttime]
Efix - list of lists, each containing [starttime, endtime, duration, endx, endy]
Esac - EMPTY! list of lists, each containing [starttime, endtime, duration, startx, starty, endx, endy]
Eblk - list of lists, each containing [starttime, endtime, duration]
msg - list of lists, each containing [time, message]
NOTE: timing is in EyeTribe time!
arguments
filename - path to the file that has to be read
start - trial start string
keyword arguments
stop - trial ending string (default = None)
missing - value to be used for missing data (default = 0.0)
debug - Boolean indicating if DEBUG mode should be on or off;
if DEBUG mode is on, information on what the script
currently is doing will be printed to the console
(default = False)
returns
data - a list with a dict for every trial (see above)
"""
# # # # #
# debug mode
if debug:
def message(msg):
print(msg)
else:
def message(msg):
pass
# # # # #
# file handling
# check if the file exists
if os.path.isfile(filename):
# open file
message("opening file '%s'" % filename)
f = open(filename, 'r')
# raise exception if the file does not exist
else:
raise Exception("Error in read_eyetribe: file '%s' does not exist" % filename)
# read file contents
message("reading file '%s'" % filename)
raw = f.readlines()
# close file
message("closing file '%s'" % filename)
f.close()
# # # # #
# parse lines
# variables
data = []
x = []
y = []
size = []
time = []
trackertime = []
events = {'Sfix':[],'Ssac':[],'Sblk':[],'Efix':[],'Esac':[],'Eblk':[],'msg':[]}
starttime = 0
started = False
trialend = False
# loop through all lines
for i in range(len(raw)):
# string to list
line = raw[i].replace('\n','').replace('\r','').split('\t')
# check if trial has already started
if started:
# only check for stop if there is one
if stop != None:
if (line[0] == 'MSG' and stop in line[3]) or i == len(raw)-1:
started = False
trialend = True
# check for new start otherwise
else:
if start in line:
started = True
trialend = True
# # # # #
# trial ending
if trialend:
message("trialend %d; %d samples found" % (len(data),len(x)))
# trial dict
trial = {}
trial['x'] = numpy.array(x)
trial['y'] = numpy.array(y)
trial['size'] = numpy.array(size)
trial['time'] = numpy.array(time)
trial['trackertime'] = numpy.array(trackertime)
trial['events'] = copy.deepcopy(events)
# events
trial['events']['Sblk'], trial['events']['Eblk'] = blink_detection(trial['x'],trial['y'],trial['trackertime'],missing=missing)
trial['events']['Sfix'], trial['events']['Efix'] = fixation_detection(trial['x'],trial['y'],trial['trackertime'],missing=missing)
trial['events']['Ssac'], trial['events']['Esac'] = saccade_detection(trial['x'],trial['y'],trial['trackertime'],missing=missing)
# add trial to data
data.append(trial)
# reset stuff
x = []
y = []
size = []
time = []
trackertime = []
events = {'Sfix':[],'Ssac':[],'Sblk':[],'Efix':[],'Esac':[],'Eblk':[],'msg':[]}
trialend = False
# check if the current line contains start message
else:
if line[0] == "MSG":
if start in line[3]:
message("trialstart %d" % len(data))
# set started to True
started = True
# find starting time
starttime = int(line[2])
# # # # #
# parse line
if started:
# message lines will start with MSG, followed by a tab, then a
# timestamp, a tab, the time, a tab and the message, e.g.:
# "MSG\t2014-07-01 17:02:33.770\t853589802\tsomething of importance here"
if line[0] == "MSG":
t = int(line[2]) # time
m = line[3] # message
events['msg'].append([t,m])
# regular lines will contain tab separated values, beginning with
# a timestamp, follwed by the values that were asked to be stored
# in the data file. Usually, this comes down to
# timestamp, time, fix, state, rawx, rawy, avgx, avgy, psize,
# Lrawx, Lrawy, Lavgx, Lavgy, Lpsize, Lpupilx, Lpupily,
# Rrawx, Rrawy, Ravgx, Ravgy, Rpsize, Rpupilx, Rpupily
# e.g.:
# '2014-07-01 17:02:33.770, 853589802, False, 7, 512.5897, 510.8104, 614.6975, 614.3327, 16.8657,
# 523.3592, 475.2756, 511.1529, 492.7412, 16.9398, 0.4037, 0.5209,
# 501.8202, 546.3453, 609.3405, 623.2287, 16.7916, 0.5539, 0.5209'
else:
# see if current line contains relevant data
try:
# extract data
x.append(float(line[6]))
y.append(float(line[7]))
size.append(float(line[8]))
time.append(int(line[1])-starttime)
trackertime.append(int(line[1]))
except:
message("line '%s' could not be parsed" % line)
continue # skip this line
# # # # #
# return
return data
def read_eyetribe(filename, start, stop=None, missing=0.0, debug=False):
"""Returns a list with dicts for every trial. A trial dict contains the
following keys:
x - numpy array of x positions
y - numpy array of y positions
size - numpy array of pupil size
time - numpy array of timestamps, t=0 at trialstart
trackertime- numpy array of timestamps, according to the tracker
events - dict with the following keys:
Sfix - list of lists, each containing [starttime]
Ssac - EMPTY! list of lists, each containing [starttime]
Sblk - list of lists, each containing [starttime]
Efix - list of lists, each containing [starttime, endtime, duration, endx, endy]
Esac - EMPTY! list of lists, each containing [starttime, endtime, duration, startx, starty, endx, endy]
Eblk - list of lists, each containing [starttime, endtime, duration]
msg - list of lists, each containing [time, message]
NOTE: timing is in EyeTribe time!
arguments
filename - path to the file that has to be read
start - trial start string
keyword arguments
stop - trial ending string (default = None)
missing - value to be used for missing data (default = 0.0)
debug - Boolean indicating if DEBUG mode should be on or off;
if DEBUG mode is on, information on what the script
currently is doing will be printed to the console
(default = False)
returns
data - a list with a dict for every trial (see above)
"""
# # # # #
# debug mode
if debug:
def message(msg):
print(msg)
else:
def message(msg):
pass
# # # # #
# file handling
# check if the file exists
if os.path.isfile(filename):
# open file
message("opening file '%s'" % filename)
f = open(filename, 'r')
# raise exception if the file does not exist
else:
raise Exception("Error in read_eyetribe: file '%s' does not exist" %
filename)
# read file contents
message("reading file '%s'" % filename)
raw = f.readlines()
# close file
message("closing file '%s'" % filename)
f.close()
# # # # #
# parse lines
# variables
data = []
x = []
y = []
size = []
time = []
trackertime = []
events = {
'Sfix': [],
'Ssac': [],
'Sblk': [],
'Efix': [],
'Esac': [],
'Eblk': [],
'msg': []
}
starttime = 0
started = False
trialend = False
# loop through all lines
for i in range(len(raw)):
# string to list
line = raw[i].replace('\n', '').replace('\r', '').split('\t')
# check if trial has already started
if started:
# only check for stop if there is one
if stop != None:
if (line[0] == 'MSG' and stop in line[3]) or i == len(raw) - 1:
started = False
trialend = True
# check for new start otherwise
else:
if start in line:
started = True
trialend = True
# # # # #
# trial ending
if trialend:
message("trialend %d; %d samples found" % (len(data), len(x)))
# trial dict
trial = {}
trial['x'] = numpy.array(x)
trial['y'] = numpy.array(y)
trial['size'] = numpy.array(size)
trial['time'] = numpy.array(time)
trial['trackertime'] = numpy.array(trackertime)
trial['events'] = copy.deepcopy(events)
# events
trial['events']['Sblk'], trial['events'][
'Eblk'] = blink_detection(trial['x'],
trial['y'],
trial['trackertime'],
missing=missing)
trial['events']['Sfix'], trial['events'][
'Efix'] = fixation_detection(trial['x'],
trial['y'],
trial['trackertime'],
missing=missing)
trial['events']['Ssac'], trial['events'][
'Esac'] = saccade_detection(trial['x'],
trial['y'],
trial['trackertime'],
missing=missing)
# add trial to data
data.append(trial)
# reset stuff
x = []
y = []
size = []
time = []
trackertime = []
events = {
'Sfix': [],
'Ssac': [],
'Sblk': [],
'Efix': [],
'Esac': [],
'Eblk': [],
'msg': []
}
trialend = False
# check if the current line contains start message
else:
if line[0] == "MSG":
if start in line[3]:
message("trialstart %d" % len(data))
# set started to True
started = True
# find starting time
starttime = int(line[2])
# # # # #
# parse line
if started:
# message lines will start with MSG, followed by a tab, then a
# timestamp, a tab, the time, a tab and the message, e.g.:
# "MSG\t2014-07-01 17:02:33.770\t853589802\tsomething of importance here"
if line[0] == "MSG":
t = int(line[2]) # time
m = line[3] # message
events['msg'].append([t, m])
# regular lines will contain tab separated values, beginning with
# a timestamp, follwed by the values that were asked to be stored
# in the data file. Usually, this comes down to
# timestamp, time, fix, state, rawx, rawy, avgx, avgy, psize,
# Lrawx, Lrawy, Lavgx, Lavgy, Lpsize, Lpupilx, Lpupily,
# Rrawx, Rrawy, Ravgx, Ravgy, Rpsize, Rpupilx, Rpupily
# e.g.:
# '2014-07-01 17:02:33.770, 853589802, False, 7, 512.5897, 510.8104, 614.6975, 614.3327, 16.8657,
# 523.3592, 475.2756, 511.1529, 492.7412, 16.9398, 0.4037, 0.5209,
# 501.8202, 546.3453, 609.3405, 623.2287, 16.7916, 0.5539, 0.5209'
else:
# see if current line contains relevant data
try:
# extract data
x.append(float(line[6]))
y.append(float(line[7]))
size.append(float(line[8]))
time.append(int(line[1]) - starttime)
trackertime.append(int(line[1]))
except:
message("line '%s' could not be parsed" % line)
continue # skip this line
# # # # #
# return
return data
def read_idf(filename, start, stop=None, missing=0.0, debug=False):
"""Returns a list with dicts for every trial. A trial dict contains the
following keys:
x - numpy array of x positions
y - numpy array of y positions
size - numpy array of pupil size
time - numpy array of timestamps, t=0 at trialstart
trackertime- numpy array of timestamps, according to the tracker
events - dict with the following keys:
Sfix - list of lists, each containing [starttime]
Ssac - EMPTY! list of lists, each containing [starttime]
Sblk - list of lists, each containing [starttime]
Efix - list of lists, each containing [starttime, endtime, duration, endx, endy]
Esac - EMPTY! list of lists, each containing [starttime, endtime, duration, startx, starty, endx, endy]
Eblk - list of lists, each containing [starttime, endtime, duration]
msg - list of lists, each containing [time, message]
NOTE: timing is in EyeTribe time!
arguments
filename - path to the file that has to be read
start - trial start string
keyword arguments
stop - trial ending string (default = None)
missing - value to be used for missing data (default = 0.0)
debug - Boolean indicating if DEBUG mode should be on or off;
if DEBUG mode is on, information on what the script
currently is doing will be printed to the console
(default = False)
returns
data - a list with a dict for every trial (see above)
"""
# # # # #
# debug mode
if debug:
def message(msg):
print(msg)
else:
def message(msg):
pass
# # # # #
# file handling
# check if the file exists
if os.path.isfile(filename):
# open file
message("opening file '%s'" % filename)
f = open(filename, 'r')
# raise exception if the file does not exist
else:
raise Exception("Error in read_eyetribe: file '%s' does not exist" % filename)
# read file contents
message("reading file '%s'" % filename)
raw = f.readlines()
# close file
message("closing file '%s'" % filename)
f.close()
# # # # #
# parse lines
# variables
data = []
x = []
y = []
size = []
time = []
trackertime = []
events = {'Sfix':[],'Ssac':[],'Sblk':[],'Efix':[],'Esac':[],'Eblk':[],'msg':[]}
starttime = 0
started = False
trialend = False
filestarted = False
# loop through all lines
for i in range(len(raw)):
# string to list
line = raw[i].replace('\n','').replace('\r','').split('\t')
# check if the line starts with '##' (denoting header)
if '##' in line[0]:
# skip processing
continue
elif '##' not in line[0] and not filestarted:
# check the indexes for several key things we want to extract
# (we need to do this, because ASCII outputs of the IDF reader
# are different, based on whatever the user wanted to extract)
timei = line.index("Time")
typei = line.index("Type")
msgi = -1
xi = {'L':None, 'R':None}
yi = {'L':None, 'R':None}
sizei = {'L':None, 'R':None}
if "L POR X [px]" in line:
xi['L'] = line.index("L POR X [px]")
if "R POR X [px]" in line:
xi['R'] = line.index("R POR X [px]")
if "L POR Y [px]" in line:
yi['L'] = line.index("L POR Y [px]")
if "R POR Y [px]" in line:
yi['R'] = line.index("R POR Y [px]")
if "L Dia X [px]" in line:
sizei['L'] = line.index("L Dia X [px]")
if "R Dia X [px]" in line:
sizei['R'] = line.index("R Dia X [px]")
# set filestarted to True, so we don't attempt to extract
# this info on all consecutive lines
filestarted = True
# check if trial has already started
if started:
# only check for stop if there is one
if stop != None:
if (line[typei] == 'MSG' and stop in line[msgi]) or i == len(raw)-1:
started = False
trialend = True
# check for new start otherwise
else:
if start in line or i == len(raw)-1:
started = True
trialend = True
# # # # #
# trial ending
if trialend:
message("trialend %d; %d samples found" % (len(data),len(x)))
# trial dict
trial = {}
trial['x'] = numpy.array(x)
trial['y'] = numpy.array(y)
trial['size'] = numpy.array(size)
trial['time'] = numpy.array(time)
trial['trackertime'] = numpy.array(trackertime)
trial['events'] = copy.deepcopy(events)
# events
trial['events']['Sblk'], trial['events']['Eblk'] = blink_detection(trial['x'],trial['y'],trial['trackertime'],missing=missing)
trial['events']['Sfix'], trial['events']['Efix'] = fixation_detection(trial['x'],trial['y'],trial['trackertime'],missing=missing)
trial['events']['Ssac'], trial['events']['Esac'] = saccade_detection(trial['x'],trial['y'],trial['trackertime'],missing=missing)
# add trial to data
data.append(trial)
# reset stuff
x = []
y = []
size = []
time = []
trackertime = []
events = {'Sfix':[],'Ssac':[],'Sblk':[],'Efix':[],'Esac':[],'Eblk':[],'msg':[]}
trialend = False
# check if the current line contains start message
else:
if line[typei] == "MSG":
if start in line[msgi]:
message("trialstart %d" % len(data))
# set started to True
started = True
# find starting time
starttime = int(line[timei])
# # # # #
# parse line
if started:
# message lines will usually start with a timestamp, followed
# by 'MSG', the trial number and the actual message, e.g.:
# "7818328012 MSG 1 # Message: 3"
if line[typei] == "MSG":
t = int(line[timei]) # time
m = line[msgi] # message
events['msg'].append([t,m])
# regular lines will contain tab separated values, beginning with
# a timestamp, follwed by the values that were chosen to be
# extracted by the IDF converter
else:
# see if current line contains relevant data
try:
# extract data on POR and pupil size
for var in ['x', 'y', 'size']:
exec("vi = %si" % var)
exec("v = %s" % var)
# nothing
if vi['L'] == None and vi['R'] == None:
val = 'not in IDF'
# only left eye
elif vi['L'] != None and vi['R'] == None:
val = float(line[vi['L']])
# only right eye
elif vi['L'] == None and vi['R'] != None:
val = float(line[vi['R']])
# average the two eyes, but only if they both
# contain valid data
elif vi['L'] != None and vi['R'] != None:
if float(line[vi['L']]) == 0:
val = float(line[vi['R']])
elif float(line[vi['R']]) == 0:
val = float(line[vi['L']])
else:
val = (float(line[vi['L']]) + float(line[vi['R']])) / 2.0
v.append(val)
# extract time data
time.append(int(line[timei])-starttime)
trackertime.append(int(line[timei]))
except:
message("line '%s' could not be parsed" % line)
continue # skip this line
# # # # #
# return
return data
import pandas as pd
import detectors, gazeplotter
df = pd.read_csv("80_rawdata.tsv", delimiter="\t", comment="#")
df = df[df["Type"] == "SMP"]
fixations = detectors.fixation_detection(df['L POR X [px]'].to_numpy(), df['L POR Y [px]'].to_numpy(), df['Time'].to_numpy(), maxdist=25, mindur=500)
print(fixations[1])
print(len(fixations[0]))
def read_opengaze(filename, start, stop=None, missing=0.0, debug=False):
"""Returns a list with dicts for every trial. A trial dict contains the
following keys:
x - numpy array of x positions
y - numpy array of y positions
size - numpy array of pupil size
time - numpy array of timestamps, t=0 at trialstart
trackertime- numpy array of timestamps, according to the tracker
events - dict with the following keys:
Sfix - list of lists, each containing [starttime]
Ssac - EMPTY! list of lists, each containing [starttime]
Sblk - list of lists, each containing [starttime]
Efix - list of lists, each containing [starttime, endtime, duration, endx, endy]
Esac - EMPTY! list of lists, each containing [starttime, endtime, duration, startx, starty, endx, endy]
Eblk - list of lists, each containing [starttime, endtime, duration]
msg - list of lists, each containing [time, message]
NOTE: timing is in EyeTribe time!
arguments
filename - path to the file that has to be read
start - trial start string
keyword arguments
stop - trial ending string (default = None)
missing - value to be used for missing data (default = 0.0)
debug - Boolean indicating if DEBUG mode should be on or off;
if DEBUG mode is on, information on what the script
currently is doing will be printed to the console
(default = False)
returns
data - a list with a dict for every trial (see above)
"""
# # # # #
# debug mode
if debug:
def message(msg):
print(msg)
else:
def message(msg):
pass
# # # # #
# file handling
# check if the file exists
if os.path.isfile(filename):
# open file
message("opening file '%s'" % filename)
f = open(filename, 'r')
# raise exception if the file does not exist
else:
raise Exception("Error in read_eyetribe: file '%s' does not exist" %
filename)
# read file contents
message("reading file '%s'" % filename)
raw = f.readlines()
# close file
message("closing file '%s'" % filename)
f.close()
# # # # #
# parse lines
# Parse the header.
header = raw.pop(0)
header = header.replace('\n', '').replace('\r', '').split('\t')
# variables
data = []
x = []
y = []
size = []
time = []
trackertime = []
events = {
'Sfix': [],
'Ssac': [],
'Sblk': [],
'Efix': [],
'Esac': [],
'Eblk': [],
'msg': []
}
starttime = 0
started = False
trialend = False
laststart = None
# loop through all lines
for i in range(len(raw)):
# string to list
line = raw[i].replace('\n', '').replace('\r', '').split('\t')
# check if trial has already started
if started:
# only check for stop if there is one
if stop != None:
if (line[header.index("USER")] != '0' and \
stop in line[header.index("USER")]) \
or i == len(raw)-1:
started = False
trialend = True
# check for new start otherwise
else:
if start in line[header.index("USER")] \
or i == len(raw)-1:
# Only start if the current start is more than 1
# sample away from the previous start.
if (laststart == None) or (i != laststart + 1):
started = True
trialend = True
laststart = i
# # # # #
# trial ending
if trialend:
message("trialend %d; %d samples found" % (len(data), len(x)))
# trial dict
trial = {}
trial['x'] = numpy.array(x)
trial['y'] = numpy.array(y)
trial['size'] = numpy.array(size)
trial['time'] = numpy.array(time)
trial['trackertime'] = numpy.array(trackertime)
trial['events'] = copy.deepcopy(events)
# events
trial['events']['Sblk'], trial['events'][
'Eblk'] = blink_detection(trial['x'],
trial['y'],
trial['trackertime'],
missing=missing)
trial['events']['Sfix'], trial['events'][
'Efix'] = fixation_detection(trial['x'],
trial['y'],
trial['trackertime'],
missing=missing)
trial['events']['Ssac'], trial['events'][
'Esac'] = saccade_detection(trial['x'],
trial['y'],
trial['trackertime'],
missing=missing)
# add trial to data
data.append(trial)
# reset stuff
x = []
y = []
size = []
time = []
trackertime = []
events = {
'Sfix': [],
'Ssac': [],
'Sblk': [],
'Efix': [],
'Esac': [],
'Eblk': [],
'msg': []
}
trialend = False
# check if the current line contains start message
else:
if line[header.index("USER")] != '0':
if start in line[header.index("USER")]:
# Only start if the current start is more than 1
# sample away from the previous start.
if (laststart == None) or (i != laststart + 1):
message("trialstart %d" % len(data))
# set started to True
started = True
# find starting time
starttime = int(1000 *
float(line[header.index("TIME")]))
laststart = i
# # # # #
# parse line
if started:
# Messages are encoded in the user variable, which otherwise
# is '0'. NOTE: Sometimes messages repeat in consecutive
# samples, by accident.
if line[header.index("USER")] != '0':
t = int(1000 * float(line[header.index("TIME")])) # time
m = line[header.index("USER")] # message
events['msg'].append([t, m])
# All lines (when obtained through PyOpenGaze or PyGaze)
# should contain the following data:
# CNT, TIME, TIME_TICK,
# FPOGX, FPOGY, FPOGS, FPOGD, FPOGID, FPOGV,
# LPOGX, LPOGY, LPOGV, RPOGX, RPOGY, RPOGV,
# BPOGX, BPOGY, BPOGV,
# LPCX, LPCY, LPD, LPS, LPV, RPCX, RPCY, RPD, RPS, RPV
# LEYEX, LEYEY, LEYEZ, LPUPILD, LPUPILV,
# REYEX, REYEY, REYEZ, RPUPILD, RPUPILV,
# CX, CY, CS, USER
try:
# Compute the size of the pupil.
left = line[header.index("LPV")] == '1'
right = line[header.index("RPV")] == '1'
if left and right:
s = (float(line[header.index("LPD")]) + \
float(line[header.index("RPD")])) / 2.0
elif left and not right:
s = float(line[header.index("LPD")])
elif not left and right:
s = float(line[header.index("RPD")])
else:
s = 0.0
# extract data
x.append(float(line[header.index("BPOGX")]))
y.append(float(line[header.index("BPOGY")]))
size.append(s)
time.append(
int(1000 * float(line[header.index("TIME")])) - starttime)
trackertime.append(
int(1000 * float(line[header.index("TIME")])))
except:
message("line '%s' could not be parsed" % line)
continue # skip this line
# # # # #
# return
return data
# required (untested).
df = pd.read_csv('/path/to/TOBII_output.tsv', sep='\t', skiprows=range(0, 17))
# Define which values to use in the PyGaze functions (corresponding to columns in the output).
x = df['GazePointX']
y = df['GazePointY']
time = df['TimeStamp']
# # # # # # # # # # # # #
# FIXATIONS
# # # # # # # # # # # # #
# Using default max dist in this example, could be updated according to aforementioned calibration report
fixations = detectors.fixation_detection(x,
y,
time,
missing=0.0,
maxdist=25,
mindur=50)
Efix = fixations[1]
df_fix = pd.DataFrame(
Efix, columns=['starttime', 'endtime', 'duration', 'endx', 'endy'])
df_fix['label'] = 'fixation' # add label
# # # # # # # # # # # # #
# SACCADES
# # # # # # # # # # # # #
# Default values except minlen - 20ms, was 5.
saccades = detectors.saccade_detection(x,
y,
time,
def read_idf(filename, start, stop=None, missing=0.0, debug=False):
"""Returns a list with dicts for every trial. A trial dict contains the
following keys:
x - numpy array of x positions
y - numpy array of y positions
size - numpy array of pupil size
time - numpy array of timestamps, t=0 at trialstart
trackertime- numpy array of timestamps, according to the tracker
events - dict with the following keys:
Sfix - list of lists, each containing [starttime]
Ssac - EMPTY! list of lists, each containing [starttime]
Sblk - list of lists, each containing [starttime]
Efix - list of lists, each containing [starttime, endtime, duration, endx, endy]
Esac - EMPTY! list of lists, each containing [starttime, endtime, duration, startx, starty, endx, endy]
Eblk - list of lists, each containing [starttime, endtime, duration]
msg - list of lists, each containing [time, message]
NOTE: timing is in EyeTribe time!
arguments
filename - path to the file that has to be read
start - trial start string
keyword arguments
stop - trial ending string (default = None)
missing - value to be used for missing data (default = 0.0)
debug - Boolean indicating if DEBUG mode should be on or off;
if DEBUG mode is on, information on what the script
currently is doing will be printed to the console
(default = False)
returns
data - a list with a dict for every trial (see above)
"""
# # # # #
# debug mode
if debug:
def message(msg):
print(msg)
else:
def message(msg):
pass
# # # # #
# file handling
# check if the file exists
if os.path.isfile(filename):
# open file
message("opening file '%s'" % filename)
f = open(filename, 'r')
# raise exception if the file does not exist
else:
raise Exception("Error in read_eyetribe: file '%s' does not exist" %
filename)
# read file contents
message("reading file '%s'" % filename)
raw = f.readlines()
# close file
message("closing file '%s'" % filename)
f.close()
# # # # #
# parse lines
# variables
data = []
x = []
y = []
size = []
time = []
trackertime = []
events = {
'Sfix': [],
'Ssac': [],
'Sblk': [],
'Efix': [],
'Esac': [],
'Eblk': [],
'msg': []
}
starttime = 0
started = False
trialend = False
filestarted = False
# loop through all lines
for i in range(len(raw)):
# string to list
line = raw[i].replace('\n', '').replace('\r', '').split('\t')
# check if the line starts with '##' (denoting header)
if '##' in line[0]:
# skip processing
continue
elif '##' not in line[0] and not filestarted:
# check the indexes for several key things we want to extract
# (we need to do this, because ASCII outputs of the IDF reader
# are different, based on whatever the user wanted to extract)
timei = line.index("Time")
typei = line.index("Type")
msgi = -1
xi = {'L': None, 'R': None}
yi = {'L': None, 'R': None}
sizei = {'L': None, 'R': None}
if "L POR X [px]" in line:
xi['L'] = line.index("L POR X [px]")
if "R POR X [px]" in line:
xi['R'] = line.index("R POR X [px]")
if "L POR Y [px]" in line:
yi['L'] = line.index("L POR Y [px]")
if "R POR Y [px]" in line:
yi['R'] = line.index("R POR Y [px]")
if "L Dia X [px]" in line:
sizei['L'] = line.index("L Dia X [px]")
if "R Dia X [px]" in line:
sizei['R'] = line.index("R Dia X [px]")
# set filestarted to True, so we don't attempt to extract
# this info on all consecutive lines
filestarted = True
# check if trial has already started
if started:
# only check for stop if there is one
if stop != None:
if (line[typei] == 'MSG'
and stop in line[msgi]) or i == len(raw) - 1:
started = False
trialend = True
# check for new start otherwise
else:
if start in line or i == len(raw) - 1:
started = True
trialend = True
# # # # #
# trial ending
if trialend:
message("trialend %d; %d samples found" % (len(data), len(x)))
# trial dict
trial = {}
trial['x'] = numpy.array(x)
trial['y'] = numpy.array(y)
trial['size'] = numpy.array(size)
trial['time'] = numpy.array(time)
trial['trackertime'] = numpy.array(trackertime)
trial['events'] = copy.deepcopy(events)
# events
trial['events']['Sblk'], trial['events'][
'Eblk'] = blink_detection(trial['x'],
trial['y'],
trial['trackertime'],
missing=missing)
trial['events']['Sfix'], trial['events'][
'Efix'] = fixation_detection(trial['x'],
trial['y'],
trial['trackertime'],
missing=missing)
trial['events']['Ssac'], trial['events'][
'Esac'] = saccade_detection(trial['x'],
trial['y'],
trial['trackertime'],
missing=missing)
# add trial to data
data.append(trial)
# reset stuff
x = []
y = []
size = []
time = []
trackertime = []
events = {
'Sfix': [],
'Ssac': [],
'Sblk': [],
'Efix': [],
'Esac': [],
'Eblk': [],
'msg': []
}
trialend = False
# check if the current line contains start message
else:
if line[typei] == "MSG":
if start in line[msgi]:
message("trialstart %d" % len(data))
# set started to True
started = True
# find starting time
starttime = int(line[timei])
# # # # #
# parse line
if started:
# message lines will usually start with a timestamp, followed
# by 'MSG', the trial number and the actual message, e.g.:
# "7818328012 MSG 1 # Message: 3"
if line[typei] == "MSG":
t = int(line[timei]) # time
m = line[msgi] # message
events['msg'].append([t, m])
# regular lines will contain tab separated values, beginning with
# a timestamp, follwed by the values that were chosen to be
# extracted by the IDF converter
else:
# see if current line contains relevant data
try:
# extract data on POR and pupil size
for var in ['x', 'y', 'size']:
exec("vi = %si" % var)
exec("v = %s" % var)
# nothing
if vi['L'] == None and vi['R'] == None:
val = 'not in IDF'
# only left eye
elif vi['L'] != None and vi['R'] == None:
val = float(line[vi['L']])
# only right eye
elif vi['L'] == None and vi['R'] != None:
val = float(line[vi['R']])
# average the two eyes, but only if they both
# contain valid data
elif vi['L'] != None and vi['R'] != None:
if float(line[vi['L']]) == 0:
val = float(line[vi['R']])
elif float(line[vi['R']]) == 0:
val = float(line[vi['L']])
else:
val = (float(line[vi['L']]) +
float(line[vi['R']])) / 2.0
v.append(val)
# extract time data
time.append(int(line[timei]) - starttime)
trackertime.append(int(line[timei]))
except:
message("line '%s' could not be parsed" % line)
continue # skip this line
# # # # #
# return
return data
def read_opengaze(filename, start, stop=None, missing=0.0, debug=False):
"""Returns a list with dicts for every trial. A trial dict contains the
following keys:
x - numpy array of x positions
y - numpy array of y positions
size - numpy array of pupil size
time - numpy array of timestamps, t=0 at trialstart
trackertime- numpy array of timestamps, according to the tracker
events - dict with the following keys:
Sfix - list of lists, each containing [starttime]
Ssac - EMPTY! list of lists, each containing [starttime]
Sblk - list of lists, each containing [starttime]
Efix - list of lists, each containing [starttime, endtime, duration, endx, endy]
Esac - EMPTY! list of lists, each containing [starttime, endtime, duration, startx, starty, endx, endy]
Eblk - list of lists, each containing [starttime, endtime, duration]
msg - list of lists, each containing [time, message]
NOTE: timing is in EyeTribe time!
arguments
filename - path to the file that has to be read
start - trial start string
keyword arguments
stop - trial ending string (default = None)
missing - value to be used for missing data (default = 0.0)
debug - Boolean indicating if DEBUG mode should be on or off;
if DEBUG mode is on, information on what the script
currently is doing will be printed to the console
(default = False)
returns
data - a list with a dict for every trial (see above)
"""
# # # # #
# debug mode
if debug:
def message(msg):
print(msg)
else:
def message(msg):
pass
# # # # #
# file handling
# check if the file exists
if os.path.isfile(filename):
# open file
message("opening file '%s'" % filename)
f = open(filename, 'r')
# raise exception if the file does not exist
else:
raise Exception("Error in read_eyetribe: file '%s' does not exist" % filename)
# read file contents
message("reading file '%s'" % filename)
raw = f.readlines()
# close file
message("closing file '%s'" % filename)
f.close()
# # # # #
# parse lines
# Parse the header.
header = raw.pop(0)
header = header.replace('\n','').replace('\r','').split('\t')
# variables
data = []
x = []
y = []
size = []
time = []
trackertime = []
events = {'Sfix':[],'Ssac':[],'Sblk':[],'Efix':[],'Esac':[],'Eblk':[],'msg':[]}
starttime = 0
started = False
trialend = False
laststart = None
# loop through all lines
for i in range(len(raw)):
# string to list
line = raw[i].replace('\n','').replace('\r','').split('\t')
# check if trial has already started
if started:
# only check for stop if there is one
if stop != None:
if (line[header.index("USER")] != '0' and \
stop in line[header.index("USER")]) \
or i == len(raw)-1:
started = False
trialend = True
# check for new start otherwise
else:
if start in line[header.index("USER")] \
or i == len(raw)-1:
# Only start if the current start is more than 1
# sample away from the previous start.
if (laststart == None) or (i != laststart + 1):
started = True
trialend = True
laststart = i
# # # # #
# trial ending
if trialend:
message("trialend %d; %d samples found" % (len(data),len(x)))
# trial dict
trial = {}
trial['x'] = numpy.array(x)
trial['y'] = numpy.array(y)
trial['size'] = numpy.array(size)
trial['time'] = numpy.array(time)
trial['trackertime'] = numpy.array(trackertime)
trial['events'] = copy.deepcopy(events)
# events
trial['events']['Sblk'], trial['events']['Eblk'] = blink_detection(trial['x'],trial['y'],trial['trackertime'],missing=missing)
trial['events']['Sfix'], trial['events']['Efix'] = fixation_detection(trial['x'],trial['y'],trial['trackertime'],missing=missing)
trial['events']['Ssac'], trial['events']['Esac'] = saccade_detection(trial['x'],trial['y'],trial['trackertime'],missing=missing)
# add trial to data
data.append(trial)
# reset stuff
x = []
y = []
size = []
time = []
trackertime = []
events = {'Sfix':[],'Ssac':[],'Sblk':[],'Efix':[],'Esac':[],'Eblk':[],'msg':[]}
trialend = False
# check if the current line contains start message
else:
if line[header.index("USER")] != '0':
if start in line[header.index("USER")]:
# Only start if the current start is more than 1
# sample away from the previous start.
if (laststart == None) or (i != laststart + 1):
message("trialstart %d" % len(data))
# set started to True
started = True
# find starting time
starttime = int(1000 * float(line[header.index("TIME")]))
laststart = i
# # # # #
# parse line
if started:
# Messages are encoded in the user variable, which otherwise
# is '0'. NOTE: Sometimes messages repeat in consecutive
# samples, by accident.
if line[header.index("USER")] != '0':
t = int(1000 * float(line[header.index("TIME")])) # time
m = line[header.index("USER")] # message
events['msg'].append([t,m])
# All lines (when obtained through PyOpenGaze or PyGaze)
# should contain the following data:
# CNT, TIME, TIME_TICK,
# FPOGX, FPOGY, FPOGS, FPOGD, FPOGID, FPOGV,
# LPOGX, LPOGY, LPOGV, RPOGX, RPOGY, RPOGV,
# BPOGX, BPOGY, BPOGV,
# LPCX, LPCY, LPD, LPS, LPV, RPCX, RPCY, RPD, RPS, RPV
# LEYEX, LEYEY, LEYEZ, LPUPILD, LPUPILV,
# REYEX, REYEY, REYEZ, RPUPILD, RPUPILV,
# CX, CY, CS, USER
try:
# Compute the size of the pupil.
left = line[header.index("LPV")] == '1'
right = line[header.index("RPV")] == '1'
if left and right:
s = (float(line[header.index("LPD")]) + \
float(line[header.index("RPD")])) / 2.0
elif left and not right:
s = float(line[header.index("LPD")])
elif not left and right:
s = float(line[header.index("RPD")])
else:
s = 0.0
# extract data
x.append(float(line[header.index("BPOGX")]))
y.append(float(line[header.index("BPOGY")]))
size.append(s)
time.append(int(1000 * float(line[header.index("TIME")]))-starttime)
trackertime.append(int(1000 * float(line[header.index("TIME")])))
except:
message("line '%s' could not be parsed" % line)
continue # skip this line
# # # # #
# return
return data
