def __init__(self,
subject,
experiment_name,
experiment_nr,
version,
project_directory,
loggingLevel=logging.DEBUG,
sample_rate_new=50):
self.subject = subject
self.experiment_name = experiment_name
self.experiment = experiment_nr
self.version = version
try:
os.mkdir(os.path.join(project_directory, experiment_name))
os.mkdir(
os.path.join(project_directory, experiment_name, self.subject))
except OSError:
pass
self.project_directory = project_directory
self.base_directory = os.path.join(self.project_directory,
self.experiment_name, self.subject)
self.create_folder_hierarchy()
self.hdf5_filename = os.path.join(self.base_directory, 'processed',
self.subject + '.hdf5')
self.ho = hedfpy.HDFEyeOperator(self.hdf5_filename)
self.sample_rate_new = int(sample_rate_new)
self.downsample_rate = int(1000 / sample_rate_new)
def convert_session(sj, ses, indir, outdir, pupil_hp, pupil_lp):
if type(sj) == int:
sj, ses = str(sj).zfill(2), str(ses).zfill(2)
globstr = os.path.join(
indir,
'sub-{sj}/ses-{ses}/eyetrack/sub-{sj}_ses-{ses}_task-*_run-*_eyetrack.edf'
.format(sj=sj, ses=ses))
edf_files = glob.glob(globstr)
edf_files.sort(
) #list of absolute paths to all edf files in that session for that subject
#single hdf5 file that contains all eye data for the runs of that session
hdf_file = os.path.join(
outdir, 'sub-{sj}/ses-{ses}/eyetrack.h5'.format(sj=sj, ses=ses))
if not os.path.exists(os.path.split(hdf_file)
[0]): # check if path to save hdf5 files exists
os.makedirs(os.path.split(hdf_file)[0]) # if not create it
ho = hedfpy.HDFEyeOperator(hdf_file)
for ef in edf_files:
alias = os.path.splitext(
os.path.split(ef)[1])[0] #name of data for that run
ho.add_edf_file(ef)
ho.edf_message_data_to_hdf(
alias=alias) #write messages ex_events to hdf5
ho.edf_gaze_data_to_hdf(
alias=alias, pupil_hp=pupil_hp,
pupil_lp=pupil_lp) #add raw and preprocessed data to hdf5
else:
print('%s already exists, skipping' % hdf_file)
def preprocess_subjects(subjects, task, output_dir, analysis_params):
for subject in subjects:
# data:
edfs = glob.glob(
os.path.join(raw_dir, subject, '{}*{}*.edf'.format(subject, task)))
# folder hierarchy:
preprocess_dir = os.path.join(output_dir, task, subject)
try:
os.makedirs(os.path.join(preprocess_dir, 'raw'))
except OSError:
pass
# hdf5 filename:
hdf5_filename = os.path.join(preprocess_dir,
'{}_{}.hdf5'.format(subject, task))
try:
os.remove(hdf5_filename)
except OSError:
pass
# initialize hdf5 HDFEyeOperator:
ho = hedfpy.HDFEyeOperator(hdf5_filename)
# variables:
session_nrs = [r.split('ses-')[1][0:2] for r in edfs]
run_nrs = [r.split('run-')[1][0:1] for r in edfs]
aliases = []
for i in range(len(session_nrs)):
aliases.append('{}_{}_{}'.format(task, session_nrs[i], run_nrs[i]))
# preprocessing:
for edf_file, alias in zip(edfs, aliases):
shutil.copy(
edf_file,
os.path.join(preprocess_dir, 'raw', '{}.edf'.format(alias)))
ho.add_edf_file(
os.path.join(preprocess_dir, 'raw', '{}.edf'.format(alias)))
ho.edf_message_data_to_hdf(alias=alias)
ho.edf_gaze_data_to_hdf(
alias=alias,
sample_rate=analysis_params['sample_rate'],
pupil_lp=analysis_params['lp'],
pupil_hp=analysis_params['hp'],
normalization=analysis_params['normalization'],
regress_blinks=analysis_params['regress_blinks'],
regress_sacs=analysis_params['regress_sacs'],
use_standard_blinksac_kernels=analysis_params[
'use_standard_blinksac_kernels'],
)
def __init__(self,
subjects,
experiment_name,
project_directory,
sample_rate_new=50):
self.subjects = subjects
self.nr_subjects = len(self.subjects)
self.experiment_name = experiment_name
self.project_directory = project_directory
self.sample_rate_new = int(sample_rate_new)
self.downsample_rate = int(1000 / sample_rate_new)
parameters = []
for s in self.subjects:
self.base_directory = os.path.join(self.project_directory,
self.experiment_name, s)
self.hdf5_filename = os.path.join(self.base_directory, 'processed',
s + '.hdf5')
self.ho = hedfpy.HDFEyeOperator(self.hdf5_filename)
try:
parameters.append(
self.ho.read_session_data('', 'parameters_joined'))
except:
shell()
self.parameters_joined = pd.concat(parameters)
self.omissions = np.array(self.parameters_joined['omissions'])
self.omissions = self.omissions + (np.array(
self.parameters_joined['correct']) == -1)
self.omissions = self.omissions + (np.array(
self.parameters_joined['trial']) == 0)
# self.omissions = self.omissions + (np.array(self.parameters_joined['missing_data']) > 0.25)
# regress out RT per session:
for subj in np.unique(self.parameters_joined.subject):
for s in np.unique(self.parameters_joined.session[
self.parameters_joined.subject == subj]):
ind = (self.parameters_joined.subject
== subj) * (self.parameters_joined.session == s)
rt = np.array(self.parameters_joined['rt'][ind]) / 1000.0
pupil_d = np.array(self.parameters_joined['pupil_d'][ind])
pupil_d = myfuncs.lin_regress_resid(pupil_d,
[rt]) + pupil_d.mean()
self.parameters_joined['pupil_d'][ind] = pupil_d
self.parameters_joined = self.parameters_joined[-self.omissions]
self.rt = np.array(self.parameters_joined['rt'])
self.hit = np.array(self.parameters_joined['hit'], dtype=bool)
self.fa = np.array(self.parameters_joined['fa'], dtype=bool)
self.miss = np.array(self.parameters_joined['miss'], dtype=bool)
self.cr = np.array(self.parameters_joined['cr'], dtype=bool)
self.yes = np.array(self.parameters_joined['yes'], dtype=bool)
self.no = -np.array(self.parameters_joined['yes'], dtype=bool)
self.run = np.array(self.parameters_joined['run'], dtype=int)
self.session = np.array(self.parameters_joined['session'], dtype=int)
try:
self.present = np.array(self.parameters_joined['signal_present'],
dtype=bool)
except:
self.present = np.array(
self.parameters_joined['target_present_in_stimulus'],
dtype=bool)
self.absent = -self.present
self.correct = np.array(self.parameters_joined['correct'], dtype=bool)
self.error = -np.array(self.parameters_joined['correct'], dtype=bool)
self.pupil_b = np.array(self.parameters_joined['pupil_b'])
self.pupil_d = np.array(self.parameters_joined['pupil_d'])
self.pupil_t = np.array(self.parameters_joined['pupil_t'])
self.subj_idx = np.concatenate(
np.array([
np.repeat(
i,
sum(self.parameters_joined['subject'] == self.subjects[i]))
for i in range(len(self.subjects))
]))
self.criterion = np.array([
np.array(self.parameters_joined[self.parameters_joined['subject']
== subj]['criterion'])[0]
for subj in self.subjects
])
#########
# pupil split:
self.pupil_l_ind = []
self.pupil_h_ind = []
for subj_idx in self.subjects:
d = self.parameters_joined[self.parameters_joined.subject ==
subj_idx]
p_h = []
p_l = []
for s in np.array(np.unique(d['session']), dtype=int):
pupil = np.array(d['pupil_d'])[np.array(d.session) == s]
p_l.append(pupil <= np.percentile(pupil, 40))
p_h.append(pupil >= np.percentile(pupil, 60))
self.pupil_l_ind.append(np.concatenate(p_l))
self.pupil_h_ind.append(np.concatenate(p_h))
self.pupil_l_ind = np.concatenate(self.pupil_l_ind)
self.pupil_h_ind = np.concatenate(self.pupil_h_ind)
self.pupil_rest_ind = -(self.pupil_h_ind + self.pupil_l_ind)
# self.pupil_l_ind = []
# self.pupil_h_ind = []
# for subj_idx in self.subjects:
# d = self.parameters_joined[self.parameters_joined.subject == subj_idx]
# p_h = []
# p_l = []
# for r in np.array(np.unique(d['run']), dtype=int):
# pupil = np.array(d['pupil_b_lp'])[np.array(d.run) ==r]
# p_l.append( (pupil <= np.percentile(pupil, 25)) + (pupil >= np.percentile(pupil, 75)) )
# p_h.append( (pupil > np.percentile(pupil, 25)) & (pupil < np.percentile(pupil, 75)) )
# self.pupil_l_ind.append(np.concatenate(p_l))
# self.pupil_h_ind.append(np.concatenate(p_h))
# self.pupil_l_ind = np.concatenate(self.pupil_l_ind)
# self.pupil_h_ind = np.concatenate(self.pupil_h_ind)
# self.pupil_rest_ind = -(self.pupil_h_ind + self.pupil_l_ind)
# initialize behavior operator:
d = {
'subj_idx': pd.Series(self.subj_idx),
'choice_a': pd.Series(np.array(self.yes, dtype=int)),
'stimulus': pd.Series(np.array(self.present, dtype=int)),
'rt': pd.Series(np.array(self.rt)) / 1000.0,
'pupil_b': pd.Series(np.array(self.pupil_b)),
'pupil_d': pd.Series(np.array(self.pupil_d)),
'pupil_t': pd.Series(np.array(self.pupil_t)),
'pupil_high': pd.Series(self.pupil_h_ind),
'run': pd.Series(np.array(self.run, dtype=int)),
'session': pd.Series(np.array(self.session, dtype=int)),
}
self.df = pd.DataFrame(d)
self.behavior = myfuncs.behavior(self.df)
def main(subject, sourcedata, rawdata):
behavior = pd.read_csv(op.join(rawdata, 'magData.csv')).set_index(['subcode', 'run']).loc[subject]
behavior = behavior.rename(columns={'sure_bet':'stim1', 'prob_bet':'stim2', 'leftright':'response'})
behavior['accuracy'] = behavior['correct'] == behavior['response']
template = op.join(rawdata, 'garcia_ruff_risk_precision_behdata', f'sub_{subject:02d}_*')
results = glob.glob(template)
print(template)
assert(len(results) == 1)
base_dir = results[0]
der_dir = op.join(sourcedata, 'derivatives', 'pupil', f'sub-{subject}', 'pupil')
if not op.exists(der_dir):
os.makedirs(der_dir)
hdf5_file = op.join(der_dir, f'sub-{subject}_pupil.hdf5')
if op.exists(hdf5_file):
os.remove(hdf5_file)
ho = hedfpy.HDFEyeOperator(hdf5_file)
for run in range(1, 7):
hedf_key = f'sub-{subject}_run-{run}'
fn = op.join(base_dir, f'Ms{subject:02d}rn{run:02d}.edf')
ho.add_edf_file(fn)
ho.edf_message_data_to_hdf(hedf_key)
ho.edf_gaze_data_to_hdf(alias=hedf_key,
sample_rate=analysis_params['sample_rate'],
pupil_lp=analysis_params['lp'],
pupil_hp=analysis_params['hp'],
normalization=analysis_params['normalization'],
regress_blinks=analysis_params['regress_blinks'],
regress_sacs=analysis_params['regress_sacs'],
use_standard_blinksac_kernels=analysis_params['use_standard_blinksac_kernels'],
)
properties = ho.block_properties(hedf_key)
assert(analysis_params['sample_rate'] == properties.loc[0, 'sample_rate'])
# Detect behavioral messages
generic_messages = pd.DataFrame(ho.edf_operator.read_generic_events())
start_ix = (generic_messages.message == '1_trialon').argmax()
start_ts = float(generic_messages.loc[start_ix].EL_timestamp)
events = generic_messages.loc[start_ix+2:]
start_ts = generic_messages.loc[start_ix].EL_timestamp
events['onset'] = (events['EL_timestamp'] - start_ts) / 1000
events['trial'] = events.message.apply(lambda x: int(x.split('_')[0]))
events['message'] = events.message.apply(lambda x: x.split('_')[1])
b = behavior.loc[run]
b['trial'] = np.arange(1, 37)
events = events.merge(b[['trial', 'stim1', 'stim2', 'correct', 'response', 'accuracy']])
last_ts = float(events.iloc[-1].EL_timestamp)
# detect saccades
saccades = ho.saccades_from_message_file_during_period([start_ts, last_ts+5000], hedf_key)
saccades['onset'] = (saccades['start_timestamp'] - start_ts) / 1000.
saccades['duration'] /= 1000.
saccades = saccades[['duration', 'onset', 'start_timestamp']]
# Detect blinks
blinks = ho.blinks_during_period([start_ts, last_ts + 5000], hedf_key)
blinks['onset'] = (blinks['start_timestamp'] - start_ts) / 1000.
blinks['duration'] /= 1000.
blinks = blinks[['onset', 'duration']]
eye = ho.block_properties(hedf_key).loc[0, 'eye_recorded']
# Get data
d = ho.data_from_time_period([start_ts, last_ts+5000], hedf_key)
d['time'] = (d['time'] - start_ts) / 1000. - 1./analysis_params['sample_rate']
d = d.set_index(pd.Index(d['time'], name='time'))
d['interpolated'] = d[f'{eye}_interpolated_timepoints'].astype(bool)
d['pupil'] = d[f'{eye}_pupil_bp']
d = d[['interpolated', 'pupil']]
# Save everything
saccades.to_csv(op.join(der_dir, f'sub-{subject}_run-{run}_saccades.tsv'), sep='\t', index=False)
blinks.to_csv(op.join(der_dir, f'sub-{subject}_run-{run}_blinks.tsv'), sep='\t', index=False)
d.to_csv(op.join(der_dir, f'sub-{subject}_run-{run}_pupil.tsv.gz'), sep='\t')
d.to_csv(op.join(der_dir, f'sub-{subject}_run-{run}_pupil.tsv'), sep='\t')
analysis_params['eyetrack_dir'],
analysis_params['high_pass_pupil_f'],
analysis_params['low_pass_pupil_f'])
# plot density over image for all runs
for run in [1, 2, 3, 4]:
if sj == 5 and run == 3:
print('no run %d' % run)
else:
# define hdf operator for specific run
hdf_file = os.path.join(
analysis_params['eyetrack_dir'],
'sub-{sj}/ses-{ses}/eyetrack.h5'.format(sj=str(sj).zfill(2),
ses=str(ses).zfill(2)))
ho = hedfpy.HDFEyeOperator(hdf_file)
alias = 'sub-{sj}_ses-{ses}_task-{task}_run-{run}_bold_eyetrack'.format(
sj=str(sj).zfill(2),
ses=str(ses).zfill(2),
task=task,
run=str(run).zfill(2))
# load table with timestamps for run
with pd.HDFStore(ho.input_object) as h5_file:
table_trl = h5_file['%s/trials' %
alias] # load table with timestamps for run
# compute array with all timestamps of start and end of trial, for whole run
parameters = ho.read_session_data(alias, 'parameters')
trl_str_end = []