def from_path(session_path, force=False, save=True):
"""
Extract a session from full ALF path (ex: '/scratch/witten/ibl_witten_01/2018-12-18/001')
:param force: (False) overwrite existing files
:param save: (True) boolean or list of ALF file names to extract
:return: None
"""
extractor_type = get_session_extractor_type(session_path)
logger_.info(f"Extracting {session_path} as {extractor_type}")
if is_extracted(session_path) and not force:
logger_.info(f"Session {session_path} already extracted.")
return
if extractor_type == 'training':
settings, data = rawio.load_bpod(session_path)
logger_.info('training session on ' + settings['PYBPOD_BOARD'])
training_trials.extract_all(session_path, data=data, save=save)
training_wheel.extract_all(session_path, bp_data=data, save=save)
logger_.info('session extracted \n') # timing info in log
if extractor_type == 'biased':
settings, data = rawio.load_bpod(session_path)
logger_.info('biased session on ' + settings['PYBPOD_BOARD'])
biased_trials.extract_all(session_path, data=data, save=save)
biased_wheel.extract_all(session_path, bp_data=data, save=save)
logger_.info('session extracted \n') # timing info in log
if extractor_type == 'ephys':
data = rawio.load_data(session_path)
logger_.info('extract BPOD for ephys session')
ephys_trials.extract_all(session_path, data=data, save=save)
logger_.info('extract FPGA information for ephys session')
tmax = data[-1]['behavior_data']['States timestamps']['exit_state'][0][-1] + 60
ephys_fpga.extract_all(session_path, save=save, tmax=tmax)
if extractor_type == 'sync_ephys':
ephys_fpga.extract_sync(session_path, save=save)
def from_path(session_path, force=False, save=True):
"""
Extract a session from full ALF path (ex: '/scratch/witten/ibl_witten_01/2018-12-18/001')
:param force: (False) overwrite existing files
:param save: (True) boolean or list of ALF file names to extract
:return: None
"""
logger_.info('Extracting ' + str(session_path))
extractor_type = extractors_exist(session_path)
if is_extracted(session_path) and not force:
logger_.info(f"Session {session_path} already extracted.")
return
if extractor_type == 'training':
settings, data = raw.load_bpod(session_path)
logger_.info('training session on ' + settings['PYBPOD_BOARD'])
training_trials.extract_all(session_path, data=data, save=save)
training_wheel.extract_all(session_path, bp_data=data, save=save)
logger_.info('session extracted \n') # timing info in log
if extractor_type == 'biased':
settings, data = raw.load_bpod(session_path)
logger_.info('biased session on ' + settings['PYBPOD_BOARD'])
biased_trials.extract_all(session_path, data=data, save=save)
biased_wheel.extract_all(session_path, bp_data=data, save=save)
logger_.info('session extracted \n') # timing info in log
if extractor_type == 'ephys':
data = raw.load_data(session_path)
ephys_trials.extract_all(session_path, data=data, save=save)
ephys_fpga.extract_all(session_path, save=save)
def mouse_data_loader(rootdir):
'''
rootdir (str): mouse directory
variables (list): list containing the keys of the variables of interest
Will extract and load data from the whole life of animal
'''
mouse_df = pd.DataFrame()
counter = 0
for file in sorted(os.listdir(rootdir)):
d = os.path.join(rootdir, file)
if os.path.isdir(d):
print(d)
day_df = pd.DataFrame()
counter += 1
if counter > 70:
break
for ses in sorted(os.listdir(d)):
s = os.path.join(d, ses)
if os.path.isdir(s):
try:
if Path(s + '/alf').is_dir() == False:
extract_all(s, save=True)
if Path(s + '/alf/probe00').is_dir() == False:
full_bandit_fix(s)
ses_df = pd.DataFrame()
sesdata = alf.io.load_object(s + '/alf', 'trials')
del sesdata['intervals']
ses_df = pd.DataFrame.from_dict(sesdata)
protocol = load_settings(s)['_PROTOCOL']
if protocol == '_bandit_100_0_biasedChoiceWorld': # check GABOR file for shaping step
with zipfile.ZipFile(
s +
'/raw_behavior_data/_iblrig_taskCodeFiles.raw.zip'
) as gaborzip:
with gaborzip.open(
'GaborIBLTask/Gabor2D.bonsai'
) as bonsaifile:
bonsaicode = bonsaifile.read().find(
b'it * 5<'
) #Line of code that makes the GABORs different
if bonsaicode == -1:
protocol = '_bandit_100_0_biasedChoiceWorld' + '_equal_stim'
else:
protocol = '_bandit_100_0_biasedChoiceWorld' + '_different_stim'
ses_df['protocol'] = protocol
day_df = pd.concat([day_df, ses_df])
except:
continue
day_df['day'] = counter
mouse_df = pd.concat([mouse_df, day_df])
return mouse_df
def test_extract_all(self):
# TRAINING SESSIONS
out, files = training_trials.extract_all(
self.training_lt5['path'],
settings={'IBLRIG_VERSION_TAG': '4.9.9'},
save=True)
# -- version >= 5.0.0
out, files = training_trials.extract_all(self.training_ge5['path'],
save=True)
# BIASED SESSIONS
out, files = biased_trials.extract_all(
self.biased_lt5['path'],
settings={'IBLRIG_VERSION_TAG': '4.9.9'},
save=True)
# -- version >= 5.0.0
out, files = biased_trials.extract_all(self.biased_ge5['path'],
save=True)
def extract_all(session_path, save=True, bpod_trials=None, settings=None):
"""
Extracts a training session from its path. NB: Wheel must be extracted first in order to
extract trials.firstMovement_times.
:param session_path: the path to the session to be extracted
:param save: if true a subset of the extracted data are saved as ALF
:param bpod_trials: list of Bpod trial data
:param settings: the Bpod session settings
:return: trials: Bunch/dict of trials
:return: wheel: Bunch/dict of wheel positions
:return: out_Files: list of output files
"""
extractor_type = ibllib.io.extractors.base.get_session_extractor_type(
session_path)
_logger.info(f"Extracting {session_path} as {extractor_type}")
bpod_trials = bpod_trials or rawio.load_data(session_path)
settings = settings or rawio.load_settings(session_path)
if extractor_type == 'training':
_logger.info('training session on ' + settings['PYBPOD_BOARD'])
wheel, files_wheel = training_wheel.extract_all(
session_path,
bpod_trials=bpod_trials,
settings=settings,
save=save)
trials, files_trials = training_trials.extract_all(
session_path,
bpod_trials=bpod_trials,
settings=settings,
save=save)
elif extractor_type == 'biased' or extractor_type == 'ephys':
_logger.info('biased session on ' + settings['PYBPOD_BOARD'])
wheel, files_wheel = training_wheel.extract_all(
session_path,
bpod_trials=bpod_trials,
settings=settings,
save=save)
trials, files_trials = biased_trials.extract_all(
session_path,
bpod_trials=bpod_trials,
settings=settings,
save=save)
elif extractor_type == 'habituation':
from ibllib.misc import version
_logger.info('habituation session on ' + settings['PYBPOD_BOARD'])
if version.le(settings['IBLRIG_VERSION_TAG'], '5.0.0'):
_logger.warning("No extraction of legacy habituation sessions")
return None, None, None
trials, files_trials = habituation_trials.extract_all(
session_path,
bpod_trials=bpod_trials,
settings=settings,
save=save)
wheel = None
files_wheel = []
else:
raise ValueError(f"No extractor for task {extractor_type}")
_logger.info('session extracted \n') # timing info in log
return trials, wheel, (files_trials + files_wheel) if save else None
def test_extract_all(self):
# TRAINING SESSIONS
out, files = training_trials.extract_all(
self.training_lt5['path'], settings={'IBLRIG_VERSION_TAG': '4.9.9'}, save=True)
# -- version >= 5.0.0
out, files = training_trials.extract_all(
self.training_ge5['path'], save=True)
# BIASED SESSIONS
# The new trials extractor additionally extracts the wheel data and this fails for the < 5.0
# test data so we will stub the wheel extractor
with unittest.mock.patch('ibllib.io.extractors.biased_trials.Wheel') as Wheel:
Wheel.var_names = tuple()
Wheel().extract.return_value = ({}, [])
out, files = biased_trials.extract_all(
self.biased_lt5['path'], settings={'IBLRIG_VERSION_TAG': '4.9.9'}, save=True)
# -- version >= 5.0.0
out, files = biased_trials.extract_all(
self.biased_ge5['path'], save=True)
def extract_all(*args, extra_classes=None, **kwargs):
"""
Extracts the biased trials for a training session
"""
if extra_classes is not None:
extra_classes.append(LaserBool)
else:
extra_classes = [LaserBool]
return biased_trials.extract_all(*args,
**kwargs,
extra_classes=extra_classes)
def extract_all(session_path, save=True, bpod_trials=None, settings=None):
"""
Extracts a training session from its path. NB: Wheel must be extracted first in order to
extract trials.firstMovement_times.
:param session_path: the path to the session to be extracted
:param save: if true a subset of the extracted data are saved as ALF
:param bpod_trials: list of Bpod trial data
:param settings: the Bpod session settings
:return: trials: Bunch/dict of trials
:return: wheel: Bunch/dict of wheel positions
:return: out_Files: list of output files
"""
extractor_type = ibllib.io.extractors.base.get_session_extractor_type(session_path)
_logger.info(f"Extracting {session_path} as {extractor_type}")
bpod_trials = bpod_trials or rawio.load_data(session_path)
settings = settings or rawio.load_settings(session_path)
_logger.info(f'{extractor_type} session on {settings["PYBPOD_BOARD"]}')
# Determine which additional extractors are required
extra = []
if extractor_type == 'ephys': # Should exclude 'ephys_biased'
_logger.debug('Engaging biased TrialsTableEphys')
extra.append(biased_trials.TrialsTableEphys)
if extractor_type in ['biased_opto', 'ephys_biased_opto']:
_logger.debug('Engaging opto_trials LaserBool')
extra.append(opto_trials.LaserBool)
# Determine base extraction
if extractor_type in ['training', 'ephys_training']:
trials, files_trials = training_trials.extract_all(
session_path, bpod_trials=bpod_trials, settings=settings, save=save)
# This is hacky but avoids extracting the wheel twice.
# files_trials should contain wheel files at the end.
files_wheel = []
wheel = OrderedDict({k: trials.pop(k) for k in tuple(trials.keys()) if 'wheel' in k})
elif 'biased' in extractor_type or 'ephys' in extractor_type:
trials, files_trials = biased_trials.extract_all(
session_path, bpod_trials=bpod_trials, settings=settings, save=save, extra_classes=extra)
files_wheel = []
wheel = OrderedDict({k: trials.pop(k) for k in tuple(trials.keys()) if 'wheel' in k})
elif extractor_type == 'habituation':
if settings['IBLRIG_VERSION_TAG'] and version.le(settings['IBLRIG_VERSION_TAG'], '5.0.0'):
_logger.warning("No extraction of legacy habituation sessions")
return None, None, None
trials, files_trials = habituation_trials.extract_all(
session_path, bpod_trials=bpod_trials, settings=settings, save=save)
wheel = None
files_wheel = []
else:
raise ValueError(f"No extractor for task {extractor_type}")
_logger.info('session extracted \n') # timing info in log
return trials, wheel, (files_trials + files_wheel) if save else None
def _extract(self, sync=None, chmap=None, **kwargs):
"""Extracts ephys trials by combining Bpod and FPGA sync pulses"""
# extract the behaviour data from bpod
if sync is None or chmap is None:
_sync, _chmap = get_main_probe_sync(self.session_path, bin_exists=False)
sync = sync or _sync
chmap = chmap or _chmap
# load the bpod data and performs a biased choice world training extraction
bpod_raw = raw_data_loaders.load_data(self.session_path)
assert bpod_raw is not None, "No task trials data in raw_behavior_data - Exit"
bpod_trials, _ = biased_trials.extract_all(
session_path=self.session_path, save=False, bpod_trials=bpod_raw)
bpod_trials['intervals_bpod'] = np.copy(bpod_trials['intervals'])
fpga_trials = extract_behaviour_sync(sync=sync, chmap=chmap, bpod_trials=bpod_trials,
tmax=bpod_trials['intervals'][-1, -1] + 60)
# checks consistency and compute dt with bpod
self.bpod2fpga, drift_ppm, ibpod, ifpga = dsp.utils.sync_timestamps(
bpod_trials['intervals_bpod'][:, 0], fpga_trials.pop('intervals')[:, 0],
return_indices=True)
nbpod = bpod_trials['intervals_bpod'].shape[0]
npfga = fpga_trials['feedback_times'].shape[0]
nsync = len(ibpod)
_logger.info(f"N trials: {nbpod} bpod, {npfga} FPGA, {nsync} merged, sync {drift_ppm} ppm")
if drift_ppm > BPOD_FPGA_DRIFT_THRESHOLD_PPM:
_logger.warning('BPOD/FPGA synchronization shows values greater than %i ppm',
BPOD_FPGA_DRIFT_THRESHOLD_PPM)
# those fields get directly in the output
bpod_fields = ['feedbackType', 'choice', 'rewardVolume', 'intervals_bpod']
# those fields have to be resynced
bpod_rsync_fields = ['intervals', 'response_times', 'goCueTrigger_times',
'stimOnTrigger_times', 'stimOffTrigger_times',
'stimFreezeTrigger_times', 'errorCueTrigger_times']
# build trials output
out = OrderedDict()
out.update({k: bpod_trials[k][ibpod] for k in bpod_fields})
out.update({k: self.bpod2fpga(bpod_trials[k][ibpod]) for k in bpod_rsync_fields})
out.update({k: fpga_trials[k][ifpga] for k in sorted(fpga_trials.keys())})
# extract the wheel data
from ibllib.io.extractors.training_wheel import extract_first_movement_times
ts, pos = extract_wheel_sync(sync=sync, chmap=chmap)
moves = extract_wheel_moves(ts, pos)
settings = raw_data_loaders.load_settings(session_path=self.session_path)
min_qt = settings.get('QUIESCENT_PERIOD', None)
first_move_onsets, *_ = extract_first_movement_times(moves, out, min_qt=min_qt)
out.update({'firstMovement_times': first_move_onsets})
assert tuple(filter(lambda x: 'wheel' not in x, self.var_names)) == tuple(out.keys())
return [out[k] for k in out] + [ts, pos, moves['intervals'], moves['peakAmplitude']]
def extract_training(session_path, save=True):
"""
Extracts a training session from its path. NB: Wheel must be extracted first in order to
extract trials.firstMovement_times.
:param session_path:
:param save:
:return: trials: Bunch/dict of trials
:return: wheel: Bunch/dict of wheel positions
:return: out_Files: list of output files
"""
extractor_type = rawio.get_session_extractor_type(session_path)
_logger.info(f"Extracting {session_path} as {extractor_type}")
settings, bpod_trials = rawio.load_bpod(session_path)
if extractor_type == 'training':
_logger.info('training session on ' + settings['PYBPOD_BOARD'])
wheel, files_wheel = training_wheel.extract_all(
session_path,
bpod_trials=bpod_trials,
settings=settings,
save=save)
trials, files_trials = training_trials.extract_all(
session_path,
bpod_trials=bpod_trials,
settings=settings,
save=save)
elif extractor_type == 'biased':
_logger.info('biased session on ' + settings['PYBPOD_BOARD'])
wheel, files_wheel = training_wheel.extract_all(
session_path,
bpod_trials=bpod_trials,
settings=settings,
save=save)
trials, files_trials = biased_trials.extract_all(
session_path,
bpod_trials=bpod_trials,
settings=settings,
save=save)
else:
raise ValueError(f"No extractor for task {extractor_type}")
_logger.info('session extracted \n') # timing info in log
return trials, wheel, files_trials + files_wheel
def _extract(self, sync=None, chmap=None):
# extracts trials
# extract the behaviour data from bpod
if sync is None or chmap is None:
_sync, _chmap = _get_main_probe_sync(self.session_path,
bin_exists=False)
sync = sync or _sync
chmap = chmap or _chmap
bpod_raw = raw_data_loaders.load_data(self.session_path)
tmax = bpod_raw[-1]['behavior_data']['States timestamps'][
'exit_state'][0][-1] + 60
bpod_trials, _ = biased_trials.extract_all(
session_path=self.session_path, save=False, bpod_trials=bpod_raw)
bpod_trials['intervals_bpod'] = np.copy(bpod_trials['intervals'])
fpga_trials = extract_behaviour_sync(sync=sync, chmap=chmap, tmax=tmax)
# checks consistency and compute dt with bpod
ibpod, ifpga, fcn_bpod2fpga = bpod_fpga_sync(
bpod_trials['intervals_bpod'], fpga_trials['intervals'])
# those fields get directly in the output
bpod_fields = [
'feedbackType', 'choice', 'rewardVolume', 'intervals_bpod'
]
# those fields have to be resynced
bpod_rsync_fields = [
'intervals', 'response_times', 'goCueTrigger_times'
]
# ephys fields to save in the output
fpga_fields = [
'stimOn_times', 'stimOff_times', 'goCue_times', 'feedback_times'
]
# get ('probabilityLeft', 'contrastLeft', 'contrastRight') from the custom ephys extractors
pclcr, _ = ProbaContrasts(self.session_path).extract(
bpod_trials=bpod_raw, save=False)
# build trials output
out = OrderedDict()
out.update({
k: pclcr[i][ifpga]
for i, k in enumerate(ProbaContrasts.var_names)
})
out.update({k: bpod_trials[k][ibpod] for k in bpod_fields})
out.update({
k: fcn_bpod2fpga(bpod_trials[k][ibpod])
for k in bpod_rsync_fields
})
out.update({k: fpga_trials[k][ifpga] for k in fpga_fields})
# extract the wheel data
from ibllib.io.extractors.training_wheel import extract_first_movement_times
ts, pos = extract_wheel_sync(sync=sync, chmap=chmap)
moves = extract_wheel_moves(ts, pos)
settings = raw_data_loaders.load_settings(
session_path=self.session_path)
min_qt = settings.get('QUIESCENT_PERIOD', None)
first_move_onsets, *_ = extract_first_movement_times(moves,
out,
min_qt=min_qt)
out.update({'firstMovement_times': first_move_onsets})
assert tuple(filter(lambda x: 'wheel' not in x,
self.var_names)) == tuple(out.keys())
return [out[k] for k in out
] + [ts, pos, moves['intervals'], moves['peakAmplitude']]
os.chdir(dataPath)
days = os.listdir(subject)
for day in days:
os.chdir(dataPath)
os.chdir(subject)
runs = os.listdir(day)
training = []
laserData = []
for run in runs:
os.chdir(os.path.join(dataPath, subject, day, run))
try:
currpath = os.getcwd()
training.append(extract_all(currpath)) #get behavior data
except:
pass
# os.chdir(run)
laserData.append(np.load("laserData")) #get laser data
useMax = []
for el in training:
useMax.append(len(el['feedbackType']))
training = training[useMax.index(
max(useMax))] #only take the longest training session
laserData = laserData[useMax.index(max(useMax))] #same for laser
if max(useMax) <= min(useMax) * 10 and max(useMax) != min(useMax):
print(
'WARNING, there are two files that are close to the same length in folder:'
ses_df['previous_laser_block_1'] = ses_df['laser_block'].shift(1)
ses_df['previous_laser_block_2'] = ses_df['laser_block'].shift(2)
ses_df['previous_laser_block_3'] = ses_df['laser_block'].shift(3)
ses_df['previous_laser_block_4'] = ses_df['laser_block'].shift(4)
ses_df['previous_laser_block_5'] = ses_df['laser_block'].shift(5)
ses_df['previous_laser_block_6'] = ses_df['laser_block'].shift(6)
ses_df['previous_laser_block_7'] = ses_df['laser_block'].shift(7)
ses_df['previous_laser_block_8'] = ses_df['laser_block'].shift(8)
ses_df['previous_laser_block_9'] = ses_df['laser_block'].shift(9)
ses_df['previous_laser_block_10'] = ses_df['laser_block'].shift(10)
return ses_df
######################################################################################################
if __name__ == "__main__":
ses = sys.argv[1]
extract_all(ses, save=True)
extract_all_wheel(ses, save=True)
full_bandit_fix(ses)
ses_df=pd.DataFrame()
ses_df= load_session_dataframe(ses)
ses_df = ses_df.reset_index()
ses_df = ses_df.iloc[:,1:]
# Create new variables
ses_df['repeated'] = ses_df['choice']==ses_df['previous_choice_1']
sns.barplot(data=ses_df, x='laser_block', y='repeated', hue='previous_outcome_1', ci=68)
plt.savefig(ses+'/stay_summary.png')
plt.close()
# Plot session
plot_session(ses_df)
plt.savefig(ses+'/session.png')
