def from_path(session_path, force=False, save=True): """ Extract a session from full ALF path (ex: '/scratch/witten/ibl_witten_01/2018-12-18/001') :param force: (False) overwrite existing files :param save: (True) boolean or list of ALF file names to extract :return: None """ extractor_type = get_session_extractor_type(session_path) logger_.info(f"Extracting {session_path} as {extractor_type}") if is_extracted(session_path) and not force: logger_.info(f"Session {session_path} already extracted.") return if extractor_type == 'training': settings, data = rawio.load_bpod(session_path) logger_.info('training session on ' + settings['PYBPOD_BOARD']) training_trials.extract_all(session_path, data=data, save=save) training_wheel.extract_all(session_path, bp_data=data, save=save) logger_.info('session extracted \n') # timing info in log if extractor_type == 'biased': settings, data = rawio.load_bpod(session_path) logger_.info('biased session on ' + settings['PYBPOD_BOARD']) biased_trials.extract_all(session_path, data=data, save=save) biased_wheel.extract_all(session_path, bp_data=data, save=save) logger_.info('session extracted \n') # timing info in log if extractor_type == 'ephys': data = rawio.load_data(session_path) logger_.info('extract BPOD for ephys session') ephys_trials.extract_all(session_path, data=data, save=save) logger_.info('extract FPGA information for ephys session') tmax = data[-1]['behavior_data']['States timestamps']['exit_state'][0][-1] + 60 ephys_fpga.extract_all(session_path, save=save, tmax=tmax) if extractor_type == 'sync_ephys': ephys_fpga.extract_sync(session_path, save=save)
def from_path(session_path, force=False, save=True): """ Extract a session from full ALF path (ex: '/scratch/witten/ibl_witten_01/2018-12-18/001') :param force: (False) overwrite existing files :param save: (True) boolean or list of ALF file names to extract :return: None """ logger_.info('Extracting ' + str(session_path)) extractor_type = extractors_exist(session_path) if is_extracted(session_path) and not force: logger_.info(f"Session {session_path} already extracted.") return if extractor_type == 'training': settings, data = raw.load_bpod(session_path) logger_.info('training session on ' + settings['PYBPOD_BOARD']) training_trials.extract_all(session_path, data=data, save=save) training_wheel.extract_all(session_path, bp_data=data, save=save) logger_.info('session extracted \n') # timing info in log if extractor_type == 'biased': settings, data = raw.load_bpod(session_path) logger_.info('biased session on ' + settings['PYBPOD_BOARD']) biased_trials.extract_all(session_path, data=data, save=save) biased_wheel.extract_all(session_path, bp_data=data, save=save) logger_.info('session extracted \n') # timing info in log if extractor_type == 'ephys': data = raw.load_data(session_path) ephys_trials.extract_all(session_path, data=data, save=save) ephys_fpga.extract_all(session_path, save=save)
def mouse_data_loader(rootdir): ''' rootdir (str): mouse directory variables (list): list containing the keys of the variables of interest Will extract and load data from the whole life of animal ''' mouse_df = pd.DataFrame() counter = 0 for file in sorted(os.listdir(rootdir)): d = os.path.join(rootdir, file) if os.path.isdir(d): print(d) day_df = pd.DataFrame() counter += 1 if counter > 70: break for ses in sorted(os.listdir(d)): s = os.path.join(d, ses) if os.path.isdir(s): try: if Path(s + '/alf').is_dir() == False: extract_all(s, save=True) if Path(s + '/alf/probe00').is_dir() == False: full_bandit_fix(s) ses_df = pd.DataFrame() sesdata = alf.io.load_object(s + '/alf', 'trials') del sesdata['intervals'] ses_df = pd.DataFrame.from_dict(sesdata) protocol = load_settings(s)['_PROTOCOL'] if protocol == '_bandit_100_0_biasedChoiceWorld': # check GABOR file for shaping step with zipfile.ZipFile( s + '/raw_behavior_data/_iblrig_taskCodeFiles.raw.zip' ) as gaborzip: with gaborzip.open( 'GaborIBLTask/Gabor2D.bonsai' ) as bonsaifile: bonsaicode = bonsaifile.read().find( b'it * 5<' ) #Line of code that makes the GABORs different if bonsaicode == -1: protocol = '_bandit_100_0_biasedChoiceWorld' + '_equal_stim' else: protocol = '_bandit_100_0_biasedChoiceWorld' + '_different_stim' ses_df['protocol'] = protocol day_df = pd.concat([day_df, ses_df]) except: continue day_df['day'] = counter mouse_df = pd.concat([mouse_df, day_df]) return mouse_df
def test_extract_all(self): # TRAINING SESSIONS out, files = training_trials.extract_all( self.training_lt5['path'], settings={'IBLRIG_VERSION_TAG': '4.9.9'}, save=True) # -- version >= 5.0.0 out, files = training_trials.extract_all(self.training_ge5['path'], save=True) # BIASED SESSIONS out, files = biased_trials.extract_all( self.biased_lt5['path'], settings={'IBLRIG_VERSION_TAG': '4.9.9'}, save=True) # -- version >= 5.0.0 out, files = biased_trials.extract_all(self.biased_ge5['path'], save=True)
def extract_all(session_path, save=True, bpod_trials=None, settings=None): """ Extracts a training session from its path. NB: Wheel must be extracted first in order to extract trials.firstMovement_times. :param session_path: the path to the session to be extracted :param save: if true a subset of the extracted data are saved as ALF :param bpod_trials: list of Bpod trial data :param settings: the Bpod session settings :return: trials: Bunch/dict of trials :return: wheel: Bunch/dict of wheel positions :return: out_Files: list of output files """ extractor_type = ibllib.io.extractors.base.get_session_extractor_type( session_path) _logger.info(f"Extracting {session_path} as {extractor_type}") bpod_trials = bpod_trials or rawio.load_data(session_path) settings = settings or rawio.load_settings(session_path) if extractor_type == 'training': _logger.info('training session on ' + settings['PYBPOD_BOARD']) wheel, files_wheel = training_wheel.extract_all( session_path, bpod_trials=bpod_trials, settings=settings, save=save) trials, files_trials = training_trials.extract_all( session_path, bpod_trials=bpod_trials, settings=settings, save=save) elif extractor_type == 'biased' or extractor_type == 'ephys': _logger.info('biased session on ' + settings['PYBPOD_BOARD']) wheel, files_wheel = training_wheel.extract_all( session_path, bpod_trials=bpod_trials, settings=settings, save=save) trials, files_trials = biased_trials.extract_all( session_path, bpod_trials=bpod_trials, settings=settings, save=save) elif extractor_type == 'habituation': from ibllib.misc import version _logger.info('habituation session on ' + settings['PYBPOD_BOARD']) if version.le(settings['IBLRIG_VERSION_TAG'], '5.0.0'): _logger.warning("No extraction of legacy habituation sessions") return None, None, None trials, files_trials = habituation_trials.extract_all( session_path, bpod_trials=bpod_trials, settings=settings, save=save) wheel = None files_wheel = [] else: raise ValueError(f"No extractor for task {extractor_type}") _logger.info('session extracted \n') # timing info in log return trials, wheel, (files_trials + files_wheel) if save else None
def test_extract_all(self): # TRAINING SESSIONS out, files = training_trials.extract_all( self.training_lt5['path'], settings={'IBLRIG_VERSION_TAG': '4.9.9'}, save=True) # -- version >= 5.0.0 out, files = training_trials.extract_all( self.training_ge5['path'], save=True) # BIASED SESSIONS # The new trials extractor additionally extracts the wheel data and this fails for the < 5.0 # test data so we will stub the wheel extractor with unittest.mock.patch('ibllib.io.extractors.biased_trials.Wheel') as Wheel: Wheel.var_names = tuple() Wheel().extract.return_value = ({}, []) out, files = biased_trials.extract_all( self.biased_lt5['path'], settings={'IBLRIG_VERSION_TAG': '4.9.9'}, save=True) # -- version >= 5.0.0 out, files = biased_trials.extract_all( self.biased_ge5['path'], save=True)
def extract_all(*args, extra_classes=None, **kwargs): """ Extracts the biased trials for a training session """ if extra_classes is not None: extra_classes.append(LaserBool) else: extra_classes = [LaserBool] return biased_trials.extract_all(*args, **kwargs, extra_classes=extra_classes)
def extract_all(session_path, save=True, bpod_trials=None, settings=None): """ Extracts a training session from its path. NB: Wheel must be extracted first in order to extract trials.firstMovement_times. :param session_path: the path to the session to be extracted :param save: if true a subset of the extracted data are saved as ALF :param bpod_trials: list of Bpod trial data :param settings: the Bpod session settings :return: trials: Bunch/dict of trials :return: wheel: Bunch/dict of wheel positions :return: out_Files: list of output files """ extractor_type = ibllib.io.extractors.base.get_session_extractor_type(session_path) _logger.info(f"Extracting {session_path} as {extractor_type}") bpod_trials = bpod_trials or rawio.load_data(session_path) settings = settings or rawio.load_settings(session_path) _logger.info(f'{extractor_type} session on {settings["PYBPOD_BOARD"]}') # Determine which additional extractors are required extra = [] if extractor_type == 'ephys': # Should exclude 'ephys_biased' _logger.debug('Engaging biased TrialsTableEphys') extra.append(biased_trials.TrialsTableEphys) if extractor_type in ['biased_opto', 'ephys_biased_opto']: _logger.debug('Engaging opto_trials LaserBool') extra.append(opto_trials.LaserBool) # Determine base extraction if extractor_type in ['training', 'ephys_training']: trials, files_trials = training_trials.extract_all( session_path, bpod_trials=bpod_trials, settings=settings, save=save) # This is hacky but avoids extracting the wheel twice. # files_trials should contain wheel files at the end. files_wheel = [] wheel = OrderedDict({k: trials.pop(k) for k in tuple(trials.keys()) if 'wheel' in k}) elif 'biased' in extractor_type or 'ephys' in extractor_type: trials, files_trials = biased_trials.extract_all( session_path, bpod_trials=bpod_trials, settings=settings, save=save, extra_classes=extra) files_wheel = [] wheel = OrderedDict({k: trials.pop(k) for k in tuple(trials.keys()) if 'wheel' in k}) elif extractor_type == 'habituation': if settings['IBLRIG_VERSION_TAG'] and version.le(settings['IBLRIG_VERSION_TAG'], '5.0.0'): _logger.warning("No extraction of legacy habituation sessions") return None, None, None trials, files_trials = habituation_trials.extract_all( session_path, bpod_trials=bpod_trials, settings=settings, save=save) wheel = None files_wheel = [] else: raise ValueError(f"No extractor for task {extractor_type}") _logger.info('session extracted \n') # timing info in log return trials, wheel, (files_trials + files_wheel) if save else None
def _extract(self, sync=None, chmap=None, **kwargs): """Extracts ephys trials by combining Bpod and FPGA sync pulses""" # extract the behaviour data from bpod if sync is None or chmap is None: _sync, _chmap = get_main_probe_sync(self.session_path, bin_exists=False) sync = sync or _sync chmap = chmap or _chmap # load the bpod data and performs a biased choice world training extraction bpod_raw = raw_data_loaders.load_data(self.session_path) assert bpod_raw is not None, "No task trials data in raw_behavior_data - Exit" bpod_trials, _ = biased_trials.extract_all( session_path=self.session_path, save=False, bpod_trials=bpod_raw) bpod_trials['intervals_bpod'] = np.copy(bpod_trials['intervals']) fpga_trials = extract_behaviour_sync(sync=sync, chmap=chmap, bpod_trials=bpod_trials, tmax=bpod_trials['intervals'][-1, -1] + 60) # checks consistency and compute dt with bpod self.bpod2fpga, drift_ppm, ibpod, ifpga = dsp.utils.sync_timestamps( bpod_trials['intervals_bpod'][:, 0], fpga_trials.pop('intervals')[:, 0], return_indices=True) nbpod = bpod_trials['intervals_bpod'].shape[0] npfga = fpga_trials['feedback_times'].shape[0] nsync = len(ibpod) _logger.info(f"N trials: {nbpod} bpod, {npfga} FPGA, {nsync} merged, sync {drift_ppm} ppm") if drift_ppm > BPOD_FPGA_DRIFT_THRESHOLD_PPM: _logger.warning('BPOD/FPGA synchronization shows values greater than %i ppm', BPOD_FPGA_DRIFT_THRESHOLD_PPM) # those fields get directly in the output bpod_fields = ['feedbackType', 'choice', 'rewardVolume', 'intervals_bpod'] # those fields have to be resynced bpod_rsync_fields = ['intervals', 'response_times', 'goCueTrigger_times', 'stimOnTrigger_times', 'stimOffTrigger_times', 'stimFreezeTrigger_times', 'errorCueTrigger_times'] # build trials output out = OrderedDict() out.update({k: bpod_trials[k][ibpod] for k in bpod_fields}) out.update({k: self.bpod2fpga(bpod_trials[k][ibpod]) for k in bpod_rsync_fields}) out.update({k: fpga_trials[k][ifpga] for k in sorted(fpga_trials.keys())}) # extract the wheel data from ibllib.io.extractors.training_wheel import extract_first_movement_times ts, pos = extract_wheel_sync(sync=sync, chmap=chmap) moves = extract_wheel_moves(ts, pos) settings = raw_data_loaders.load_settings(session_path=self.session_path) min_qt = settings.get('QUIESCENT_PERIOD', None) first_move_onsets, *_ = extract_first_movement_times(moves, out, min_qt=min_qt) out.update({'firstMovement_times': first_move_onsets}) assert tuple(filter(lambda x: 'wheel' not in x, self.var_names)) == tuple(out.keys()) return [out[k] for k in out] + [ts, pos, moves['intervals'], moves['peakAmplitude']]
def extract_training(session_path, save=True): """ Extracts a training session from its path. NB: Wheel must be extracted first in order to extract trials.firstMovement_times. :param session_path: :param save: :return: trials: Bunch/dict of trials :return: wheel: Bunch/dict of wheel positions :return: out_Files: list of output files """ extractor_type = rawio.get_session_extractor_type(session_path) _logger.info(f"Extracting {session_path} as {extractor_type}") settings, bpod_trials = rawio.load_bpod(session_path) if extractor_type == 'training': _logger.info('training session on ' + settings['PYBPOD_BOARD']) wheel, files_wheel = training_wheel.extract_all( session_path, bpod_trials=bpod_trials, settings=settings, save=save) trials, files_trials = training_trials.extract_all( session_path, bpod_trials=bpod_trials, settings=settings, save=save) elif extractor_type == 'biased': _logger.info('biased session on ' + settings['PYBPOD_BOARD']) wheel, files_wheel = training_wheel.extract_all( session_path, bpod_trials=bpod_trials, settings=settings, save=save) trials, files_trials = biased_trials.extract_all( session_path, bpod_trials=bpod_trials, settings=settings, save=save) else: raise ValueError(f"No extractor for task {extractor_type}") _logger.info('session extracted \n') # timing info in log return trials, wheel, files_trials + files_wheel
def _extract(self, sync=None, chmap=None): # extracts trials # extract the behaviour data from bpod if sync is None or chmap is None: _sync, _chmap = _get_main_probe_sync(self.session_path, bin_exists=False) sync = sync or _sync chmap = chmap or _chmap bpod_raw = raw_data_loaders.load_data(self.session_path) tmax = bpod_raw[-1]['behavior_data']['States timestamps'][ 'exit_state'][0][-1] + 60 bpod_trials, _ = biased_trials.extract_all( session_path=self.session_path, save=False, bpod_trials=bpod_raw) bpod_trials['intervals_bpod'] = np.copy(bpod_trials['intervals']) fpga_trials = extract_behaviour_sync(sync=sync, chmap=chmap, tmax=tmax) # checks consistency and compute dt with bpod ibpod, ifpga, fcn_bpod2fpga = bpod_fpga_sync( bpod_trials['intervals_bpod'], fpga_trials['intervals']) # those fields get directly in the output bpod_fields = [ 'feedbackType', 'choice', 'rewardVolume', 'intervals_bpod' ] # those fields have to be resynced bpod_rsync_fields = [ 'intervals', 'response_times', 'goCueTrigger_times' ] # ephys fields to save in the output fpga_fields = [ 'stimOn_times', 'stimOff_times', 'goCue_times', 'feedback_times' ] # get ('probabilityLeft', 'contrastLeft', 'contrastRight') from the custom ephys extractors pclcr, _ = ProbaContrasts(self.session_path).extract( bpod_trials=bpod_raw, save=False) # build trials output out = OrderedDict() out.update({ k: pclcr[i][ifpga] for i, k in enumerate(ProbaContrasts.var_names) }) out.update({k: bpod_trials[k][ibpod] for k in bpod_fields}) out.update({ k: fcn_bpod2fpga(bpod_trials[k][ibpod]) for k in bpod_rsync_fields }) out.update({k: fpga_trials[k][ifpga] for k in fpga_fields}) # extract the wheel data from ibllib.io.extractors.training_wheel import extract_first_movement_times ts, pos = extract_wheel_sync(sync=sync, chmap=chmap) moves = extract_wheel_moves(ts, pos) settings = raw_data_loaders.load_settings( session_path=self.session_path) min_qt = settings.get('QUIESCENT_PERIOD', None) first_move_onsets, *_ = extract_first_movement_times(moves, out, min_qt=min_qt) out.update({'firstMovement_times': first_move_onsets}) assert tuple(filter(lambda x: 'wheel' not in x, self.var_names)) == tuple(out.keys()) return [out[k] for k in out ] + [ts, pos, moves['intervals'], moves['peakAmplitude']]
os.chdir(dataPath) days = os.listdir(subject) for day in days: os.chdir(dataPath) os.chdir(subject) runs = os.listdir(day) training = [] laserData = [] for run in runs: os.chdir(os.path.join(dataPath, subject, day, run)) try: currpath = os.getcwd() training.append(extract_all(currpath)) #get behavior data except: pass # os.chdir(run) laserData.append(np.load("laserData")) #get laser data useMax = [] for el in training: useMax.append(len(el['feedbackType'])) training = training[useMax.index( max(useMax))] #only take the longest training session laserData = laserData[useMax.index(max(useMax))] #same for laser if max(useMax) <= min(useMax) * 10 and max(useMax) != min(useMax): print( 'WARNING, there are two files that are close to the same length in folder:'
ses_df['previous_laser_block_1'] = ses_df['laser_block'].shift(1) ses_df['previous_laser_block_2'] = ses_df['laser_block'].shift(2) ses_df['previous_laser_block_3'] = ses_df['laser_block'].shift(3) ses_df['previous_laser_block_4'] = ses_df['laser_block'].shift(4) ses_df['previous_laser_block_5'] = ses_df['laser_block'].shift(5) ses_df['previous_laser_block_6'] = ses_df['laser_block'].shift(6) ses_df['previous_laser_block_7'] = ses_df['laser_block'].shift(7) ses_df['previous_laser_block_8'] = ses_df['laser_block'].shift(8) ses_df['previous_laser_block_9'] = ses_df['laser_block'].shift(9) ses_df['previous_laser_block_10'] = ses_df['laser_block'].shift(10) return ses_df ###################################################################################################### if __name__ == "__main__": ses = sys.argv[1] extract_all(ses, save=True) extract_all_wheel(ses, save=True) full_bandit_fix(ses) ses_df=pd.DataFrame() ses_df= load_session_dataframe(ses) ses_df = ses_df.reset_index() ses_df = ses_df.iloc[:,1:] # Create new variables ses_df['repeated'] = ses_df['choice']==ses_df['previous_choice_1'] sns.barplot(data=ses_df, x='laser_block', y='repeated', hue='previous_outcome_1', ci=68) plt.savefig(ses+'/stay_summary.png') plt.close() # Plot session plot_session(ses_df) plt.savefig(ses+'/session.png')