def from_path(session_path, force=False, save=True): """ Extract a session from full ALF path (ex: '/scratch/witten/ibl_witten_01/2018-12-18/001') :param force: (False) overwrite existing files :param save: (True) boolean or list of ALF file names to extract :return: None """ logger_.info('Extracting ' + str(session_path)) extractor_type = extractors_exist(session_path) if is_extracted(session_path) and not force: logger_.info(f"Session {session_path} already extracted.") return if extractor_type == 'training': settings, data = raw.load_bpod(session_path) logger_.info('training session on ' + settings['PYBPOD_BOARD']) training_trials.extract_all(session_path, data=data, save=save) training_wheel.extract_all(session_path, bp_data=data, save=save) logger_.info('session extracted \n') # timing info in log if extractor_type == 'biased': settings, data = raw.load_bpod(session_path) logger_.info('biased session on ' + settings['PYBPOD_BOARD']) biased_trials.extract_all(session_path, data=data, save=save) biased_wheel.extract_all(session_path, bp_data=data, save=save) logger_.info('session extracted \n') # timing info in log if extractor_type == 'ephys': data = raw.load_data(session_path) ephys_trials.extract_all(session_path, data=data, save=save) ephys_fpga.extract_all(session_path, save=save)
def from_path(session_path, force=False, save=True): """ Extract a session from full ALF path (ex: '/scratch/witten/ibl_witten_01/2018-12-18/001') :param force: (False) overwrite existing files :param save: (True) boolean or list of ALF file names to extract :return: None """ extractor_type = get_session_extractor_type(session_path) logger_.info(f"Extracting {session_path} as {extractor_type}") if is_extracted(session_path) and not force: logger_.info(f"Session {session_path} already extracted.") return if extractor_type == 'training': settings, data = rawio.load_bpod(session_path) logger_.info('training session on ' + settings['PYBPOD_BOARD']) training_trials.extract_all(session_path, data=data, save=save) training_wheel.extract_all(session_path, bp_data=data, save=save) logger_.info('session extracted \n') # timing info in log if extractor_type == 'biased': settings, data = rawio.load_bpod(session_path) logger_.info('biased session on ' + settings['PYBPOD_BOARD']) biased_trials.extract_all(session_path, data=data, save=save) biased_wheel.extract_all(session_path, bp_data=data, save=save) logger_.info('session extracted \n') # timing info in log if extractor_type == 'ephys': data = rawio.load_data(session_path) logger_.info('extract BPOD for ephys session') ephys_trials.extract_all(session_path, data=data, save=save) logger_.info('extract FPGA information for ephys session') tmax = data[-1]['behavior_data']['States timestamps']['exit_state'][0][-1] + 60 ephys_fpga.extract_all(session_path, save=save, tmax=tmax) if extractor_type == 'sync_ephys': ephys_fpga.extract_sync(session_path, save=save)
def compare_wheel_fpga_behaviour(session_path, display=DISPLAY): alf_path = session_path.joinpath('alf') shutil.rmtree(alf_path, ignore_errors=True) sync, chmap = ephys_fpga.get_main_probe_sync(session_path, bin_exists=False) fpga_t, fpga_pos = ephys_fpga.extract_wheel_sync(sync, chmap=chmap) bpod_t, bpod_pos = training_wheel.get_wheel_position(session_path, display=display) data, _ = ephys_fpga.extract_all(session_path) bpod2fpga = scipy.interpolate.interp1d(data['intervals_bpod'][:, 0], data['intervals'][:, 0], fill_value="extrapolate") # resample both traces to the same rate and compute correlation coeff bpod_t = bpod2fpga(bpod_t) tmin = max([np.min(fpga_t), np.min(bpod_t)]) tmax = min([np.max(fpga_t), np.max(bpod_t)]) wheel = {'tscale': np.arange(tmin, tmax, 0.01)} wheel['fpga'] = scipy.interpolate.interp1d(fpga_t, fpga_pos)(wheel['tscale']) wheel['bpod'] = scipy.interpolate.interp1d(bpod_t, bpod_pos)(wheel['tscale']) if display: plt.figure() plt.plot(fpga_t - bpod2fpga(0), fpga_pos, '*') plt.plot(bpod_t - bpod2fpga(0), bpod_pos, '.') raw_wheel = { 'fpga_t': fpga_t, 'fpga_pos': fpga_pos, 'bpod_t': bpod_t, 'bpod_pos': bpod_pos } return raw_wheel, wheel
def extract_data(self): """Extracts and loads behaviour data for QC NB: partial extraction when bpod_only attribute is False requires intervals and intervals_bpod to be assigned to the data attribute before calling this function. :return: """ self.log.info(f"Extracting session: {self.session_path}") self.type = self.type or get_session_extractor_type(self.session_path) self.wheel_encoding = 'X4' if (self.type == 'ephys' and not self.bpod_only) else 'X1' if not self.raw_data: self.load_raw_data() # Run extractors if self.type == 'ephys' and not self.bpod_only: data, _ = ephys_fpga.extract_all(self.session_path) bpod2fpga = interp1d(data['intervals_bpod'][:, 0], data['table']['intervals_0'], fill_value='extrapolate') # Add Bpod wheel data re_ts, pos = get_wheel_position(self.session_path, self.raw_data) data['wheel_timestamps_bpod'] = bpod2fpga(re_ts) data['wheel_position_bpod'] = pos else: kwargs = dict(save=False, bpod_trials=self.raw_data, settings=self.settings) trials, wheel, _ = bpod_trials.extract_all(self.session_path, **kwargs) n_trials = np.unique(list(map(lambda k: trials[k].shape[0], trials)))[0] if self.type == 'habituation': data = trials data['position'] = np.array([t['position'] for t in self.raw_data]) data['phase'] = np.array([t['stim_phase'] for t in self.raw_data]) # Nasty hack to trim last trial due to stim off events happening at trial num + 1 data = {k: v[:n_trials] for k, v in data.items()} else: data = {**trials, **wheel} # Update the data attribute with extracted data self.data = self.rename_data(data)
def _qc_from_path(sess_path, display=True): WHEEL = False sess_path = Path(sess_path) temp_alf_folder = sess_path.joinpath('fpga_test', 'alf') temp_alf_folder.mkdir(parents=True, exist_ok=True) raw_trials = raw_data_loaders.load_data(sess_path) tmax = raw_trials[-1]['behavior_data']['States timestamps']['exit_state'][ 0][-1] + 60 sync, chmap = ephys_fpga._get_main_probe_sync(sess_path, bin_exists=False) _ = ephys_fpga.extract_all(sess_path, output_path=temp_alf_folder, save=True) # check that the output is complete fpga_trials = ephys_fpga.extract_behaviour_sync( sync, output_path=temp_alf_folder, tmax=tmax, chmap=chmap, save=True, display=display) # align with the bpod bpod2fpga = ephys_fpga.align_with_bpod(temp_alf_folder.parent) alf_trials = alf.io.load_object(temp_alf_folder, 'trials') shutil.rmtree(temp_alf_folder) # do the QC qcs, qct = qc_fpga_task(fpga_trials, alf_trials) # do the wheel part if WHEEL: bpod_wheel = training_wheel.get_wheel_data(sess_path, save=False) fpga_wheel = ephys_fpga.extract_wheel_sync(sync, chmap=chmap, save=False) if display: import matplotlib.pyplot as plt t0 = max(np.min(bpod2fpga(bpod_wheel['re_ts'])), np.min(fpga_wheel['re_ts'])) dy = np.interp( t0, fpga_wheel['re_ts'], fpga_wheel['re_pos']) - np.interp( t0, bpod2fpga(bpod_wheel['re_ts']), bpod_wheel['re_pos']) fix, axes = plt.subplots(nrows=2, sharex='all', sharey='all') # axes[0].plot(t, pos), axes[0].title.set_text('Extracted') axes[0].plot(bpod2fpga(bpod_wheel['re_ts']), bpod_wheel['re_pos'] + dy) axes[0].plot(fpga_wheel['re_ts'], fpga_wheel['re_pos']) axes[0].title.set_text('FPGA') axes[1].plot(bpod2fpga(bpod_wheel['re_ts']), bpod_wheel['re_pos'] + dy) axes[1].title.set_text('Bpod') return alf.io.dataframe({**fpga_trials, **alf_trials, **qct})
def __init__(self, session_path, bpod_only=False, local=False): """ Loads and extracts the QC data for a given session path :param session_path: A str or Path to a Bpod session :param bpod_only: When True all data is extracted from Bpod instead of FPGA for ephys """ super().__init__(session_path, one=one, log=_logger) if local: dsets, out_files = ephys_fpga.extract_all(session_path, save=True) self.extractor = TaskQCExtractor(session_path, lazy=True, one=one) # Extract extra datasets required for QC self.extractor.data = dsets self.extractor.extract_data() # Aggregate and update Alyx QC fields self.run(update=False) else: self.load_data(bpod_only=bpod_only) self.compute() self.n_trials = self.extractor.data['intervals'].shape[0] self.wheel_data = { 're_pos': self.extractor.data.pop('wheel_position'), 're_ts': self.extractor.data.pop('wheel_timestamps') } # Print failed outcome, results, outcomes = self.compute_session_status() map = {k: [] for k in set(outcomes.values())} for k, v in outcomes.items(): map[v].append(k[6:]) for k, v in map.items(): if k == 'PASS': continue print(f'The following checks were labelled {k}:') print('\n'.join(v), '\n') # Make DataFrame from the trail level metrics def get_trial_level_failed(d): new_dict = { k[6:]: v for k, v in d.items() if isinstance(v, Sized) and len(v) == self.n_trials } return pd.DataFrame.from_dict(new_dict) self.frame = get_trial_level_failed(self.metrics) self.frame['intervals_0'] = self.extractor.data['intervals'][:, 0] self.frame['intervals_1'] = self.extractor.data['intervals'][:, 1] self.frame.insert(loc=0, column='trial_no', value=self.frame.index)
def _run(self): dsets, out_files = ephys_fpga.extract_all(self.session_path, save=True) self._behaviour_criterion() # Run the task QC qc = TaskQC(self.session_path, one=self.one, log=_logger) qc.extractor = TaskQCExtractor(self.session_path, lazy=True, one=qc.one) # Extract extra datasets required for QC qc.extractor.data = dsets qc.extractor.extract_data() # Aggregate and update Alyx QC fields qc.run(update=True) return out_files
def test_frame2ttl_flicker(self): from ibllib.io.extractors import ephys_fpga from ibllib.qc.task_metrics import TaskQC from ibllib.qc.task_extractors import TaskQCExtractor init_path = self.data_path.joinpath("ephys", "ephys_choice_world_task") session_path = init_path.joinpath("ibl_witten_27/2021-01-21/001") dsets, out_files = ephys_fpga.extract_all(session_path, save=True) # Run the task QC qc = TaskQC(session_path, one=None) qc.extractor = TaskQCExtractor(session_path, lazy=True, one=None) # Extr+act extra datasets required for QC qc.extractor.data = dsets qc.extractor.extract_data() # Aggregate and update Alyx QC fields _, myqc = qc.run(update=False) # from ibllib.misc import pprint # pprint(myqc) assert myqc['_task_stimOn_delays'] > 0.9 # 0.6176 assert myqc["_task_stimFreeze_delays"] > 0.9 assert myqc["_task_stimOff_delays"] > 0.9 assert myqc["_task_stimOff_itiIn_delays"] > 0.9 assert myqc["_task_stimOn_delays"] > 0.9 assert myqc["_task_stimOn_goCue_delays"] > 0.9
def _task_extraction_assertions(self, session_path): alf_path = session_path.joinpath('alf') shutil.rmtree(alf_path, ignore_errors=True) # this gets the full output ephys_fpga.extract_all(session_path, save=True, bin_exists=False) # check that the output is complete for f in BPOD_FILES: self.assertTrue(alf_path.joinpath(f).exists()) # check that the output is complete for f in FPGA_FILES: self.assertTrue(alf_path.joinpath(f).exists()) # check dimensions after alf load alf_trials = alf.io.load_object(alf_path, 'trials') self.assertTrue(alf.io.check_dimensions(alf_trials) == 0) # go deeper and check the internal fpga trials structure consistency sync, chmap = ephys_fpga.get_main_probe_sync(session_path, bin_exists=False) fpga_trials = ephys_fpga.extract_behaviour_sync(sync, chmap) # check dimensions self.assertEqual(alf.io.check_dimensions(fpga_trials), 0) # check that the stimOn < stimFreeze < stimOff self.assertTrue( np.all(fpga_trials['stimOn_times'][:-1] < fpga_trials['stimOff_times'][:-1])) self.assertTrue( np.all(fpga_trials['stimFreeze_times'][:-1] < fpga_trials['stimOff_times'][:-1])) # a trial is either an error-nogo or a reward self.assertTrue( np.all( np.isnan(fpga_trials['valveOpen_times'][:-1] * fpga_trials['errorCue_times'][:-1]))) self.assertTrue( np.all( np.logical_xor(np.isnan(fpga_trials['valveOpen_times'][:-1]), np.isnan(fpga_trials['errorCue_times'][:-1])))) # do the task qc # tqc_ephys.extractor.settings['PYBPOD_PROTOCOL'] from ibllib.qc.task_extractors import TaskQCExtractor ex = TaskQCExtractor(session_path, lazy=True, one=None, bpod_only=False) ex.data = fpga_trials ex.extract_data() from ibllib.qc.task_metrics import TaskQC # '/mnt/s0/Data/IntegrationTests/ephys/ephys_choice_world_task/CSP004/2019-11-27/001' tqc_ephys = TaskQC(session_path) tqc_ephys.extractor = ex _, res_ephys = tqc_ephys.run(bpod_only=False, download_data=False) tqc_bpod = TaskQC(session_path) _, res_bpod = tqc_bpod.run(bpod_only=True, download_data=False) # for a swift comparison using variable explorer # import pandas as pd # df = pd.DataFrame([[res_bpod[k], res_ephys[k]] for k in res_ephys], index=res_ephys.keys()) ok = True for k in res_ephys: if k == "_task_response_feedback_delays": continue if (np.abs(res_bpod[k] - res_ephys[k]) > .2): ok = False print(f"{k} bpod: {res_bpod[k]}, ephys: {res_ephys[k]}") assert ok shutil.rmtree(alf_path, ignore_errors=True)
def _run(self): dsets, out_files = ephys_fpga.extract_all(self.session_path, save=True) return out_files