def _get_passive_spacers(session_path, sync=None, sync_map=None): """ load and get spacer information, do corr to find spacer timestamps returns t_passive_starts, t_starts, t_ends """ if sync is None or sync_map is None: sync, sync_map = ephys_fpga._get_main_probe_sync(session_path, bin_exists=False) meta = _load_passive_stim_meta() # t_end_ephys = passive.ephysCW_end(session_path=session_path) fttl = ephys_fpga._get_sync_fronts(sync, sync_map["frame2ttl"], tmin=None) spacer_template = ( np.array(meta["VISUAL_STIM_0"]["ttl_frame_nums"], dtype=np.float32) / FRAME_FS) jitter = 3 / FRAME_FS # allow for 3 screen refresh as jitter t_quiet = meta["VISUAL_STIM_0"]["delay_around"] spacer_times, _ = _get_spacer_times(spacer_template=spacer_template, jitter=jitter, ttl_signal=fttl["times"], t_quiet=t_quiet) # Check correct number of spacers found n_exp_spacer = np.sum(np.array( meta["STIM_ORDER"]) == 0) # Hardcoded 0 for spacer if n_exp_spacer != np.size(spacer_times) / 2: raise ValueError(f"The number of expected spacer ({n_exp_spacer}) " f"is different than the one found on the raw " f"trace ({np.size(spacer_times)/2})") spacer_times = np.r_[spacer_times.flatten(), sync["times"][-1]] return spacer_times[0], spacer_times[1::2], spacer_times[2::2]
def load_raw_data(self): """ Loads the TTLs, raw task data and task settings :return: """ self.log.info(f"Loading raw data from {self.session_path}") self.type = self.type or raw.get_session_extractor_type( self.session_path) self.settings, self.raw_data = raw.load_bpod(self.session_path) # Fetch the TTLs for the photodiode and audio if self.type != 'ephys' or self.bpod_only is True: # Extract from Bpod self.frame_ttls, self.audio_ttls = raw.load_bpod_fronts( self.session_path, data=self.raw_data) else: # Extract from FPGA sync, chmap = _get_main_probe_sync(self.session_path) def channel_events(name): """Fetches the polarities and times for a given channel""" keys = ('polarities', 'times') mask = sync['channels'] == chmap[name] return dict(zip(keys, (sync[k][mask] for k in keys))) ttls = [ channel_events(ch) for ch in ('frame2ttl', 'audio', 'bpod') ] self.frame_ttls, self.audio_ttls, self.bpod_ttls = ttls
def _qc_from_path(sess_path, display=True): WHEEL = False sess_path = Path(sess_path) temp_alf_folder = sess_path.joinpath('fpga_test', 'alf') temp_alf_folder.mkdir(parents=True, exist_ok=True) raw_trials = raw_data_loaders.load_data(sess_path) tmax = raw_trials[-1]['behavior_data']['States timestamps']['exit_state'][ 0][-1] + 60 sync, chmap = ephys_fpga._get_main_probe_sync(sess_path, bin_exists=False) _ = ephys_fpga.extract_all(sess_path, output_path=temp_alf_folder, save=True) # check that the output is complete fpga_trials = ephys_fpga.extract_behaviour_sync( sync, output_path=temp_alf_folder, tmax=tmax, chmap=chmap, save=True, display=display) # align with the bpod bpod2fpga = ephys_fpga.align_with_bpod(temp_alf_folder.parent) alf_trials = alf.io.load_object(temp_alf_folder, 'trials') shutil.rmtree(temp_alf_folder) # do the QC qcs, qct = qc_fpga_task(fpga_trials, alf_trials) # do the wheel part if WHEEL: bpod_wheel = training_wheel.get_wheel_data(sess_path, save=False) fpga_wheel = ephys_fpga.extract_wheel_sync(sync, chmap=chmap, save=False) if display: import matplotlib.pyplot as plt t0 = max(np.min(bpod2fpga(bpod_wheel['re_ts'])), np.min(fpga_wheel['re_ts'])) dy = np.interp( t0, fpga_wheel['re_ts'], fpga_wheel['re_pos']) - np.interp( t0, bpod2fpga(bpod_wheel['re_ts']), bpod_wheel['re_pos']) fix, axes = plt.subplots(nrows=2, sharex='all', sharey='all') # axes[0].plot(t, pos), axes[0].title.set_text('Extracted') axes[0].plot(bpod2fpga(bpod_wheel['re_ts']), bpod_wheel['re_pos'] + dy) axes[0].plot(fpga_wheel['re_ts'], fpga_wheel['re_pos']) axes[0].title.set_text('FPGA') axes[1].plot(bpod2fpga(bpod_wheel['re_ts']), bpod_wheel['re_pos'] + dy) axes[1].title.set_text('Bpod') return alf.io.dataframe({**fpga_trials, **alf_trials, **qct})
def extract_rfmapping(session_path: str, sync: dict = None, sync_map: dict = None, trfm: np.array = None) -> Tuple[np.array, np.array]: meta = _load_passive_stim_meta() mkey = ("VISUAL_STIM_" + {v: k for k, v in meta["VISUAL_STIMULI"].items() }["receptive_field_mapping"]) if sync is None or sync_map is None: sync, sync_map = ephys_fpga._get_main_probe_sync(session_path, bin_exists=False) if trfm is None: passivePeriods_df = extract_passive_periods(session_path, sync=sync, sync_map=sync_map) trfm = passivePeriods_df.RFM.values fttl = ephys_fpga._get_sync_fronts(sync, sync_map["frame2ttl"], tmin=trfm[0], tmax=trfm[1]) RF_file = Path().joinpath(session_path, "raw_passive_data", "_iblrig_RFMapStim.raw.bin") passiveRFM_frames, RF_ttl_trace = _reshape_RF(RF_file=RF_file, meta_stim=meta[mkey]) rf_id_up, rf_id_dw, RF_n_ttl_expected = _get_id_raisefall_from_analogttl( RF_ttl_trace) meta[mkey]["ttl_num"] = RF_n_ttl_expected rf_times_on_idx = np.where(np.diff(fttl["times"]) < 1)[0] rf_times_off_idx = rf_times_on_idx + 1 RF_times = fttl["times"][np.sort( np.concatenate([rf_times_on_idx, rf_times_off_idx]))] RF_times_1 = RF_times[0::2] # Interpolate times for RF before outputting dataset passiveRFM_times = _interpolate_rf_mapping_stimulus( idxs_up=rf_id_up, idxs_dn=rf_id_dw, times=RF_times_1, Xq=np.arange(passiveRFM_frames.shape[0]), t_bin=1 / FRAME_FS, ) return passiveRFM_times # _ibl_passiveRFM.times.npy
def load_ttl_pulses(session_path): """ Extract ttl pulses from sync signals :param session_path: absolute path of a session, i.e. /mnt/data/Subjects/ZM_1887/2019-07-10/001 :type session_path: str :return: ttl pulse times :rtype: np.ndarray """ from ibllib.io.extractors.ephys_fpga import _get_main_probe_sync sync, sync_chmap = _get_main_probe_sync(session_path, bin_exists=False) fr2ttl_ch = sync_chmap['frame2ttl'] # find times of when ttl polarity changes on fr2ttl channel sync_pol_ = sync['polarities'][sync['channels'] == fr2ttl_ch] sync_times_ = sync['times'][sync['channels'] == fr2ttl_ch] return sync_pol_, sync_times_
def extract_task_replay( session_path: str, sync: dict = None, sync_map: dict = None, treplay: np.array = None) -> Tuple[pd.DataFrame, pd.DataFrame]: if sync is None or sync_map is None: sync, sync_map = ephys_fpga._get_main_probe_sync(session_path, bin_exists=False) if treplay is None: passivePeriods_df = extract_passive_periods(session_path, sync=sync, sync_map=sync_map) treplay = passivePeriods_df.taskReplay.values fttl = ephys_fpga._get_sync_fronts(sync, sync_map["frame2ttl"], tmin=treplay[0]) passiveGabor_df = _extract_passiveGabor_df(fttl, session_path) bpod = ephys_fpga._get_sync_fronts(sync, sync_map["bpod"], tmin=treplay[0]) passiveValve_intervals = _extract_passiveValve_intervals(bpod) audio = ephys_fpga._get_sync_fronts(sync, sync_map["audio"], tmin=treplay[0]) passiveTone_intervals, passiveNoise_intervals = _extract_passiveAudio_intervals( audio) passiveStims_df = np.concatenate([ passiveValve_intervals, passiveTone_intervals, passiveNoise_intervals ], axis=1) columns = [ "valveOn", "valveOff", "toneOn", "toneOff", "noiseOn", "noiseOff" ] passiveStims_df = pd.DataFrame(passiveStims_df, columns=columns) return ( passiveGabor_df, passiveStims_df, ) # _ibl_passiveGabor.table.csv, _ibl_passiveStims.times_table.csv
def extract_passive_periods(session_path: str, sync: dict = None, sync_map: dict = None) -> pd.DataFrame: if sync is None or sync_map is None: sync, sync_map = ephys_fpga._get_main_probe_sync(session_path, bin_exists=False) t_start_passive, t_starts, t_ends = _get_passive_spacers(session_path, sync=sync, sync_map=sync_map) t_starts_col = np.insert(t_starts, 0, t_start_passive) t_ends_col = np.insert(t_ends, 0, t_ends[-1]) # tpassive_protocol = [t_start_passive, t_ends[-1]] # tspontaneous = [t_starts[0], t_ends[0]] # trfm = [t_starts[1], t_ends[1]] # treplay = [t_starts[2], t_ends[2]] passivePeriods_df = pd.DataFrame( [t_starts_col, t_ends_col], index=["start", "stop"], columns=[ "passiveProtocol", "spontaneousActivity", "RFM", "taskReplay" ], ) return passivePeriods_df # _ibl_passivePeriods.intervalsTable.csv
def validate_ttl_test(ses_path, display=False): """ For a mock session on the Ephys Choice world task, check the sync channels for all device properly connected and perform a synchronization if dual probes to check that all channels are recorded properly :param ses_path: session path :param display: show the probe synchronization plot if several probes :return: True if tests pass, errors otherwise """ def _single_test(assertion, str_ok, str_ko): if assertion: _logger.info(str_ok) return True else: _logger.error(str_ko) return False EXPECTED_RATES_HZ = {'left_camera': 60, 'right_camera': 150, 'body_camera': 30} SYNC_RATE_HZ = 1 MIN_TRIALS_NB = 6 ok = True ses_path = Path(ses_path) if not ses_path.exists(): return False rawsync, sync_map = fpga._get_main_probe_sync(ses_path) last_time = rawsync['times'][-1] # get upgoing fronts for each sync = Bunch({}) for k in sync_map: fronts = fpga._get_sync_fronts(rawsync, sync_map[k]) sync[k] = fronts['times'][fronts['polarities'] == 1] wheel = fpga.extract_wheel_sync(rawsync, chmap=sync_map, save=False) frame_rates = {'right_camera': np.round(1 / np.median(np.diff(sync.right_camera))), 'left_camera': np.round(1 / np.median(np.diff(sync.left_camera))), 'body_camera': np.round(1 / np.median(np.diff(sync.body_camera)))} # check the camera frame rates for lab in frame_rates: expect = EXPECTED_RATES_HZ[lab] ok &= _single_test(assertion=abs((1 - frame_rates[lab] / expect)) < 0.1, str_ok=f'PASS: {lab} frame rate: {frame_rates[lab]} = {expect} Hz', str_ko=f'FAILED: {lab} frame rate: {frame_rates[lab]} != {expect} Hz') # check that the wheel has a minimum rate of activity on both channels re_test = abs(1 - sync.rotary_encoder_1.size / sync.rotary_encoder_0.size) < 0.1 re_test &= len(wheel['re_pos']) / last_time > 5 ok &= _single_test(assertion=re_test, str_ok="PASS: Rotary encoder", str_ko="FAILED: Rotary encoder") # check that the frame 2 ttls has a minimum rate of activity ok &= _single_test(assertion=len(sync.frame2ttl) / last_time > 0.2, str_ok="PASS: Frame2TTL", str_ko="FAILED: Frame2TTL") # the audio has to have at least one event per trial ok &= _single_test(assertion=len(sync.bpod) > len(sync.audio) > MIN_TRIALS_NB, str_ok="PASS: audio", str_ko="FAILED: audio") # the bpod has to have at least twice the amount of min trial pulses ok &= _single_test(assertion=len(sync.bpod) > MIN_TRIALS_NB * 2, str_ok="PASS: Bpod", str_ko="FAILED: Bpod") try: # note: tried to depend as little as possible on the extraction code but for the valve... behaviour = fpga.extract_behaviour_sync(rawsync, save=False, chmap=sync_map) res = behaviour.valve_open.size > 1 except AssertionError: res = False # check that the reward valve is actionned at least once ok &= _single_test(assertion=res, str_ok="PASS: Valve open", str_ko="FAILED: Valve open not detected") _logger.info('ALL CHECKS PASSED !') # the imec sync is for 3B Probes only if sync.get('imec_sync') is not None: ok &= _single_test(assertion=np.all(1 - SYNC_RATE_HZ * np.diff(sync.imec_sync) < 0.1), str_ok="PASS: imec sync", str_ko="FAILED: imec sync") # second step is to test that we can make the sync. Assertions are whithin the synch code if sync.get('imec_sync') is not None: sync_result = sync_probes.version3B(ses_path, display=display) else: sync_result = sync_probes.version3A(ses_path, display=display) ok &= _single_test(assertion=sync_result, str_ok="PASS: synchronisation", str_ko="FAILED: probe synchronizations threshold exceeded") if not ok: raise ValueError('FAILED TTL test') return ok
def _extract(self, sync: dict = None, sync_map: dict = None, plot: bool = False, **kwargs) -> tuple: if sync is None or sync_map is None: sync, sync_map = ephys_fpga._get_main_probe_sync(self.session_path, bin_exists=False) try: # Passive periods passivePeriods_df = extract_passive_periods(self.session_path, sync=sync, sync_map=sync_map) trfm = passivePeriods_df.RFM.values treplay = passivePeriods_df.taskReplay.values except BaseException as e: log.error("Failed to extract passive periods", e) passivePeriods_df = None trfm = None treplay = None try: # RFMapping passiveRFM_times = extract_rfmapping(self.session_path, sync=sync, sync_map=sync_map, trfm=trfm) except BaseException as e: log.error("Failed to extract RFMapping datasets", e) passiveRFM_times = None try: ( passiveGabor_df, passiveStims_df, ) = extract_task_replay(self.session_path, sync=sync, sync_map=sync_map, treplay=treplay) except BaseException as e: log.error("Failed to extract task replay stimuli", e) ( passiveGabor_df, passiveStims_df, ) = ( None, None, ) if plot: f, ax = plt.subplots(1, 1) f.suptitle("/".join(str(self.session_path).split("/")[-5:])) plot_sync_channels(sync=sync, sync_map=sync_map, ax=ax) plot_passive_periods(passivePeriods_df, ax=ax) plot_rfmapping(passiveRFM_times, ax=ax) plot_gabor_times(passiveGabor_df, ax=ax) plot_stims_times(passiveStims_df, ax=ax) return ( passivePeriods_df, # _ibl_passivePeriods.intervalsTable.csv passiveRFM_times, # _ibl_passiveRFM.times.npy passiveGabor_df, # _ibl_passiveGabor.table.csv, passiveStims_df, # _ibl_passiveStims.table.csv )
def extract_replay_debug( session_path: str, sync: dict = None, sync_map: dict = None, treplay: np.array = None, ax: plt.axes = None, ) -> Tuple[pd.DataFrame, pd.DataFrame]: # Load sessions sync channels, map if sync is None or sync_map is None: sync, sync_map = ephys_fpga._get_main_probe_sync(session_path, bin_exists=False) if treplay is None: passivePeriods_df = extract_passive_periods(session_path, sync=sync, sync_map=sync_map) treplay = passivePeriods_df.taskReplay.values if ax is None: f, ax = plt.subplots(1, 1) f = ax.figure f.suptitle("/".join(str(session_path).split("/")[-5:])) plot_sync_channels(sync=sync, sync_map=sync_map, ax=ax) passivePeriods_df = extract_passive_periods(session_path, sync=sync, sync_map=sync_map) treplay = passivePeriods_df.taskReplay.values plot_passive_periods(passivePeriods_df, ax=ax) fttl = ephys_fpga._get_sync_fronts(sync, sync_map["frame2ttl"], tmin=treplay[0]) passiveGabor_df = _extract_passiveGabor_df(fttl, session_path) plot_gabor_times(passiveGabor_df, ax=ax) bpod = ephys_fpga._get_sync_fronts(sync, sync_map["bpod"], tmin=treplay[0]) passiveValve_intervals = _extract_passiveValve_intervals(bpod) plot_valve_times(passiveValve_intervals, ax=ax) audio = ephys_fpga._get_sync_fronts(sync, sync_map["audio"], tmin=treplay[0]) passiveTone_intervals, passiveNoise_intervals = _extract_passiveAudio_intervals( audio) plot_audio_times(passiveTone_intervals, passiveNoise_intervals, ax=ax) passiveStims_df = np.concatenate([ passiveValve_intervals, passiveTone_intervals, passiveNoise_intervals ], axis=1) columns = [ "valveOn", "valveOff", "toneOn", "toneOff", "noiseOn", "noiseOff" ] passiveStims_df = pd.DataFrame(passiveStims_df, columns=columns) return ( passiveGabor_df, passiveStims_df, ) # _ibl_passiveGabor.table.csv, _ibl_passiveStims.table.csv
session_path = alf.io.get_session_path(local_paths[0]) # load session fixtures settings = rawio.load_settings(session_path) ses_nb = settings['SESSION_ORDER'][settings['SESSION_IDX']] path_fixtures = Path(ephys_fpga.__file__).parent.joinpath('ephys_sessions') fixture = {'pcs': np.load(path_fixtures.joinpath(f'session_{ses_nb}_passive_pcs.npy')), 'delays': np.load(path_fixtures.joinpath(f'session_{ses_nb}_passive_stimDelays.npy')), 'ids': np.load(path_fixtures.joinpath(f'session_{ses_nb}_passive_stimIDs.npy'))} # load general metadata with open(path_fixtures.joinpath('passive_stim_meta.json'), 'r') as f: meta = json.load(f) t_end_ephys = passive.ephysCW_end(session_path=session_path) # load stimulus sequence sync, sync_map = ephys_fpga._get_main_probe_sync(session_path, bin_exists=False) fpga_sync = ephys_fpga._get_sync_fronts(sync, sync_map['frame2ttl']) fttl = ephys_fpga._get_sync_fronts(sync, sync_map['frame2ttl'], tmin=t_end_ephys) def get_spacers(): """ load and get spacer information, do corr to find spacer timestamps returns t_passive_starts, t_starts, t_ends """ spacer_template = np.array(meta['VISUAL_STIM_0']['ttl_frame_nums'], dtype=np.float32) / FRAME_FS jitter = 3 / FRAME_FS # allow for 3 screen refresh as jitter t_quiet = meta['VISUAL_STIM_0']['delay_around'] spacer_times, _ = passive.get_spacer_times(spacer_template=spacer_template, jitter=jitter, ttl_signal=fttl['times'], t_quiet=t_quiet)