def get_stimOn_times(session_path, save=False, data=False, settings=False): """ Find the time of the statemachine command to turn on hte stim (state stim_on start or rotary_encoder_event2) Find the next frame change from the photodiodeafter that TS. Screen is not displaying anything until then. (Frame changes are in BNC1High and BNC1Low) """ if not data: data = raw.load_data(session_path) if not settings: settings = raw.load_settings(session_path) if settings is None or settings['IBLRIG_VERSION_TAG'] == '': settings = {'IBLRIG_VERSION_TAG': '100.0.0'} # Version check if version.ge(settings['IBLRIG_VERSION_TAG'], '5.0.0'): stimOn_times = get_stimOn_times_ge5(session_path, data=data) else: stimOn_times = get_stimOn_times_lt5(session_path, data=data) if raw.save_bool(save, '_ibl_trials.stimOn_times.npy'): check_alf_folder(session_path) fpath = os.path.join(session_path, 'alf', '_ibl_trials.stimOn_times.npy') np.save(fpath, np.array(stimOn_times)) return np.array(stimOn_times)
def extract_all(session_path, save=False, bpod_trials=None, settings=None): if not bpod_trials: bpod_trials = raw.load_data(session_path) if not settings: settings = raw.load_settings(session_path) if settings is None or settings['IBLRIG_VERSION_TAG'] == '': settings = {'IBLRIG_VERSION_TAG': '100.0.0'} base = [ FeedbackType, ContrastLR, ProbabilityLeft, Choice, RepNum, RewardVolume, FeedbackTimes, Intervals, ResponseTimes, GoCueTriggerTimes, GoCueTimes ] # Version check if version.ge(settings['IBLRIG_VERSION_TAG'], '5.0.0'): base.extend([ StimOnTriggerTimes, StimOnOffFreezeTimes, ItiInTimes, StimOffTriggerTimes, StimFreezeTriggerTimes, ErrorCueTriggerTimes ]) else: base.extend([IncludedTrials, ItiDuration, StimOnTimes]) out, fil = run_extractor_classes(base, save=save, session_path=session_path, bpod_trials=bpod_trials, settings=settings) return out, fil
def get_feedback_times(session_path, save=False, data=False, settings=False): """ Get the times the water or error tone was delivered to the animal. **Optional:** saves _ibl_trials.feedback_times.npy Gets reward and error state init times vectors, checks if theintersection of nans is empty, then merges the 2 vectors. :param session_path: Absolute path of session folder :type session_path: str :param save: wether to save the corresponding alf file to the alf folder, defaults to False :param save: bool, optional :return: numpy.ndarray :rtype: dtype('float64') """ if not data: data = raw.load_data(session_path) if not settings: settings = raw.load_settings(session_path) if settings is None or settings['IBLRIG_VERSION_TAG'] == '': settings = {'IBLRIG_VERSION_TAG': '100.0.0'} # Version check if version.ge(settings['IBLRIG_VERSION_TAG'], '5.0.0'): merge = get_feedback_times_ge5(session_path, data=data) else: merge = get_feedback_times_lt5(session_path, data=data) if raw.save_bool(save, '_ibl_trials.feedback_times.npy'): check_alf_folder(session_path) fpath = os.path.join(session_path, 'alf', '_ibl_trials.feedback_times.npy') np.save(fpath, merge) return np.array(merge)
def _extract(self): if version.ge(self.settings['IBLRIG_VERSION_TAG'], '5.0.0'): trials_included = self.get_included_trials_ge5( data=self.bpod_trials, settings=self.settings) else: trials_included = self.get_included_trials_lt5(data=self.bpod_trials) return trials_included
def test_trials_extraction(self): # extract all sessions with tempfile.TemporaryDirectory() as tdir: subjects_path = Path(tdir).joinpath('Subjects') shutil.copytree(self.INIT_FOLDER, subjects_path) for fil in subjects_path.rglob('_iblrig_taskData.raw*.jsonable'): session_path = fil.parents[1] # task running part job = TrainingTrials(session_path, one=self.one) job.run() # check the trials objects trials = alf.io.load_object(session_path / 'alf', 'trials') self.assertTrue(alf.io.check_dimensions(trials) == 0) settings = rawio.load_settings(session_path) if version.ge(settings['IBLRIG_VERSION_TAG'], '5.0.0'): tkeys = TRIAL_KEYS_ge5 else: tkeys = TRIAL_KEYS_lt5 self.assertTrue(set(trials.keys()) == set(tkeys)) # check the wheel object if the extraction didn't fail if job.status != -1: wheel = alf.io.load_object(session_path / 'alf', 'wheel') self.assertTrue(alf.io.check_dimensions(wheel) == 0) self.assertTrue(set(wheel.keys()) == set(WHEEL_KEYS)) """ For this session only the downgoing front of a trial was detected, resulting in an error for the gocuetime. The fix was to extract the downgoing front and subtract 100ms. """ session_path = subjects_path / "CSHL_007/2019-07-31/001" trials = alf.io.load_object(session_path / 'alf', 'trials') self.assertTrue(np.all(np.logical_not(np.isnan(trials.goCue_times))))
def _extract(self): # Version check if version.ge(self.settings['IBLRIG_VERSION_TAG'], '5.0.0'): merge = self.get_feedback_times_ge5(self.session_path, data=self.bpod_trials) else: merge = self.get_feedback_times_lt5(self.session_path, data=self.bpod_trials) return np.array(merge)
def _extract(self): if version.ge(self.settings['IBLRIG_VERSION_TAG'], '5.0.0'): goCue = np.array([tr['behavior_data']['States timestamps'] ['play_tone'][0][0] for tr in self.bpod_trials]) else: goCue = np.array([tr['behavior_data']['States timestamps'] ['closed_loop'][0][0] for tr in self.bpod_trials]) return goCue
def _extract(self): """ Find the time of the state machine command to turn on the stim (state stim_on start or rotary_encoder_event2) Find the next frame change from the photodiode after that TS. Screen is not displaying anything until then. (Frame changes are in BNC1 High and BNC1 Low) """ # Version check if version.ge(self.settings['IBLRIG_VERSION_TAG'], '5.0.0'): stimOn_times = self.get_stimOn_times_ge5(self.session_path, data=self.bpod_trials) else: stimOn_times = self.get_stimOn_times_lt5(self.session_path, data=self.bpod_trials) return np.array(stimOn_times)
def extract_all(session_path, save=False, data=False, settings=False): if not data: data = raw.load_data(session_path) if not settings: settings = raw.load_settings(session_path) if settings is None or settings['IBLRIG_VERSION_TAG'] == '': settings = {'IBLRIG_VERSION_TAG': '100.0.0'} # Common to all versions feedbackType = get_feedbackType(session_path, save=save, data=data, settings=settings) contrastLeft, contrastRight = get_contrastLR( session_path, save=save, data=data, settings=settings) probabilityLeft = get_probabilityLeft( session_path, save=save, data=data, settings=settings) choice = get_choice(session_path, save=save, data=data, settings=settings) rewardVolume = get_rewardVolume(session_path, save=save, data=data, settings=settings) feedback_times = get_feedback_times(session_path, save=save, data=data, settings=settings) stimOn_times = get_stimOn_times(session_path, save=save, data=data, settings=settings) intervals = get_intervals(session_path, save=save, data=data, settings=settings) response_times = get_response_times(session_path, save=save, data=data, settings=settings) go_cue_trig_times = get_goCueTrigger_times( session_path, save=save, data=data, settings=settings) go_cue_times = get_goCueOnset_times(session_path, save=save, data=data, settings=settings) camera_timestamps = get_camera_timestamps( session_path, save=save, data=data, settings=settings) out = {'feedbackType': feedbackType, 'contrastLeft': contrastLeft, 'contrastRight': contrastRight, 'probabilityLeft': probabilityLeft, 'session_path': session_path, 'choice': choice, 'rewardVolume': rewardVolume, 'feedback_times': feedback_times, 'stimOn_times': stimOn_times, 'intervals': intervals, 'response_times': response_times, 'camera_timestamps': camera_timestamps, 'goCue_times': go_cue_times, 'goCueTrigger_times': go_cue_trig_times} # Version specific extractions if version.ge(settings['IBLRIG_VERSION_TAG'], '5.0.0'): out['stimOnTrigger_times'] = get_stimOnTrigger_times( session_path, save=save, data=data, settings=settings) out['included'] = get_included_trials( session_path, save=save, data=data, settings=settings) else: out['iti_dur'] = get_iti_duration(session_path, save=save, data=data, settings=settings) return out
def load_encoder_positions(session_path, settings=False): """ Load Rotary Encoder (RE) positions from raw data file within a session path. Assumes that a folder called "raw_behavior_data" exists in folder. Positions are RE ticks [-512, 512] == [-180º, 180º] 0 == trial stim init position Positive nums are rightwards movements (mouse) or RE CW (mouse) Variable line number, depends on movements. Raw datafile Columns: Position, RE timestamp, RE Position, Bonsai Timestamp Position is always equal to 'Position' so this column was dropped. >>> data.columns >>> ['re_ts', # Rotary Encoder Timestamp (ms) 'numpy.int64' 're_pos', # Rotary Encoder position (ticks) 'numpy.int64' 'bns_ts'] # Bonsai Timestamp 'pandas.Timestamp' # pd.to_datetime(data.bns_ts) to work in datetimes :param session_path: Absolute path of session folder :type session_path: str :return: dataframe w/ 3 cols and N positions :rtype: Pandas.DataFrame """ if session_path is None: return path = Path(session_path).joinpath("raw_behavior_data") path = next(path.glob("_iblrig_encoderPositions.raw*.ssv"), None) if not settings: settings = load_settings(session_path) if settings is None or settings['IBLRIG_VERSION_TAG'] == '': settings = {'IBLRIG_VERSION_TAG': '100.0.0'} # auto-detect old files when version is not labeled with open(path) as fid: line = fid.readline() if line.startswith('Position'): settings = {'IBLRIG_VERSION_TAG': '0.0.0'} if not path: _logger.warning("No data loaded: could not find raw encoderPositions file") return None if version.ge(settings['IBLRIG_VERSION_TAG'], '5.0.0'): return _load_encoder_positions_file_ge5(path) else: return _load_encoder_positions_file_lt5(path)
def get_included_trials(session_path, save=False, data=False, settings=False): if not data: data = raw.load_data(session_path) if not settings: settings = raw.load_settings(session_path) if settings is None or settings['IBLRIG_VERSION_TAG'] == '': settings = {'IBLRIG_VERSION_TAG': '100.0.0'} if version.ge(settings['IBLRIG_VERSION_TAG'], '5.0.0'): trials_included = get_included_trials_ge5(session_path, data=data, settings=settings) else: trials_included = get_included_trials_lt5(session_path, data=data) if raw.save_bool(save, '_ibl_trials.included'): fpath = Path(session_path).joinpath('alf', '_ibl_trials.included.npy') np.save(fpath, trials_included) return trials_included
def get_goCueTrigger_times(session_path, save=False, data=False, settings=False): """ Get trigger times of goCue from state machine. Current software solution for triggering sounds uses PyBpod soft codes. Delays can be in the order of 10's of ms. This is the time when the command to play the sound was executed. To measure accurate time, either getting the sound onset from xonar soundcard sync pulse (latencies may vary). :param session_path: Absolute path of session folder :type session_path: str :param save: wether to save the corresponding alf file to the alf folder, defaults to False :param save: bool, optional :return: numpy.ndarray :rtype: dtype('float64') """ if not data: data = raw.load_data(session_path) if not settings: settings = raw.load_settings(session_path) if settings is None or settings['IBLRIG_VERSION_TAG'] == '': settings = {'IBLRIG_VERSION_TAG': '100.0.0'} # Version check if version.ge(settings['IBLRIG_VERSION_TAG'], '5.0.0'): goCue = np.array([ tr['behavior_data']['States timestamps']['play_tone'][0][0] for tr in data ]) else: goCue = np.array([ tr['behavior_data']['States timestamps']['closed_loop'][0][0] for tr in data ]) if raw.save_bool(save, '_ibl_trials.goCue_times.npy'): check_alf_folder(session_path) fpath = os.path.join(session_path, 'alf', '_ibl_trials.goCueTrigger_times.npy') np.save(fpath, goCue) return goCue
def extract_all(session_path, save=False, bpod_trials=None, settings=None): """Extract trials and wheel data. For task versions >= 5.0.0, outputs wheel data and trials.table dataset (+ some extra datasets) Parameters ---------- session_path : str, pathlib.Path The path to the session save : bool If true save the data files to ALF bpod_trials : list of dicts The Bpod trial dicts loaded from the _iblrig_taskData.raw dataset settings : dict The Bpod settings loaded from the _iblrig_taskSettings.raw dataset Returns ------- A list of extracted data and a list of file paths if save is True (otherwise None) """ if not bpod_trials: bpod_trials = raw.load_data(session_path) if not settings: settings = raw.load_settings(session_path) if settings is None or settings['IBLRIG_VERSION_TAG'] == '': settings = {'IBLRIG_VERSION_TAG': '100.0.0'} base = [RepNum, GoCueTriggerTimes] # Version check if version.ge(settings['IBLRIG_VERSION_TAG'], '5.0.0'): # We now extract a single trials table base.extend([ StimOnTriggerTimes, ItiInTimes, StimOffTriggerTimes, StimFreezeTriggerTimes, ErrorCueTriggerTimes, TrialsTable, PhasePosQuiescence ]) else: base.extend([ Intervals, Wheel, FeedbackType, ContrastLR, ProbabilityLeft, Choice, IncludedTrials, ItiDuration, StimOnTimes_deprecated, RewardVolume, FeedbackTimes, ResponseTimes, GoCueTimes, PhasePosQuiescence ]) out, fil = run_extractor_classes( base, save=save, session_path=session_path, bpod_trials=bpod_trials, settings=settings) return out, fil
def load_encoder_events(session_path, settings=False): """ Load Rotary Encoder (RE) events raw data file. Assumes that a folder called "raw_behavior_data" exists in folder. On each trial the RE sends 3 events to Bonsai 1 - meaning trial start/turn off the stim; 2 - meaning show the current trial stimulus; and 3 - meaning begin the closed loop making the stim move with the RE. These events are triggered by the state machine in the corrensponding states: trial_start, stim_on, closed_loop Raw datafile Columns: Event, RE timestamp, Source, data, Bonsai Timestamp Event is always equal 'Event' Source is always equal 'StateMachine'. For this reason these columns are dropped. >>> data.columns >>> ['re_ts', # Rotary Encoder Timestamp (ms) 'numpy.int64' 'sm_ev', # State Machine Event 'numpy.int64' 'bns_ts'] # Bonsai Timestamp (int) 'pandas.Timestamp' # pd.to_datetime(data.bns_ts) to work in datetimes :param session_path: [description] :type session_path: [type] :return: dataframe w/ 3 cols and (ntrials * 3) lines :rtype: Pandas.DataFrame """ if session_path is None: return path = Path(session_path).joinpath("raw_behavior_data") path = next(path.glob("_iblrig_encoderEvents.raw*.ssv"), None) if not settings: settings = load_settings(session_path) if settings is None or settings['IBLRIG_VERSION_TAG'] == '': settings = {'IBLRIG_VERSION_TAG': '100.0.0'} if not path: return None if version.ge(settings['IBLRIG_VERSION_TAG'], '5.0.0'): return _load_encoder_events_file_ge5(path) else: return _load_encoder_events_file_lt5(path)
def test_partial_extraction(self): ex = TaskQCExtractor(self.session_path, lazy=True, one=self.one, bpod_only=True) ex.extract_data() expected = [ 'contrastLeft', 'contrastRight', 'phase', 'position', 'probabilityLeft', 'quiescence', 'stimOn_times' ] expected_5up = [ 'contrast', 'errorCueTrigger_times', 'itiIn_times', 'stimFreezeTrigger_times', 'stimFreeze_times', 'stimOffTrigger_times', 'stimOff_times', 'stimOnTrigger_times' ] if version.ge(ex.settings['IBLRIG_VERSION_TAG'], '5.0.0'): expected += expected_5up self.assertTrue(set(expected).issubset(set(ex.data.keys())))
def load_encoder_events(session_path, settings=False): """ Load Rotary Encoder (RE) events raw data file. Assumes that a folder called "raw_behavior_data" exists in folder. Events number correspond to following bpod states: 1: correct / hide_stim 2: stim_on 3: closed_loop 4: freeze_error / freeze_correct >>> data.columns >>> ['re_ts', # Rotary Encoder Timestamp (ms) 'numpy.int64' 'sm_ev', # State Machine Event 'numpy.int64' 'bns_ts'] # Bonsai Timestamp (int) 'pandas.Timestamp' # pd.to_datetime(data.bns_ts) to work in datetimes :param session_path: [description] :type session_path: [type] :return: dataframe w/ 3 cols and (ntrials * 3) lines :rtype: Pandas.DataFrame """ if session_path is None: return path = Path(session_path).joinpath("raw_behavior_data") path = next(path.glob("_iblrig_encoderEvents.raw*.ssv"), None) if not settings: settings = load_settings(session_path) if settings is None or settings['IBLRIG_VERSION_TAG'] == '': settings = {'IBLRIG_VERSION_TAG': '100.0.0'} # auto-detect old files when version is not labeled with open(path) as fid: line = fid.readline() if line.startswith('Event') and 'StateMachine' in line: settings = {'IBLRIG_VERSION_TAG': '0.0.0'} if not path: return None if version.ge(settings['IBLRIG_VERSION_TAG'], '5.0.0'): return _load_encoder_events_file_ge5(path) else: return _load_encoder_events_file_lt5(path)
def extract_all(session_path, save=False, bpod_trials=False, settings=False, extra_classes=None): """ Same as training_trials.extract_all except... - there is no RepNum - ContrastLR is extracted differently - IncludedTrials is only extracted for 5.0.0 or greater :param session_path: :param save: :param bpod_trials: :param settings: :param extra_classes: additional BaseBpodTrialsExtractor subclasses for custom extractions :return: """ if not bpod_trials: bpod_trials = raw.load_data(session_path) if not settings: settings = raw.load_settings(session_path) if settings is None or settings['IBLRIG_VERSION_TAG'] == '': settings = {'IBLRIG_VERSION_TAG': '100.0.0'} base = [GoCueTriggerTimes] # Version check if version.ge(settings['IBLRIG_VERSION_TAG'], '5.0.0'): # We now extract a single trials table base.extend([ StimOnTriggerTimes, ItiInTimes, StimOffTriggerTimes, StimFreezeTriggerTimes, ErrorCueTriggerTimes, TrialsTableBiased, IncludedTrials, PhasePosQuiescence ]) else: base.extend([ Intervals, Wheel, FeedbackType, ContrastLR, ProbabilityLeft, Choice, ItiDuration, StimOnTimes_deprecated, RewardVolume, FeedbackTimes, ResponseTimes, GoCueTimes, PhasePosQuiescence ]) if extra_classes: base.extend(extra_classes) out, fil = run_extractor_classes( base, save=save, session_path=session_path, bpod_trials=bpod_trials, settings=settings) return out, fil
def assert_g(self, v0, v_): self.assertFalse(version.eq(v0, v_)) self.assertTrue(version.ge(v0, v_)) self.assertFalse(version.le(v0, v_)) self.assertTrue(version.gt(v0, v_)) self.assertFalse(version.lt(v0, v_))