def test_get_probabilityLeft(self): # TRAINING SESSIONS pl = extractors.training_trials.ProbabilityLeft( self.training_lt5['path']).extract()[0] self.assertTrue(isinstance(pl, np.ndarray)) # -- version >= 5.0.0 pl = extractors.training_trials.ProbabilityLeft( self.training_ge5['path']).extract()[0] self.assertTrue(isinstance(pl, np.ndarray)) # BIASED SESSIONS pl = extractors.biased_trials.ProbabilityLeft( self.biased_lt5['path']).extract()[0] self.assertTrue(isinstance(pl, np.ndarray)) # Test if only probs that are in prob set md = raw.load_settings(self.biased_lt5['path']) if md: probs = md['BLOCK_PROBABILITY_SET'] probs.append(0.5) self.assertTrue(sum([x in probs for x in pl]) == len(pl)) # -- version >= 5.0.0 pl = extractors.biased_trials.ProbabilityLeft( self.biased_ge5['path']).extract()[0] self.assertTrue(isinstance(pl, np.ndarray)) # Test if only probs that are in prob set md = raw.load_settings(self.biased_ge5['path']) probs = md['BLOCK_PROBABILITY_SET'] probs.append(0.5) self.assertTrue(sum([x in probs for x in pl]) == len(pl))
def patch_settings_file(sess_or_file: str, patch: dict) -> None: sess_or_file = Path(sess_or_file) if sess_or_file.is_file() and sess_or_file.name.endswith( "_iblrig_taskSettings.raw.json"): session = sess_or_file.parent.parent file = sess_or_file elif sess_or_file.is_dir() and sess_or_file.name.isdecimal(): file = sess_or_file / "raw_behavior_data" / "_iblrig_taskSettings.raw.json" session = sess_or_file else: print("not a settings file or a session folder") return settings = raw.load_settings(session) settings.update(patch) # Rename file on disk keeps pathlib ref to "file" intact file.rename(file.with_suffix(".json_bk")) with open(file, "w") as f: f.write(json.dumps(settings, indent=1)) f.write("\n") f.flush() # Check if properly saved saved_settings = raw.load_settings(session) if settings == saved_settings: file.with_suffix(".json_bk").unlink() return
def get_probabilityLeft(session_path, save=False, data=False, settings=False): if not data: data = raw.load_data(session_path) if not settings: settings = raw.load_settings(session_path) if settings is None: settings = {"IBLRIG_VERSION_TAG": "100.0.0"} elif settings["IBLRIG_VERSION_TAG"] == "": settings.update({"IBLRIG_VERSION_TAG": "100.0.0"}) num = settings.get("PRELOADED_SESSION_NUM", None) if num is None: num = settings.get("PREGENERATED_SESSION_NUM", None) if num is None: fn = settings.get('SESSION_LOADED_FILE_PATH', None) fn = PureWindowsPath(fn).name num = ''.join([d for d in fn if d.isdigit()]) if num == '': raise ValueError("Can't extract left probability behaviour.") # Load the pregenerated file sessions_folder = Path(raw.__file__).parent.joinpath( 'extractors', 'ephys_sessions') fname = f"session_{num}_ephys_pcqs.npy" pcqsp = np.load(sessions_folder.joinpath(fname)) pLeft = pcqsp[:, 4] pLeft = pLeft[:len(data)] if raw.save_bool(save, "_ibl_trials.probabilityLeft.npy"): lpath = Path(session_path).joinpath("alf", "_ibl_trials.probabilityLeft.npy") np.save(lpath, pLeft) return pLeft
def _previous_data_files(self, typ='data'): logger.debug(f"Looking for previous files of type: {typ}") prev_data_files = [] prev_session_files = [] data_fname = self.BASE_FILENAME + 'Data.raw.jsonable' settings_fname = self.BASE_FILENAME + 'Settings.raw.json' logger.debug(f"Looking for files:{data_fname} AND {settings_fname}") for prev_sess_path in self._previous_session_folders(): prev_sess_path = Path(prev_sess_path) / 'raw_behavior_data' # Get all data and settings file if they both exist if ((prev_sess_path / data_fname).exists() and (prev_sess_path / settings_fname).exists()): prev_data_files.append(prev_sess_path / data_fname) prev_session_files.append(prev_sess_path / settings_fname) logger.debug(f"Found {len(prev_data_files)} file pairs") # Remove empty files ds_out = [(d, s) for d, s in zip(prev_data_files, prev_session_files) if d.stat().st_size != 0 and s.stat().st_size != 0] logger.debug(f"Found {len(ds_out)} non empty file pairs") # Remove sessions of different task protocols ds_out = [(d, s) for d, s in ds_out if self._PROTOCOL in raw.load_settings(str(s.parent.parent))['PYBPOD_PROTOCOL']] logger.debug( f"Found {len(ds_out)} file pairs for protocol {self._PROTOCOL}") data_out = [str(d) for d, s in ds_out] settings_out = [str(s) for d, s in ds_out] if not data_out: logger.debug( f'NOT FOUND: Previous data files for task {self._PROTOCOL}') if not settings_out: logger.debug( f'NOT FOUND: Previous settings files for task {self._PROTOCOL}') logger.debug(f"Reurning {typ} files") return data_out if typ == 'data' else settings_out
def audio_training(root_data_folder, dry=False, max_sessions=False): from ibllib.io.extractors import training_audio as audio audio_flags = Path(root_data_folder).rglob('audio_training.flag') c = 0 for flag in audio_flags: c += 1 if max_sessions and c > max_sessions: return _logger.info(flag) if dry: continue session_path = flag.parent try: settings = raw_data_loaders.load_settings(session_path) typ = extract_session.get_task_extractor_type( settings.get('PYBPOD_PROTOCOL')) except json.decoder.JSONDecodeError: typ = 'unknown' # this extractor is only for biased and training sessions if typ not in ['biased', 'training', 'habituation']: flag.unlink() continue audio.extract_sound(session_path, save=True, delete=True) flag.unlink() session_path.joinpath('register_me.flag').touch()
def extract_all(session_path, save=False, bpod_trials=None, settings=None): if not bpod_trials: bpod_trials = raw.load_data(session_path) if not settings: settings = raw.load_settings(session_path) if settings is None or settings['IBLRIG_VERSION_TAG'] == '': settings = {'IBLRIG_VERSION_TAG': '100.0.0'} base = [ FeedbackType, ContrastLR, ProbabilityLeft, Choice, RepNum, RewardVolume, FeedbackTimes, Intervals, ResponseTimes, GoCueTriggerTimes, GoCueTimes ] # Version check if version.ge(settings['IBLRIG_VERSION_TAG'], '5.0.0'): base.extend([ StimOnTriggerTimes, StimOnOffFreezeTimes, ItiInTimes, StimOffTriggerTimes, StimFreezeTriggerTimes, ErrorCueTriggerTimes ]) else: base.extend([IncludedTrials, ItiDuration, StimOnTimes]) out, fil = run_extractor_classes(base, save=save, session_path=session_path, bpod_trials=bpod_trials, settings=settings) return out, fil
def get_feedback_times(session_path, save=False, data=False, settings=False): """ Get the times the water or error tone was delivered to the animal. **Optional:** saves _ibl_trials.feedback_times.npy Gets reward and error state init times vectors, checks if theintersection of nans is empty, then merges the 2 vectors. :param session_path: Absolute path of session folder :type session_path: str :param save: wether to save the corresponding alf file to the alf folder, defaults to False :param save: bool, optional :return: numpy.ndarray :rtype: dtype('float64') """ if not data: data = raw.load_data(session_path) if not settings: settings = raw.load_settings(session_path) if settings is None or settings['IBLRIG_VERSION_TAG'] == '': settings = {'IBLRIG_VERSION_TAG': '100.0.0'} # Version check if version.ge(settings['IBLRIG_VERSION_TAG'], '5.0.0'): merge = get_feedback_times_ge5(session_path, data=data) else: merge = get_feedback_times_lt5(session_path, data=data) if raw.save_bool(save, '_ibl_trials.feedback_times.npy'): check_alf_folder(session_path) fpath = os.path.join(session_path, 'alf', '_ibl_trials.feedback_times.npy') np.save(fpath, merge) return np.array(merge)
def get_rewardVolume(session_path, save=False, data=False, settings=False): """ Load reward volume delivered for each trial. **Optional:** saves _ibl_trials.rewardVolume.npy Uses reward_current to accumulate the amount of :param session_path: Absolute path of session folder :type session_path: str :param save: wether to save the corresponding alf file to the alf folder, defaults to False :param save: bool, optional :return: numpy.ndarray :rtype: dtype('int64') """ if not data: data = raw.load_data(session_path) if not settings: settings = raw.load_settings(session_path) if settings is None or settings['IBLRIG_VERSION_TAG'] == '': settings = {'IBLRIG_VERSION_TAG': '100.0.0'} trial_volume = [ x['reward_amount'] if x['trial_correct'] else 0 for x in data ] rewardVolume = np.array(trial_volume).astype(np.float64) assert len(rewardVolume) == len(data) if raw.save_bool(save, '_ibl_trials.rewardVolume.npy'): check_alf_folder(session_path) fpath = os.path.join(session_path, 'alf', '_ibl_trials.rewardVolume.npy') np.save(fpath, rewardVolume) return rewardVolume
def create_current_running_session(session_folder): one = get_one() settings = raw.load_settings(session_folder) subject = one.alyx.rest( 'subjects?nickname=' + settings['PYBPOD_SUBJECTS'][0], 'list')[0] ses_ = { 'subject': subject['nickname'], 'users': [settings['PYBPOD_CREATOR'][0]], 'location': settings['PYBPOD_BOARD'], 'procedures': ['Behavior training/tasks'], 'lab': subject['lab'], 'type': 'Experiment', 'task_protocol': settings['PYBPOD_PROTOCOL'] + settings['IBLRIG_VERSION_TAG'], 'number': settings['SESSION_NUMBER'], 'start_time': settings['SESSION_DATETIME'], 'end_time': None, 'n_correct_trials': None, 'n_trials': None, 'json': None, } session = one.alyx.rest('sessions', 'create', data=ses_) return session
def test_trials_extraction(self): # extract all sessions with tempfile.TemporaryDirectory() as tdir: subjects_path = Path(tdir).joinpath('Subjects') shutil.copytree(self.INIT_FOLDER, subjects_path) for fil in subjects_path.rglob('_iblrig_taskData.raw*.jsonable'): session_path = fil.parents[1] # task running part job = TrainingTrials(session_path, one=self.one) job.run() # check the trials objects trials = alf.io.load_object(session_path / 'alf', 'trials') self.assertTrue(alf.io.check_dimensions(trials) == 0) settings = rawio.load_settings(session_path) if version.ge(settings['IBLRIG_VERSION_TAG'], '5.0.0'): tkeys = TRIAL_KEYS_ge5 else: tkeys = TRIAL_KEYS_lt5 self.assertTrue(set(trials.keys()) == set(tkeys)) # check the wheel object if the extraction didn't fail if job.status != -1: wheel = alf.io.load_object(session_path / 'alf', 'wheel') self.assertTrue(alf.io.check_dimensions(wheel) == 0) self.assertTrue(set(wheel.keys()) == set(WHEEL_KEYS)) """ For this session only the downgoing front of a trial was detected, resulting in an error for the gocuetime. The fix was to extract the downgoing front and subtract 100ms. """ session_path = subjects_path / "CSHL_007/2019-07-31/001" trials = alf.io.load_object(session_path / 'alf', 'trials') self.assertTrue(np.all(np.logical_not(np.isnan(trials.goCue_times))))
def main(): passive_sessions = IBLRIG_DATA_PATH.rglob("passive_data_for_ephys.flag") # For each passive session found look into passiveSettings to find ephysSession name # search for the ephys session session in the rglobbed ephys sessions # If you find it just rename and move the folder raw_behavior_data -> raw_passive_data, # If no find search specifically for that session from the metadata and try to copy the folder # If folder exists throw an error for ps in passive_sessions: sett = raw.load_settings(str(ps.parent)) esess = sett["CORRESPONDING_EPHYS_SESSION"] if not esess or esess is None: log.warning( "Corresponding ephys session NOT FOUND in settings - data not moved" ) return if not Path(esess).exists(): log.warning( f"Ephys session {esess}: NOT FOUND on filesystem - data not moved" ) return # Fails if dst_folder exists! misc.transfer_folder( str(ps.parent / "raw_behavior_data"), str(Path(esess) / "raw_passive_data"), force=False, ) ps.unlink()
def get_task_extractor_type(task_name): """ Splits the task name according to naming convention: - ignores everything _iblrig_tasks_biasedChoiceWorld3.7.0 returns "biased" _iblrig_tasks_trainingChoiceWorld3.6.0 returns "training' :param task_name: :return: """ if isinstance(task_name, Path): try: settings = raw.load_settings(get_session_path(task_name)) except json.decoder.JSONDecodeError: return if settings: task_name = settings.get('PYBPOD_PROTOCOL', None) else: return if '_biasedChoiceWorld' in task_name: return 'biased' elif 'biasedScanningChoiceWorld' in task_name: return 'biased' elif '_habituationChoiceWorld' in task_name: return 'habituation' elif '_trainingChoiceWorld' in task_name: return 'training' elif 'ephysChoiceWorld' in task_name: return 'ephys' elif task_name and task_name.startswith( '_iblrig_tasks_ephys_certification'): return 'sync_ephys'
def _load_passive_session_fixtures(session_path: str) -> dict: """load_passive_session_fixtures Loads corresponding ephys session fixtures :param session_path: the path to a session :type session_path: str :return: position contrast phase delays and stim id's :rtype: dict """ settings = rawio.load_settings(session_path) ses_nb = settings["SESSION_ORDER"][settings["SESSION_IDX"]] path_fixtures = Path(ephys_fpga.__file__).parent.joinpath("ephys_sessions") fixture = { "pcs": np.load(path_fixtures.joinpath(f"session_{ses_nb}_passive_pcs.npy")), "delays": np.load( path_fixtures.joinpath( f"session_{ses_nb}_passive_stimDelays.npy")), "ids": np.load( path_fixtures.joinpath(f"session_{ses_nb}_passive_stimIDs.npy")), } return fixture
def get_response_times(session_path, save=False, data=False, settings=False): """ Time (in absolute seconds from session start) when a response was recorded. **Optional:** saves _ibl_trials.response_times.npy Uses the timestamp of the end of the closed_loop state. :param session_path: Absolute path of session folder :type session_path: str :param save: wether to save the corresponding alf file to the alf folder, defaults to False :param save: bool, optional :return: numpy.ndarray :rtype: dtype('float64') """ if not data: data = raw.load_data(session_path) if not settings: settings = raw.load_settings(session_path) if settings is None or settings['IBLRIG_VERSION_TAG'] == '': settings = {'IBLRIG_VERSION_TAG': '100.0.0'} rt = np.array([ tr['behavior_data']['States timestamps']['closed_loop'][0][1] for tr in data ]) if raw.save_bool(save, '_ibl_trials.response_times.npy'): check_alf_folder(session_path) fpath = os.path.join(session_path, 'alf', '_ibl_trials.response_times.npy') np.save(fpath, rt) return rt
def extract_all(session_path, save=False, bpod_trials=False, settings=False): """Extract all datasets from habituationChoiceWorld Note: only the datasets from the HabituationTrials extractor will be saved to disc. :param session_path: The session path where the raw data are saved :param save: If True, the datasets that are considered standard are saved to the session path :param bpod_trials: The raw Bpod trial data :param settings: The raw Bpod sessions :returns: a dict of datasets and a corresponding list of file names """ if not bpod_trials: bpod_trials = raw.load_data(session_path) if not settings: settings = raw.load_settings(session_path) # Standard datasets that may be saved as ALFs params = dict(session_path=session_path, bpod_trials=bpod_trials, settings=settings) out, fil = run_extractor_classes(HabituationTrials, save=save, **params) # The extra datasets non_standard = [ ItiInTimes, StimOffTriggerTimes, StimCenterTriggerTimes, StimCenterTimes ] data, _ = run_extractor_classes(non_standard, save=False, **params) # Merge the extracted data out.update(data) fil.extend([None for _ in data.keys()]) return out, fil
def create_current_running_session(session_folder): one = get_one() settings = raw.load_settings(session_folder) subject = one.alyx.rest( "subjects?nickname=" + settings["PYBPOD_SUBJECTS"][0], "list")[0] ses_ = { "subject": subject["nickname"], "users": [settings["PYBPOD_CREATOR"][0]], "location": settings["PYBPOD_BOARD"], "procedures": ["Behavior training/tasks"], "lab": subject["lab"], "type": "Experiment", "task_protocol": settings["PYBPOD_PROTOCOL"] + settings["IBLRIG_VERSION_TAG"], "number": settings["SESSION_NUMBER"], "start_time": settings["SESSION_DATETIME"], "end_time": None, "n_correct_trials": None, "n_trials": None, "json": None, } session = one.alyx.rest("sessions", "create", data=ses_) return session
def get_iti_duration(session_path, save=False, data=False, settings=False): """ Calculate duration of iti from state timestamps. **Optional:** saves _ibl_trials.iti_duration.npy Uses Trial end timestamp and get_response_times to calculate iti. :param session_path: Absolute path of session folder :type session_path: str :param save: wether to save the corresponding alf file to the alf folder, defaults to False :param save: bool, optional :return: numpy.ndarray :rtype: dtype('float64') """ if not data: data = raw.load_data(session_path) if not settings: settings = raw.load_settings(session_path) if settings is None or settings['IBLRIG_VERSION_TAG'] == '': settings = {'IBLRIG_VERSION_TAG': '100.0.0'} rt = get_response_times(session_path, save=False, data=False) ends = np.array([t['behavior_data']['Trial end timestamp'] for t in data]) iti_dur = ends - rt if raw.save_bool(save, '_ibl_trials.itiDuration.npy'): check_alf_folder(session_path) fpath = os.path.join(session_path, 'alf', '_ibl_trials.itiDuration.npy') np.save(fpath, iti_dur) return iti_dur
def get_intervals(session_path, save=False, data=False, settings=False): """ Trial start to trial end. Trial end includes 1 or 2 seconds after feedback, (depending on the feedback) and 0.5 seconds of iti. **Optional:** saves _ibl_trials.intervals.npy Uses the corrected Trial start and Trial end timpestamp values form PyBpod. :param session_path: Absolute path of session folder :type session_path: str :param save: wether to save the corresponding alf file to the alf folder, defaults to False :param save: bool, optional :return: 2D numpy.ndarray (col0 = start, col1 = end) :rtype: dtype('float64') """ if not data: data = raw.load_data(session_path) if not settings: settings = raw.load_settings(session_path) if settings is None or settings['IBLRIG_VERSION_TAG'] == '': settings = {'IBLRIG_VERSION_TAG': '100.0.0'} starts = [t['behavior_data']['Trial start timestamp'] for t in data] ends = [t['behavior_data']['Trial end timestamp'] for t in data] intervals = np.array([starts, ends]).T if raw.save_bool(save, '_ibl_trials.intervals.npy'): check_alf_folder(session_path) fpath = os.path.join(session_path, 'alf', '_ibl_trials.intervals.npy') np.save(fpath, intervals) return intervals
def get_stimOn_times(session_path, save=False, data=False, settings=False): """ Find the time of the statemachine command to turn on hte stim (state stim_on start or rotary_encoder_event2) Find the next frame change from the photodiodeafter that TS. Screen is not displaying anything until then. (Frame changes are in BNC1High and BNC1Low) """ if not data: data = raw.load_data(session_path) if not settings: settings = raw.load_settings(session_path) if settings is None or settings['IBLRIG_VERSION_TAG'] == '': settings = {'IBLRIG_VERSION_TAG': '100.0.0'} # Version check if version.ge(settings['IBLRIG_VERSION_TAG'], '5.0.0'): stimOn_times = get_stimOn_times_ge5(session_path, data=data) else: stimOn_times = get_stimOn_times_lt5(session_path, data=data) if raw.save_bool(save, '_ibl_trials.stimOn_times.npy'): check_alf_folder(session_path) fpath = os.path.join(session_path, 'alf', '_ibl_trials.stimOn_times.npy') np.save(fpath, np.array(stimOn_times)) return np.array(stimOn_times)
def get_goCueOnset_times(session_path, save=False, data=False, settings=False): """ Get trigger times of goCue from state machine. Current software solution for triggering sounds uses PyBpod soft codes. Delays can be in the order of 10's of ms. This is the time when the command to play the sound was executed. To measure accurate time, either getting the sound onset from the future microphone OR the new xonar soundcard and setup developed by Sanworks guarantees a set latency (in testing). :param session_path: Absolute path of session folder :type session_path: str :param save: bool, optional :return: numpy.ndarray :rtype: dtype('float64') """ if not data: data = raw.load_data(session_path) if not settings: settings = raw.load_settings(session_path) if settings is None or settings['IBLRIG_VERSION_TAG'] == '': settings = {'IBLRIG_VERSION_TAG': '100.0.0'} go_cue_times = np.zeros([ len(data), ]) for ind, tr in enumerate(data): if get_port_events(tr, 'BNC2'): bnchigh = tr['behavior_data']['Events timestamps'].get( 'BNC2High', None) if bnchigh: go_cue_times[ind] = bnchigh[0] continue bnclow = tr['behavior_data']['Events timestamps'].get( 'BNC2Low', None) if bnclow: go_cue_times[ind] = bnclow[0] - 0.1 continue go_cue_times[ind] = np.nan else: go_cue_times[ind] = np.nan nmissing = np.sum(np.isnan(go_cue_times)) # Check if all stim_syncs have failed to be detected if np.all(np.isnan(go_cue_times)): logger_.error( f'{session_path}: Missing ALL BNC2 stimulus ({nmissing} trials') # Check if any stim_sync has failed be detected for every trial if np.any(np.isnan(go_cue_times)): logger_.warning( f'{session_path}: Missing BNC2 stimulus on {nmissing} trials') if raw.save_bool(save, '_ibl_trials.goCue_times.npy'): check_alf_folder(session_path) fpath = Path(session_path).joinpath('alf', '_ibl_trials.goCue_times.npy') np.save(fpath, go_cue_times) return go_cue_times
def extract_all(session_path, save=True, bpod_trials=None, settings=None): """ Extracts a training session from its path. NB: Wheel must be extracted first in order to extract trials.firstMovement_times. :param session_path: the path to the session to be extracted :param save: if true a subset of the extracted data are saved as ALF :param bpod_trials: list of Bpod trial data :param settings: the Bpod session settings :return: trials: Bunch/dict of trials :return: wheel: Bunch/dict of wheel positions :return: out_Files: list of output files """ extractor_type = ibllib.io.extractors.base.get_session_extractor_type( session_path) _logger.info(f"Extracting {session_path} as {extractor_type}") bpod_trials = bpod_trials or rawio.load_data(session_path) settings = settings or rawio.load_settings(session_path) if extractor_type == 'training': _logger.info('training session on ' + settings['PYBPOD_BOARD']) wheel, files_wheel = training_wheel.extract_all( session_path, bpod_trials=bpod_trials, settings=settings, save=save) trials, files_trials = training_trials.extract_all( session_path, bpod_trials=bpod_trials, settings=settings, save=save) elif extractor_type == 'biased' or extractor_type == 'ephys': _logger.info('biased session on ' + settings['PYBPOD_BOARD']) wheel, files_wheel = training_wheel.extract_all( session_path, bpod_trials=bpod_trials, settings=settings, save=save) trials, files_trials = biased_trials.extract_all( session_path, bpod_trials=bpod_trials, settings=settings, save=save) elif extractor_type == 'habituation': from ibllib.misc import version _logger.info('habituation session on ' + settings['PYBPOD_BOARD']) if version.le(settings['IBLRIG_VERSION_TAG'], '5.0.0'): _logger.warning("No extraction of legacy habituation sessions") return None, None, None trials, files_trials = habituation_trials.extract_all( session_path, bpod_trials=bpod_trials, settings=settings, save=save) wheel = None files_wheel = [] else: raise ValueError(f"No extractor for task {extractor_type}") _logger.info('session extracted \n') # timing info in log return trials, wheel, (files_trials + files_wheel) if save else None
def main(local_folder: str, remote_folder: str, force: bool = False) -> None: local_folder = Path(local_folder) remote_folder = Path(remote_folder) src_session_paths = [ x.parent for x in local_folder.rglob("transfer_me.flag") ] if not src_session_paths: log.info("Nothing to transfer, exiting...") return # Create all dst paths dst_session_paths = [] for s in src_session_paths: mouse = s.parts[-3] date = s.parts[-2] sess = s.parts[-1] d = remote_folder / mouse / date / sess dst_session_paths.append(d) for src, dst in zip(src_session_paths, dst_session_paths): src_flag_file = src / "transfer_me.flag" flag = flags.read_flag_file(src_flag_file) if isinstance(flag, list): raise NotImplementedError else: if force: shutil.rmtree(dst, ignore_errors=True) log.info(f"Copying {src}...") shutil.copytree(src, dst, ignore=ig(str(src_flag_file.name))) # finally if folder was created delete the src flag_file and create compress_me.flag if dst.exists(): task_type = ibllib.io.extractors.base.get_session_extractor_type( Path(src)) if task_type not in ['ephys', 'ephys_sync', 'ephys_mock']: flags.write_flag_file(dst.joinpath('raw_session.flag')) settings = raw.load_settings(dst) if 'ephys' in settings[ 'PYBPOD_BOARD']: # Any traing task on an ephys rig dst.joinpath('raw_session.flag').unlink() log.info( f"Copied to {remote_folder}: Session {src_flag_file.parent}") src_flag_file.unlink() # Cleanup src_video_file = src / 'raw_video_data' / '_iblrig_leftCamera.raw.avi' dst_video_file = dst / 'raw_video_data' / '_iblrig_leftCamera.raw.avi' src_audio_file = src / 'raw_behavior_data' / '_iblrig_micData.raw.wav' dst_audio_file = dst / 'raw_behavior_data' / '_iblrig_micData.raw.wav' if src_audio_file.exists() and \ src_audio_file.stat().st_size == dst_audio_file.stat().st_size: src_audio_file.unlink() if src_video_file.exists() and \ src_video_file.stat().st_size == dst_video_file.stat().st_size: src_video_file.unlink()
def save_session_settings(sph: object) -> None: save_this = json.dumps(sph, cls=ComplexEncoder, indent=1) with open(sph.SETTINGS_FILE_PATH, 'a') as f: f.write(save_this) f.write('\n') save_this = json.loads(save_this) settings = raw.load_settings(sph.SESSION_FOLDER) assert (save_this == settings)
def get_task_protocol(session_path): try: settings = load_settings(get_session_path(session_path)) except json.decoder.JSONDecodeError: _logger.error(f"Can't read settings for {session_path}") return if settings: return settings.get('PYBPOD_PROTOCOL', None) else: return
def probes_description(ses_path, bin_exists=True): """ Aggregate probes information into ALF files Input: raw_ephys_data/probeXX/ Output: alf/probes.description.npy alf/probes.trajecory.npy """ ses_path = Path(ses_path) ephys_files = spikeglx.glob_ephys_files(ses_path, bin_exists=bin_exists) subdirs, labels, efiles_sorted = zip( *sorted([(ep.ap.parent, ep.label, ep) for ep in ephys_files if ep.get('ap')])) """Ouputs the probes description file""" probe_description = [] for label, ef in zip(labels, efiles_sorted): md = spikeglx.read_meta_data(ef.ap.with_suffix('.meta')) probe_description.append({'label': label, 'model': md.neuropixelVersion, 'serial': int(md.serial), 'raw_file_name': md.fileName, }) alf_path = ses_path.joinpath('alf') alf_path.mkdir(exist_ok=True, parents=True) probe_description_file = alf_path.joinpath('probes.description.json') with open(probe_description_file, 'w+') as fid: fid.write(json.dumps(probe_description)) """Ouputs the probes trajectory file""" bpod_meta = raw_data_loaders.load_settings(ses_path) if not bpod_meta.get('PROBE_DATA'): _logger.error('No probe information in settings JSON. Skipping probes.trajectory') return def prb2alf(prb, label): return {'label': label, 'x': prb['X'], 'y': prb['Y'], 'z': prb['Z'], 'phi': prb['A'], 'theta': prb['P'], 'depth': prb['D'], 'beta': prb['T']} # the labels may not match, in which case throw a warning and work in alphabetical order if labels != ('probe00', 'probe01'): _logger.warning("Probe names do not match the json settings files. Will match coordinates" " per alphabetical order !") _ = [_logger.warning(f" probe0{i} ---------- {lab} ") for i, lab in enumerate(labels)] trajs = [] keys = sorted(bpod_meta['PROBE_DATA'].keys()) for i, k in enumerate(keys): if i >= len(labels): break trajs.append(prb2alf(bpod_meta['PROBE_DATA'][f'probe0{i}'], labels[i])) probe_trajectory_file = alf_path.joinpath('probes.trajectory.json') with open(probe_trajectory_file, 'w+') as fid: fid.write(json.dumps(trajs)) return [probe_trajectory_file, probe_description_file]
def extractors_exist(session_path): settings = raw.load_settings(session_path) task_name = settings['PYBPOD_PROTOCOL'] task_name = task_name.split('_')[-1] extractor_type = task_name[:task_name.find('ChoiceWorld')] if any([extractor_type in x for x in globals()]): return extractor_type else: logger_.warning( f"No extractors were found for {extractor_type}ChoiceWorld") return False
def _load_task_protocol(session_path: str) -> str: """Find the IBL rig version used for the session :param session_path: the path to a session :type session_path: str :return: ibl rig task protocol version :rtype: str """ settings = rawio.load_settings(session_path) ses_ver = settings["IBLRIG_VERSION_TAG"] return ses_ver
def load_raw_data(self): _logger.info(f"Loading raw data from {self.session_path}") self.raw_data = raw.load_data(self.session_path) self.details = raw.load_settings(self.session_path) self.BNC1, self.BNC2 = raw.load_bpod_fronts(self.session_path, data=self.raw_data) # NOTE: wheel_position is actually an extractor needs _iblrig_encoderPositions.raw # to be there but not as input... FIXME: we should have the extractor use the data # without assuming it's there ts, pos = get_wheel_position(self.session_path, bp_data=self.raw_data) self.wheel_data = {'re_ts': ts, 're_pos': pos} assert np.all(np.diff(self.wheel_data["re_ts"]) > 0)
def get_feedbackType(session_path, save=False, data=False, settings=False): """ Get the feedback that was delivered to subject. **Optional:** saves _ibl_trials.feedbackType.npy Checks in raw datafile for error and reward state. Will raise an error if more than one of the mutually exclusive states have been triggered. Sets feedbackType to -1 if error state was trigered (applies to no-go) Sets feedbackType to +1 if reward state was triggered :param session_path: absolute path of session folder :type session_path: str :param save: wether to save the corresponding alf file to the alf folder, defaults to False :type save: bool, optional :return: numpy.ndarray :rtype: dtype('int64') """ if not data: data = raw.load_data(session_path) if not settings: settings = raw.load_settings(session_path) if settings is None or settings['IBLRIG_VERSION_TAG'] == '': settings = {'IBLRIG_VERSION_TAG': '100.0.0'} feedbackType = np.empty(len(data)) feedbackType.fill(np.nan) reward = [] error = [] no_go = [] for t in data: reward.append( ~np.isnan(t['behavior_data']['States timestamps']['reward'][0][0])) error.append( ~np.isnan(t['behavior_data']['States timestamps']['error'][0][0])) no_go.append( ~np.isnan(t['behavior_data']['States timestamps']['no_go'][0][0])) if not all(np.sum([reward, error, no_go], axis=0) == np.ones(len(data))): raise ValueError feedbackType[reward] = 1 feedbackType[error] = -1 feedbackType[no_go] = -1 feedbackType = feedbackType.astype('int64') if raw.save_bool(save, '_ibl_trials.feedbackType.npy'): check_alf_folder(session_path) fpath = os.path.join(session_path, 'alf', '_ibl_trials.feedbackType.npy') np.save(fpath, feedbackType) return feedbackType
def extractors_exist(session_path): settings = raw.load_settings(session_path) if settings is None: logger_.error( f'ABORT: No data found in "raw_behavior_data" folder {session_path}' ) return False task_name = settings['PYBPOD_PROTOCOL'] task_name = task_name.split('_')[-1] extractor_type = task_name[:task_name.find('ChoiceWorld')] return extractor_type