def test_get_probabilityLeft(self):
# TRAINING SESSIONS
pl = extractors.training_trials.ProbabilityLeft(
self.training_lt5['path']).extract()[0]
self.assertTrue(isinstance(pl, np.ndarray))
# -- version >= 5.0.0
pl = extractors.training_trials.ProbabilityLeft(
self.training_ge5['path']).extract()[0]
self.assertTrue(isinstance(pl, np.ndarray))
# BIASED SESSIONS
pl = extractors.biased_trials.ProbabilityLeft(
self.biased_lt5['path']).extract()[0]
self.assertTrue(isinstance(pl, np.ndarray))
# Test if only probs that are in prob set
md = raw.load_settings(self.biased_lt5['path'])
if md:
probs = md['BLOCK_PROBABILITY_SET']
probs.append(0.5)
self.assertTrue(sum([x in probs for x in pl]) == len(pl))
# -- version >= 5.0.0
pl = extractors.biased_trials.ProbabilityLeft(
self.biased_ge5['path']).extract()[0]
self.assertTrue(isinstance(pl, np.ndarray))
# Test if only probs that are in prob set
md = raw.load_settings(self.biased_ge5['path'])
probs = md['BLOCK_PROBABILITY_SET']
probs.append(0.5)
self.assertTrue(sum([x in probs for x in pl]) == len(pl))
def patch_settings_file(sess_or_file: str, patch: dict) -> None:
sess_or_file = Path(sess_or_file)
if sess_or_file.is_file() and sess_or_file.name.endswith(
"_iblrig_taskSettings.raw.json"):
session = sess_or_file.parent.parent
file = sess_or_file
elif sess_or_file.is_dir() and sess_or_file.name.isdecimal():
file = sess_or_file / "raw_behavior_data" / "_iblrig_taskSettings.raw.json"
session = sess_or_file
else:
print("not a settings file or a session folder")
return
settings = raw.load_settings(session)
settings.update(patch)
# Rename file on disk keeps pathlib ref to "file" intact
file.rename(file.with_suffix(".json_bk"))
with open(file, "w") as f:
f.write(json.dumps(settings, indent=1))
f.write("\n")
f.flush()
# Check if properly saved
saved_settings = raw.load_settings(session)
if settings == saved_settings:
file.with_suffix(".json_bk").unlink()
return
def get_probabilityLeft(session_path, save=False, data=False, settings=False):
if not data:
data = raw.load_data(session_path)
if not settings:
settings = raw.load_settings(session_path)
if settings is None:
settings = {"IBLRIG_VERSION_TAG": "100.0.0"}
elif settings["IBLRIG_VERSION_TAG"] == "":
settings.update({"IBLRIG_VERSION_TAG": "100.0.0"})
num = settings.get("PRELOADED_SESSION_NUM", None)
if num is None:
num = settings.get("PREGENERATED_SESSION_NUM", None)
if num is None:
fn = settings.get('SESSION_LOADED_FILE_PATH', None)
fn = PureWindowsPath(fn).name
num = ''.join([d for d in fn if d.isdigit()])
if num == '':
raise ValueError("Can't extract left probability behaviour.")
# Load the pregenerated file
sessions_folder = Path(raw.__file__).parent.joinpath(
'extractors', 'ephys_sessions')
fname = f"session_{num}_ephys_pcqs.npy"
pcqsp = np.load(sessions_folder.joinpath(fname))
pLeft = pcqsp[:, 4]
pLeft = pLeft[:len(data)]
if raw.save_bool(save, "_ibl_trials.probabilityLeft.npy"):
lpath = Path(session_path).joinpath("alf",
"_ibl_trials.probabilityLeft.npy")
np.save(lpath, pLeft)
return pLeft
def _previous_data_files(self, typ='data'):
logger.debug(f"Looking for previous files of type: {typ}")
prev_data_files = []
prev_session_files = []
data_fname = self.BASE_FILENAME + 'Data.raw.jsonable'
settings_fname = self.BASE_FILENAME + 'Settings.raw.json'
logger.debug(f"Looking for files:{data_fname} AND {settings_fname}")
for prev_sess_path in self._previous_session_folders():
prev_sess_path = Path(prev_sess_path) / 'raw_behavior_data'
# Get all data and settings file if they both exist
if ((prev_sess_path / data_fname).exists() and
(prev_sess_path / settings_fname).exists()):
prev_data_files.append(prev_sess_path / data_fname)
prev_session_files.append(prev_sess_path / settings_fname)
logger.debug(f"Found {len(prev_data_files)} file pairs")
# Remove empty files
ds_out = [(d, s) for d, s in zip(prev_data_files, prev_session_files)
if d.stat().st_size != 0 and s.stat().st_size != 0]
logger.debug(f"Found {len(ds_out)} non empty file pairs")
# Remove sessions of different task protocols
ds_out = [(d, s) for d, s in ds_out if self._PROTOCOL in
raw.load_settings(str(s.parent.parent))['PYBPOD_PROTOCOL']]
logger.debug(
f"Found {len(ds_out)} file pairs for protocol {self._PROTOCOL}")
data_out = [str(d) for d, s in ds_out]
settings_out = [str(s) for d, s in ds_out]
if not data_out:
logger.debug(
f'NOT FOUND: Previous data files for task {self._PROTOCOL}')
if not settings_out:
logger.debug(
f'NOT FOUND: Previous settings files for task {self._PROTOCOL}')
logger.debug(f"Reurning {typ} files")
return data_out if typ == 'data' else settings_out
def audio_training(root_data_folder, dry=False, max_sessions=False):
from ibllib.io.extractors import training_audio as audio
audio_flags = Path(root_data_folder).rglob('audio_training.flag')
c = 0
for flag in audio_flags:
c += 1
if max_sessions and c > max_sessions:
return
_logger.info(flag)
if dry:
continue
session_path = flag.parent
try:
settings = raw_data_loaders.load_settings(session_path)
typ = extract_session.get_task_extractor_type(
settings.get('PYBPOD_PROTOCOL'))
except json.decoder.JSONDecodeError:
typ = 'unknown'
# this extractor is only for biased and training sessions
if typ not in ['biased', 'training', 'habituation']:
flag.unlink()
continue
audio.extract_sound(session_path, save=True, delete=True)
flag.unlink()
session_path.joinpath('register_me.flag').touch()
def extract_all(session_path, save=False, bpod_trials=None, settings=None):
if not bpod_trials:
bpod_trials = raw.load_data(session_path)
if not settings:
settings = raw.load_settings(session_path)
if settings is None or settings['IBLRIG_VERSION_TAG'] == '':
settings = {'IBLRIG_VERSION_TAG': '100.0.0'}
base = [
FeedbackType, ContrastLR, ProbabilityLeft, Choice, RepNum,
RewardVolume, FeedbackTimes, Intervals, ResponseTimes,
GoCueTriggerTimes, GoCueTimes
]
# Version check
if version.ge(settings['IBLRIG_VERSION_TAG'], '5.0.0'):
base.extend([
StimOnTriggerTimes, StimOnOffFreezeTimes, ItiInTimes,
StimOffTriggerTimes, StimFreezeTriggerTimes, ErrorCueTriggerTimes
])
else:
base.extend([IncludedTrials, ItiDuration, StimOnTimes])
out, fil = run_extractor_classes(base,
save=save,
session_path=session_path,
bpod_trials=bpod_trials,
settings=settings)
return out, fil
def get_feedback_times(session_path, save=False, data=False, settings=False):
"""
Get the times the water or error tone was delivered to the animal.
**Optional:** saves _ibl_trials.feedback_times.npy
Gets reward and error state init times vectors,
checks if theintersection of nans is empty, then
merges the 2 vectors.
:param session_path: Absolute path of session folder
:type session_path: str
:param save: wether to save the corresponding alf file
to the alf folder, defaults to False
:param save: bool, optional
:return: numpy.ndarray
:rtype: dtype('float64')
"""
if not data:
data = raw.load_data(session_path)
if not settings:
settings = raw.load_settings(session_path)
if settings is None or settings['IBLRIG_VERSION_TAG'] == '':
settings = {'IBLRIG_VERSION_TAG': '100.0.0'}
# Version check
if version.ge(settings['IBLRIG_VERSION_TAG'], '5.0.0'):
merge = get_feedback_times_ge5(session_path, data=data)
else:
merge = get_feedback_times_lt5(session_path, data=data)
if raw.save_bool(save, '_ibl_trials.feedback_times.npy'):
check_alf_folder(session_path)
fpath = os.path.join(session_path, 'alf', '_ibl_trials.feedback_times.npy')
np.save(fpath, merge)
return np.array(merge)
def get_rewardVolume(session_path, save=False, data=False, settings=False):
"""
Load reward volume delivered for each trial.
**Optional:** saves _ibl_trials.rewardVolume.npy
Uses reward_current to accumulate the amount of
:param session_path: Absolute path of session folder
:type session_path: str
:param save: wether to save the corresponding alf file
to the alf folder, defaults to False
:param save: bool, optional
:return: numpy.ndarray
:rtype: dtype('int64')
"""
if not data:
data = raw.load_data(session_path)
if not settings:
settings = raw.load_settings(session_path)
if settings is None or settings['IBLRIG_VERSION_TAG'] == '':
settings = {'IBLRIG_VERSION_TAG': '100.0.0'}
trial_volume = [
x['reward_amount'] if x['trial_correct'] else 0 for x in data
]
rewardVolume = np.array(trial_volume).astype(np.float64)
assert len(rewardVolume) == len(data)
if raw.save_bool(save, '_ibl_trials.rewardVolume.npy'):
check_alf_folder(session_path)
fpath = os.path.join(session_path, 'alf',
'_ibl_trials.rewardVolume.npy')
np.save(fpath, rewardVolume)
return rewardVolume
def create_current_running_session(session_folder):
one = get_one()
settings = raw.load_settings(session_folder)
subject = one.alyx.rest(
'subjects?nickname=' + settings['PYBPOD_SUBJECTS'][0], 'list')[0]
ses_ = {
'subject':
subject['nickname'],
'users': [settings['PYBPOD_CREATOR'][0]],
'location':
settings['PYBPOD_BOARD'],
'procedures': ['Behavior training/tasks'],
'lab':
subject['lab'],
'type':
'Experiment',
'task_protocol':
settings['PYBPOD_PROTOCOL'] + settings['IBLRIG_VERSION_TAG'],
'number':
settings['SESSION_NUMBER'],
'start_time':
settings['SESSION_DATETIME'],
'end_time':
None,
'n_correct_trials':
None,
'n_trials':
None,
'json':
None,
}
session = one.alyx.rest('sessions', 'create', data=ses_)
return session
def test_trials_extraction(self):
# extract all sessions
with tempfile.TemporaryDirectory() as tdir:
subjects_path = Path(tdir).joinpath('Subjects')
shutil.copytree(self.INIT_FOLDER, subjects_path)
for fil in subjects_path.rglob('_iblrig_taskData.raw*.jsonable'):
session_path = fil.parents[1]
# task running part
job = TrainingTrials(session_path, one=self.one)
job.run()
# check the trials objects
trials = alf.io.load_object(session_path / 'alf', 'trials')
self.assertTrue(alf.io.check_dimensions(trials) == 0)
settings = rawio.load_settings(session_path)
if version.ge(settings['IBLRIG_VERSION_TAG'], '5.0.0'):
tkeys = TRIAL_KEYS_ge5
else:
tkeys = TRIAL_KEYS_lt5
self.assertTrue(set(trials.keys()) == set(tkeys))
# check the wheel object if the extraction didn't fail
if job.status != -1:
wheel = alf.io.load_object(session_path / 'alf', 'wheel')
self.assertTrue(alf.io.check_dimensions(wheel) == 0)
self.assertTrue(set(wheel.keys()) == set(WHEEL_KEYS))
"""
For this session only the downgoing front of a trial was detected, resulting in
an error for the gocuetime. The fix was to extract the downgoing front and
subtract 100ms.
"""
session_path = subjects_path / "CSHL_007/2019-07-31/001"
trials = alf.io.load_object(session_path / 'alf', 'trials')
self.assertTrue(np.all(np.logical_not(np.isnan(trials.goCue_times))))
def main():
passive_sessions = IBLRIG_DATA_PATH.rglob("passive_data_for_ephys.flag")
# For each passive session found look into passiveSettings to find ephysSession name
# search for the ephys session session in the rglobbed ephys sessions
# If you find it just rename and move the folder raw_behavior_data -> raw_passive_data,
# If no find search specifically for that session from the metadata and try to copy the folder
# If folder exists throw an error
for ps in passive_sessions:
sett = raw.load_settings(str(ps.parent))
esess = sett["CORRESPONDING_EPHYS_SESSION"]
if not esess or esess is None:
log.warning(
"Corresponding ephys session NOT FOUND in settings - data not moved"
)
return
if not Path(esess).exists():
log.warning(
f"Ephys session {esess}: NOT FOUND on filesystem - data not moved"
)
return
# Fails if dst_folder exists!
misc.transfer_folder(
str(ps.parent / "raw_behavior_data"),
str(Path(esess) / "raw_passive_data"),
force=False,
)
ps.unlink()
def get_task_extractor_type(task_name):
"""
Splits the task name according to naming convention:
- ignores everything
_iblrig_tasks_biasedChoiceWorld3.7.0 returns "biased"
_iblrig_tasks_trainingChoiceWorld3.6.0 returns "training'
:param task_name:
:return:
"""
if isinstance(task_name, Path):
try:
settings = raw.load_settings(get_session_path(task_name))
except json.decoder.JSONDecodeError:
return
if settings:
task_name = settings.get('PYBPOD_PROTOCOL', None)
else:
return
if '_biasedChoiceWorld' in task_name:
return 'biased'
elif 'biasedScanningChoiceWorld' in task_name:
return 'biased'
elif '_habituationChoiceWorld' in task_name:
return 'habituation'
elif '_trainingChoiceWorld' in task_name:
return 'training'
elif 'ephysChoiceWorld' in task_name:
return 'ephys'
elif task_name and task_name.startswith(
'_iblrig_tasks_ephys_certification'):
return 'sync_ephys'
def _load_passive_session_fixtures(session_path: str) -> dict:
"""load_passive_session_fixtures Loads corresponding ephys session fixtures
:param session_path: the path to a session
:type session_path: str
:return: position contrast phase delays and stim id's
:rtype: dict
"""
settings = rawio.load_settings(session_path)
ses_nb = settings["SESSION_ORDER"][settings["SESSION_IDX"]]
path_fixtures = Path(ephys_fpga.__file__).parent.joinpath("ephys_sessions")
fixture = {
"pcs":
np.load(path_fixtures.joinpath(f"session_{ses_nb}_passive_pcs.npy")),
"delays":
np.load(
path_fixtures.joinpath(
f"session_{ses_nb}_passive_stimDelays.npy")),
"ids":
np.load(
path_fixtures.joinpath(f"session_{ses_nb}_passive_stimIDs.npy")),
}
return fixture
def get_response_times(session_path, save=False, data=False, settings=False):
"""
Time (in absolute seconds from session start) when a response was recorded.
**Optional:** saves _ibl_trials.response_times.npy
Uses the timestamp of the end of the closed_loop state.
:param session_path: Absolute path of session folder
:type session_path: str
:param save: wether to save the corresponding alf file
to the alf folder, defaults to False
:param save: bool, optional
:return: numpy.ndarray
:rtype: dtype('float64')
"""
if not data:
data = raw.load_data(session_path)
if not settings:
settings = raw.load_settings(session_path)
if settings is None or settings['IBLRIG_VERSION_TAG'] == '':
settings = {'IBLRIG_VERSION_TAG': '100.0.0'}
rt = np.array([
tr['behavior_data']['States timestamps']['closed_loop'][0][1]
for tr in data
])
if raw.save_bool(save, '_ibl_trials.response_times.npy'):
check_alf_folder(session_path)
fpath = os.path.join(session_path, 'alf',
'_ibl_trials.response_times.npy')
np.save(fpath, rt)
return rt
def extract_all(session_path, save=False, bpod_trials=False, settings=False):
"""Extract all datasets from habituationChoiceWorld
Note: only the datasets from the HabituationTrials extractor will be saved to disc.
:param session_path: The session path where the raw data are saved
:param save: If True, the datasets that are considered standard are saved to the session path
:param bpod_trials: The raw Bpod trial data
:param settings: The raw Bpod sessions
:returns: a dict of datasets and a corresponding list of file names
"""
if not bpod_trials:
bpod_trials = raw.load_data(session_path)
if not settings:
settings = raw.load_settings(session_path)
# Standard datasets that may be saved as ALFs
params = dict(session_path=session_path,
bpod_trials=bpod_trials,
settings=settings)
out, fil = run_extractor_classes(HabituationTrials, save=save, **params)
# The extra datasets
non_standard = [
ItiInTimes, StimOffTriggerTimes, StimCenterTriggerTimes,
StimCenterTimes
]
data, _ = run_extractor_classes(non_standard, save=False, **params)
# Merge the extracted data
out.update(data)
fil.extend([None for _ in data.keys()])
return out, fil
def create_current_running_session(session_folder):
one = get_one()
settings = raw.load_settings(session_folder)
subject = one.alyx.rest(
"subjects?nickname=" + settings["PYBPOD_SUBJECTS"][0], "list")[0]
ses_ = {
"subject":
subject["nickname"],
"users": [settings["PYBPOD_CREATOR"][0]],
"location":
settings["PYBPOD_BOARD"],
"procedures": ["Behavior training/tasks"],
"lab":
subject["lab"],
"type":
"Experiment",
"task_protocol":
settings["PYBPOD_PROTOCOL"] + settings["IBLRIG_VERSION_TAG"],
"number":
settings["SESSION_NUMBER"],
"start_time":
settings["SESSION_DATETIME"],
"end_time":
None,
"n_correct_trials":
None,
"n_trials":
None,
"json":
None,
}
session = one.alyx.rest("sessions", "create", data=ses_)
return session
def get_iti_duration(session_path, save=False, data=False, settings=False):
"""
Calculate duration of iti from state timestamps.
**Optional:** saves _ibl_trials.iti_duration.npy
Uses Trial end timestamp and get_response_times to calculate iti.
:param session_path: Absolute path of session folder
:type session_path: str
:param save: wether to save the corresponding alf file
to the alf folder, defaults to False
:param save: bool, optional
:return: numpy.ndarray
:rtype: dtype('float64')
"""
if not data:
data = raw.load_data(session_path)
if not settings:
settings = raw.load_settings(session_path)
if settings is None or settings['IBLRIG_VERSION_TAG'] == '':
settings = {'IBLRIG_VERSION_TAG': '100.0.0'}
rt = get_response_times(session_path, save=False, data=False)
ends = np.array([t['behavior_data']['Trial end timestamp'] for t in data])
iti_dur = ends - rt
if raw.save_bool(save, '_ibl_trials.itiDuration.npy'):
check_alf_folder(session_path)
fpath = os.path.join(session_path, 'alf',
'_ibl_trials.itiDuration.npy')
np.save(fpath, iti_dur)
return iti_dur
def get_intervals(session_path, save=False, data=False, settings=False):
"""
Trial start to trial end. Trial end includes 1 or 2 seconds after feedback,
(depending on the feedback) and 0.5 seconds of iti.
**Optional:** saves _ibl_trials.intervals.npy
Uses the corrected Trial start and Trial end timpestamp values form PyBpod.
:param session_path: Absolute path of session folder
:type session_path: str
:param save: wether to save the corresponding alf file
to the alf folder, defaults to False
:param save: bool, optional
:return: 2D numpy.ndarray (col0 = start, col1 = end)
:rtype: dtype('float64')
"""
if not data:
data = raw.load_data(session_path)
if not settings:
settings = raw.load_settings(session_path)
if settings is None or settings['IBLRIG_VERSION_TAG'] == '':
settings = {'IBLRIG_VERSION_TAG': '100.0.0'}
starts = [t['behavior_data']['Trial start timestamp'] for t in data]
ends = [t['behavior_data']['Trial end timestamp'] for t in data]
intervals = np.array([starts, ends]).T
if raw.save_bool(save, '_ibl_trials.intervals.npy'):
check_alf_folder(session_path)
fpath = os.path.join(session_path, 'alf', '_ibl_trials.intervals.npy')
np.save(fpath, intervals)
return intervals
def get_stimOn_times(session_path, save=False, data=False, settings=False):
"""
Find the time of the statemachine command to turn on hte stim
(state stim_on start or rotary_encoder_event2)
Find the next frame change from the photodiodeafter that TS.
Screen is not displaying anything until then.
(Frame changes are in BNC1High and BNC1Low)
"""
if not data:
data = raw.load_data(session_path)
if not settings:
settings = raw.load_settings(session_path)
if settings is None or settings['IBLRIG_VERSION_TAG'] == '':
settings = {'IBLRIG_VERSION_TAG': '100.0.0'}
# Version check
if version.ge(settings['IBLRIG_VERSION_TAG'], '5.0.0'):
stimOn_times = get_stimOn_times_ge5(session_path, data=data)
else:
stimOn_times = get_stimOn_times_lt5(session_path, data=data)
if raw.save_bool(save, '_ibl_trials.stimOn_times.npy'):
check_alf_folder(session_path)
fpath = os.path.join(session_path, 'alf',
'_ibl_trials.stimOn_times.npy')
np.save(fpath, np.array(stimOn_times))
return np.array(stimOn_times)
def get_goCueOnset_times(session_path, save=False, data=False, settings=False):
"""
Get trigger times of goCue from state machine.
Current software solution for triggering sounds uses PyBpod soft codes.
Delays can be in the order of 10's of ms. This is the time when the command
to play the sound was executed. To measure accurate time, either getting the
sound onset from the future microphone OR the new xonar soundcard and
setup developed by Sanworks guarantees a set latency (in testing).
:param session_path: Absolute path of session folder
:type session_path: str
:param save: bool, optional
:return: numpy.ndarray
:rtype: dtype('float64')
"""
if not data:
data = raw.load_data(session_path)
if not settings:
settings = raw.load_settings(session_path)
if settings is None or settings['IBLRIG_VERSION_TAG'] == '':
settings = {'IBLRIG_VERSION_TAG': '100.0.0'}
go_cue_times = np.zeros([
len(data),
])
for ind, tr in enumerate(data):
if get_port_events(tr, 'BNC2'):
bnchigh = tr['behavior_data']['Events timestamps'].get(
'BNC2High', None)
if bnchigh:
go_cue_times[ind] = bnchigh[0]
continue
bnclow = tr['behavior_data']['Events timestamps'].get(
'BNC2Low', None)
if bnclow:
go_cue_times[ind] = bnclow[0] - 0.1
continue
go_cue_times[ind] = np.nan
else:
go_cue_times[ind] = np.nan
nmissing = np.sum(np.isnan(go_cue_times))
# Check if all stim_syncs have failed to be detected
if np.all(np.isnan(go_cue_times)):
logger_.error(
f'{session_path}: Missing ALL BNC2 stimulus ({nmissing} trials')
# Check if any stim_sync has failed be detected for every trial
if np.any(np.isnan(go_cue_times)):
logger_.warning(
f'{session_path}: Missing BNC2 stimulus on {nmissing} trials')
if raw.save_bool(save, '_ibl_trials.goCue_times.npy'):
check_alf_folder(session_path)
fpath = Path(session_path).joinpath('alf',
'_ibl_trials.goCue_times.npy')
np.save(fpath, go_cue_times)
return go_cue_times
def extract_all(session_path, save=True, bpod_trials=None, settings=None):
"""
Extracts a training session from its path. NB: Wheel must be extracted first in order to
extract trials.firstMovement_times.
:param session_path: the path to the session to be extracted
:param save: if true a subset of the extracted data are saved as ALF
:param bpod_trials: list of Bpod trial data
:param settings: the Bpod session settings
:return: trials: Bunch/dict of trials
:return: wheel: Bunch/dict of wheel positions
:return: out_Files: list of output files
"""
extractor_type = ibllib.io.extractors.base.get_session_extractor_type(
session_path)
_logger.info(f"Extracting {session_path} as {extractor_type}")
bpod_trials = bpod_trials or rawio.load_data(session_path)
settings = settings or rawio.load_settings(session_path)
if extractor_type == 'training':
_logger.info('training session on ' + settings['PYBPOD_BOARD'])
wheel, files_wheel = training_wheel.extract_all(
session_path,
bpod_trials=bpod_trials,
settings=settings,
save=save)
trials, files_trials = training_trials.extract_all(
session_path,
bpod_trials=bpod_trials,
settings=settings,
save=save)
elif extractor_type == 'biased' or extractor_type == 'ephys':
_logger.info('biased session on ' + settings['PYBPOD_BOARD'])
wheel, files_wheel = training_wheel.extract_all(
session_path,
bpod_trials=bpod_trials,
settings=settings,
save=save)
trials, files_trials = biased_trials.extract_all(
session_path,
bpod_trials=bpod_trials,
settings=settings,
save=save)
elif extractor_type == 'habituation':
from ibllib.misc import version
_logger.info('habituation session on ' + settings['PYBPOD_BOARD'])
if version.le(settings['IBLRIG_VERSION_TAG'], '5.0.0'):
_logger.warning("No extraction of legacy habituation sessions")
return None, None, None
trials, files_trials = habituation_trials.extract_all(
session_path,
bpod_trials=bpod_trials,
settings=settings,
save=save)
wheel = None
files_wheel = []
else:
raise ValueError(f"No extractor for task {extractor_type}")
_logger.info('session extracted \n') # timing info in log
return trials, wheel, (files_trials + files_wheel) if save else None
def main(local_folder: str, remote_folder: str, force: bool = False) -> None:
local_folder = Path(local_folder)
remote_folder = Path(remote_folder)
src_session_paths = [
x.parent for x in local_folder.rglob("transfer_me.flag")
]
if not src_session_paths:
log.info("Nothing to transfer, exiting...")
return
# Create all dst paths
dst_session_paths = []
for s in src_session_paths:
mouse = s.parts[-3]
date = s.parts[-2]
sess = s.parts[-1]
d = remote_folder / mouse / date / sess
dst_session_paths.append(d)
for src, dst in zip(src_session_paths, dst_session_paths):
src_flag_file = src / "transfer_me.flag"
flag = flags.read_flag_file(src_flag_file)
if isinstance(flag, list):
raise NotImplementedError
else:
if force:
shutil.rmtree(dst, ignore_errors=True)
log.info(f"Copying {src}...")
shutil.copytree(src, dst, ignore=ig(str(src_flag_file.name)))
# finally if folder was created delete the src flag_file and create compress_me.flag
if dst.exists():
task_type = ibllib.io.extractors.base.get_session_extractor_type(
Path(src))
if task_type not in ['ephys', 'ephys_sync', 'ephys_mock']:
flags.write_flag_file(dst.joinpath('raw_session.flag'))
settings = raw.load_settings(dst)
if 'ephys' in settings[
'PYBPOD_BOARD']: # Any traing task on an ephys rig
dst.joinpath('raw_session.flag').unlink()
log.info(
f"Copied to {remote_folder}: Session {src_flag_file.parent}")
src_flag_file.unlink()
# Cleanup
src_video_file = src / 'raw_video_data' / '_iblrig_leftCamera.raw.avi'
dst_video_file = dst / 'raw_video_data' / '_iblrig_leftCamera.raw.avi'
src_audio_file = src / 'raw_behavior_data' / '_iblrig_micData.raw.wav'
dst_audio_file = dst / 'raw_behavior_data' / '_iblrig_micData.raw.wav'
if src_audio_file.exists() and \
src_audio_file.stat().st_size == dst_audio_file.stat().st_size:
src_audio_file.unlink()
if src_video_file.exists() and \
src_video_file.stat().st_size == dst_video_file.stat().st_size:
src_video_file.unlink()
def save_session_settings(sph: object) -> None:
save_this = json.dumps(sph, cls=ComplexEncoder, indent=1)
with open(sph.SETTINGS_FILE_PATH, 'a') as f:
f.write(save_this)
f.write('\n')
save_this = json.loads(save_this)
settings = raw.load_settings(sph.SESSION_FOLDER)
assert (save_this == settings)
def get_task_protocol(session_path):
try:
settings = load_settings(get_session_path(session_path))
except json.decoder.JSONDecodeError:
_logger.error(f"Can't read settings for {session_path}")
return
if settings:
return settings.get('PYBPOD_PROTOCOL', None)
else:
return
def probes_description(ses_path, bin_exists=True):
"""
Aggregate probes information into ALF files
Input:
raw_ephys_data/probeXX/
Output:
alf/probes.description.npy
alf/probes.trajecory.npy
"""
ses_path = Path(ses_path)
ephys_files = spikeglx.glob_ephys_files(ses_path, bin_exists=bin_exists)
subdirs, labels, efiles_sorted = zip(
*sorted([(ep.ap.parent, ep.label, ep) for ep in ephys_files if ep.get('ap')]))
"""Ouputs the probes description file"""
probe_description = []
for label, ef in zip(labels, efiles_sorted):
md = spikeglx.read_meta_data(ef.ap.with_suffix('.meta'))
probe_description.append({'label': label,
'model': md.neuropixelVersion,
'serial': int(md.serial),
'raw_file_name': md.fileName,
})
alf_path = ses_path.joinpath('alf')
alf_path.mkdir(exist_ok=True, parents=True)
probe_description_file = alf_path.joinpath('probes.description.json')
with open(probe_description_file, 'w+') as fid:
fid.write(json.dumps(probe_description))
"""Ouputs the probes trajectory file"""
bpod_meta = raw_data_loaders.load_settings(ses_path)
if not bpod_meta.get('PROBE_DATA'):
_logger.error('No probe information in settings JSON. Skipping probes.trajectory')
return
def prb2alf(prb, label):
return {'label': label, 'x': prb['X'], 'y': prb['Y'], 'z': prb['Z'], 'phi': prb['A'],
'theta': prb['P'], 'depth': prb['D'], 'beta': prb['T']}
# the labels may not match, in which case throw a warning and work in alphabetical order
if labels != ('probe00', 'probe01'):
_logger.warning("Probe names do not match the json settings files. Will match coordinates"
" per alphabetical order !")
_ = [_logger.warning(f" probe0{i} ---------- {lab} ") for i, lab in enumerate(labels)]
trajs = []
keys = sorted(bpod_meta['PROBE_DATA'].keys())
for i, k in enumerate(keys):
if i >= len(labels):
break
trajs.append(prb2alf(bpod_meta['PROBE_DATA'][f'probe0{i}'], labels[i]))
probe_trajectory_file = alf_path.joinpath('probes.trajectory.json')
with open(probe_trajectory_file, 'w+') as fid:
fid.write(json.dumps(trajs))
return [probe_trajectory_file, probe_description_file]
def extractors_exist(session_path):
settings = raw.load_settings(session_path)
task_name = settings['PYBPOD_PROTOCOL']
task_name = task_name.split('_')[-1]
extractor_type = task_name[:task_name.find('ChoiceWorld')]
if any([extractor_type in x for x in globals()]):
return extractor_type
else:
logger_.warning(
f"No extractors were found for {extractor_type}ChoiceWorld")
return False
def _load_task_protocol(session_path: str) -> str:
"""Find the IBL rig version used for the session
:param session_path: the path to a session
:type session_path: str
:return: ibl rig task protocol version
:rtype: str
"""
settings = rawio.load_settings(session_path)
ses_ver = settings["IBLRIG_VERSION_TAG"]
return ses_ver
def load_raw_data(self):
_logger.info(f"Loading raw data from {self.session_path}")
self.raw_data = raw.load_data(self.session_path)
self.details = raw.load_settings(self.session_path)
self.BNC1, self.BNC2 = raw.load_bpod_fronts(self.session_path,
data=self.raw_data)
# NOTE: wheel_position is actually an extractor needs _iblrig_encoderPositions.raw
# to be there but not as input... FIXME: we should have the extractor use the data
# without assuming it's there
ts, pos = get_wheel_position(self.session_path, bp_data=self.raw_data)
self.wheel_data = {'re_ts': ts, 're_pos': pos}
assert np.all(np.diff(self.wheel_data["re_ts"]) > 0)
def get_feedbackType(session_path, save=False, data=False, settings=False):
"""
Get the feedback that was delivered to subject.
**Optional:** saves _ibl_trials.feedbackType.npy
Checks in raw datafile for error and reward state.
Will raise an error if more than one of the mutually exclusive states have
been triggered.
Sets feedbackType to -1 if error state was trigered (applies to no-go)
Sets feedbackType to +1 if reward state was triggered
:param session_path: absolute path of session folder
:type session_path: str
:param save: wether to save the corresponding alf file
to the alf folder, defaults to False
:type save: bool, optional
:return: numpy.ndarray
:rtype: dtype('int64')
"""
if not data:
data = raw.load_data(session_path)
if not settings:
settings = raw.load_settings(session_path)
if settings is None or settings['IBLRIG_VERSION_TAG'] == '':
settings = {'IBLRIG_VERSION_TAG': '100.0.0'}
feedbackType = np.empty(len(data))
feedbackType.fill(np.nan)
reward = []
error = []
no_go = []
for t in data:
reward.append(
~np.isnan(t['behavior_data']['States timestamps']['reward'][0][0]))
error.append(
~np.isnan(t['behavior_data']['States timestamps']['error'][0][0]))
no_go.append(
~np.isnan(t['behavior_data']['States timestamps']['no_go'][0][0]))
if not all(np.sum([reward, error, no_go], axis=0) == np.ones(len(data))):
raise ValueError
feedbackType[reward] = 1
feedbackType[error] = -1
feedbackType[no_go] = -1
feedbackType = feedbackType.astype('int64')
if raw.save_bool(save, '_ibl_trials.feedbackType.npy'):
check_alf_folder(session_path)
fpath = os.path.join(session_path, 'alf',
'_ibl_trials.feedbackType.npy')
np.save(fpath, feedbackType)
return feedbackType
def extractors_exist(session_path):
settings = raw.load_settings(session_path)
if settings is None:
logger_.error(
f'ABORT: No data found in "raw_behavior_data" folder {session_path}'
)
return False
task_name = settings['PYBPOD_PROTOCOL']
task_name = task_name.split('_')[-1]
extractor_type = task_name[:task_name.find('ChoiceWorld')]
return extractor_type
