def make(self, key):
eID = str((acquisition.Session & key).fetch1('session_uuid'))
key['passive_beep_times'], key['passive_valve_click_times'], \
key['passive_white_noise_times'] = \
ONE().load(eID, dataset_types=['_ibl_passiveBeeps.times',
'_ibl_passiveValveClick.times',
'_ibl_passiveWhiteNoise.times'])
self.insert1(key)
def __init__(self, eid, one=None, bpod_ntrials=None, lazy=False):
self.one = one or ONE()
self.eid = eid
self.details = self.one.get_details(self.eid, full=True)
self.bpod_ntrials = bpod_ntrials or self.details["n_trials"]
self.metrics = None
self.passed = None
if not lazy:
self.compute()
def criteria_opto_eids(eids,
max_lapse=0.2,
max_bias=0.3,
min_trials=200,
one=None):
if one is None:
one = ONE()
use_eids = []
for j, eid in enumerate(eids):
try:
trials = load_trials(eid, laser_stimulation=True)
lapse_l = 1 - (
np.sum(trials.loc[trials['signed_contrast'] == -1,
'choice'] == 1) /
trials.loc[trials['signed_contrast'] == -1, 'choice'].shape[0])
lapse_r = 1 - (
np.sum(trials.loc[trials['signed_contrast'] == 1,
'choice'] == -1) /
trials.loc[trials['signed_contrast'] == 1, 'choice'].shape[0])
bias = np.abs(0.5 -
(np.sum(trials.loc[trials['signed_contrast'] == 0,
'choice'] == 1) /
np.shape(trials.loc[trials['signed_contrast'] == 0,
'choice'] == 1)[0]))
details = one.get_details(eid)
if ((lapse_l < max_lapse) & (lapse_r < max_lapse) &
(trials.shape[0] > min_trials)
& (bias < max_bias) &
('laser_stimulation' in trials.columns)):
use_eids.append(eid)
elif 'laser_stimulation' not in trials.columns:
print('No laser_stimulation data for %s %s' %
(details['subject'], details['start_time'][:10]))
else:
print(
'%s %s excluded (n_trials: %d, lapse_l: %.2f, lapse_r: %.2f, bias: %.2f)'
% (details['subject'], details['start_time'][:10],
trials.shape[0], lapse_l, lapse_r, bias))
except Exception:
print('Could not load session %s' % eid)
return use_eids
def make(self, key):
eID = (acquisition.Session & key).fetch1('session_uuid')
extra_rewards_times = \
ONE().load(eID, dataset_types=['_ibl_extraRewards.times'])
key['extra_rewards_times'] = extra_rewards_times
self.insert1(key)
logger.info('Populated an ExtraRewards tuple for subject {subject_nickname} in session started at {session_start_time}'.format(**key))
def test_sync_extract(self):
session_path = self.session_path
dry = False
ibllib.io.flags.create_other_flags(session_path,
'extract_ephys.flag',
force=True)
iblrig_pipeline.extract_ephys(session_path, dry=dry)
one = ONE(base_url='https://test.alyx.internationalbrainlab.org',
username='******',
password='******')
iblrig_pipeline.register(session_path, one=one)
def setUp(self):
# Init connection to the database
one = ONE(base_url='https://test.alyx.internationalbrainlab.org',
username='******',
password='******')
eids = [
'cf264653-2deb-44cb-aa84-89b82507028a',
'4e0b3320-47b7-416e-b842-c34dc9004cf8'
]
self.eid = eids[0]
self.eid2 = eids[1]
self.One = one
def get_one_data(tmp_path):
eid_temp = 'da188f2c-553c-4e04-879b-c9ea2d1b9a93'
metadata_converter = Alyx2NWBMetadata(eid=eid_temp, one_obj=ONE())
nwbsaveloc = str(tmp_path/'test.nwb')
converter_nwb = Alyx2NWBConverter(
metadata_obj=metadata_converter,
saveloc=nwbsaveloc,
save_raw=False,
save_camera_raw=False)
converter_nwb.run_conversion()
converter_nwb.write_nwb()
yield nwbsaveloc
def get_all_sess_with_ME():
one = ONE()
# get all bwm sessions with dlc
all_sess = one.alyx.rest('sessions',
'list',
project='ibl_neuropixel_brainwide_01',
task_protocol="ephys",
dataset_types='camera.ROIMotionEnergy')
eids = [s['url'].split('/')[-1] for s in all_sess]
return eids
def get_micro_manipulator_data(subject, one=None, force_extract=False):
"""
Looks for all ephys sessions for a given subject and get the probe micro-manipulator
trajectories.
If probes ALF object not on flat-iron, attempts to perform the extraction from meta-data
and task settings file.
"""
if not one:
one = ONE()
eids, sessions = one.search(subject=subject,
task_protocol='ephys',
details=True)
dtypes = [
'probes.description',
'probes.trajectory',
]
probes = alf.io.AlfBunch({})
for ses in sessions:
sess_path = Path(ses['local_path'])
probe = None
if not force_extract:
probe = one.load_object(ses['url'], 'probes')
if not probe:
_logger.warning(f"Re-extraction probe info for {sess_path}")
dtypes = ['_iblrig_taskSettings.raw', 'ephysData.raw.meta']
raw_files = one.load(ses['url'],
dataset_types=dtypes,
download_only=True)
if all([rf is None for rf in raw_files]):
_logger.warning(
f"no raw settings files nor ephys data found for"
f" {ses['local_path']}. Skip this session.")
continue
extract_probes(sess_path, bin_exists=False)
probe = alf.io.load_object(sess_path.joinpath('alf'), 'probes')
one.load(ses['url'],
dataset_types='channels.localCoordinates',
download_only=True)
# get for each insertion the sites local mapping: if not found assumes checkerboard pattern
probe['sites_coordinates'] = []
for prb in probe.description:
chfile = Path(ses['local_path']).joinpath(
'alf', prb['label'], 'channels.localCoordinates.npy')
if chfile.exists():
probe['sites_coordinates'].append(np.load(chfile))
else:
_logger.warning(
f"no channel.localCoordinates found for {ses['local_path']}."
f"Assumes checkerboard pattern")
probe['sites_coordinates'].append(SITES_COORDINATES)
# put the session information in there
probe['session'] = [ses] * len(probe.description)
probes = probes.append(probe)
return probes
def load_wheel_reaction_times(eid, one=None):
"""
Return the calculated reaction times for session. Reaction times are defined as the time
between the go cue (onset tone) and the onset of the first substantial wheel movement. A
movement is considered sufficiently large if its peak amplitude is at least 1/3rd of the
distance to threshold (~0.1 radians).
Negative times mean the onset of the movement occurred before the go cue. Nans may occur if
there was no detected movement withing the period, or when the goCue_times or feedback_times
are nan.
Parameters
----------
eid : str
Session UUID
one : oneibl.ONE
An instance of ONE for loading data. If None a new one is instantiated using the defaults.
Returns
----------
array-like
reaction times
"""
if one is None:
one = ONE()
trials = one.load_object(eid, 'trials')
# If already extracted, load and return
if trials and 'firstMovement_times' in trials:
return trials['firstMovement_times'] - trials['goCue_times']
# Otherwise load wheelMoves object and calculate
moves = one.load_object(eid, 'wheelMoves')
# Re-extract wheel moves if necessary
if not moves or 'peakAmplitude' not in moves:
wheel = one.load_object(eid, 'wheel')
moves = extract_wheel_moves(wheel['timestamps'], wheel['position'])
assert trials and moves, 'unable to load trials and wheelMoves data'
firstMove_times, is_final_movement, ids = extract_first_movement_times(
moves, trials)
return firstMove_times - trials['goCue_times']
def make(self, key):
passive_trial_key = key.copy()
eID = (acquisition.Session & key).fetch1('session_uuid')
passive_visual_stim_contrast_left, passive_visual_stim_contrast_right = \
ONE().load(eID, dataset_types=['_ibl_passiveVisual.contrastLeft',
'_ibl_passiveVisual.contrastRight',
'_ibl_passiveVisual.times'])
assert len(
np.unique(
np.array([
len(passive_visual_stim_contrast_left),
len(passive_visual_stim_contrast_right),
len(passive_visual_stim_times)
]))
) == 1, 'Loaded passive visual files do not have the same length'
key['passive_trials_total_num'] = len(passive_visual_stim_times)
key['passive_trials_start_time'] = float(passive_visual_stim_times[0])
key['passive_trials_end_time'] = float(passive_visual_stim_times[-1])
self.insert1(key)
for idx_trial in range(len(passive_visual_stim_times)):
if np.isnan(passive_visual_stim_contrast_left[idx_trial]):
passive_stim_contrast_left = 0
else:
passive_stim_contrast_left = passive_visual_stim_contrast_left[
idx_trial]
if np.isnan(passive_visual_stim_contrast_right[idx_trial]):
passive_stim_contrast_right = 0
else:
passive_stim_contrast_right = passive_visual_stim_contrast_right[
idx_trial]
passive_trial_key['passive_trial_id'] = idx_trial + 1
passive_trial_key['passive_trial_stim_on_time'] = float(
passive_visual_stim_times[idx_trial])
passive_trial_key['passive_trial_stim_contrast_left'] = float(
passive_stim_contrast_left)
passive_trial_key['passive_trial_stim_contrast_right'] = float(
passive_stim_contrast_right)
self.PassiveTrial().insert1(passive_trial_key)
logger.info(
'Populated a PassiveTrialSet tuple, all Trial tuples and Excluded Trial tuples for subject {subject_nickname} in session started at {session_start_time}'
.format(**key))
def job_creator(root_path,
one=None,
dry=False,
rerun=False,
max_md5_size=None):
"""
Server function that will look for creation flags and for each:
1) create the sessions on Alyx
2) register the corresponding raw data files on Alyx
3) create the tasks to be run on Alyx
:param root_path: main path containing sessions or session path
:param one
:param dry
:param rerun
:param max_md5_size
:return:
"""
if not one:
one = ONE()
rc = registration.RegistrationClient(one=one)
flag_files = list(Path(root_path).glob('**/extract_me.flag'))
flag_files += list(Path(root_path).glob('**/extract_ephys.flag'))
all_datasets = []
for flag_file in flag_files:
session_path = flag_file.parent
_logger.info(f'creating session for {session_path}')
if dry:
continue
# providing a false flag stops the registration after session creation
rc.create_session(session_path)
flag_file.unlink()
files, dsets = registration.register_session_raw_data(
session_path, one=one, max_md5_size=max_md5_size)
if dsets is not None:
all_datasets.extend(dsets)
session_type = rawio.get_session_extractor_type(session_path)
if session_type in ['biased', 'habituation', 'training']:
pipe = training_preprocessing.TrainingExtractionPipeline(
session_path, one=one)
elif session_type in ['ephys']:
pipe = ephys_preprocessing.EphysExtractionPipeline(session_path,
one=one)
else:
_logger.info(
f"Session type {session_type} as no matching extractor {session_path}"
)
if rerun:
rerun__status__in = '__all__'
else:
rerun__status__in = ['Waiting']
pipe.create_alyx_tasks(rerun__status__in=rerun__status__in)
return all_datasets
def get_latest_session_eid(subject_nickname, one=None):
"""Return the eID of the latest session for Subject that has data on
Flatiron"""
one = one or ONE()
last_session = one.search(
subject=subject_nickname,
dataset_types=["_iblrig_taskData.raw", "_iblrig_taskSettings.raw"],
limit=1,
)
if last_session:
return last_session[0]
else:
return None
def make(self, key):
datasets = (data.FileRecord & key & 'repo_name LIKE "flatiron_%"' & {
'exists': 1
}).fetch('dataset_name')
is_complete = bool(
np.all([req_ds in datasets for req_ds in self.required_datasets]))
if is_complete is True:
if '_ibl_trials.stimOn_times.npy' not in datasets:
key['stim_on_times_status'] = 'Missing'
else:
eID = str((acquisition.Session & key).fetch1('session_uuid'))
lab_name = (subject.SubjectLab & key).fetch1('lab_name')
if lab_name == 'wittenlab':
stimOn_times = np.squeeze(ONE().load(
eID,
dataset_types='_ibl_trials.stimOn_times',
clobber=True))
else:
stimOn_times = ONE().load(
eID, dataset_types='_ibl_trials.stimOn_times')
if np.all(np.isnan(stimOn_times)):
key['stim_on_times_status'] = 'Missing'
elif np.any(np.isnan(stimOn_times)):
key['stim_on_times_status'] = 'Partial'
else:
key['stim_on_times_status'] = 'Complete'
if '_ibl_trials.repNum.npy' not in datasets:
key['rep_num_status'] = 'Missing'
else:
key['rep_num_status'] = 'Complete'
if '_ibl_trials.included.npy' not in datasets:
key['included_status'] = 'Missing'
else:
key['included_status'] = 'Complete'
self.insert1(key)
def download_bpodqc_raw_data(eid, one=None):
one = one or ONE()
dstypes = [
"_iblrig_taskData.raw",
"_iblrig_taskSettings.raw",
"_iblrig_encoderPositions.raw",
"_iblrig_encoderEvents.raw",
"_iblrig_stimPositionScreen.raw",
"_iblrig_syncSquareUpdate.raw",
"_iblrig_encoderTrialInfo.raw",
"_iblrig_ambientSensorData.raw",
]
one.load(eid, dataset_types=dstypes, download_only=True)
def ref2path(ref: Union[str, Mapping, Iter], one=None, offline: bool = False) -> Union[Path, List]:
"""
Convert one or more experiment references to session path(s)
:param ref: One or more objects with keys ('subject', 'date', 'sequence'), or strings with the
form yyyy-mm-dd_n_subject
:param one: An instance of ONE
:param offline: Return path without connecting to database (unimplemented)
:return: a Path object for the experiment session
Examples:
>>> base = 'https://test.alyx.internationalbrainlab.org'
>>> one = ONE(username='******', password='******', base_url=base)
Connected to...
>>> ref = {'subject': 'flowers', 'date': datetime(2018, 7, 13).date(), 'sequence': 1}
>>> ref2path(ref, one=one)
WindowsPath('E:/FlatIron/zadorlab/Subjects/flowers/2018-07-13/001')
>>> ref2path(['2018-07-13_1_flowers', '2019-04-11_1_KS005'], one=one)
[WindowsPath('E:/FlatIron/zadorlab/Subjects/flowers/2018-07-13/001'),
WindowsPath('E:/FlatIron/cortexlab/Subjects/KS005/2019-04-11/001')]
"""
if not one:
one = ONE()
if offline:
raise NotImplementedError # Requires lab name :(
# root = Path(one._get_cache_dir(None))
# path = root / ref.subject / str(ref.date) / ('%03d' % ref.sequence)
else:
ref = ref2dict(ref, parse=False)
eid, (d,) = one.search(
subjects=ref['subject'],
date_range=(str(ref['date']), str(ref['date'])),
number=ref['sequence'],
details=True)
path = d.get('local_path')
if not path:
root = Path(one._get_cache_dir(None)) / 'Subjects' / d['lab']
return root / d['subject'] / d['start_time'][:10] / ('%03d' % d['number'])
else:
return Path(path)
def download_raw_video(eid, cameras=None):
"""
Downloads the raw video from FlatIron or cache dir.
This allows you to download just one of the
three videos
:param cameras: the specific camera to load
(i.e. 'left', 'right', or 'body') If None all
three videos are downloaded.
:return: the file path(s) of the raw videos
"""
one = ONE()
if cameras:
cameras = [cameras] if isinstance(cameras, str) else cameras
cam_files = ['_iblrig_{}Camera.raw.mp4'.format(cam) for cam in cameras]
datasets = one._alyxClient.get('sessions/' +
eid)['data_dataset_session_related']
urls = [ds['data_url'] for ds in datasets if ds['name'] in cam_files]
cache_dir = one.path_from_eid(eid).joinpath('raw_video_data')
if not os.path.exists(str(cache_dir)):
os.mkdir(str(cache_dir))
else: # Check if file already downloaded
# cam_files = [fi[:-4] for fi in cam_files] # Remove ext
filenames = [
f for f in os.listdir(str(cache_dir))
if any([cam in f for cam in cam_files])
]
if filenames:
return [cache_dir.joinpath(file) for file in filenames]
http_download_file_list(urls,
username=one._par.HTTP_DATA_SERVER_LOGIN,
password=one._par.HTTP_DATA_SERVER_PWD,
cache_dir=str(cache_dir))
return
else:
return one.load(eid, ['_iblrig_Camera.raw'], download_only=True)
def load_spike_sorting(eid, one=None, dataset_types=None):
"""
From an eid, hits the Alyx database and downloads a standard default set of dataset types
From a local session Path (pathlib.Path), loads a standard default set of dataset types
to perform analysis:
'clusters.channels',
'clusters.depths',
'clusters.metrics',
'spikes.clusters',
'spikes.times',
'probes.description'
:param eid: experiment UUID or pathlib.Path of the local session
:param one:
:param dataset_types: additional spikes/clusters objects to add to the standard default list
:return: spikes, clusters (dict of bunch, 1 bunch per probe)
"""
if isinstance(eid, Path):
return _load_spike_sorting_local(eid)
if not one:
one = ONE()
# This is a first draft, no safeguard, no error handling and a draft dataset list.
session_path = one.path_from_eid(eid)
if not session_path:
print("no session path")
return (None, None), 'no session path'
dtypes_default = [
'clusters.channels', 'clusters.depths', 'clusters.metrics',
'spikes.clusters', 'spikes.times', 'probes.description'
]
if dataset_types is None:
dtypes = dtypes_default
else:
# Append extra optional DS
dtypes = list(set(dataset_types + dtypes_default))
one.load(eid, dataset_types=dtypes, download_only=True)
return _load_spike_sorting_local(session_path)
def purge_local_data(local_folder, file_name, lab=None, dry=False):
# Figure out datasetType from file_name or file path
file_name = Path(file_name).name
alf_parts = file_name.split('.')
dstype = '.'.join(alf_parts[:2])
print(f'Looking for file <{file_name}> in folder <{local_folder}>')
# Get all paths for file_name in local folder
local_folder = Path(local_folder)
files = list(local_folder.rglob(f'*{file_name}'))
print(f'Found {len(files)} files')
print(f'Checking on Flatiron for datsetType: {dstype}...')
# Get all sessions and details from Alyx that have the dstype
one = ONE()
if lab is None:
eid, det = one.search(dataset_types=[dstype], details=True)
else:
eid, det = one.search(dataset_types=[dstype], lab=lab, details=True)
urls = []
for d in det:
urls.extend([x['data_url'] for x in d['data_dataset_session_related']
if x['dataset_type'] == dstype])
# Remove None answers when session is registered but dstype not htere yet
urls = [u for u in urls if u is not None]
print(f'Found files on Flatiron: {len(urls)}')
to_remove = []
for f in files:
sess_name = session_name(f)
for u in urls:
if sess_name in u:
to_remove.append(f)
print(f'Local files to remove: {len(to_remove)}')
for f in to_remove:
print(f)
if dry:
continue
else:
f.unlink()
return
def tasks_runner(subjects_path,
tasks_dict,
one=None,
dry=False,
count=5,
time_out=None,
**kwargs):
"""
Function to run a list of tasks (task dictionary from Alyx query) on a local server
:param subjects_path:
:param tasks_dict:
:param one:
:param dry:
:param count: maximum number of tasks to run
:param time_out: between each task, if time elapsed is greater than time out, returns (seconds)
:param kwargs:
:return: list of dataset dictionaries
"""
if one is None:
one = ONE()
import time
tstart = time.time()
c = 0
last_session = None
all_datasets = []
for tdict in tasks_dict:
# if the count is reached or if the time_out has been elapsed, break the loop and return
if c >= count or (time_out and time.time() - tstart > time_out):
break
# reconstruct the session local path. As many jobs belong to the same session
# cache the result
if last_session != tdict['session']:
ses = one.alyx.rest('sessions',
'list',
django=f"pk,{tdict['session']}")[0]
session_path = Path(subjects_path).joinpath(
Path(ses['subject'], ses['start_time'][:10],
str(ses['number']).zfill(3)))
last_session = tdict['session']
if dry:
print(session_path, tdict['name'])
else:
task, dsets = tasks.run_alyx_task(tdict=tdict,
session_path=session_path,
one=one,
**kwargs)
if dsets:
all_datasets.extend(dsets)
c += 1
return all_datasets
def make(self, key):
eID = (acquisition.Session & key).fetch1('session_uuid')
sparse_noise_positions, sparse_noise_times = \
ONE().load(eID, dataset_types=['_ibl_sparseNoise.positions', '_ns_sparseNoise.times'])
assert len(np.unique(np.array([len(sparse_noise_positions), len(sparse_noise_times)]))) == 1, 'Loaded sparse noise files do not have the same length'
key['sparse_noise_x_pos'] = sparse_noise_positions[:, 0],
key['sparse_noise_y_pos'] = sparse_noise_positions[:, 1],
key['sparse_noise_times'] = sparse_noise_times
self.insert1(key)
logger.info('Populated a SparseNoise tuple for subject {subject_nickname} in session started at {session_start_time}'.format(**key))
def sessions_with_region(acronym, one=None):
if one is None:
one = ONE()
query_str = f'channels__brain_region__acronym__icontains,{acronym},' \
'probe_insertion__session__project__name__icontains,ibl_neuropixel_brainwide_01,' \
'probe_insertion__session__qc__lt,50,' \
'~probe_insertion__json__qc,CRITICAL'
traj = one.alyx.rest('trajectories',
'list',
provenance='Ephys aligned histology track',
django=query_str)
eids = np.array([i['session']['id'] for i in traj])
probes = np.array([i['probe_name'] for i in traj])
return eids, probes
def create_alyx_probe_insertions(session_path: str,
force: bool = False,
one: object = None,
model: str = None,
labels: list = None):
if one is None:
one = ONE()
if is_uuid_string(session_path):
eid = session_path
else:
eid = one.eid_from_path(session_path)
if eid is None:
print(
'Session not found on Alyx: please create session before creating insertions'
)
if model is None:
probe_model = spikeglx.get_neuropixel_version_from_folder(session_path)
pmodel = '3B2' if probe_model == '3B' else probe_model
else:
pmodel = model
raw_ephys_data_path = Path(session_path) / 'raw_ephys_data'
if labels is None:
probe_labels = [
x.name for x in Path(raw_ephys_data_path).glob('*')
if x.is_dir() and ('00' in x.name or '01' in x.name)
]
else:
probe_labels = labels
# create the dictionary
for plabel in probe_labels:
insdict = {'session': eid, 'name': plabel, 'model': pmodel}
# search for the corresponding insertion in Alyx
alyx_insertion = one.alyx.rest('insertions',
'list',
session=insdict['session'],
name=insdict['name'])
# if it doesn't exist, create it
if len(alyx_insertion) == 0:
alyx_insertion = one.alyx.rest('insertions',
'create',
data=insdict)
else:
iid = alyx_insertion[0]['id']
if force:
alyx_insertion = one.alyx.rest('insertions',
'update',
id=iid,
data=insdict)
else:
alyx_insertion = alyx_insertion[0]
def job_creator(root_path, one=None, dry=False, rerun=False, max_md5_size=None):
"""
Server function that will look for creation flags and for each:
1) create the sessions on Alyx
2) register the corresponding raw data files on Alyx
3) create the tasks to be run on Alyx
:param root_path: main path containing sessions or session path
:param one
:param dry
:param rerun
:param max_md5_size
:return:
"""
if not one:
one = ONE()
rc = registration.RegistrationClient(one=one)
flag_files = list(Path(root_path).glob('**/raw_session.flag'))
all_datasets = []
for flag_file in flag_files:
session_path = flag_file.parent
_logger.info(f'creating session for {session_path}')
if dry:
continue
# if the subject doesn't exist in the database, skip
try:
rc.create_session(session_path)
files, dsets = registration.register_session_raw_data(
session_path, one=one, max_md5_size=max_md5_size)
except BaseException:
_logger.error(traceback.format_exc())
_logger.info(f"Creating session / registering raw datasets {session_path} errored")
continue
if dsets is not None:
all_datasets.extend(dsets)
session_type = get_session_extractor_type(session_path)
if session_type in ['biased', 'habituation', 'training']:
pipe = training_preprocessing.TrainingExtractionPipeline(session_path, one=one)
# only start extracting ephys on a raw_session.flag
elif session_type in ['ephys'] and flag_file.name == 'raw_session.flag':
pipe = ephys_preprocessing.EphysExtractionPipeline(session_path, one=one)
else:
_logger.info(f"Session type {session_type} as no matching extractor {session_path}")
return
if rerun:
rerun__status__in = '__all__'
else:
rerun__status__in = ['Waiting']
pipe.create_alyx_tasks(rerun__status__in=rerun__status__in)
flag_file.unlink()
return all_datasets
def load_spike_sorting(eid, one=None, dataset_types=None):
"""
From an eid, hits the Alyx database and downloads a standard set of dataset types to perform
analysis.
:param eid:
:param dataset_types: additional spikes/clusters objects to add to the standard list
:return:
"""
if not one:
one = ONE()
# This is a first draft, no safeguard, no error handling and a draft dataset list.
session_path = one.path_from_eid(eid)
dtypes = [
'clusters.channels',
'clusters.depths',
'clusters.metrics',
'spikes.clusters',
'spikes.times',
'probes.description',
]
if dataset_types:
dtypes = list(set(dataset_types + dtypes))
_ = one.load(eid, dataset_types=dtypes, download_only=True)
probes = alf.io.load_object(session_path.joinpath('alf'), 'probes')
spikes = {}
clusters = {}
for i, _ in enumerate(probes['description']):
probe_path = session_path.joinpath('alf',
probes['description'][i]['label'])
cluster = alf.io.load_object(probe_path, object='clusters')
spike = alf.io.load_object(probe_path, object='spikes')
label = probes['description'][i]['label']
clusters[label] = cluster
spikes[label] = spike
return spikes, clusters
def get_example_images(eid):
eids = get_repeated_sites()
# eid = eids[23]
# video_type = 'body'
#eids = ['15f742e1-1043-45c9-9504-f1e8a53c1744']
eids = ['4a45c8ba-db6f-4f11-9403-56e06a33dfa4']
frts = {'body': 30, 'left': 60, 'right': 150}
one = ONE()
#for eid in eids:
for video_type in frts:
frame_idx = [20 * 60 * frts[video_type]]
try:
r = one.list(eid, 'dataset_types')
recs = [
x for x in r if f'{video_type}Camera.raw.mp4' in x['name']
][0]['file_records']
video_path = [
x['data_url'] for x in recs if x['data_url'] is not None
][0]
frames = get_video_frames_preload(video_path,
frame_idx,
mask=np.s_[:, :, 0])
np.save(
'/home/mic/reproducible_dlc/example_images/'
f'{eid}_{video_type}.npy', frames)
print(eid, video_type, 'done')
except:
print(eid, video_type, 'error')
continue
def get_repeated_sites():
one = ONE()
STR_QUERY = 'probe_insertion__session__project__name__icontains,ibl_neuropixel_brainwide_01,' \
'probe_insertion__session__qc__lt,50,' \
'~probe_insertion__json__qc,CRITICAL,' \
'probe_insertion__session__n_trials__gte,400'
all_sess = one.alyx.rest('trajectories',
'list',
provenance='Planned',
x=-2243,
y=-2000,
theta=15,
django=STR_QUERY)
eids = [s['session']['id'] for s in all_sess]
return eids
def make(self, key):
trial_key = key.copy()
eID = str((acquisition.Session & key).fetch1('session_uuid'))
asd = ONE().load(eID, dataset_types='_iblrig_ambientSensorData.raw')
if not len(TrialSet.Trial & key) == len(asd[0]):
print(
'Size of ambient sensor data does not match the trial number')
return
for idx_trial, asd_trial in enumerate(asd[0]):
trial_key['trial_id'] = idx_trial + 1
trial_key['temperature_c'] = asd_trial['Temperature_C'][0]
trial_key['air_pressure_mb'] = asd_trial['AirPressure_mb'][0]
trial_key['relative_humidity'] = asd_trial['RelativeHumidity'][0]
self.insert1(trial_key)
def list_current_sessions(one=None):
"""
Get the set of session eids used in integration tests. When writing new tests, this can be
a useful way of choosing which sessions to use.
:param one: An ONE object for fetching session eid from path
:return: Set of integration session eids
"""
def not_null(itr):
return filter(lambda x: x is not None, itr)
one = one or ONE()
root = IntegrationTest.default_data_root()
folders = set(alf.folders.session_path(x[0]) for x in os.walk(root))
eids = not_null(one.eid_from_path(x) for x in not_null(folders))
return set(eids)
def plot_saturation():
'''
plot the number of segments that are quiet in terms of spikes
'''
plt.ion()
results_folder = '/home/mic/saturation_scan/'
t = list(os.walk(results_folder))[0][-1]
sess_info = []
sat_segs = []
for ii in t:
try:
a = np.load(results_folder + ii)
except:
print("could't load %s" % ii)
continue
sess_info.append(a[:, 0])
sat_segs.append(a[:, 1])
flat_sess_info = [item for sublist in sess_info for item in sublist]
flat_sat_segs = [int(item) for sublist in sat_segs for item in sublist]
maxes = np.where(np.array(flat_sat_segs) > 10)
height = np.array(flat_sat_segs)[maxes] #flat_sat_segs
bars = np.array(flat_sess_info)[maxes]
one = ONE()
#flat_sess_info
y_pos = np.arange(len(bars))
# Create horizontal bars
plt.barh(y_pos, height)
# Create names on the y-axis
plt.yticks(y_pos, bars, fontsize=10)
plt.xlabel('number of saturated 200 ms segments')
plt.title(
'sessions with histology that meet behavior criterion for the BWM')
sess_info = bars
seg_info = height
return seg_info, sess_info
