def path_from_eid(self, eid: str) -> Optional[Listable(Path)]:
"""
From an experiment id or a list of experiment ids, gets the local cache path
:param eid: eid (UUID) or list of UUIDs
:return: eid or list of eids
"""
# If eid is a list of eIDs recurse through list and return the results
if isinstance(eid, list):
path_list = []
for p in eid:
path_list.append(self.path_from_eid(p))
return path_list
# If not valid return None
if not alfio.is_uuid_string(eid):
print(eid, " is not a valid eID/UUID string")
return
if self._cache.size == 0:
return
# load path from cache
ic = find_first_2d(self._cache[['eid_0', 'eid_1']].to_numpy(),
parquet.str2np(eid))
if ic is not None:
ses = self._cache.iloc[ic]
return Path(self._par.CACHE_DIR).joinpath(
ses['lab'], 'Subjects', ses['subject'],
ses['start_time'].isoformat()[:10],
str(ses['number']).zfill(3))
def get_details(self, eid: str, full: bool = False):
""" Returns details of eid like from one.search, optional return full
session details.
"""
# If eid is a list of eIDs recurse through list and return the results
if isinstance(eid, list):
details_list = []
for p in eid:
details_list.append(self.get_details(p, full=full))
return details_list
# If not valid return None
if not alfio.is_uuid_string(eid):
print(eid, " is not a valid eID/UUID string")
return
# load all details
dets = self.alyx.rest("sessions", "read", eid)
if full:
return dets
# If it's not full return the normal output like from a one.search
det_fields = [
"subject", "start_time", "number", "lab", "project", "url",
"task_protocol", "local_path"
]
out = {k: v for k, v in dets.items() if k in det_fields}
out.update({'local_path': self.path_from_eid(eid)})
return out
def to_eid(self,
id: Listable(Union[str, Path, UUID, dict]) = None,
cache_dir: Optional[Union[str, Path]] = None) -> Listable(str):
if isinstance(id, (list, tuple)): # Recurse
return [self.to_eid(i, cache_dir) for i in id]
if isinstance(id, UUID):
return str(id)
# elif is_exp_ref(id):
# return ref2eid(id, one=self)
elif isinstance(id, dict):
assert {'subject', 'number', 'start_time', 'lab'}.issubset(id)
root = Path(self._get_cache_dir(cache_dir))
id = root.joinpath(id['lab'], 'Subjects', id['subject'],
id['start_time'][:10], ('%03d' % id['number']))
if alfio.is_session_path(id):
return self.eid_from_path(id)
elif isinstance(id, str):
if len(id) > 36:
id = id[-36:]
if not alfio.is_uuid_string(id):
raise ValueError('Invalid experiment ID')
else:
return id
else:
raise ValueError('Unrecognized experiment ID')
def path_from_eid(self,
eid: str,
use_cache: bool = True) -> Listable(Path):
"""
From an experiment id or a list of experiment ids, gets the local cache path
:param eid: eid (UUID) or list of UUIDs
:param use_cache: if set to False, will force database connection
:return: eid or list of eids
"""
# If eid is a list of eIDs recurse through list and return the results
if isinstance(eid, list):
path_list = []
for p in eid:
path_list.append(self.path_from_eid(p))
return path_list
# If not valid return None
if not alfio.is_uuid_string(eid):
print(eid, " is not a valid eID/UUID string")
return
# first try avoid hitting the database
if self._cache.size > 0 and use_cache:
cache_path = super().path_from_eid(eid)
if cache_path:
return cache_path
# if it wasn't successful, query Alyx
ses = self.alyx.rest('sessions', 'list', django=f'pk,{eid}')
if len(ses) == 0:
return None
else:
return Path(self._par.CACHE_DIR).joinpath(
ses[0]['lab'], 'Subjects', ses[0]['subject'],
ses[0]['start_time'][:10],
str(ses[0]['number']).zfill(3))
def _set_eid_or_path(self, session_path_or_eid):
"""Overloading base: session path not supported"""
if is_uuid_string(str(session_path_or_eid)):
self.eid = session_path_or_eid
# Try to set session_path if data is found locally
self.session_path = self.one.path_from_eid(self.eid)
else:
self.log.error("Cannot run ONE QC: an experiment uuid requried")
raise ValueError("'session' must be a valid session uuid")
def _load(self, eid, dataset_types=None, dclass_output=False, dry_run=False, cache_dir=None,
download_only=False, clobber=False, offline=False, keep_uuid=False):
"""
From a Session ID and dataset types, queries Alyx database, downloads the data
from Globus, and loads into numpy array. Single session only
"""
# if the input as an UUID, add the beginning of URL to it
cache_dir = self._get_cache_dir(cache_dir)
if is_uuid_string(eid):
eid = '/sessions/' + eid
eid_str = eid[-36:]
# get session json information as a dictionary from the alyx API
try:
ses = self.alyx.get('/sessions/' + eid_str)
except requests.HTTPError:
raise requests.HTTPError('Session ' + eid_str + ' does not exist')
# ses = ses[0]
# if no dataset_type is provided:
# a) force the output to be a dictionary that provides context to the data
# b) download all types that have a data url specified whithin the alf folder
dataset_types = [dataset_types] if isinstance(dataset_types, str) else dataset_types
if not dataset_types or dataset_types == ['__all__']:
dclass_output = True
dc = SessionDataInfo.from_session_details(ses, dataset_types=dataset_types, eid=eid_str)
# loop over each dataset and download if necessary
for ind in range(len(dc)):
if dc.url[ind] and not dry_run:
relpath = PurePath(dc.url[ind].replace(self._par.HTTP_DATA_SERVER, '.')).parents[0]
cache_dir_file = PurePath(cache_dir, relpath)
Path(cache_dir_file).mkdir(parents=True, exist_ok=True)
dc.local_path[ind] = self._download_file(
dc.url[ind], str(cache_dir_file), clobber=clobber, offline=offline,
keep_uuid=keep_uuid, file_size=dc.file_size[ind], hash=dc.hash[ind])
# load the files content in variables if requested
if not download_only:
for ind, fil in enumerate(dc.local_path):
dc.data[ind] = load_file_content(fil)
# parse output arguments
if dclass_output:
return dc
# if required, parse the output as a list that matches dataset_types requested
list_out = []
for dt in dataset_types:
if dt not in dc.dataset_type:
_logger.warning('dataset ' + dt + ' not found for session: ' + eid_str)
list_out.append(None)
continue
for i, x, in enumerate(dc.dataset_type):
if dt == x:
if dc.data[i] is not None:
list_out.append(dc.data[i])
else:
list_out.append(dc.local_path[i])
return list_out
def __init__(self, one=None, eid=None, lazy=True):
self.one = one or ONE()
self.eid = eid if is_uuid_string(eid) else None
self.bpodqc = None
self.oneqc = None
self.frame = None
if not lazy:
self.compute_all_qc()
self.build_extended_qc_frame()
def wrapper(eid, *args, **kwargs):
if eid is None:
log.warning("Input eid or session_path is None")
return
# Check if first arg is path or eid
if is_uuid_string(str(eid)):
session_path = one.path_from_eid(eid)
else:
session_path = Path(eid)
if dl:
_dl_raw_behavior(session_path, full=full, dry=dry, force=force)
return func(session_path, *args, **kwargs)
def _set_eid_or_path(self, session_path_or_eid):
if session_path_or_eid is None:
log.error(
"Cannot run BpodQC: Plese insert a valid session path or eid")
if is_uuid_string(str(session_path_or_eid)):
self.eid = session_path_or_eid
# Try to setsession_path if data is found locally
self.session_path = self.one.path_from_eid(self.eid)
elif is_session_path(session_path_or_eid):
self.session_path = session_path_or_eid
else:
log.error(
"Cannot run BpodQC: Plese insert a valid session path or eid")
def _check_inputs(self, endpoint: str, uuid: str) -> None:
# make sure the queryied endpoint exists, if not throw an informative error
if endpoint not in self._rest_schemes.keys():
av = [
k for k in self._rest_schemes.keys()
if not k.startswith('_') and k
]
raise ValueError('REST endpoint "' + endpoint +
'" does not exist. Available ' +
'endpoints are \n ' + '\n '.join(av))
# make sure the uuid is a valid UUID4
if is_uuid_string(uuid) is False:
raise ValueError(f"{uuid} is not a valid uuid")
return
def create_alyx_probe_insertions(session_path: str,
force: bool = False,
one: object = None,
model: str = None,
labels: list = None):
if one is None:
one = ONE()
if is_uuid_string(session_path):
eid = session_path
else:
eid = one.eid_from_path(session_path)
if eid is None:
print(
'Session not found on Alyx: please create session before creating insertions'
)
if model is None:
probe_model = spikeglx.get_neuropixel_version_from_folder(session_path)
pmodel = '3B2' if probe_model == '3B' else probe_model
else:
pmodel = model
raw_ephys_data_path = Path(session_path) / 'raw_ephys_data'
if labels is None:
probe_labels = [
x.name for x in Path(raw_ephys_data_path).glob('*')
if x.is_dir() and ('00' in x.name or '01' in x.name)
]
else:
probe_labels = labels
# create the dictionary
for plabel in probe_labels:
insdict = {'session': eid, 'name': plabel, 'model': pmodel}
# search for the corresponding insertion in Alyx
alyx_insertion = one.alyx.rest('insertions',
'list',
session=insdict['session'],
name=insdict['name'])
# if it doesn't exist, create it
if len(alyx_insertion) == 0:
alyx_insertion = one.alyx.rest('insertions',
'create',
data=insdict)
else:
iid = alyx_insertion[0]['id']
if force:
alyx_insertion = one.alyx.rest('insertions',
'update',
id=iid,
data=insdict)
else:
alyx_insertion = alyx_insertion[0]
def _load(self,
eid,
dataset_types=None,
dclass_output=False,
download_only=False,
offline=False,
**kwargs):
"""
From a Session ID and dataset types, queries Alyx database, downloads the data
from Globus, and loads into numpy array. Single session only
"""
if alfio.is_uuid_string(eid):
eid = '/sessions/' + eid
eid_str = eid[-36:]
# if no dataset_type is provided:
# a) force the output to be a dictionary that provides context to the data
# b) download all types that have a data url specified whithin the alf folder
dataset_types = [dataset_types] if isinstance(dataset_types,
str) else dataset_types
if not dataset_types or dataset_types == ['__all__']:
dclass_output = True
if offline:
dc = self._make_dataclass_offline(eid_str, dataset_types, **kwargs)
else:
dc = self._make_dataclass(eid_str, dataset_types, **kwargs)
# load the files content in variables if requested
if not download_only:
for ind, fil in enumerate(dc.local_path):
dc.data[ind] = alfio.load_file_content(fil)
# parse output arguments
if dclass_output:
return dc
# if required, parse the output as a list that matches dataset_types requested
list_out = []
for dt in dataset_types:
if dt not in dc.dataset_type:
_logger.warning('dataset ' + dt + ' not found for session: ' +
eid_str)
list_out.append(None)
continue
for i, x, in enumerate(dc.dataset_type):
if dt == x:
if dc.data[i] is not None:
list_out.append(dc.data[i])
else:
list_out.append(dc.local_path[i])
return list_out
def _to_eid(invar):
""" get eid from: details, path, or lists of details or paths
"""
outvar = []
if isinstance(invar, list) or isinstance(invar, tuple):
for i in invar:
outvar.append(_to_eid(i))
return outvar
elif isinstance(invar, dict) and is_details_dict(invar):
return invar["url"][-36:]
elif isinstance(invar, str) and is_session_path(invar):
return one.eid_from_path(invar)
elif isinstance(invar, str) and is_uuid_string(invar):
return invar
else:
log.warning("Unknown input type: please input a valid path or details object")
return
def _set_eid_or_path(self, session_path_or_eid):
"""Parse a given eID or session path
If a session UUID is given, resolves and stores the local path and vice versa
:param session_path_or_eid: A session eid or path
:return:
"""
self.eid = None
if is_uuid_string(str(session_path_or_eid)):
self.eid = session_path_or_eid
# Try to set session_path if data is found locally
self.session_path = self.one.path_from_eid(self.eid)
elif is_session_path(session_path_or_eid):
self.session_path = Path(session_path_or_eid)
if self.one is not None:
self.eid = self.one.eid_from_path(self.session_path)
if not self.eid:
self.log.warning('Failed to determine eID from session path')
else:
self.log.error('Cannot run QC: an experiment uuid or session path is required')
raise ValueError("'session' must be a valid session path or uuid")
def path_from_eid(self, eid: str, grep_str=None) -> Path:
# If eid is a list of eIDs recurse through list and return the results
if isinstance(eid, list):
path_list = []
for p in eid:
path_list.append(self.path_from_eid(p, grep_str=grep_str))
return path_list
# If not valid return None
if not is_uuid_string(eid):
print(eid, " is not a valid eID/UUID string")
return
# Load data, if no data present on disk return None
data = self._load(eid, download_only=True, offline=True)
if not data.local_path:
return None
# If user defined a grep list of specific files return paths to files
if grep_str is not None:
files = [x for x in data.local_path if grep_str in str(x)]
return files
# If none of the above happen return the session path of the first file you find
session_path = get_session_path(data.local_path[0])
return session_path
def path_from_eid(self, eid: str, use_cache=True) -> Path:
"""
From an experiment id or a list of experiment ids, gets the local cache path
:param eid: eid (UUID) or list of UUIDs
:param use_cache: if set to False, will force database connection
:return: eid or list of eids
"""
# If eid is a list of eIDs recurse through list and return the results
if isinstance(eid, list):
path_list = []
for p in eid:
path_list.append(self.path_from_eid(p))
return path_list
# If not valid return None
if not is_uuid_string(eid):
print(eid, " is not a valid eID/UUID string")
return
# first try avoid hitting the database
if self._cache.size > 0 and use_cache:
ic = parquet.find_first_2d(
self._cache[['eid_0', 'eid_1']].to_numpy(), parquet.str2np(eid))
if ic is not None:
ses = self._cache.iloc[ic]
return Path(self._par.CACHE_DIR).joinpath(
ses['lab'], 'Subjects', ses['subject'], ses['start_time'].isoformat()[:10],
str(ses['number']).zfill(3))
# if it wasn't successful, query Alyx
ses = self.alyx.rest('sessions', 'list', django=f'pk,{eid}')
if len(ses) == 0:
return None
else:
return Path(self._par.CACHE_DIR).joinpath(
ses[0]['lab'], 'Subjects', ses[0]['subject'], ses[0]['start_time'][:10],
str(ses[0]['number']).zfill(3))
def _load(self,
eid,
dataset_types=None,
dclass_output=False,
dry_run=False,
cache_dir=None,
download_only=False,
clobber=False,
offline=False,
keep_uuid=False):
"""
From a Session ID and dataset types, queries Alyx database, downloads the data
from Globus, and loads into numpy array. Single session only
"""
# if the input as an UUID, add the beginning of URL to it
cache_dir = self._get_cache_dir(cache_dir)
if is_uuid_string(eid):
eid = '/sessions/' + eid
eid_str = eid[-36:]
# get session json information as a dictionary from the alyx API
try:
ses = self.alyx.get('/sessions/' + eid_str)
except requests.HTTPError:
raise requests.HTTPError('Session ' + eid_str + ' does not exist')
# ses = ses[0]
# if no dataset_type is provided:
# a) force the output to be a dictionary that provides context to the data
# b) download all types that have a data url specified whithin the alf folder
dataset_types = [dataset_types] if isinstance(dataset_types,
str) else dataset_types
if not dataset_types or dataset_types == ['__all__']:
dclass_output = True
# this performs the filtering
dc = SessionDataInfo.from_session_details(ses,
dataset_types=dataset_types,
eid=eid_str)
# loop over each dataset and download if necessary
with concurrent.futures.ThreadPoolExecutor(
max_workers=NTHREADS) as executor:
futures = []
for ind in range(len(dc)):
if dc.url[ind] is None or dry_run:
futures.append(None)
else:
futures.append(
executor.submit(self.download_dataset,
dc.url[ind],
cache_dir=cache_dir,
clobber=clobber,
offline=offline,
keep_uuid=keep_uuid,
file_size=dc.file_size[ind],
hash=dc.hash[ind]))
concurrent.futures.wait(
list(filter(lambda x: x is not None, futures)))
for ind, future in enumerate(futures):
if future is None:
continue
dc.local_path[ind] = future.result()
# load the files content in variables if requested
if not download_only:
for ind, fil in enumerate(dc.local_path):
dc.data[ind] = load_file_content(fil)
# parse output arguments
if dclass_output:
return dc
# if required, parse the output as a list that matches dataset_types requested
list_out = []
for dt in dataset_types:
if dt not in dc.dataset_type:
_logger.warning('dataset ' + dt + ' not found for session: ' +
eid_str)
list_out.append(None)
continue
for i, x, in enumerate(dc.dataset_type):
if dt == x:
if dc.data[i] is not None:
list_out.append(dc.data[i])
else:
list_out.append(dc.local_path[i])
return list_out
def __init__(self,
eid=None,
trial=None,
camera='left',
dlc_features=None,
quick_load=True,
t_win=3,
one=None,
start=True):
"""
Plot the wheel trace alongside the video frames. Below is list of key bindings:
:key n: plot movements of next trial
:key p: plot movements of previous trial
:key r: plot movements of a random trial
:key t: prompt for a trial number to plot
:key l: toggle between legend for wheel and trial events
:key space: pause/play frames
:key left: move to previous frame
:key right: move to next frame
:param eid: uuid of experiment session to load
:param trial: the trial id to plot
:param camera: the camera position to load, options: 'left' (default), 'right', 'body'
:param plot_dlc: tuple of dlc features overlay onto frames
:param quick_load: when true, move onset detection is performed on individual trials
instead of entire session
:param t_win: the window in seconds over which to plot the wheel trace
:param start: if False, the Viewer must be started by calling the `run` method
:return: Viewer object
"""
self._logger = logging.getLogger('ibllib')
self.t_win = t_win # Time window of wheel plot
self.one = one or ONE()
self.quick_load = quick_load
# Input validation
if camera not in ['left', 'right', 'body']:
raise ValueError(
"camera must be one of 'left', 'right', or 'body'")
# If None, randomly pick a session to load
if not eid:
self._logger.info('Finding random session')
eids = self.find_sessions(dlc=dlc_features is not None)
eid = random.choice(eids)
ref = eid2ref(eid, as_dict=False, one=self.one)
self._logger.info('using session %s (%s)', eid, ref)
elif not is_uuid_string(eid):
raise ValueError('f"{eid}" is not a valid session uuid')
# Store complete session data: trials, timestamps, etc.
ref = eid2ref(eid, one=self.one, parse=False)
self._session_data = {'eid': eid, 'ref': ref, 'dlc': None}
self._plot_data = {
} # Holds data specific to current plot, namely data for single trial
# Download the DLC data if required
if dlc_features:
self._session_data['dlc'] = self.get_dlc(dlc_features,
camera=camera)
# These are for the dict returned by ONE
trial_data = self.get_trial_data('ONE')
total_trials = trial_data['intervals'].shape[0]
trial = random.randint(0, total_trials) if not trial else trial
self._session_data['total_trials'] = total_trials
self._session_data['trials'] = trial_data
# Check for local first movement times
first_moves = self.one.path_from_eid(
eid) / 'alf' / '_ibl_trials.firstMovement_times.npy'
if first_moves.exists() and 'firstMovement_times' not in trial_data:
# Load file if exists locally
self._session_data['trials']['firstMovement_times'] = np.load(
first_moves)
# Download the raw video for left camera only
self.video_path, = self.download_raw_video(camera)
cam_ts = self.one.load(self._session_data['eid'], ['camera.times'],
dclass_output=True)
cam_ts, = [
ts for ts, url in zip(cam_ts.data, cam_ts.url) if camera in url
]
# _, cam_ts, _ = one.load(eid, ['camera.times']) # leftCamera is in the middle of the list
Fs = 1 / np.diff(
cam_ts).mean() # Approx. frequency of camera timestamps
# Verify video frames and timestamps agree
_, fps, count = get_video_frames_preload(self.video_path, [])
if count != cam_ts.size:
assert count <= cam_ts.size, 'fewer camera timestamps than frames'
msg = 'number of timestamps does not match number video file frames: '
self._logger.warning(msg + '%i more timestamps than frames',
cam_ts.size - count)
assert Fs - fps < 1, 'camera timestamps do not match reported frame rate'
self._logger.info("Frame rate = %.0fHz", fps)
# cam_ts = cam_ts[-count:] # Remove extraneous timestamps
self._session_data['camera_ts'] = cam_ts
# Load wheel data
self._session_data['wheel'] = self.one.load_object(
self._session_data['eid'], 'wheel')
if 'firstMovement_times' in self._session_data['trials']:
pos, t = wh.interpolate_position(
self._session_data['wheel']['timestamps'],
self._session_data['wheel']['position'],
freq=1000)
# Plot the first frame in the upper subplot
fig, axes = plt.subplots(nrows=2)
fig.canvas.mpl_disconnect(
fig.canvas.manager.key_press_handler_id) # Disable defaults
fig.canvas.mpl_connect(
'key_press_event',
self.process_key) # Connect our own key press fn
self._plot_data['figure'] = fig
self._plot_data['axes'] = axes
self._trial_num = trial
self.anim = animation.FuncAnimation(fig,
self.animate,
init_func=self.init_plot,
frames=cycle(range(60)),
interval=20,
blit=False,
repeat=True,
cache_frame_data=False)
self.anim.running = False
self.trial_num = trial # Set trial and prepare plot/frame data
if start:
self.run()
def create_alyx_probe_insertions(
session_path: str,
force: bool = False,
one: object = None,
model: str = None,
labels: list = None,
):
if one is None:
one = ONE()
if is_uuid_string(session_path):
eid = session_path
else:
eid = one.eid_from_path(session_path)
if eid is None:
print(
"Session not found on Alyx: please create session before creating insertions"
)
if model is None:
probe_model = spikeglx.get_neuropixel_version_from_folder(session_path)
pmodel = "3B2" if probe_model == "3B" else probe_model
else:
pmodel = model
raw_ephys_data_path = Path(session_path) / "raw_ephys_data"
if labels is None:
probe_labels = [
x.name for x in Path(raw_ephys_data_path).glob("*")
if x.is_dir() and ("00" in x.name or "01" in x.name)
]
else:
probe_labels = labels
# create the qc fields in the json field
qc_dict = {}
qc_dict.update({"qc": "NOT_SET"})
qc_dict.update({"extended_qc": {}})
# create the dictionary
for plabel in probe_labels:
insdict = {
"session": eid,
"name": plabel,
"model": pmodel,
"json": qc_dict
}
# search for the corresponding insertion in Alyx
alyx_insertion = one.alyx.rest("insertions",
"list",
session=insdict["session"],
name=insdict["name"])
# if it doesn't exist, create it
if len(alyx_insertion) == 0:
alyx_insertion = one.alyx.rest("insertions",
"create",
data=insdict)
else:
iid = alyx_insertion[0]["id"]
if force:
alyx_insertion = one.alyx.rest("insertions",
"update",
id=iid,
data=insdict)
else:
alyx_insertion = alyx_insertion[0]
