def make(self, key):
wheel_move_key = key.copy()
eID = str((acquisition.Session & key).fetch1('session_uuid'))
wheel_moves_intervals, wheel_moves_types = \
ONE().load(eID, dataset_types=['_ibl_wheelMoves.intervals',
'_ibl_wheelMoves.type'])
wheel_moves_types = wheel_moves_types.columns
assert len(np.unique(np.array([len(wheel_moves_intervals),
len(wheel_moves_types)]))) == 1, \
'Loaded wheel move files do not have the same length'
key['wheel_move_number'] = len(wheel_moves_types)
self.insert1(key)
for idx_move in range(len(wheel_moves_types)):
wheel_move_key['wheel_move_id'] = idx_move + 1
wheel_move_key['wheel_move_start_time'] = \
wheel_moves_intervals[idx_move, 0]
wheel_move_key['wheel_move_end_time'] = \
wheel_moves_intervals[idx_move, 1]
wheel_move_type = wheel_moves_types[idx_move]
if 'CCW' in wheel_move_type:
wheel_move_key['wheel_move_type'] = 'CCW'
elif 'flinch' in wheel_move_type:
wheel_move_key['wheel_move_type'] = 'flinch'
else:
wheel_move_key['wheel_move_type'] = 'CW'
self.WheelMove().insert1(wheel_move_key)
logger.info('Populated a WheelMoveSet and all WheelMove tuples for \
subject {subject_uuid} in session started at \
{session_start_time}'.format(**key))
def load_spike_sorting(eid, one=None, dataset_types=None):
"""
From an eid, hits the Alyx database and downloads a standard set of dataset types to perform
analysis.
:param eid:
:param dataset_types: additional spikes/clusters objects to add to the standard list
:return:
"""
if not one:
one = ONE()
# This is a first draft, no safeguard, no error handling and a draft dataset list.
session_path = one.path_from_eid(eid)
dtypes = [
'clusters.channels',
'clusters.depths',
'clusters.metrics',
'spikes.clusters',
'spikes.times',
'probes.description',
]
if dataset_types:
dtypes = list(set(dataset_types + dtypes))
_ = one.load(eid, dataset_types=dtypes, download_only=True)
probes = alf.io.load_object(session_path.joinpath('alf'), 'probes')
spikes = {}
clusters = {}
for i, _ in enumerate(probes['description']):
probe_path = session_path.joinpath('alf',
probes['description'][i]['label'])
cluster = alf.io.load_object(probe_path, object='clusters')
spike = alf.io.load_object(probe_path, object='spikes')
label = probes['description'][i]['label']
clusters[label] = cluster
spikes[label] = spike
return spikes, clusters
def get_example_images(eid):
eids = get_repeated_sites()
# eid = eids[23]
# video_type = 'body'
#eids = ['15f742e1-1043-45c9-9504-f1e8a53c1744']
eids = ['4a45c8ba-db6f-4f11-9403-56e06a33dfa4']
frts = {'body': 30, 'left': 60, 'right': 150}
one = ONE()
#for eid in eids:
for video_type in frts:
frame_idx = [20 * 60 * frts[video_type]]
try:
r = one.list(eid, 'dataset_types')
recs = [
x for x in r if f'{video_type}Camera.raw.mp4' in x['name']
][0]['file_records']
video_path = [
x['data_url'] for x in recs if x['data_url'] is not None
][0]
frames = get_video_frames_preload(video_path,
frame_idx,
mask=np.s_[:, :, 0])
np.save(
'/home/mic/reproducible_dlc/example_images/'
f'{eid}_{video_type}.npy', frames)
print(eid, video_type, 'done')
except:
print(eid, video_type, 'error')
continue
def get_ME(eid, video_type):
#video_type = 'left'
one = ONE()
dataset_types = ['camera.ROIMotionEnergy', 'camera.times']
a = one.list(eid, 'dataset-types')
# for newer iblib version do [x['dataset_type'] for x in a]
# if not all([(u in [x['dataset_type'] for x in a]) for u in dataset_types]):
# print('not all data available')
# return
one.load(eid, dataset_types=dataset_types)
local_path = one.path_from_eid(eid)
alf_path = local_path / 'alf'
cam0 = alf.io.load_object(alf_path,
'%sCamera' % video_type,
namespace='ibl')
ME = np.load(alf_path / f'{video_type}Camera.ROIMotionEnergy.npy')
Times = cam0['times']
return Times, ME
def get_dlc_XYs(eid, video_type):
#video_type = 'left'
one = ONE()
dataset_types = ['camera.dlc', 'camera.times']
a = one.list(eid, 'dataset-types')
# for newer iblib version do [x['dataset_type'] for x in a]
# if not all([(u in [x['dataset_type'] for x in a]) for u in dataset_types]):
# print('not all data available')
# return
one.load(eid, dataset_types=dataset_types) #clobber=True # force download
local_path = one.path_from_eid(eid)
alf_path = local_path / 'alf'
cam0 = alf.io.load_object(alf_path,
'%sCamera' % video_type,
namespace='ibl')
Times = cam0['times']
cam = cam0['dlc']
points = np.unique(['_'.join(x.split('_')[:-1]) for x in cam.keys()])
# Set values to nan if likelyhood is too low # for pqt: .to_numpy()
XYs = {}
for point in points:
x = np.ma.masked_where(cam[point + '_likelihood'] < 0.9,
cam[point + '_x'])
x = x.filled(np.nan)
y = np.ma.masked_where(cam[point + '_likelihood'] < 0.9,
cam[point + '_y'])
y = y.filled(np.nan)
XYs[point] = np.array([x, y])
return Times, XYs
def make(self, key):
eID = str((acquisition.Session & key).fetch1('session_uuid'))
wheel_position, wheel_velocity, wheel_timestamps = \
ONE().load(eID, dataset_types=['_ibl_wheel.position',
'_ibl_wheel.velocity',
'_ibl_wheel.timestamps'])
wheel_sample_ids = wheel_timestamps[:, 0]
wheel_timestamps = wheel_timestamps[:, 1]
wheel_sampling_rate = 1 / np.median(np.diff(wheel_timestamps))
key['wheel_position'] = wheel_position
key['wheel_velocity'] = wheel_velocity
key['wheel_sample_ids'] = wheel_sample_ids
key['wheel_timestamps'] = wheel_timestamps
key['wheel_start_time'] = wheel_timestamps[0]
key['wheel_end_time'] = wheel_timestamps[-1]
key['wheel_duration'] = wheel_timestamps[-1] - wheel_timestamps[0]
key['wheel_sampling_rate'] = wheel_sampling_rate
self.insert1(key)
logger.info('Populated a Wheel tuple for subject {subject_uuid} \
in session on {session_start_time}'.format(**key))
def test_training_session(self, mock_ext, mock_meta):
"""
Tests the full QC process for a training session. Mock a load of things so we don't need
the ful video file.
:param mock_ext: mock cv.VideoCapture in camera extractor module
:param mock_meta: mock get_video_meta
:return:
"""
n_samples = 100
length = 107913
eid, session_path = self.training
self.frames = np.load(self.data_path / 'camera' / f'{eid}_frame_samples.npy')
mock_meta.return_value = \
Bunch({'length': length, **CameraQC.video_meta['training']['left']})
mock_ext().get.return_value = length
mock_ext().read.side_effect = self.side_effect()
qc = CameraQC(session_path, 'left',
stream=False, n_samples=n_samples, one=ONE(offline=True))
qc.load_data(download_data=False, extract_times=True)
outcome, extended = qc.run(update=False)
self.assertEqual('FAIL', outcome)
expected = {
'_videoLeft_brightness': 'FAIL',
'_videoLeft_camera_times': ('PASS', 0),
'_videoLeft_dropped_frames': ('PASS', 1, 0),
'_videoLeft_file_headers': 'PASS',
'_videoLeft_focus': 'FAIL',
'_videoLeft_framerate': ('FAIL', 32.895),
'_videoLeft_pin_state': ('WARNING', 1151, 0),
'_videoLeft_position': 'FAIL',
'_videoLeft_resolution': 'PASS',
'_videoLeft_timestamps': 'PASS',
'_videoLeft_wheel_alignment': ('FAIL', -95)
}
self.assertEqual(expected, extended)
def make(self, key):
eID = (acquisition.Session & key).fetch1('session_uuid')
lick_times, lick_piezo_raw, lick_piezo_timestamps = \
ONE().load(eID, dataset_types=['_ibl_licks.times', '_ibl_lickPiezo.raw', '_ibl_lickPiezo.timestamps'])
lick_sample_ids = lick_piezo_timestamps[:, 0]
lick_piezo_timestamps = lick_piezo_timestamps[:, 1]
key['lick_times'] = lick_times
key['lick_piezo_raw'] = lick_piezo_raw
key['lick_sample_ids'] = lick_sample_ids
key['lick_piezo_timestamps'] = lick_piezo_timestamps
key['lick_start_time'] = lick_piezo_timestamps[0]
key['lick_end_time'] = lick_piezo_timestamps[-1]
key['lick_sampling_rate'] = 1 / np.median(
np.diff(lick_piezo_timestamps))
self.insert1(key)
logger.info(
'Populated a Lick tuple for subject {subject_nickname} in session started at {session_start_time}'
.format(**key))
def make(self, key):
spon_time_key = key.copy()
eID = str((acquisition.Session & key).fetch1('session_uuid'))
spontaneous_intervals = \
ONE().load(eID, dataset_types=['_ibl_spontaneous.intervals'])
key['spontaneous_time_total_num'] = len(spontaneous_intervals)
self.insert1(key)
for idx_spon in range(len(spontaneous_intervals)):
spon_time_key['spontaneous_time_id'] = idx_spon + 1
spon_time_key['spontaneous_start_time'] = \
spontaneous_intervals[idx_spon, 0]
spon_time_key['spontaneous_end_time'] = \
spontaneous_intervals[idx_spon, 1]
spon_time_key['spontaneous_time_duration'] = \
float(np.diff(spontaneous_intervals[idx_spon, :]))
self.SpontaneousTime().insert1(spon_time_key)
logger.info('Populated a SpontaneousTimeSet tuple and all \
Spontaneoustime tuples for subject {subject_uuid} in \
session started at {session_start_time}'.format(**key))
def setUpClass(cls) -> None:
"""Load a few 10 second videos for testing the various video QC checks"""
data_path = base.IntegrationTest.default_data_root()
video_path = data_path.joinpath('camera')
videos = sorted(video_path.rglob('*.mp4'))
# Instantiate using session with a video path to fool constructor.
# To remove once we use ONE cache file
one = ONE(base_url='https://test.alyx.internationalbrainlab.org',
username='******',
password='******')
dummy_id = 'd3372b15-f696-4279-9be5-98f15783b5bb'
qc = CameraQC(dummy_id, 'left',
n_samples=10, stream=False, download_data=False, one=one)
qc.one = None
qc._type = 'ephys' # All videos come from ephys sessions
qcs = OrderedDict()
for video in videos:
qc.video_path = video
qc.label = vidio.label_from_path(video)
qc.n_samples = 10
qc.load_video_data()
qcs[video] = qc.data.copy()
cls.qc = qc
cls.data = qcs
With EXAMPLE_OVERWRITE = True, the script downloads an example dataset and runs
the registration (used for automatic testing of the example).
'''
# Author: Olivier, Gaelle
from ibllib.pipes import histology
from oneibl.one import ONE
from pathlib import Path
# ======== EDIT FOR USERS ====
# Edit so as to reflect the directory containing your electrode tracks
path_tracks = "/Users/gaelle/Downloads/Flatiron/examples/00_to_add"
EXAMPLE_OVERWRITE = True # Put to False when wanting to run the script on your data
ALYX_URL = "https://dev.alyx.internationalbrainlab.org" # FOR TESTING
# ALYX_URL = "https://alyx.internationalbrainlab.org" # UNCOMMENT WHEN READY
# ======== DO NOT EDIT BELOW ====
one = ONE(base_url=ALYX_URL)
if EXAMPLE_OVERWRITE:
# TODO Olivier : Function to download examples folder
cachepath = Path(one._par.CACHE_DIR)
path_tracks = cachepath.joinpath('examples', 'histology', 'tracks_to_add')
histology.register_track_files(path_tracks=path_tracks, one=one, overwrite=True)
histology.detect_missing_histology_tracks(path_tracks=path_tracks, one=one)
# loading and plotting functions
from define_paths import fig_path
from behavior_plots import *
# from load_mouse_data import * # this has all plotting functions
## INITIALIZE A FEW THINGS
sns.set_style("darkgrid", {
'xtick.bottom': True,
'ytick.left': True,
'lines.markeredgewidth': 0
})
sns.set_context(context="paper")
sns.set_palette("colorblind") # palette for water types
## CONNECT TO ONE
one = ONE() # initialize
# get folder to save plots
path = fig_path()
path = os.path.join(path, 'per_lab/')
if not os.path.exists(path):
os.mkdir(path)
users = [
'danlab', 'angelakilab', 'wittenlab', 'churchlandlab', 'mainenlab',
'mrsicflogellab'
]
# ============================================= #
# START BIG OVERVIEW PLOT
# ============================================= #
from . import acquisition, reference, behavior
import numpy as np
try:
from oneibl.one import ONE
except:
pass
mode = environ.get('MODE')
if mode == 'update':
schema = dj.schema('ibl_ephys')
else:
schema = dj.schema(dj.config.get('database.prefix', '') + 'ibl_ephys')
try:
one = ONE()
except:
pass
@schema
class Ephys(dj.Imported):
definition = """
-> acquisition.Session
---
ephys_raw_dir=null: varchar(256) # Path of Raw ephys file: array of size nSamples * nChannels. Channels from all probes are included. NOTE: this is huge, and hardly even used. To allow people to load it, we need to add slice capabilities to ONE
ephys_timestamps=null: longblob # Timestamps for raw ephys timeseries (seconds)
ephys_start_time=null: float # (seconds)
ephys_stop_time=null: float # (seconds)
ephys_duration=null: float # (seconds)
ephys_sampling_rate=null: float # samples per second
def load_trials_df(eid,
one=None,
maxlen=None,
t_before=0.,
t_after=0.,
ret_wheel=False,
ret_abswheel=False,
wheel_binsize=0.02):
"""
Generate a pandas dataframe of per-trial timing information about a given session.
Each row in the frame will correspond to a single trial, with timing values indicating timing
session-wide (i.e. time in seconds since session start). Can optionally return a resampled
wheel velocity trace of either the signed or absolute wheel velocity.
The resulting dataframe will have a new set of columns, trial_start and trial_end, which define
via t_before and t_after the span of time assigned to a given trial.
(useful for bb.modeling.glm)
Parameters
----------
eid : str
Session UUID string to pass to ONE
one : oneibl.one.OneAlyx, optional
one object to use for loading. Will generate internal one if not used, by default None
maxlen : float, optional
Maximum trial length for inclusion in df. Trials where feedback - response is longer
than this value will not be included in the dataframe, by default None
t_before : float, optional
Time before stimulus onset to include for a given trial, as defined by the trial_start
column of the dataframe. If zero, trial_start will be identical to stimOn, by default 0.
t_after : float, optional
Time after feedback to include in the trail, as defined by the trial_end
column of the dataframe. If zero, trial_end will be identical to feedback, by default 0.
ret_wheel : bool, optional
Whether to return the time-resampled wheel velocity trace, by default False
ret_abswheel : bool, optional
Whether to return the time-resampled absolute wheel velocity trace, by default False
wheel_binsize : float, optional
Time bins to resample wheel velocity to, by default 0.02
Returns
-------
pandas.DataFrame
Dataframe with trial-wise information. Indices are the actual trial order in the original
data, preserved even if some trials do not meet the maxlen criterion. As a result will not
have a monotonic index. Has special columns trial_start and trial_end which define start
and end times via t_before and t_after
"""
if not one:
one = ONE()
if ret_wheel and ret_abswheel:
raise ValueError('ret_wheel and ret_abswheel cannot both be true.')
# Define which datatypes we want to pull out
trialstypes = [
'trials.choice',
'trials.probabilityLeft',
'trials.feedbackType',
'trials.feedback_times',
'trials.contrastLeft',
'trials.contrastRight',
'trials.goCue_times',
'trials.stimOn_times',
]
# A quick function to remap probabilities in those sessions where it was not computed correctly
def remap_trialp(probs):
# Block probabilities in trial data aren't accurate and need to be remapped
validvals = np.array([0.2, 0.5, 0.8])
diffs = np.abs(np.array([x - validvals for x in probs]))
maps = diffs.argmin(axis=1)
return validvals[maps]
starttimes = one.load(eid, dataset_types=['trials.stimOn_times'])[0]
endtimes = one.load(eid, dataset_types=['trials.feedback_times'])[0]
tmp = one.load(eid, dataset_types=trialstypes)
if maxlen is not None:
with np.errstate(invalid='ignore'):
keeptrials = (endtimes - starttimes) <= maxlen
else:
keeptrials = range(len(starttimes))
trialdata = {
x.split('.')[1]: tmp[i][keeptrials]
for i, x in enumerate(trialstypes)
}
trialdata['probabilityLeft'] = remap_trialp(trialdata['probabilityLeft'])
trialsdf = pd.DataFrame(trialdata)
if maxlen is not None:
trialsdf.set_index(np.nonzero(keeptrials)[0], inplace=True)
trialsdf['trial_start'] = trialsdf['stimOn_times'] - t_before
trialsdf['trial_end'] = trialsdf['feedback_times'] + t_after
if not ret_wheel and not ret_abswheel:
return trialsdf
wheel = one.load_object(eid, 'wheel')
whlpos, whlt = wheel.position, wheel.timestamps
starttimes = trialsdf['trial_start']
endtimes = trialsdf['trial_end']
wh_endlast = 0
trials = []
for (start, end) in np.vstack((starttimes, endtimes)).T:
wh_startind = np.searchsorted(whlt[wh_endlast:], start) + wh_endlast
wh_endind = np.searchsorted(whlt[wh_endlast:], end,
side='right') + wh_endlast + 4
wh_endlast = wh_endind
tr_whlpos = whlpos[wh_startind - 1:wh_endind + 1]
tr_whlt = whlt[wh_startind - 1:wh_endind + 1] - start
tr_whlt[0] = 0. # Manual previous-value interpolation
whlseries = TimeSeries(tr_whlt, tr_whlpos, columns=['whlpos'])
whlsync = sync(wheel_binsize, timeseries=whlseries, interp='previous')
trialstartind = np.searchsorted(whlsync.times, 0)
trialendind = np.ceil((end - start) / wheel_binsize).astype(int)
trpos = whlsync.values[trialstartind:trialendind + trialstartind]
whlvel = trpos[1:] - trpos[:-1]
whlvel = np.insert(whlvel, 0, 0)
if np.abs((trialendind - len(whlvel))) > 0:
raise IndexError(
'Mismatch between expected length of wheel data and actual.')
if ret_wheel:
trials.append(whlvel)
elif ret_abswheel:
trials.append(np.abs(whlvel))
trialsdf['wheel_velocity'] = trials
return trialsdf
def load_spike_sorting(eid,
one=None,
probe=None,
dataset_types=None,
force=False):
"""
From an eid, hits the Alyx database and downloads a standard default set of dataset types
From a local session Path (pathlib.Path), loads a standard default set of dataset types
to perform analysis:
'clusters.channels',
'clusters.depths',
'clusters.metrics',
'spikes.clusters',
'spikes.times',
'probes.description'
:param eid: experiment UUID or pathlib.Path of the local session
:param one:
:param probe: name of probe to load in, if not given all probes for session will be loaded
:param dataset_types: additional spikes/clusters objects to add to the standard default list
:param force: by default function looks for data on local computer and loads this in. If you
want to connect to database and make sure files are still the same set force=True
:return: spikes, clusters (dict of bunch, 1 bunch per probe)
"""
if isinstance(eid, Path):
# Do everything locally without ONE
session_path = eid
if isinstance(probe, str):
labels = [probe]
else:
probes = alf.io.load_object(session_path.joinpath('alf'), 'probes')
labels = [pr['label'] for pr in probes['description']]
spikes = Bunch({})
clusters = Bunch({})
for label in labels:
_spikes, _clusters = _load_spike_sorting_local(session_path, label)
spikes[label] = _spikes
clusters[label] = _clusters
return spikes, clusters
else:
session_path = one.path_from_eid(eid)
if not session_path:
logger.warning('Session not found')
return (None, None), 'no session path'
one = one or ONE()
dtypes_default = [
'clusters.channels', 'clusters.depths', 'clusters.metrics',
'spikes.clusters', 'spikes.times', 'probes.description'
]
if dataset_types is None:
dtypes = dtypes_default
else:
# Append extra optional DS
dtypes = list(set(dataset_types + dtypes_default))
if isinstance(probe, str):
labels = [probe]
else:
insertions = one.alyx.rest('insertions', 'list', session=eid)
labels = [ins['name'] for ins in insertions]
spikes = Bunch({})
clusters = Bunch({})
for label in labels:
_spikes, _clusters = _load_spike_sorting_local(session_path, label)
spike_dtypes = [sp for sp in dtypes if 'spikes.' in sp]
spike_local = ['spikes.' + sp for sp in list(_spikes.keys())]
spike_exists = all([sp in spike_local for sp in spike_dtypes])
cluster_dtypes = [cl for cl in dtypes if 'clusters.' in cl]
cluster_local = ['clusters.' + cl for cl in list(_clusters.keys())]
cluster_exists = all(
[cl in cluster_local for cl in cluster_dtypes])
if not spike_exists or not cluster_exists or force:
logger.info(
f'Did not find local files for spikes and clusters for {session_path} '
f'and {label}. Downloading....')
one.load(eid, dataset_types=dtypes, download_only=True)
_spikes, _clusters = _load_spike_sorting_local(
session_path, label)
if not _spikes:
logger.warning(
f'Could not load spikes datasets for session {session_path} and {label}. '
f'Spikes for {probe} will return an empty dict')
else:
spikes[label] = _spikes
if not _clusters:
logger.warning(
f'Could not load clusters datasets for session {session_path} and {label}.'
f' Clusters for {label} will return an empty dict')
else:
clusters[label] = _clusters
else:
logger.info(
f'Local files for spikes and clusters for {session_path} '
f'and {label} found. To re-download set force=True')
spikes[label] = _spikes
clusters[label] = _clusters
return spikes, clusters
def setUpClass(cls) -> None:
cls.one = ONE(
base_url="https://test.alyx.internationalbrainlab.org",
username="******",
password="******"
)
def check_wheel_angle(eid):
Plot = True
one = ONE()
#eid = 'e1023140-50c1-462a-b80e-5e05626d7f0e' # at least 9 bad cases
#eid = one.search(subject='ZM_2104', date='2019-09-19', number=1)
Dataset_types = [
'wheel.position', 'wheel.timestamps', 'trials.feedback_times',
'trials.feedbackType'
]
D = one.load(eid,
dataset_types=Dataset_types,
clobber=False,
download_only=True)
session_path = Path(D[0]).parent
wheel = alf.io.load_object(session_path, 'wheel')
trials = alf.io.load_object(session_path, 'trials')
reward_success = trials['feedback_times'][trials['feedbackType'] == 1]
reward_failure = trials['feedback_times'][trials['feedbackType'] == -1]
if Plot:
plt.plot(wheel['times'], wheel['position'], linestyle='', marker='o')
#iblplt.vertical_lines(trials['stimOn_times'], ymin=-100, ymax=100,
# color='r', linewidth=0.5, label='stimOn_times')
#iblplt.vertical_lines(reward_failure, ymin=-100, ymax=100,
# color='b', linewidth=0.5, label='reward_failure')
iblplt.vertical_lines(reward_success,
ymin=-100,
ymax=100,
color='k',
linewidth=0.5,
label='reward_success')
plt.legend()
plt.xlabel('time [sec]')
plt.ylabel('wheel linear displacement [cm]')
plt.show()
# get fraction of reward deliveries with silent wheel time_delay before the reward
time_delay = 0.5
bad_cases1 = []
for rew in reward_success:
left = wheel['times'][find_nearest(wheel['times'], rew - time_delay)]
right = wheel['times'][find_nearest(wheel['times'], rew)]
if left == right:
if left < rew - time_delay:
bad_cases1.append(rew)
if len(bad_cases1) == 0:
print('Good news, no impossible case found.')
else:
print('Bad news, at least one impossible case found.')
return len(bad_cases1)
def make(self, key):
trial_key = key.copy()
eID = str((acquisition.Session & key).fetch1('session_uuid'))
trials_feedback_times, trials_feedback_types, trials_intervals, \
trials_response_choice, trials_response_times, \
trials_contrast_left, trials_contrast_right, trials_p_left = \
ONE().load(eID, dataset_types=['_ibl_trials.feedback_times',
'_ibl_trials.feedbackType',
'_ibl_trials.intervals',
'_ibl_trials.choice',
'_ibl_trials.response_times',
'_ibl_trials.contrastLeft',
'_ibl_trials.contrastRight',
'_ibl_trials.probabilityLeft'])
stim_on_times_status, rep_num_status, included_status = \
(CompleteTrialSession & key).fetch1(
'stim_on_times_status', 'rep_num_status', 'included_status')
lab_name = (subject.SubjectLab & key).fetch1('lab_name')
if stim_on_times_status != 'Missing':
if lab_name == 'wittenlab':
trials_visual_stim_times = np.squeeze(ONE().load(
eID,
dataset_types='_ibl_trials.stimOn_times',
clobber=True))
else:
trials_visual_stim_times = ONE().load(
eID, dataset_types='_ibl_trials.stimOn_times')
if len(trials_visual_stim_times) == 1:
trials_visual_stim_times = np.squeeze(trials_visual_stim_times)
if rep_num_status != 'Missing':
trials_rep_num = np.squeeze(ONE().load(
eID, dataset_types='_ibl_trials.repNum'))
if included_status != 'Missing':
trials_included = np.squeeze(ONE().load(
eID, dataset_types='_ibl_trials.included'))
assert len(
np.unique(
np.array([
len(trials_feedback_times),
len(trials_feedback_types),
len(trials_intervals),
# len(trials_rep_num),
len(trials_response_choice),
len(trials_response_times),
len(trials_contrast_left),
len(trials_contrast_right),
# len(trials_visual_stim_times),
# len(trials_included),
len(trials_p_left)
]))) == 1
'Loaded trial files do not have the same length'
key['n_trials'] = len(trials_response_choice)
key_session = dict()
key_session['model'] = 'actions.session'
key_session['uuid'] = (acquisition.Session
& key).fetch1('session_uuid')
n_correct_trials = grf(key_session, 'n_correct_trials')
if n_correct_trials != 'None':
key['n_correct_trials'] = n_correct_trials
else:
key['n_correct_trials'] = \
sum((np.squeeze(trials_response_choice) == 1) &
(np.squeeze(trials_contrast_left) > 0)) \
+ sum((np.squeeze(trials_response_choice) == -1) &
(np.squeeze(trials_contrast_right) > 0))
key['trials_start_time'] = trials_intervals[0, 0]
key['trials_end_time'] = trials_intervals[-1, 1]
self.insert1(key)
for idx_trial in range(len(trials_response_choice)):
if np.isnan(trials_contrast_left[idx_trial]):
trial_stim_contrast_left = 0
else:
trial_stim_contrast_left = trials_contrast_left[idx_trial]
if np.isnan(trials_contrast_right[idx_trial]):
trial_stim_contrast_right = 0
else:
trial_stim_contrast_right = trials_contrast_right[idx_trial]
if trials_response_choice[idx_trial] == -1:
trial_response_choice = "CCW"
elif trials_response_choice[idx_trial] == 0:
trial_response_choice = "No Go"
elif trials_response_choice[idx_trial] == 1:
trial_response_choice = "CW"
else:
raise ValueError('Invalid reponse choice.')
trial_key['trial_id'] = idx_trial + 1
trial_key['trial_start_time'] = trials_intervals[idx_trial, 0]
trial_key['trial_end_time'] = trials_intervals[idx_trial, 1]
trial_key['trial_response_time'] = float(
trials_response_times[idx_trial])
trial_key['trial_response_choice'] = trial_response_choice
if stim_on_times_status != 'Missing':
trial_key['trial_stim_on_time'] = trials_visual_stim_times[
idx_trial]
trial_key['trial_stim_contrast_left'] = float(
trial_stim_contrast_left)
trial_key['trial_stim_contrast_right'] = float(
trial_stim_contrast_right)
trial_key['trial_feedback_time'] = float(
trials_feedback_times[idx_trial])
trial_key['trial_feedback_type'] = int(
trials_feedback_types[idx_trial])
if rep_num_status != 'Missing':
trial_key['trial_rep_num'] = int(trials_rep_num[idx_trial])
trial_key['trial_stim_prob_left'] = float(trials_p_left[idx_trial])
if included_status != 'Missing':
trial_key['trial_included'] = bool(trials_included[idx_trial])
self.Trial().insert1(trial_key)
logger.info('Populated a TrialSet tuple, \
all Trial tuples and Excluded Trial tuples for \
subject {subject_uuid} in session started at \
{session_start_time}'.format(**key))
def register(root_data_folder, dry=False):
# registration part
one = ONE(base_url='https://alyx.internationalbrainlab.org')
rc = RegistrationClient(one=one)
rc.register_sync(root_data_folder, dry=dry)
def create_alyx_probe_insertions(
session_path: str,
force: bool = False,
one: object = None,
model: str = None,
labels: list = None,
):
if one is None:
one = ONE()
if is_uuid_string(session_path):
eid = session_path
else:
eid = one.eid_from_path(session_path)
if eid is None:
print(
"Session not found on Alyx: please create session before creating insertions"
)
if model is None:
probe_model = spikeglx.get_neuropixel_version_from_folder(session_path)
pmodel = "3B2" if probe_model == "3B" else probe_model
else:
pmodel = model
raw_ephys_data_path = Path(session_path) / "raw_ephys_data"
if labels is None:
probe_labels = [
x.name for x in Path(raw_ephys_data_path).glob("*")
if x.is_dir() and ("00" in x.name or "01" in x.name)
]
else:
probe_labels = labels
# create the qc fields in the json field
qc_dict = {}
qc_dict.update({"qc": "NOT_SET"})
qc_dict.update({"extended_qc": {}})
# create the dictionary
for plabel in probe_labels:
insdict = {
"session": eid,
"name": plabel,
"model": pmodel,
"json": qc_dict
}
# search for the corresponding insertion in Alyx
alyx_insertion = one.alyx.rest("insertions",
"list",
session=insdict["session"],
name=insdict["name"])
# if it doesn't exist, create it
if len(alyx_insertion) == 0:
alyx_insertion = one.alyx.rest("insertions",
"create",
data=insdict)
else:
iid = alyx_insertion[0]["id"]
if force:
alyx_insertion = one.alyx.rest("insertions",
"update",
id=iid,
data=insdict)
else:
alyx_insertion = alyx_insertion[0]
import unittest
import numpy as np
import requests
from pathlib import Path
from alf.io import remove_uuid_file
from oneibl.one import ONE
one = ONE(base_url='https://test.alyx.internationalbrainlab.org',
username='******',
password='******')
class TestSearch(unittest.TestCase):
def setUp(self):
# Init connection to the database
self.One = one
def test_search_simple(self):
one = self.One
# Test users
usr = ['olivier', 'nbonacchi']
sl, sd = one.search(users=usr, details=True)
self.assertTrue(isinstance(sl, list) and isinstance(sd, list))
self.assertTrue(
all([set(usr).issubset(set(u)) for u in [s['users'] for s in sd]]))
# when the user is a string instead of a list
sl1, sd1 = one.search(users=['olivier'], details=True)
sl2, sd2 = one.search(users='olivier', details=True)
self.assertTrue(sl1 == sl2 and sd1 == sd2)
ses = one.search(users='olivier', number=6)
(behavior.TrialSet.Trial & 'trial_go_cue_time is NULL') & \
'go_cue_times_status = "Complete"'
n_go_cue = len(trial_sets_go_cue)
trial_sets_go_cue_trigger = behavior.CompleteTrialSession & \
(behavior.TrialSet.Trial & 'trial_go_cue_trigger_time is NULL') & \
'go_cue_trigger_times_status = "Complete"'
n_go_cue_trigger = len(trial_sets_go_cue_trigger)
if len(trial_sets_go_cue):
for ikey, key in enumerate(trial_sets_go_cue.fetch('KEY')):
# download data
try:
eID = str((acquisition.Session & key).fetch1('session_uuid'))
go_cue_times = np.squeeze(ONE().load(
eID, dataset_types='_ibl_trials.goCue_times'))
# update go cue time value
trials = behavior.TrialSet.Trial & key & \
'trial_go_cue_time is NULL'
for trial in trials.fetch('KEY'):
dj.Table._update(
behavior.TrialSet.Trial & trial, 'trial_go_cue_time',
go_cue_times[trial['trial_id']-1])
if ikey % 100 == 0:
print('Go Cue entry number: {}/{}'.format(ikey, n_go_cue))
except:
print(key)
rot_mat[0, 1] = sca_rot_mat[0, 1] / sy
rot_mat[1, 1] = sca_rot_mat[1, 1] / sy
rot_mat[2, 1] = sca_rot_mat[2, 1] / sy
rot_mat[0, 2] = sca_rot_mat[0, 2] / sz
rot_mat[1, 2] = sca_rot_mat[1, 2] / sz
rot_mat[2, 2] = sca_rot_mat[2, 2] / sz
# extract individual rotations in x y z:
rx = math.atan2(rot_mat[2, 1], rot_mat[2, 2])
ry = math.atan2((-rot_mat[2, 0]),
math.sqrt(((rot_mat[2, 1]**2) + (rot_mat[2, 2]**2))))
rz = math.atan2(rot_mat[1, 0], rot_mat[0, 0])
# Upload to IBL alyx:
one = ONE(base_url='https://dev.alyx.internationalbrainlab.org')
TASK_PROTOCOL = 'SWC_Histology_Serial2P_v0.0.1'
json_note = {
'sample_reception_date': ibllib.time.date2isostr(sample_reception_date),
'elastix_affine_transform': affine_matrix,
'tilt': rx,
'yaw': ry,
'roll': rz,
'dv_scale': sy,
'ap_scale': sz,
'ml_scale': sx
}
# use dump() to properly encode np array:
def setUp(self):
# Init connection to the database
one = ONE(base_url='https://test.alyx.internationalbrainlab.org',
username='******',
password='******')
self.One = one
def load_channel_locations(eid, one=None, probe=None, aligned=False):
"""
From an eid, get brain locations from Alyx database
analysis.
:param eid: session eid or dictionary returned by one.alyx.rest('sessions', 'read', id=eid)
:param dataset_types: additional spikes/clusters objects to add to the standard list
:return: channels
"""
if isinstance(eid, dict):
ses = eid
eid = ses['url'][-36:]
one = one or ONE()
# When a specific probe has been requested
if isinstance(probe, str):
insertions = one.alyx.rest('insertions',
'list',
session=eid,
name=probe)[0]
labels = [probe]
if not insertions['json']:
tracing = [False]
resolved = [False]
counts = [0]
else:
tracing = [(insertions.get('json', {
'temp': 0
}).get('extended_qc', {
'temp': 0
}).get('tracing_exists', False))]
resolved = [(insertions.get('json', {
'temp': 0
}).get('extended_qc', {
'temp': 0
}).get('alignment_resolved', False))]
counts = [(insertions.get('json', {
'temp': 0
}).get('extended_qc', {
'temp': 0
}).get('alignment_count', 0))]
probe_id = [insertions['id']]
# No specific probe specified, load any that is available
# Need to catch for the case where we have two of the same probe insertions
else:
insertions = one.alyx.rest('insertions', 'list', session=eid)
labels = [ins['name'] for ins in insertions]
try:
tracing = [
ins.get('json', {
'temp': 0
}).get('extended_qc', {
'temp': 0
}).get('tracing_exists', False) for ins in insertions
]
resolved = [
ins.get('json', {
'temp': 0
}).get('extended_qc', {
'temp': 0
}).get('alignment_resolved', False) for ins in insertions
]
counts = [
ins.get('json', {
'temp': 0
}).get('extended_qc', {
'temp': 0
}).get('alignment_count', 0) for ins in insertions
]
except Exception:
tracing = [False for ins in insertions]
resolved = [False for ins in insertions]
counts = [0 for ins in insertions]
probe_id = [ins['id'] for ins in insertions]
channels = Bunch({})
r = BrainRegions()
for label, trace, resol, count, id in zip(labels, tracing, resolved,
counts, probe_id):
if trace:
if resol:
logger.info(
f'Channel locations for {label} have been resolved. '
f'Channel and cluster locations obtained from ephys aligned histology '
f'track.')
# download the data
chans = one.load_object(eid,
'channels',
collection=f'alf/{label}')
# If we have successfully downloaded the data
if 'brainLocationIds_ccf_2017' in chans.keys():
channels[label] = Bunch({
'atlas_id':
chans['brainLocationIds_ccf_2017'],
'acronym':
r.get(chans['brainLocationIds_ccf_2017'])['acronym'],
'x':
chans['mlapdv'][:, 0] / 1e6,
'y':
chans['mlapdv'][:, 1] / 1e6,
'z':
chans['mlapdv'][:, 2] / 1e6,
'axial_um':
chans['localCoordinates'][:, 1],
'lateral_um':
chans['localCoordinates'][:, 0]
})
# Otherwise we just get the channels from alyx. Shouldn't happen often, only if
# data is still inbetween ftp and flatiron after being resolved
else:
traj_id = one.alyx.rest(
'trajectories',
'list',
session=eid,
probe=label,
provenance='Ephys aligned histology track')[0]['id']
chans = one.alyx.rest('channels',
'list',
trajectory_estimate=traj_id)
channels[label] = Bunch({
'atlas_id':
np.array([ch['brain_region'] for ch in chans]),
'x':
np.array([ch['x'] for ch in chans]) / 1e6,
'y':
np.array([ch['y'] for ch in chans]) / 1e6,
'z':
np.array([ch['z'] for ch in chans]) / 1e6,
'axial_um':
np.array([ch['axial'] for ch in chans]),
'lateral_um':
np.array([ch['lateral'] for ch in chans])
})
channels[label]['acronym'] = r.get(
channels[label]['atlas_id'])['acronym']
elif count > 0 and aligned:
logger.info(
f'Channel locations for {label} have not been '
f'resolved. However, alignment flag set to True so channel and cluster'
f' locations will be obtained from latest available ephys aligned '
f'histology track.')
# get the latest user aligned channels
traj_id = one.alyx.rest(
'trajectories',
'list',
session=eid,
probe=label,
provenance='Ephys aligned histology track')[0]['id']
chans = one.alyx.rest('channels',
'list',
trajectory_estimate=traj_id)
channels[label] = Bunch({
'atlas_id':
np.array([ch['brain_region'] for ch in chans]),
'x':
np.array([ch['x'] for ch in chans]) / 1e6,
'y':
np.array([ch['y'] for ch in chans]) / 1e6,
'z':
np.array([ch['z'] for ch in chans]) / 1e6,
'axial_um':
np.array([ch['axial'] for ch in chans]),
'lateral_um':
np.array([ch['lateral'] for ch in chans])
})
channels[label]['acronym'] = r.get(
channels[label]['atlas_id'])['acronym']
else:
logger.info(
f'Channel locations for {label} have not been resolved. '
f'Channel and cluster locations obtained from histology track.'
)
# get the channels from histology tracing
traj_id = one.alyx.rest('trajectories',
'list',
session=eid,
probe=label,
provenance='Histology track')[0]['id']
chans = one.alyx.rest('channels',
'list',
trajectory_estimate=traj_id)
channels[label] = Bunch({
'atlas_id':
np.array([ch['brain_region'] for ch in chans]),
'x':
np.array([ch['x'] for ch in chans]) / 1e6,
'y':
np.array([ch['y'] for ch in chans]) / 1e6,
'z':
np.array([ch['z'] for ch in chans]) / 1e6,
'axial_um':
np.array([ch['axial'] for ch in chans]),
'lateral_um':
np.array([ch['lateral'] for ch in chans])
})
channels[label]['acronym'] = r.get(
channels[label]['atlas_id'])['acronym']
else:
logger.warning(f'Histology tracing for {label} does not exist. '
f'No channels for {label}')
return channels
def Viewer(eid, video_type, trial_range, save_video=True, eye_zoom=False):
'''
eid: session id, e.g. '3663d82b-f197-4e8b-b299-7b803a155b84'
video_type: one of 'left', 'right', 'body'
trial_range: first and last trial number of range to be shown, e.g. [5,7]
save_video: video is displayed and saved in local folder
Example usage to view and save labeled video with wheel angle:
Viewer('3663d82b-f197-4e8b-b299-7b803a155b84', 'left', [5,7])
3D example: 'cb2ad999-a6cb-42ff-bf71-1774c57e5308', [5,7]
'''
save_vids_here = '/home/mic/'
if save_vids_here[-1] != '/':
return 'Last character of save_vids_here must be slash'
one = ONE()
dataset_types = [
'camera.times', 'wheel.position', 'wheel.timestamps',
'trials.intervals', 'camera.dlc'
]
a = one.list(eid, 'dataset-types')
assert all([i in a for i in dataset_types
]), 'For this eid, not all data available'
D = one.load(eid, dataset_types=dataset_types, dclass_output=True)
alf_path = Path(D.local_path[0]).parent.parent / 'alf'
# Download a single video
video_data = alf_path.parent / 'raw_video_data'
download_raw_video(eid, cameras=[video_type])
video_path = list(video_data.rglob('_iblrig_%sCamera.raw.*' %
video_type))[0]
print(video_path)
# that gives cam time stamps and DLC output (change to alf_path eventually)
cam = alf.io.load_object(alf_path,
'%sCamera' % video_type,
namespace='ibl')
# just to read in times for newer data (which has DLC results in pqt format
# cam = alf.io.load_object(alf_path, '_ibl_%sCamera' % video_type)
# set where to read and save video and get video info
cap = cv2.VideoCapture(video_path.as_uri())
length = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
fps = cap.get(cv2.CAP_PROP_FPS)
size = (int(cap.get(3)), int(cap.get(4)))
assert length < len(cam['times']), '#frames > #stamps'
print(eid, ', ', video_type, ', fsp:', fps, ', #frames:', length,
', #stamps:', len(cam['times']), ', #frames - #stamps = ',
length - len(cam['times']))
# pick trial range for which to display stuff
trials = alf.io.load_object(alf_path, 'trials', namespace='ibl')
num_trials = len(trials['intervals'])
if trial_range[-1] > num_trials - 1:
print('There are only %s trials' % num_trials)
frame_start = find_nearest(cam['times'],
[trials['intervals'][trial_range[0]][0]])
frame_stop = find_nearest(cam['times'],
[trials['intervals'][trial_range[-1]][1]])
'''
wheel related stuff
'''
wheel = alf.io.load_object(alf_path, 'wheel', namespace='ibl')
import brainbox.behavior.wheel as wh
try:
pos, t = wh.interpolate_position(wheel['timestamps'],
wheel['position'],
freq=1000)
except BaseException:
pos, t = wh.interpolate_position(wheel['times'],
wheel['position'],
freq=1000)
w_start = find_nearest(t, trials['intervals'][trial_range[0]][0])
w_stop = find_nearest(t, trials['intervals'][trial_range[-1]][1])
# confine to interval
pos_int = pos[w_start:w_stop]
t_int = t[w_start:w_stop]
# alignment of cam stamps and interpolated wheel stamps
wheel_pos = []
kk = 0
for wt in cam['times'][frame_start:frame_stop]:
wheel_pos.append(pos_int[find_nearest(t_int, wt)])
kk += 1
if kk % 3000 == 0:
print('iteration', kk)
'''
DLC related stuff
'''
Times = cam['times'][frame_start:frame_stop]
del cam['times']
# some exception for inconsisitent data formats
try:
dlc_name = '_ibl_%sCamera.dlc.pqt' % video_type
dlc_path = alf_path / dlc_name
cam = pd.read_parquet(dlc_path, engine="fastparquet")
print('it is pqt')
except BaseException:
raw_vid_path = alf_path.parent / 'raw_video_data'
cam = alf.io.load_object(raw_vid_path,
'%sCamera' % video_type,
namespace='ibl')
points = np.unique(['_'.join(x.split('_')[:-1]) for x in cam.keys()])
if len(points) == 1:
cam = cam['dlc']
points = np.unique(['_'.join(x.split('_')[:-1]) for x in cam.keys()])
if video_type != 'body':
d = list(points)
d.remove('tube_top')
d.remove('tube_bottom')
points = np.array(d)
# Set values to nan if likelyhood is too low # for pqt: .to_numpy()
XYs = {}
for point in points:
x = np.ma.masked_where(cam[point + '_likelihood'] < 0.9,
cam[point + '_x'])
x = x.filled(np.nan)
y = np.ma.masked_where(cam[point + '_likelihood'] < 0.9,
cam[point + '_y'])
y = y.filled(np.nan)
XYs[point] = np.array(
[x[frame_start:frame_stop], y[frame_start:frame_stop]])
# Just for 3D testing
# return XYs
# Zoom at eye
if eye_zoom:
pivot = np.nanmean(XYs['pupil_top_r'], axis=1)
x0 = int(pivot[0]) - 33
x1 = int(pivot[0]) + 33
y0 = int(pivot[1]) - 28
y1 = int(pivot[1]) + 38
size = (66, 66)
dot_s = 1 # [px] for painting DLC dots
else:
x0 = 0
x1 = size[0]
y0 = 0
y1 = size[1]
if video_type == 'left':
dot_s = 10 # [px] for painting DLC dots
else:
dot_s = 5
if save_video:
loc = save_vids_here + '%s_trials_%s_%s_%s.mp4' % (
eid, trial_range[0], trial_range[-1], video_type)
out = cv2.VideoWriter(loc, cv2.VideoWriter_fourcc(*'mp4v'), fps,
size) # put , 0 if grey scale
# writing stuff on frames
font = cv2.FONT_HERSHEY_SIMPLEX
if video_type == 'left':
bottomLeftCornerOfText = (20, 1000)
fontScale = 4
else:
bottomLeftCornerOfText = (10, 500)
fontScale = 2
lineType = 2
# assign a color to each DLC point (now: all points red)
cmap = matplotlib.cm.get_cmap('Spectral')
CR = np.arange(len(points)) / len(points)
block = np.ones((2 * dot_s, 2 * dot_s, 3))
# set start frame
cap.set(1, frame_start)
k = 0
while (cap.isOpened()):
ret, frame = cap.read()
gray = frame
# print wheel angle
fontColor = (255, 255, 255)
Angle = round(wheel_pos[k], 2)
Time = round(Times[k], 3)
cv2.putText(gray, 'Wheel angle: ' + str(Angle), bottomLeftCornerOfText,
font, fontScale / 2, fontColor, lineType)
a, b = bottomLeftCornerOfText
bottomLeftCornerOfText0 = (int(a * 10 + b / 2), b)
cv2.putText(gray, ' time: ' + str(Time), bottomLeftCornerOfText0,
font, fontScale / 2, fontColor, lineType)
# print DLC dots
ll = 0
for point in points:
# Put point color legend
fontColor = (np.array([cmap(CR[ll])]) * 255)[0][:3]
a, b = bottomLeftCornerOfText
if video_type == 'right':
bottomLeftCornerOfText2 = (a, a * 2 * (1 + ll))
else:
bottomLeftCornerOfText2 = (b, a * 2 * (1 + ll))
fontScale2 = fontScale / 4
cv2.putText(gray, point, bottomLeftCornerOfText2, font, fontScale2,
fontColor, lineType)
X0 = XYs[point][0][k]
Y0 = XYs[point][1][k]
# transform for opencv?
X = Y0
Y = X0
if not np.isnan(X) and not np.isnan(Y):
col = (np.array([cmap(CR[ll])]) * 255)[0][:3]
# col = np.array([0, 0, 255]) # all points red
X = X.astype(int)
Y = Y.astype(int)
gray[X - dot_s:X + dot_s, Y - dot_s:Y + dot_s] = block * col
ll += 1
gray = gray[y0:y1, x0:x1]
if save_video:
out.write(gray)
cv2.imshow('frame', gray)
cv2.waitKey(1)
k += 1
if k == (frame_stop - frame_start) - 1:
break
if save_video:
out.release()
cap.release()
cv2.destroyAllWindows()
# Mock dataset
import unittest
import numpy as np
from ibllib.qc import BpodQCExtractor
from oneibl.one import ONE
one = ONE(
base_url="https://test.alyx.internationalbrainlab.org",
username="******",
password="******",
)
dstypes = [
"_iblrig_taskData.raw",
"_iblrig_taskSettings.raw",
"_iblrig_encoderPositions.raw",
"_iblrig_encoderEvents.raw",
"_iblrig_stimPositionScreen.raw",
"_iblrig_syncSquareUpdate.raw",
"_iblrig_encoderTrialInfo.raw",
"_iblrig_ambientSensorData.raw",
]
class TestBpodQCExtractors(unittest.TestCase):
"""TODO: should be an integration test in iblscripts"""
def setUp(self):
self.one = one
self.eid = "b1c968ad-4874-468d-b2e4-5ffa9b9964e9"
def __init__(self,
saveloc=None,
nwb_metadata_file=None,
metadata_obj: Alyx2NWBMetadata = None,
one_object: ONE = None,
save_raw=False,
save_camera_raw=False,
complevel=4,
shuffle=False,
buffer_size=1):
"""
Retrieve all Alyx session, subject metadata, raw data for eid using the one apis load method
Map that to nwb supported datatypes and create an nwb file.
Parameters
----------
saveloc: str, Path
save location of nwbfile
nwb_metadata_file: [dict, str]
output of Alyx2NWBMetadata as a dict/json location str
metadata_obj: Alyx2NWBMetadata
one_object: ONE()
save_raw: bool
will load and save large raw files: ecephys.raw.ap/lf.cbin to nwb
save_camera_raw: bool
will load and save mice camera movie .mp4: _iblrig_Camera.raw
complevel: int
level of compression to apply to raw datasets
(0-9)>(low,high). https://docs.h5py.org/en/latest/high/dataset.html
shuffle: bool
Enable shuffle I/O filter. http://docs.h5py.org/en/latest/high/dataset.html#dataset-shuffle
"""
self.buffer_size = buffer_size
self.complevel = complevel
self.shuffle = shuffle
if nwb_metadata_file is not None:
if isinstance(nwb_metadata_file, dict):
self.nwb_metadata = nwb_metadata_file
elif isinstance(nwb_metadata_file, str):
with open(nwb_metadata_file, 'r') as f:
self.nwb_metadata = json.load(f)
elif metadata_obj is not None:
self.nwb_metadata = metadata_obj.complete_metadata
else:
raise Exception(
'required one of argument: nwb_metadata_file OR metadata_obj')
if one_object is not None:
self.one_object = one_object
elif metadata_obj is not None:
self.one_object = metadata_obj.one_obj
else:
Warning('creating a ONE object and continuing')
self.one_object = ONE()
if saveloc is None:
Warning('saving nwb file in current working directory')
self.saveloc = str(Path.cwd())
else:
self.saveloc = str(saveloc)
self.eid = self.nwb_metadata["eid"]
if not isinstance(self.nwb_metadata['NWBFile']['session_start_time'],
datetime):
self.nwb_metadata['NWBFile']['session_start_time'] = \
datetime.strptime(self.nwb_metadata['NWBFile']['session_start_time'], '%Y-%m-%dT%X').replace(
tzinfo=pytz.utc)
self.nwb_metadata['IBLSubject']['date_of_birth'] = \
datetime.strptime(self.nwb_metadata['IBLSubject']['date_of_birth'], '%Y-%m-%dT%X').replace(
tzinfo=pytz.utc)
# create nwbfile:
self.initialize_nwbfile()
self.no_probes = len(self.nwb_metadata['Probes'])
if self.no_probes == 0:
warnings.warn(
'could not find probe information, will create trials, behavior, acquisition'
)
self.electrode_table_exist = False
self._one_data = _OneData(self.one_object,
self.eid,
self.no_probes,
self.nwb_metadata,
save_raw=save_raw,
save_camera_raw=save_camera_raw)
def __init__(self, one=None):
# one object
if one is None:
self.one = ONE()
else:
self.one = one
"""
Get video frames and metadata
===================================
Video frames and meta data can be loaded using the ibllib.io.video module, which contains
functions for loading individual or groups of frames efficiently. The video may be streamed
remotely or loaded from a local file. In these examples a remote URL is used.
"""
import numpy as np
import ibllib.io.video as vidio
from oneibl.one import ONE
one = ONE(silent=True)
eid = 'edd22318-216c-44ff-bc24-49ce8be78374' # 2020-08-19_1_CSH_ZAD_019
# Example 1: get the remote video URL from eid
urls = vidio.url_from_eid(eid, one=one)
# Without the `label` kwarg, returns a dictionary of camera URLs
url = urls['left'] # URL for the left camera
# Example 2: get the video label from a video file path or URL
label = vidio.label_from_path(url)
print(f'Using URL for the {label} camera')
# Example 3: loading a single frame
frame_n = 1000 # Frame number to fetch. Indexing starts from 0.
frame = vidio.get_video_frame(url, frame_n)
assert frame is not None, 'failed to load frame'
# Example 4: loading multiple frames
"""
