(example taken for nidq.cbin file, but also applicable for lf.cbin and ap.cbin files)
"""
# Author: Olivier, Gaelle, Mayo
from pprint import pprint
from ibllib.io import spikeglx
from one.api import ONE
one = ONE(base_url='https://openalyx.internationalbrainlab.org', silent=True)
# Download a dataset of interest
eid = one.search(subject='KS023', date_range='2019-12-10')[0]
# Optionally list the raw ephys data for this session
pprint([x for x in one.list_datasets(eid) if 'ephysData' in x])
files = one.load_object(eid,
'ephysData_g0_t0',
attribute='nidq',
collection='raw_ephys_data',
download_only=True)
# Get file path of interest
efile = next(x for x in files if str(x).endswith('.cbin'))
# Read the files and get the data
# Enough to do analysis
sr = spikeglx.Reader(efile)
# Decompress the data
"""
Get single probe label and directory, using the probes description dataset.
"""
# Author: Gaelle Chapuis, Miles Wells
from one.api import ONE
one = ONE()
eid = 'da188f2c-553c-4e04-879b-c9ea2d1b9a93'
# --- Get single probe directory filename either by
# 1. getting probe description in alf
# 2. using alyx rest end point
# Option 1.
prob_des = one.load_dataset(eid, 'probes.description.json')
labels = [x['label'] for x in prob_des]
# You can then use this label into dict, e.g. channels[label[0]]
# -- Load single probe data with probe-level collection
# List datsets for first probe
collection = f'alf/{labels[0]}'
datasets = one.list_datasets(eid, collection=collection)
def dlc_qc_plot(session_path, one=None):
"""
Creates DLC QC plot.
Data is searched first locally, then on Alyx. Panels that lack required data are skipped.
Required data to create all panels
'raw_video_data/_iblrig_bodyCamera.raw.mp4',
'raw_video_data/_iblrig_leftCamera.raw.mp4',
'raw_video_data/_iblrig_rightCamera.raw.mp4',
'alf/_ibl_bodyCamera.dlc.pqt',
'alf/_ibl_leftCamera.dlc.pqt',
'alf/_ibl_rightCamera.dlc.pqt',
'alf/_ibl_bodyCamera.times.npy',
'alf/_ibl_leftCamera.times.npy',
'alf/_ibl_rightCamera.times.npy',
'alf/_ibl_leftCamera.features.pqt',
'alf/_ibl_rightCamera.features.pqt',
'alf/rightROIMotionEnergy.position.npy',
'alf/leftROIMotionEnergy.position.npy',
'alf/bodyROIMotionEnergy.position.npy',
'alf/_ibl_trials.choice.npy',
'alf/_ibl_trials.feedbackType.npy',
'alf/_ibl_trials.feedback_times.npy',
'alf/_ibl_trials.stimOn_times.npy',
'alf/_ibl_wheel.position.npy',
'alf/_ibl_wheel.timestamps.npy',
'alf/licks.times.npy',
:params session_path: Path to session data on disk
:params one: ONE instance, if None is given, default ONE is instantiated
:returns: Matplotlib figure
"""
one = one or ONE()
# hack for running on cortexlab local server
if one.alyx.base_url == 'https://alyx.cortexlab.net':
one = ONE(base_url='https://alyx.internationalbrainlab.org')
data = {}
cams = ['left', 'right', 'body']
session_path = Path(session_path)
# Load data for each camera
for cam in cams:
# Load a single frame for each video
# Check if video data is available locally,if yes, load a single frame
video_path = session_path.joinpath('raw_video_data',
f'_iblrig_{cam}Camera.raw.mp4')
if video_path.exists():
data[f'{cam}_frame'] = get_video_frame(video_path,
frame_number=5 * 60 *
SAMPLING[cam])[:, :, 0]
# If not, try to stream a frame (try three times)
else:
try:
video_url = url_from_eid(one.path2eid(session_path),
one=one)[cam]
for tries in range(3):
try:
data[f'{cam}_frame'] = get_video_frame(
video_url,
frame_number=5 * 60 * SAMPLING[cam])[:, :, 0]
break
except BaseException:
if tries < 2:
tries += 1
logger.info(
f"Streaming {cam} video failed, retrying x{tries}"
)
time.sleep(30)
else:
logger.warning(
f"Could not load video frame for {cam} cam. Skipping trace on frame."
)
data[f'{cam}_frame'] = None
except KeyError:
logger.warning(
f"Could not load video frame for {cam} cam. Skipping trace on frame."
)
data[f'{cam}_frame'] = None
# Other camera associated data
for feat in ['dlc', 'times', 'features', 'ROIMotionEnergy']:
# Check locally first, then try to load from alyx, if nothing works, set to None
if feat == 'features' and cam == 'body': # this doesn't exist for body cam
continue
local_file = list(
session_path.joinpath('alf').glob(f'*{cam}Camera.{feat}*'))
if len(local_file) > 0:
data[f'{cam}_{feat}'] = alfio.load_file_content(local_file[0])
else:
alyx_ds = [
ds for ds in one.list_datasets(one.path2eid(session_path))
if f'{cam}Camera.{feat}' in ds
]
if len(alyx_ds) > 0:
data[f'{cam}_{feat}'] = one.load_dataset(
one.path2eid(session_path), alyx_ds[0])
else:
logger.warning(
f"Could not load _ibl_{cam}Camera.{feat} some plots have to be skipped."
)
data[f'{cam}_{feat}'] = None
# Sometimes there is a file but the object is empty, set to None
if data[f'{cam}_{feat}'] is not None and len(
data[f'{cam}_{feat}']) == 0:
logger.warning(
f"Object loaded from _ibl_{cam}Camera.{feat} is empty, some plots have to be skipped."
)
data[f'{cam}_{feat}'] = None
# If we have no frame and/or no DLC and/or no times for all cams, raise an error, something is really wrong
assert any([data[f'{cam}_frame'] is not None
for cam in cams]), "No camera data could be loaded, aborting."
assert any([data[f'{cam}_dlc'] is not None
for cam in cams]), "No DLC data could be loaded, aborting."
assert any([data[f'{cam}_times'] is not None for cam in cams
]), "No camera times data could be loaded, aborting."
# Load session level data
for alf_object in ['trials', 'wheel', 'licks']:
try:
data[f'{alf_object}'] = alfio.load_object(
session_path.joinpath('alf'), alf_object) # load locally
continue
except ALFObjectNotFound:
pass
try:
data[f'{alf_object}'] = one.load_object(
one.path2eid(session_path), alf_object) # then try from alyx
except ALFObjectNotFound:
logger.warning(
f"Could not load {alf_object} object, some plots have to be skipped."
)
data[f'{alf_object}'] = None
# Simplify and clean up trials data
if data['trials']:
data['trials'] = pd.DataFrame({
k: data['trials'][k]
for k in
['stimOn_times', 'feedback_times', 'choice', 'feedbackType']
})
# Discard nan events and too long trials
data['trials'] = data['trials'].dropna()
data['trials'] = data['trials'].drop(
data['trials'][(data['trials']['feedback_times'] -
data['trials']['stimOn_times']) > 10].index)
# Make a list of panels, if inputs are missing, instead input a text to display
panels = []
# Panel A, B, C: Trace on frame
for cam in cams:
if data[f'{cam}_frame'] is not None and data[f'{cam}_dlc'] is not None:
panels.append((plot_trace_on_frame, {
'frame': data[f'{cam}_frame'],
'dlc_df': data[f'{cam}_dlc'],
'cam': cam
}))
else:
panels.append(
(None, f'Data missing\n{cam.capitalize()} cam trace on frame'))
# If trials data is not there, we cannot plot any of the trial average plots, skip all remaining panels
if data['trials'] is None:
panels.extend([(None, 'No trial data,\ncannot compute trial avgs')
for i in range(7)])
else:
# Panel D: Motion energy
camera_dict = {
'left': {
'motion_energy': data['left_ROIMotionEnergy'],
'times': data['left_times']
},
'right': {
'motion_energy': data['right_ROIMotionEnergy'],
'times': data['right_times']
},
'body': {
'motion_energy': data['body_ROIMotionEnergy'],
'times': data['body_times']
}
}
for cam in [
'left', 'right', 'body'
]: # Remove cameras where we don't have motion energy AND camera times
if camera_dict[cam]['motion_energy'] is None or camera_dict[cam][
'times'] is None:
_ = camera_dict.pop(cam)
if len(camera_dict) > 0:
panels.append((plot_motion_energy_hist, {
'camera_dict': camera_dict,
'trials_df': data['trials']
}))
else:
panels.append((None, 'Data missing\nMotion energy'))
# Panel E: Wheel position
if data['wheel']:
panels.append((plot_wheel_position, {
'wheel_position': data['wheel'].position,
'wheel_time': data['wheel'].timestamps,
'trials_df': data['trials']
}))
else:
panels.append((None, 'Data missing\nWheel position'))
# Panel F, G: Paw speed and nose speed
# Try if all data is there for left cam first, otherwise right
for cam in ['left', 'right']:
fail = False
if (data[f'{cam}_dlc'] is not None
and data[f'{cam}_times'] is not None
and len(data[f'{cam}_times']) >= len(data[f'{cam}_dlc'])):
break
fail = True
if not fail:
paw = 'r' if cam == 'left' else 'l'
panels.append((plot_speed_hist, {
'dlc_df': data[f'{cam}_dlc'],
'cam_times': data[f'{cam}_times'],
'trials_df': data['trials'],
'feature': f'paw_{paw}',
'cam': cam
}))
panels.append((plot_speed_hist, {
'dlc_df': data[f'{cam}_dlc'],
'cam_times': data[f'{cam}_times'],
'trials_df': data['trials'],
'feature': 'nose_tip',
'legend': False,
'cam': cam
}))
else:
panels.extend([(None, 'Data missing or corrupt\nSpeed histograms')
for i in range(2)])
# Panel H and I: Lick plots
if data['licks'] and data['licks'].times.shape[0] > 0:
panels.append((plot_lick_hist, {
'lick_times': data['licks'].times,
'trials_df': data['trials']
}))
panels.append((plot_lick_raster, {
'lick_times': data['licks'].times,
'trials_df': data['trials']
}))
else:
panels.extend([(None, 'Data missing\nLicks plots')
for i in range(2)])
# Panel J: pupil plot
# Try if all data is there for left cam first, otherwise right
for cam in ['left', 'right']:
fail = False
if (data[f'{cam}_times'] is not None
and data[f'{cam}_features'] is not None and
len(data[f'{cam}_times']) >= len(data[f'{cam}_features'])
and not np.all(
np.isnan(
data[f'{cam}_features'].pupilDiameter_smooth))):
break
fail = True
if not fail:
panels.append((plot_pupil_diameter_hist, {
'pupil_diameter': data[f'{cam}_features'].pupilDiameter_smooth,
'cam_times': data[f'{cam}_times'],
'trials_df': data['trials'],
'cam': cam
}))
else:
panels.append((None, 'Data missing or corrupt\nPupil diameter'))
# Plotting
plt.rcParams.update({'font.size': 10})
fig = plt.figure(figsize=(17, 10))
for i, panel in enumerate(panels):
ax = plt.subplot(2, 5, i + 1)
ax.text(-0.1,
1.15,
ascii_uppercase[i],
transform=ax.transAxes,
fontsize=16,
fontweight='bold')
# Check if there was in issue with inputs, if yes, print the respective text
if panel[0] is None:
ax.text(.5,
.5,
panel[1],
color='r',
fontweight='bold',
fontsize=12,
horizontalalignment='center',
verticalalignment='center',
transform=ax.transAxes)
plt.axis('off')
else:
try:
panel[0](**panel[1])
except BaseException:
logger.error(f'Error in {panel[0].__name__}\n' +
traceback.format_exc())
ax.text(.5,
.5,
f'Error while plotting\n{panel[0].__name__}',
color='r',
fontweight='bold',
fontsize=12,
horizontalalignment='center',
verticalalignment='center',
transform=ax.transAxes)
plt.axis('off')
plt.tight_layout(rect=[0, 0.03, 1, 0.95])
return fig
