def load_trials(sess_path, one):
"""
Load trials data for session. First attempts to load from local session path, if this fails will attempt to download via ONE,
if this also fails, will then attempt to re-extraxt locally
:param sess_path: session path
:param one: ONE instance
:return:
"""
# try and load trials locally
try:
trials = alfio.load_object(sess_path.joinpath('alf'), 'trials')
if 'probabilityLeft' not in trials.keys():
raise ALFObjectNotFound
except ALFObjectNotFound:
try:
# attempt to download trials using ONE
trials = one.load_object(one.path2eid(sess_path), 'trials')
if 'probabilityLeft' not in trials.keys():
raise ALFObjectNotFound
except Exception:
try:
task = get_trials_task(sess_path, one=one)
task.run()
trials = alfio.load_object(sess_path.joinpath('alf'), 'trials')
if 'probabilityLeft' not in trials.keys():
raise ALFObjectNotFound
except Exception: # TODO how can i make this more specific
trials = None
return trials
def test_size_outputs(self):
# check the output dimensions
from ibllib.io.extractors.bpod_trials import extract_all
extract_all(self.training_ge5['path'])
trials = alfio.load_object(self.training_ge5['path'] / 'alf', object='trials')
self.assertTrue(alfio.check_dimensions(trials) == 0)
extract_all(self.training_lt5['path'])
trials = alfio.load_object(self.training_lt5['path'] / 'alf', object='trials')
self.assertTrue(alfio.check_dimensions(trials) == 0)
extract_all(self.biased_ge5['path'])
trials = alfio.load_object(self.biased_ge5['path'] / 'alf', object='trials')
self.assertTrue(alfio.check_dimensions(trials) == 0)
def load_data(self):
self.spikes = alfio.load_object(self.probe_path, 'spikes')
self.trials = alfio.load_object(self.alf_path, 'trials')
self.clusters = alfio.load_object(self.probe_path, 'clusters')
try:
chan_locations = np.load(
self.probe_path.joinpath('channels.locations.npy'),
allow_pickle=True)
self.clusters.locations = chan_locations[self.clusters.channels]
except:
print('WARNING: locations not available')
self.prepare_data(self.spikes, self.clusters, self.trials)
def extract_sync(session_path, overwrite=False, ephys_files=None):
"""
Reads ephys binary file (s) and extract sync within the binary file folder
Assumes ephys data is within a `raw_ephys_data` folder
:param session_path: '/path/to/subject/yyyy-mm-dd/001'
:param overwrite: Bool on re-extraction, forces overwrite instead of loading existing files
:return: list of sync dictionaries
"""
session_path = Path(session_path)
if not ephys_files:
ephys_files = spikeglx.glob_ephys_files(session_path)
syncs = []
outputs = []
for efi in ephys_files:
bin_file = efi.get('ap', efi.get('nidq', None))
if not bin_file:
continue
alfname = dict(object='sync', namespace='spikeglx')
if efi.label:
alfname['extra'] = efi.label
file_exists = alfio.exists(bin_file.parent, **alfname)
if not overwrite and file_exists:
_logger.warning(f'Skipping raw sync: SGLX sync found for probe {efi.label} !')
sync = alfio.load_object(bin_file.parent, **alfname)
out_files, _ = alfio._ls(bin_file.parent, **alfname)
else:
sr = spikeglx.Reader(bin_file)
sync, out_files = _sync_to_alf(sr, bin_file.parent, save=True, parts=efi.label)
outputs.extend(out_files)
syncs.extend([sync])
return syncs, outputs
def _get_all_probes_sync(session_path, bin_exists=True):
# round-up of all bin ephys files in the session, infer revision and get sync map
ephys_files = spikeglx.glob_ephys_files(session_path, bin_exists=bin_exists)
version = spikeglx.get_neuropixel_version_from_files(ephys_files)
# attach the sync information to each binary file found
for ef in ephys_files:
ef['sync'] = alfio.load_object(ef.path, 'sync', namespace='spikeglx', short_keys=True)
ef['sync_map'] = get_ibl_sync_map(ef, version)
return ephys_files
def get_micro_manipulator_data(subject, one=None, force_extract=False):
"""
Looks for all ephys sessions for a given subject and get the probe micro-manipulator
trajectories.
If probes ALF object not on flat-iron, attempts to perform the extraction from meta-data
and task settings file.
"""
if not one:
one = ONE(cache_rest=None)
eids, sessions = one.search(subject=subject,
task_protocol='ephys',
details=True)
probes = alfio.AlfBunch({})
for ses in sessions:
sess_path = Path(ses['local_path'])
probe = None
if not force_extract:
probe = one.load_object(ses['url'], 'probes')
if not probe:
_logger.warning(f"Re-extraction probe info for {sess_path}")
dtypes = ['_iblrig_taskSettings.raw', 'ephysData.raw.meta']
raw_files = one.load(ses['url'],
dataset_types=dtypes,
download_only=True)
if all([rf is None for rf in raw_files]):
_logger.warning(
f"no raw settings files nor ephys data found for"
f" {ses['local_path']}. Skip this session.")
continue
extract_probes(sess_path, bin_exists=False)
probe = alfio.load_object(sess_path.joinpath('alf'), 'probes')
one.load(ses['url'],
dataset_types='channels.localCoordinates',
download_only=True)
# get for each insertion the sites local mapping: if not found assumes checkerboard pattern
probe['sites_coordinates'] = []
for prb in probe.description:
chfile = Path(ses['local_path']).joinpath(
'alf', prb['label'], 'channels.localCoordinates.npy')
if chfile.exists():
probe['sites_coordinates'].append(np.load(chfile))
else:
_logger.warning(
f"no channel.localCoordinates found for {ses['local_path']}."
f"Assumes checkerboard pattern")
probe['sites_coordinates'].append(SITES_COORDINATES)
# put the session information in there
probe['session'] = [ses] * len(probe.description)
probes = probes.append(probe)
return probes
def load_data(self, download=False):
"""
Load wheel, trial and camera timestamp data
:return: wheel, trials
"""
if download:
self.data.wheel = self.one.load_object(self.eid, 'wheel')
self.data.trials = self.one.load_object(self.eid, 'trials')
cam = self.one.load(self.eid, ['camera.times'], dclass_output=True)
self.data.camera_times = {
vidio.label_from_path(url): ts
for ts, url in zip(cam.data, cam.url)
}
else:
alf_path = self.session_path / 'alf'
self.data.wheel = alfio.load_object(alf_path,
'wheel',
short_keys=True)
self.data.trials = alfio.load_object(alf_path, 'trials')
self.data.camera_times = {
vidio.label_from_path(x): alfio.load_file_content(x)
for x in alf_path.glob('*Camera.times*')
}
assert all(x is not None for x in self.data.values())
def load_data(self, download_data: bool = None) -> None:
"""Extract the data from data files
Extracts all the required task data from the data files.
Data keys:
- camera_times (float array): camera frame timestamps extracted from frame headers
- dlc_coords (dict): keys are the points traced by dlc, items are x-y coordinates of
these points over time, those with likelihood <0.9 set to NaN
:param download_data: if True, any missing raw data is downloaded via ONE.
"""
if download_data is not None:
self.download_data = download_data
if self.one and not self.one.offline:
self._ensure_required_data()
alf_path = self.session_path / 'alf'
# Load times
self.data['camera_times'] = alfio.load_object(alf_path, f'{self.side}Camera')['times']
# Load dlc traces
dlc_df = alfio.load_object(alf_path, f'{self.side}Camera', namespace='ibl')['dlc']
targets = np.unique(['_'.join(col.split('_')[:-1]) for col in dlc_df.columns])
# Set values to nan if likelihood is too low
dlc_coords = {}
for t in targets:
idx = dlc_df.loc[dlc_df[f'{t}_likelihood'] < 0.9].index
dlc_df.loc[idx, [f'{t}_x', f'{t}_y']] = np.nan
dlc_coords[t] = np.array((dlc_df[f'{t}_x'], dlc_df[f'{t}_y']))
self.data['dlc_coords'] = dlc_coords
# load pupil diameters
if self.side in ['left', 'right']:
features = alfio.load_object(alf_path, f'{self.side}Camera', namespace='ibl')['features']
self.data['pupilDiameter_raw'] = features['pupilDiameter_raw']
self.data['pupilDiameter_smooth'] = features['pupilDiameter_smooth']
def _plot_rmsmap(outpath, typ, savefig=True):
'''
TODO document this function
'''
rmsmap = alfio.load_object(outpath,
'ephysQcTime' + typ.upper(),
namespace='spikeglx')
# hack to ensure a single key name
if 'times.probe_00' in rmsmap.keys():
rmsmap['times'] = rmsmap.pop('times.probe_00')
rmsmap['rms'] = rmsmap.pop('rms.probe_00')
elif 'times.probe_01' in rmsmap.keys():
rmsmap['times'] = rmsmap.pop('times.probe_01')
rmsmap['rms'] = rmsmap.pop('rms.probe_01')
plt.figure(figsize=[12, 4.5])
axim = plt.axes([0.2, 0.1, 0.7, 0.8])
axrms = plt.axes([0.05, 0.1, 0.15, 0.8])
axcb = plt.axes([0.92, 0.1, 0.02, 0.8])
axrms.plot(
np.median(rmsmap['rms'], axis=0)[:-1] * 1e6,
np.arange(1, rmsmap['rms'].shape[1]))
axrms.set_ylim(0, rmsmap['rms'].shape[1])
im = axim.imshow(20 * np.log10(rmsmap['rms'].T + 1e-15),
aspect='auto',
origin='lower',
extent=[
rmsmap['times'][0], rmsmap['times'][-1], 0,
rmsmap['rms'].shape[1]
])
axim.set_xlabel(r'Time (s)')
axrms.set_ylabel(r'Channel Number')
plt.colorbar(im, cax=axcb)
if typ == 'ap':
im.set_clim(-110, -90)
axrms.set_xlim(100, 0)
elif typ == 'lf':
im.set_clim(-100, -60)
axrms.set_xlim(500, 0)
axim.set_xlim(0, 4000)
axim.set_title(outpath)
if savefig:
plt.savefig(outpath / (typ + '_rms.png'), dpi=150)
def _run(self):
assert self.pid
output_files = []
if self.location != 'server':
self.histology_status = self.get_histology_status()
electrodes = self.get_channels('electrodeSites',
f'alf/{self.pname}')
# TODO need to figure out the clim range
fig, axs = plt.subplots(1,
2,
gridspec_kw={'width_ratios': [.95, .05]},
figsize=(16, 9))
ap = alfio.load_object(
self.session_path.joinpath(f'raw_ephys_data/{self.pname}'),
'ephysTimeRmsAP',
namespace='iblqc')
_, _, _ = image_rms_plot(ap.rms,
ap.timestamps,
median_subtract=False,
band='AP',
ax=axs[0],
fig_kwargs={'figsize': (8, 6)},
display=True,
title=f"{self.pid_label}")
set_axis_label_size(axs[0], cmap=True)
if self.histology_status:
plot_brain_regions(electrodes['atlas_id'],
brain_regions=self.brain_regions,
display=True,
ax=axs[1],
title=self.histology_status)
set_axis_label_size(axs[1])
else:
remove_axis_outline(axs[1])
save_path = Path(self.output_directory).joinpath("ap_rms.png")
output_files.append(save_path)
fig.savefig(save_path)
plt.close(fig)
return output_files
def get_sync_fronts(auxiliary_name):
d = Bunch({'times': [], 'nsync': np.zeros(nprobes, )})
# auxiliary_name: frame2ttl or right_camera
for ind, ephys_file in enumerate(ephys_files):
sync = alfio.load_object(ephys_file.ap.parent,
'sync',
namespace='spikeglx',
short_keys=True)
sync_map = get_ibl_sync_map(ephys_file, '3A')
# exits if sync label not found for current probe
if auxiliary_name not in sync_map:
return
isync = np.in1d(sync['channels'],
np.array([sync_map[auxiliary_name]]))
# only returns syncs if we get fronts for all probes
if np.all(~isync):
return
d.nsync[ind] = len(sync.channels)
d['times'].append(sync['times'][isync])
return d
def _plot_spectra(outpath, typ, savefig=True):
'''
TODO document this function
'''
spec = alfio.load_object(outpath,
'ephysQcFreq' + typ.upper(),
namespace='spikeglx')
# hack to ensure a single key name
if 'power.probe_00' in spec.keys():
spec['power'] = spec.pop('power.probe_00')
spec['freq'] = spec.pop('freq.probe_00')
elif 'power.probe_01' in spec.keys():
spec['power'] = spec.pop('power.probe_01')
spec['freq'] = spec.pop('freq.probe_01')
# plot
sns.set_style("whitegrid")
plt.figure(figsize=[9, 4.5])
ax = plt.axes()
ax.plot(spec['freq'],
20 * np.log10(spec['power'] + 1e-14),
linewidth=0.5,
color=[0.5, 0.5, 0.5])
ax.plot(spec['freq'],
20 * np.log10(np.median(spec['power'] + 1e-14, axis=1)),
label='median')
ax.set_xlabel(r'Frequency (Hz)')
ax.set_ylabel(r'dB rel to $V^2.$Hz$^{-1}$')
if typ == 'ap':
ax.set_ylim([-275, -125])
elif typ == 'lf':
ax.set_ylim([-260, -60])
ax.legend()
ax.set_title(outpath)
if savefig:
plt.savefig(outpath / (typ + '_spec.png'), dpi=150)
print('saved figure to %s' % (outpath / (typ + '_spec.png')))
def _qc_from_path(sess_path, display=True):
WHEEL = False
sess_path = Path(sess_path)
temp_alf_folder = sess_path.joinpath('fpga_test', 'alf')
temp_alf_folder.mkdir(parents=True, exist_ok=True)
sync, chmap = ephys_fpga.get_main_probe_sync(sess_path, bin_exists=False)
_ = ephys_fpga.extract_all(sess_path, output_path=temp_alf_folder, save=True)
# check that the output is complete
fpga_trials = ephys_fpga.extract_behaviour_sync(sync, chmap=chmap, display=display)
# align with the bpod
bpod2fpga = ephys_fpga.align_with_bpod(temp_alf_folder.parent)
alf_trials = alfio.load_object(temp_alf_folder, 'trials')
shutil.rmtree(temp_alf_folder)
# do the QC
qcs, qct = qc_fpga_task(fpga_trials, alf_trials)
# do the wheel part
if WHEEL:
bpod_wheel = training_wheel.get_wheel_data(sess_path, save=False)
fpga_wheel = ephys_fpga.extract_wheel_sync(sync, chmap=chmap, save=False)
if display:
import matplotlib.pyplot as plt
t0 = max(np.min(bpod2fpga(bpod_wheel['re_ts'])), np.min(fpga_wheel['re_ts']))
dy = np.interp(t0, fpga_wheel['re_ts'], fpga_wheel['re_pos']) - np.interp(
t0, bpod2fpga(bpod_wheel['re_ts']), bpod_wheel['re_pos'])
fix, axes = plt.subplots(nrows=2, sharex='all', sharey='all')
# axes[0].plot(t, pos), axes[0].title.set_text('Extracted')
axes[0].plot(bpod2fpga(bpod_wheel['re_ts']), bpod_wheel['re_pos'] + dy)
axes[0].plot(fpga_wheel['re_ts'], fpga_wheel['re_pos'])
axes[0].title.set_text('FPGA')
axes[1].plot(bpod2fpga(bpod_wheel['re_ts']), bpod_wheel['re_pos'] + dy)
axes[1].title.set_text('Bpod')
return alfio.dataframe({**fpga_trials, **alf_trials, **qct})
def _run(self):
output_files = []
trials = alfio.load_object(self.session_path.joinpath('alf'), 'trials')
title = '_'.join(list(self.session_path.parts[-3:]))
fig, ax = training.plot_psychometric(trials,
title=title,
figsize=(8, 6))
set_axis_label_size(ax)
save_path = Path(
self.output_directory).joinpath("psychometric_curve.png")
output_files.append(save_path)
fig.savefig(save_path)
plt.close(fig)
fig, ax = training.plot_reaction_time(trials,
title=title,
figsize=(8, 6))
set_axis_label_size(ax)
save_path = Path(
self.output_directory).joinpath("chronometric_curve.png")
output_files.append(save_path)
fig.savefig(save_path)
plt.close(fig)
fig, ax = training.plot_reaction_time_over_trials(trials,
title=title,
figsize=(8, 6))
set_axis_label_size(ax)
save_path = Path(
self.output_directory).joinpath("reaction_time_with_trials.png")
output_files.append(save_path)
fig.savefig(save_path)
plt.close(fig)
return output_files
def load_data(self,
download_data: bool = None,
extract_times: bool = False,
load_video: bool = True) -> None:
"""Extract the data from raw data files
Extracts all the required task data from the raw data files.
Data keys:
- count (int array): the sequential frame number (n, n+1, n+2...)
- pin_state (): the camera GPIO pin; records the audio TTLs; should be one per frame
- audio (float array): timestamps of audio TTL fronts
- fpga_times (float array): timestamps of camera TTLs recorded by FPGA
- timestamps (float array): extracted video timestamps (the camera.times ALF)
- bonsai_times (datetime array): system timestamps of video PC; should be one per frame
- camera_times (float array): camera frame timestamps extracted from frame headers
- wheel (Bunch): rotary encoder timestamps, position and period used for wheel motion
- video (Bunch): video meta data, including dimensions and FPS
- frame_samples (h x w x n array): array of evenly sampled frames (1 colour channel)
:param download_data: if True, any missing raw data is downloaded via ONE.
Missing data will raise an AssertionError
:param extract_times: if True, the camera.times are re-extracted from the raw data
:param load_video: if True, calls the load_video_data method
"""
assert self.session_path, 'no session path set'
if download_data is not None:
self.download_data = download_data
if self.download_data and self.eid and self.one and not self.one.offline:
self.ensure_required_data()
_log.info('Gathering data for QC')
# Get frame count and pin state
self.data['count'], self.data['pin_state'] = \
raw.load_embedded_frame_data(self.session_path, self.label, raw=True)
# Load the audio and raw FPGA times
if self.type == 'ephys':
sync, chmap = ephys_fpga.get_main_probe_sync(self.session_path)
audio_ttls = ephys_fpga.get_sync_fronts(sync, chmap['audio'])
self.data['audio'] = audio_ttls['times'] # Get rises
# Load raw FPGA times
cam_ts = extract_camera_sync(sync, chmap)
self.data['fpga_times'] = cam_ts[self.label]
else:
bpod_data = raw.load_data(self.session_path)
_, audio_ttls = raw.load_bpod_fronts(self.session_path, bpod_data)
self.data['audio'] = audio_ttls['times']
# Load extracted frame times
alf_path = self.session_path / 'alf'
try:
assert not extract_times
self.data['timestamps'] = alfio.load_object(
alf_path, f'{self.label}Camera', short_keys=True)['times']
except AssertionError: # Re-extract
kwargs = dict(video_path=self.video_path, labels=self.label)
if self.type == 'ephys':
kwargs = {**kwargs, 'sync': sync, 'chmap': chmap} # noqa
outputs, _ = extract_all(self.session_path,
self.type,
save=False,
**kwargs)
self.data['timestamps'] = outputs[
f'{self.label}_camera_timestamps']
except ALFObjectNotFound:
_log.warning('no camera.times ALF found for session')
# Get audio and wheel data
wheel_keys = ('timestamps', 'position')
try:
self.data['wheel'] = alfio.load_object(alf_path,
'wheel',
short_keys=True)
except ALFObjectNotFound:
# Extract from raw data
if self.type == 'ephys':
wheel_data = ephys_fpga.extract_wheel_sync(sync, chmap)
else:
wheel_data = training_wheel.get_wheel_position(
self.session_path)
self.data['wheel'] = Bunch(zip(wheel_keys, wheel_data))
# Find short period of wheel motion for motion correlation.
if data_for_keys(
wheel_keys,
self.data['wheel']) and self.data['timestamps'] is not None:
self.data['wheel'].period = self.get_active_wheel_period(
self.data['wheel'])
# Load Bonsai frame timestamps
try:
ssv_times = raw.load_camera_ssv_times(self.session_path,
self.label)
self.data['bonsai_times'], self.data['camera_times'] = ssv_times
except AssertionError:
_log.warning('No Bonsai video timestamps file found')
# Gather information from video file
if load_video:
_log.info('Inspecting video file...')
self.load_video_data()
def dlc_qc_plot(session_path, one=None):
"""
Creates DLC QC plot.
Data is searched first locally, then on Alyx. Panels that lack required data are skipped.
Required data to create all panels
'raw_video_data/_iblrig_bodyCamera.raw.mp4',
'raw_video_data/_iblrig_leftCamera.raw.mp4',
'raw_video_data/_iblrig_rightCamera.raw.mp4',
'alf/_ibl_bodyCamera.dlc.pqt',
'alf/_ibl_leftCamera.dlc.pqt',
'alf/_ibl_rightCamera.dlc.pqt',
'alf/_ibl_bodyCamera.times.npy',
'alf/_ibl_leftCamera.times.npy',
'alf/_ibl_rightCamera.times.npy',
'alf/_ibl_leftCamera.features.pqt',
'alf/_ibl_rightCamera.features.pqt',
'alf/rightROIMotionEnergy.position.npy',
'alf/leftROIMotionEnergy.position.npy',
'alf/bodyROIMotionEnergy.position.npy',
'alf/_ibl_trials.choice.npy',
'alf/_ibl_trials.feedbackType.npy',
'alf/_ibl_trials.feedback_times.npy',
'alf/_ibl_trials.stimOn_times.npy',
'alf/_ibl_wheel.position.npy',
'alf/_ibl_wheel.timestamps.npy',
'alf/licks.times.npy',
:params session_path: Path to session data on disk
:params one: ONE instance, if None is given, default ONE is instantiated
:returns: Matplotlib figure
"""
one = one or ONE()
# hack for running on cortexlab local server
if one.alyx.base_url == 'https://alyx.cortexlab.net':
one = ONE(base_url='https://alyx.internationalbrainlab.org')
data = {}
cams = ['left', 'right', 'body']
session_path = Path(session_path)
# Load data for each camera
for cam in cams:
# Load a single frame for each video
# Check if video data is available locally,if yes, load a single frame
video_path = session_path.joinpath('raw_video_data',
f'_iblrig_{cam}Camera.raw.mp4')
if video_path.exists():
data[f'{cam}_frame'] = get_video_frame(video_path,
frame_number=5 * 60 *
SAMPLING[cam])[:, :, 0]
# If not, try to stream a frame (try three times)
else:
try:
video_url = url_from_eid(one.path2eid(session_path),
one=one)[cam]
for tries in range(3):
try:
data[f'{cam}_frame'] = get_video_frame(
video_url,
frame_number=5 * 60 * SAMPLING[cam])[:, :, 0]
break
except BaseException:
if tries < 2:
tries += 1
logger.info(
f"Streaming {cam} video failed, retrying x{tries}"
)
time.sleep(30)
else:
logger.warning(
f"Could not load video frame for {cam} cam. Skipping trace on frame."
)
data[f'{cam}_frame'] = None
except KeyError:
logger.warning(
f"Could not load video frame for {cam} cam. Skipping trace on frame."
)
data[f'{cam}_frame'] = None
# Other camera associated data
for feat in ['dlc', 'times', 'features', 'ROIMotionEnergy']:
# Check locally first, then try to load from alyx, if nothing works, set to None
if feat == 'features' and cam == 'body': # this doesn't exist for body cam
continue
local_file = list(
session_path.joinpath('alf').glob(f'*{cam}Camera.{feat}*'))
if len(local_file) > 0:
data[f'{cam}_{feat}'] = alfio.load_file_content(local_file[0])
else:
alyx_ds = [
ds for ds in one.list_datasets(one.path2eid(session_path))
if f'{cam}Camera.{feat}' in ds
]
if len(alyx_ds) > 0:
data[f'{cam}_{feat}'] = one.load_dataset(
one.path2eid(session_path), alyx_ds[0])
else:
logger.warning(
f"Could not load _ibl_{cam}Camera.{feat} some plots have to be skipped."
)
data[f'{cam}_{feat}'] = None
# Sometimes there is a file but the object is empty, set to None
if data[f'{cam}_{feat}'] is not None and len(
data[f'{cam}_{feat}']) == 0:
logger.warning(
f"Object loaded from _ibl_{cam}Camera.{feat} is empty, some plots have to be skipped."
)
data[f'{cam}_{feat}'] = None
# If we have no frame and/or no DLC and/or no times for all cams, raise an error, something is really wrong
assert any([data[f'{cam}_frame'] is not None
for cam in cams]), "No camera data could be loaded, aborting."
assert any([data[f'{cam}_dlc'] is not None
for cam in cams]), "No DLC data could be loaded, aborting."
assert any([data[f'{cam}_times'] is not None for cam in cams
]), "No camera times data could be loaded, aborting."
# Load session level data
for alf_object in ['trials', 'wheel', 'licks']:
try:
data[f'{alf_object}'] = alfio.load_object(
session_path.joinpath('alf'), alf_object) # load locally
continue
except ALFObjectNotFound:
pass
try:
data[f'{alf_object}'] = one.load_object(
one.path2eid(session_path), alf_object) # then try from alyx
except ALFObjectNotFound:
logger.warning(
f"Could not load {alf_object} object, some plots have to be skipped."
)
data[f'{alf_object}'] = None
# Simplify and clean up trials data
if data['trials']:
data['trials'] = pd.DataFrame({
k: data['trials'][k]
for k in
['stimOn_times', 'feedback_times', 'choice', 'feedbackType']
})
# Discard nan events and too long trials
data['trials'] = data['trials'].dropna()
data['trials'] = data['trials'].drop(
data['trials'][(data['trials']['feedback_times'] -
data['trials']['stimOn_times']) > 10].index)
# Make a list of panels, if inputs are missing, instead input a text to display
panels = []
# Panel A, B, C: Trace on frame
for cam in cams:
if data[f'{cam}_frame'] is not None and data[f'{cam}_dlc'] is not None:
panels.append((plot_trace_on_frame, {
'frame': data[f'{cam}_frame'],
'dlc_df': data[f'{cam}_dlc'],
'cam': cam
}))
else:
panels.append(
(None, f'Data missing\n{cam.capitalize()} cam trace on frame'))
# If trials data is not there, we cannot plot any of the trial average plots, skip all remaining panels
if data['trials'] is None:
panels.extend([(None, 'No trial data,\ncannot compute trial avgs')
for i in range(7)])
else:
# Panel D: Motion energy
camera_dict = {
'left': {
'motion_energy': data['left_ROIMotionEnergy'],
'times': data['left_times']
},
'right': {
'motion_energy': data['right_ROIMotionEnergy'],
'times': data['right_times']
},
'body': {
'motion_energy': data['body_ROIMotionEnergy'],
'times': data['body_times']
}
}
for cam in [
'left', 'right', 'body'
]: # Remove cameras where we don't have motion energy AND camera times
if camera_dict[cam]['motion_energy'] is None or camera_dict[cam][
'times'] is None:
_ = camera_dict.pop(cam)
if len(camera_dict) > 0:
panels.append((plot_motion_energy_hist, {
'camera_dict': camera_dict,
'trials_df': data['trials']
}))
else:
panels.append((None, 'Data missing\nMotion energy'))
# Panel E: Wheel position
if data['wheel']:
panels.append((plot_wheel_position, {
'wheel_position': data['wheel'].position,
'wheel_time': data['wheel'].timestamps,
'trials_df': data['trials']
}))
else:
panels.append((None, 'Data missing\nWheel position'))
# Panel F, G: Paw speed and nose speed
# Try if all data is there for left cam first, otherwise right
for cam in ['left', 'right']:
fail = False
if (data[f'{cam}_dlc'] is not None
and data[f'{cam}_times'] is not None
and len(data[f'{cam}_times']) >= len(data[f'{cam}_dlc'])):
break
fail = True
if not fail:
paw = 'r' if cam == 'left' else 'l'
panels.append((plot_speed_hist, {
'dlc_df': data[f'{cam}_dlc'],
'cam_times': data[f'{cam}_times'],
'trials_df': data['trials'],
'feature': f'paw_{paw}',
'cam': cam
}))
panels.append((plot_speed_hist, {
'dlc_df': data[f'{cam}_dlc'],
'cam_times': data[f'{cam}_times'],
'trials_df': data['trials'],
'feature': 'nose_tip',
'legend': False,
'cam': cam
}))
else:
panels.extend([(None, 'Data missing or corrupt\nSpeed histograms')
for i in range(2)])
# Panel H and I: Lick plots
if data['licks'] and data['licks'].times.shape[0] > 0:
panels.append((plot_lick_hist, {
'lick_times': data['licks'].times,
'trials_df': data['trials']
}))
panels.append((plot_lick_raster, {
'lick_times': data['licks'].times,
'trials_df': data['trials']
}))
else:
panels.extend([(None, 'Data missing\nLicks plots')
for i in range(2)])
# Panel J: pupil plot
# Try if all data is there for left cam first, otherwise right
for cam in ['left', 'right']:
fail = False
if (data[f'{cam}_times'] is not None
and data[f'{cam}_features'] is not None and
len(data[f'{cam}_times']) >= len(data[f'{cam}_features'])
and not np.all(
np.isnan(
data[f'{cam}_features'].pupilDiameter_smooth))):
break
fail = True
if not fail:
panels.append((plot_pupil_diameter_hist, {
'pupil_diameter': data[f'{cam}_features'].pupilDiameter_smooth,
'cam_times': data[f'{cam}_times'],
'trials_df': data['trials'],
'cam': cam
}))
else:
panels.append((None, 'Data missing or corrupt\nPupil diameter'))
# Plotting
plt.rcParams.update({'font.size': 10})
fig = plt.figure(figsize=(17, 10))
for i, panel in enumerate(panels):
ax = plt.subplot(2, 5, i + 1)
ax.text(-0.1,
1.15,
ascii_uppercase[i],
transform=ax.transAxes,
fontsize=16,
fontweight='bold')
# Check if there was in issue with inputs, if yes, print the respective text
if panel[0] is None:
ax.text(.5,
.5,
panel[1],
color='r',
fontweight='bold',
fontsize=12,
horizontalalignment='center',
verticalalignment='center',
transform=ax.transAxes)
plt.axis('off')
else:
try:
panel[0](**panel[1])
except BaseException:
logger.error(f'Error in {panel[0].__name__}\n' +
traceback.format_exc())
ax.text(.5,
.5,
f'Error while plotting\n{panel[0].__name__}',
color='r',
fontweight='bold',
fontsize=12,
horizontalalignment='center',
verticalalignment='center',
transform=ax.transAxes)
plt.axis('off')
plt.tight_layout(rect=[0, 0.03, 1, 0.95])
return fig
def _run(self, collection=None):
"""runs for initiated PID, streams data, destripe and check bad channels"""
def plot_driftmap(self,
spikes,
clusters,
channels,
collection,
ylim=(0, 3840)):
fig, axs = plt.subplots(1,
2,
gridspec_kw={'width_ratios': [.95, .05]},
figsize=(16, 9))
driftmap(spikes.times,
spikes.depths,
t_bin=0.007,
d_bin=10,
vmax=0.5,
ax=axs[0])
title_str = f"{self.pid_label}, {collection}, {self.pid} \n " \
f"{spikes.clusters.size:_} spikes, {clusters.depths.size:_} clusters"
axs[0].set(ylim=ylim, title=title_str)
run_label = str(Path(collection).relative_to(f'alf/{self.pname}'))
run_label = "ks2matlab" if run_label == '.' else run_label
outfile = self.output_directory.joinpath(
f"spike_sorting_raster_{run_label}.png")
set_axis_label_size(axs[0])
if self.histology_status:
plot_brain_regions(channels['atlas_id'],
channel_depths=channels['axial_um'],
brain_regions=self.brain_regions,
display=True,
ax=axs[1],
title=self.histology_status)
axs[1].set(ylim=ylim)
set_axis_label_size(axs[1])
else:
remove_axis_outline(axs[1])
fig.savefig(outfile)
plt.close(fig)
return outfile, fig, axs
output_files = []
if self.location == 'server':
assert collection
spikes = alfio.load_object(self.session_path.joinpath(collection),
'spikes')
clusters = alfio.load_object(
self.session_path.joinpath(collection), 'clusters')
channels = alfio.load_object(
self.session_path.joinpath(collection), 'channels')
out, fig, axs = plot_driftmap(self, spikes, clusters, channels,
collection)
output_files.append(out)
else:
self.histology_status = self.get_histology_status()
all_here, output_files = self.assert_expected(self.output_files,
silent=True)
spike_sorting_runs = self.one.list_datasets(
self.eid,
filename='spikes.times.npy',
collection=f'alf/{self.pname}*')
if all_here and len(output_files) == len(spike_sorting_runs):
return output_files
logger.info(self.output_directory)
for run in spike_sorting_runs:
collection = str(Path(run).parent.as_posix())
spikes, clusters, channels = load_spike_sorting_fast(
eid=self.eid,
probe=self.pname,
one=self.one,
nested=False,
collection=collection,
dataset_types=['spikes.depths'],
brain_regions=self.brain_regions)
if 'atlas_id' not in channels.keys():
channels = self.get_channels('channels', collection)
out, fig, axs = plot_driftmap(self, spikes, clusters, channels,
collection)
output_files.append(out)
return output_files
def version3B(ses_path, display=True, type=None, tol=2.5):
"""
From a session path with _spikeglx_sync arrays extraccted, locate ephys files for 3A and
outputs one sync.timestamps.probeN.npy file per acquired probe. By convention the reference
probe is the one with the most synchronisation pulses.
Assumes the _spikeglx_sync datasets are already extracted from binary data
:param ses_path:
:param type: linear, exact or smooth
:return: None
"""
DEFAULT_TYPE = 'smooth'
ephys_files = spikeglx.glob_ephys_files(ses_path,
ext='meta',
bin_exists=False)
for ef in ephys_files:
ef['sync'] = alfio.load_object(ef.path,
'sync',
namespace='spikeglx',
short_keys=True)
ef['sync_map'] = get_ibl_sync_map(ef, '3B')
nidq_file = [ef for ef in ephys_files if ef.get('nidq')]
ephys_files = [ef for ef in ephys_files if not ef.get('nidq')]
# should have at least 2 probes and only one nidq
assert (len(nidq_file) == 1)
nidq_file = nidq_file[0]
sync_nidq = get_sync_fronts(nidq_file.sync,
nidq_file.sync_map['imec_sync'])
qc_all = True
out_files = []
for ef in ephys_files:
sync_probe = get_sync_fronts(ef.sync, ef.sync_map['imec_sync'])
sr = _get_sr(ef)
try:
# we say that the number of pulses should be within 10 %
assert (np.isclose(sync_nidq.times.size,
sync_probe.times.size,
rtol=0.1))
except AssertionError:
raise Neuropixel3BSyncFrontsNonMatching(f"{ses_path}")
# Find the indexes in case the sizes don't match
if sync_nidq.times.size != sync_probe.times.size:
_logger.warning(
f'Sync mismatch by {np.abs(sync_nidq.times.size - sync_probe.times.size)} '
f'NIDQ sync times: {sync_nidq.times.size}, Probe sync times {sync_probe.times.size}'
)
sync_idx = np.min([sync_nidq.times.size, sync_probe.times.size])
# if the qc of the diff finds anomalies, do not attempt to smooth the interp function
qcdiff = _check_diff_3b(sync_probe)
if not qcdiff:
qc_all = False
type_probe = type or 'exact'
else:
type_probe = type or DEFAULT_TYPE
timestamps, qc = sync_probe_front_times(sync_probe.times[:sync_idx],
sync_nidq.times[:sync_idx],
sr,
display=display,
type=type_probe,
tol=tol)
qc_all &= qc
out_files.extend(_save_timestamps_npy(ef, timestamps, sr))
return qc_all, out_files