""" Get single probe label and directory, using the probes description dataset. """ # Author: Gaelle Chapuis, Miles Wells from one.api import ONE one = ONE() eid = 'da188f2c-553c-4e04-879b-c9ea2d1b9a93' # --- Get single probe directory filename either by # 1. getting probe description in alf # 2. using alyx rest end point # Option 1. prob_des = one.load_dataset(eid, 'probes.description.json') labels = [x['label'] for x in prob_des] # You can then use this label into dict, e.g. channels[label[0]] # -- Load single probe data with probe-level collection # List datsets for first probe collection = f'alf/{labels[0]}' datasets = one.list_datasets(eid, collection=collection)
plt.tight_layout() if __name__ == "__main__": one = ONE() # one.list(None, 'dataset-types') # to check dataset types, 'camera.times'? # one.search(dataset=['camera.dlc.pqt', 'camera.times.npy']) eid = '61393bca-f1ff-4e7d-b2d8-da7475219866' camera_label = 'left' trials = one.load_object(eid, 'trials', collection='alf') add_stim_off_times(trials) times = one.load_dataset(eid, f'_ibl_{camera_label}Camera.times.npy', collection='alf') dlc_path = one.load_dataset(eid, f'_ibl_{camera_label}Camera.dlc.pqt', collection='alf', download_only=True) diameter = get_pupil_diameter(one.eid2path / 'alf') # get trial number for each time bin trial_numbers = np.digitize(times, trials['goCue_times']) print('Range of trials: ', [trial_numbers[0], trial_numbers[-1]]) # get a raster plot for a particular trial # plot_pupil_diameter_single_trial(trial_numbers, # 15, diameter, times, trials) plot_mean_std_around_event('stimOn_times', diameter, times, eid) plot_mean_std_around_event('feedback_times', diameter, times, eid)
are in black and white the values of each color channel are identical. Therefore to save on memory you can provide a slice that returns only one of the three channels for each frame. The resulting shape will be (l, h, w). NB: Any slice or boolean array may be provided which is useful for cropping to an ROI. If you don't need to apply operations over all the fetched frames you can use the `as_list` kwarg to return the frames as a list. This is slightly faster than fetching as an ndarray. A warning is printed if fetching a frame fails. The affected frames will be returned as zeros or None if `as_list` is True. """ frames = vidio.get_video_frames_preload(url, range(10), mask=np.s_[:, :, 0]) # Example 5: load video meta-data """ You can load all the information for a given video. In order to load the video size from a URL an instance of ONE must be provided, otherwise this entry will be blank. An Bunch is returned with a number of fields. """ meta = vidio.get_video_meta(url, one=one) for k, v in meta.items(): print(f'The video {k} = {v}') # Example 6: load video timestamps ts = one.load_dataset(eid, f'_ibl_{label}Camera.times.npy', collection='alf') # Example 7: load dlc cam = one.load_object(eid, f'{label}Camera', collection='alf') print(cam.keys()) print(cam.dlc.columns)
""" import numpy as np from one.api import ONE from brainbox.ephys_plots import scatter_raster_plot from brainbox.plot_base import plot_scatter one = ONE(base_url='https://openalyx.internationalbrainlab.org', silent=True) eid = '4ecb5d24-f5cc-402c-be28-9d0f7cb14b3a' probe = 'probe00' spikes = one.load_object(eid, obj='spikes', collection=f'alf/{probe}') metrics = one.load_dataset(eid, dataset='clusters.metrics.pqt', collection=f'alf/{probe}') # Find the clusters that have been labelled as good and their corresponding spike indices good_clusters = np.where(metrics.label == 1) spike_idx = np.where(np.isin(spikes['clusters'], good_clusters))[0] # Also filter for nans in amplitude and depth kp_idx = spike_idx[np.where(~np.isnan(spikes['depths'][spike_idx]) & ~np.isnan(spikes['amps'][spike_idx]))[0]] # Get ScatterPlot object data = scatter_raster_plot(spikes['amps'][kp_idx], spikes['depths'][kp_idx], spikes['times'][kp_idx]) # Add v lines 10s after start and 10s before end or recording
def download_ch_disp_data(x, y, provenance='Planned', project='ibl_neuropixel_brainwide_01'): """Download channels displacement data for all given probes at the planned insertion coord [x,y] from Alyx Downloads the most up-to-date data from Alyx for all recordings at [x,y], including their channel positionss, and the orthogonal points of each channel from the planned trajectory. Saves this data to a standard location in the file system. Also returns this data as a pandas DataFrame object with following: * subject, eid, probe - the subject, eid and probe IDs * chan_loc - xyz coords of all channels * planned_orth_proj - xyz coord of orthogonal line from chan_loc onto planned proj * dist - the euclidean distance between chan_loc xyz and planned_orth_proj xyz Parameters ---------- x : int x planned insertion coord in µm. Eg. repeated site is -2243 y : int y planned insertion coord in µm. Eg. repeated site is -2000. provenance : str, optional Probe provenance to list trajectories from: Planned, Micro-manipulator, Histology, E-phys aligned. The default is 'Planned'. project : str, optional Project to gather all trajectories from. The default is 'ibl_neuropixel_brainwide_01'. Returns ------- data_frame : pandas DataFrame Dataframe containing: subject, eid, probe; ins_x, ins_y; chan_loc; planned_orth_proj; dist. """ from one.api import ONE from ibllib.atlas import Insertion import brainbox.io.one as bbone #from ibllib.io import params - deprecated! # for catching errors in collecting datasets from ibllib.exceptions import IblError from urllib.error import HTTPError import numpy as np import pandas as pd from pathlib import Path # in format -2000_-2243 prefix = str(str(x) + "_" + str(y)) # connect to ONE one = ONE() # get the planned trajectory for repeated site: x=2243, y=2000 traj = one.alyx.rest('trajectories', 'list', provenance=provenance, x=x, y=y, project=project) # from this collect all eids, probes, subjects that use repeated site: eids = [sess['session']['id'] for sess in traj] probes = [sess['probe_name'] for sess in traj] subj = [sess['session']['subject'] for sess in traj] # new dict to store data from loop: # chan_loc - xyz coord of channels # planned_orth_proj - xyz coord of orthogonal line from chan_loc to planned # proj # dist - the 3D distance between chan_loc xyz and planned_orth_proj xyz data = { 'subject': [], 'lab': [], 'eid': [], 'probe': [], 'ins_x': [], 'ins_y': [], 'chan_loc_x': [], 'chan_loc_y': [], 'chan_loc_z': [], 'planned_orth_proj_x': [], 'planned_orth_proj_y': [], 'planned_orth_proj_z': [], 'dist': [], } # Fetch Repeated Site planned trajectory metadata: planned = one.alyx.rest('trajectories', 'list', session=eids[0], probe=probes[0], provenance='planned') # create insertion object of probe from planned trajectory: ins = Insertion.from_dict(planned[0]) # create a trajectory object of Planned Repeated Site from this insertion: traj = ins.trajectory subindex = 0 # loop through each eid/probe: for eid, probe in zip(eids, probes): print( "==================================================================" ) print(eids.index(eid)) print(eid) print(probe) print(subj[subindex]) subindex = subindex + 1 # get the eid/probe as insertion insertion = one.alyx.rest('insertions', 'list', session=eid, name=probe) if insertion: print(" insertion exists") # check if histology has been traced and loaded tracing = np.array(insertion[0]['json']) if tracing: print(" tracing exists") # For this insertion which has histology tracing, retrieve the # channels in xyz coords: # check the localCoordinates EXIST for this eid/probe # run in a try..except statement to continue over the eid/probe # if localCoordinates dataset does not exist try: channel_coord = one.load_dataset( eid, 'channels.localCoordinates.npy', collection='alf/' + probe) except IblError: print("ALFObjectNotFound") print("") continue except HTTPError: print("HTTPError") print("") continue except: print("ERROR - generic") continue # only proceed if channel_coord is not None if channel_coord is None: continue print(" channel_coords exist") if one.alyx.rest('trajectories', 'list', session=eid, probe=probe, provenance='Histology track') == []: print("ERROR - no Histology Track..") continue chan_loc = bbone.load_channel_locations(eid, one=one, probe=probe) print("chan_loc") # only proceed if channel locations could be retrieved if not chan_loc: continue # Next, create a representation of the planned trajectory as a # line: plannedTraj = one.alyx.rest('trajectories', 'list', session=eid, probe=probe, provenance='planned') print("plannedTraj") # create insertion object from planned trajectory: #ins = Insertion.from_dict(planned[0]) # create a trajectory object from this insertion: #traj = ins.trajectory # NEXT - compute the projected coord for each channel coord onto the # line defined by traj: for ch_ind in range(len(chan_loc[probe]['x'])): cl_x = chan_loc[probe]['x'][ch_ind] cl_y = chan_loc[probe]['y'][ch_ind] cl_z = chan_loc[probe]['z'][ch_ind] # create numpy array from chan_loc coords: ch_loc = np.array([cl_x, cl_y, cl_z]) # project the current chan_loc to the PLANNED trajectory: proj = traj.project(ch_loc) # calculate the distance between proj and chan_loc: dist = np.linalg.norm(ch_loc - proj) data['subject'].append( plannedTraj[0]['session']['subject']) data['lab'].append(plannedTraj[0]['session']['lab']) data['eid'].append(eid) data['probe'].append(probe) data['ins_x'].append(probe) data['ins_y'].append(probe) data['chan_loc_x'].append(cl_x) data['chan_loc_y'].append(cl_y) data['chan_loc_z'].append(cl_z) data['planned_orth_proj_x'].append(proj[0]) data['planned_orth_proj_y'].append(proj[1]) data['planned_orth_proj_z'].append(proj[2]) data['dist'].append(dist) # convert data to a Pandas DataFrame: data_frame = pd.DataFrame.from_dict(data) save_ch_disp_data(data_frame, prefix) return data_frame
For a given session eid (ephys session), plot spectrogram of sound recorded via the microphone. Example of using soundfile to read in .flac file extensions """ # Author: Gaelle Chapuis from ibllib.io.extractors.training_audio import welchogram import soundfile as sf import numpy as np import matplotlib.pyplot as plt from one.api import ONE eid = '4ecb5d24-f5cc-402c-be28-9d0f7cb14b3a' # TEST EXAMPLE one = ONE(base_url='https://openalyx.internationalbrainlab.org', silent=True) # -- Get raw data filename = one.load_dataset(eid, '_iblrig_micData.raw.flac', download_only=True) with open(filename, 'rb') as f: wav, fs = sf.read(f) # -- Compute spectrogram over first 2 minutes t_idx = 120 * fs tscale, fscale, W, detect = welchogram(fs, wav[:t_idx]) # -- Put data into single variable TF = {} TF['power'] = W.astype(np.single) TF['frequencies'] = fscale[None, :].astype(np.single) TF['onset_times'] = detect TF['times_mic'] = tscale[:, None].astype(np.single)
import ibllib.atlas as atlas # Instantiate brain atlas and one brain_atlas = atlas.AllenAtlas(25) one = ONE() # Find eid of interest subject = 'CSHL047' date = '2020-01-22' sess_no = 2 probe_label = 'probe01' eid = one.search(subject=subject, date=date, number=sess_no)[0] # Load in channels.localCoordinates dataset type chn_coords = one.load_dataset(eid, 'channels.localCoordinates.npy', collection=f'alf/{probe_label}') depths = chn_coords[:, 1] # Find the ephys aligned trajectory for eid probe combination trajectory = one.alyx.rest('trajectories', 'list', provenance='Ephys aligned histology track', session=eid, probe=probe_label) # Extract all alignments from the json field of object alignments = trajectory[0]['json'] # Load in the initial user xyz_picks obtained from track traccing insertion = one.alyx.rest('insertions', 'list', session=eid, name=probe_label) xyz_picks = np.array(insertion[0]['json']['xyz_picks']) / 1e6
def dlc_qc_plot(session_path, one=None): """ Creates DLC QC plot. Data is searched first locally, then on Alyx. Panels that lack required data are skipped. Required data to create all panels 'raw_video_data/_iblrig_bodyCamera.raw.mp4', 'raw_video_data/_iblrig_leftCamera.raw.mp4', 'raw_video_data/_iblrig_rightCamera.raw.mp4', 'alf/_ibl_bodyCamera.dlc.pqt', 'alf/_ibl_leftCamera.dlc.pqt', 'alf/_ibl_rightCamera.dlc.pqt', 'alf/_ibl_bodyCamera.times.npy', 'alf/_ibl_leftCamera.times.npy', 'alf/_ibl_rightCamera.times.npy', 'alf/_ibl_leftCamera.features.pqt', 'alf/_ibl_rightCamera.features.pqt', 'alf/rightROIMotionEnergy.position.npy', 'alf/leftROIMotionEnergy.position.npy', 'alf/bodyROIMotionEnergy.position.npy', 'alf/_ibl_trials.choice.npy', 'alf/_ibl_trials.feedbackType.npy', 'alf/_ibl_trials.feedback_times.npy', 'alf/_ibl_trials.stimOn_times.npy', 'alf/_ibl_wheel.position.npy', 'alf/_ibl_wheel.timestamps.npy', 'alf/licks.times.npy', :params session_path: Path to session data on disk :params one: ONE instance, if None is given, default ONE is instantiated :returns: Matplotlib figure """ one = one or ONE() # hack for running on cortexlab local server if one.alyx.base_url == 'https://alyx.cortexlab.net': one = ONE(base_url='https://alyx.internationalbrainlab.org') data = {} cams = ['left', 'right', 'body'] session_path = Path(session_path) # Load data for each camera for cam in cams: # Load a single frame for each video # Check if video data is available locally,if yes, load a single frame video_path = session_path.joinpath('raw_video_data', f'_iblrig_{cam}Camera.raw.mp4') if video_path.exists(): data[f'{cam}_frame'] = get_video_frame(video_path, frame_number=5 * 60 * SAMPLING[cam])[:, :, 0] # If not, try to stream a frame (try three times) else: try: video_url = url_from_eid(one.path2eid(session_path), one=one)[cam] for tries in range(3): try: data[f'{cam}_frame'] = get_video_frame( video_url, frame_number=5 * 60 * SAMPLING[cam])[:, :, 0] break except BaseException: if tries < 2: tries += 1 logger.info( f"Streaming {cam} video failed, retrying x{tries}" ) time.sleep(30) else: logger.warning( f"Could not load video frame for {cam} cam. Skipping trace on frame." ) data[f'{cam}_frame'] = None except KeyError: logger.warning( f"Could not load video frame for {cam} cam. Skipping trace on frame." ) data[f'{cam}_frame'] = None # Other camera associated data for feat in ['dlc', 'times', 'features', 'ROIMotionEnergy']: # Check locally first, then try to load from alyx, if nothing works, set to None if feat == 'features' and cam == 'body': # this doesn't exist for body cam continue local_file = list( session_path.joinpath('alf').glob(f'*{cam}Camera.{feat}*')) if len(local_file) > 0: data[f'{cam}_{feat}'] = alfio.load_file_content(local_file[0]) else: alyx_ds = [ ds for ds in one.list_datasets(one.path2eid(session_path)) if f'{cam}Camera.{feat}' in ds ] if len(alyx_ds) > 0: data[f'{cam}_{feat}'] = one.load_dataset( one.path2eid(session_path), alyx_ds[0]) else: logger.warning( f"Could not load _ibl_{cam}Camera.{feat} some plots have to be skipped." ) data[f'{cam}_{feat}'] = None # Sometimes there is a file but the object is empty, set to None if data[f'{cam}_{feat}'] is not None and len( data[f'{cam}_{feat}']) == 0: logger.warning( f"Object loaded from _ibl_{cam}Camera.{feat} is empty, some plots have to be skipped." ) data[f'{cam}_{feat}'] = None # If we have no frame and/or no DLC and/or no times for all cams, raise an error, something is really wrong assert any([data[f'{cam}_frame'] is not None for cam in cams]), "No camera data could be loaded, aborting." assert any([data[f'{cam}_dlc'] is not None for cam in cams]), "No DLC data could be loaded, aborting." assert any([data[f'{cam}_times'] is not None for cam in cams ]), "No camera times data could be loaded, aborting." # Load session level data for alf_object in ['trials', 'wheel', 'licks']: try: data[f'{alf_object}'] = alfio.load_object( session_path.joinpath('alf'), alf_object) # load locally continue except ALFObjectNotFound: pass try: data[f'{alf_object}'] = one.load_object( one.path2eid(session_path), alf_object) # then try from alyx except ALFObjectNotFound: logger.warning( f"Could not load {alf_object} object, some plots have to be skipped." ) data[f'{alf_object}'] = None # Simplify and clean up trials data if data['trials']: data['trials'] = pd.DataFrame({ k: data['trials'][k] for k in ['stimOn_times', 'feedback_times', 'choice', 'feedbackType'] }) # Discard nan events and too long trials data['trials'] = data['trials'].dropna() data['trials'] = data['trials'].drop( data['trials'][(data['trials']['feedback_times'] - data['trials']['stimOn_times']) > 10].index) # Make a list of panels, if inputs are missing, instead input a text to display panels = [] # Panel A, B, C: Trace on frame for cam in cams: if data[f'{cam}_frame'] is not None and data[f'{cam}_dlc'] is not None: panels.append((plot_trace_on_frame, { 'frame': data[f'{cam}_frame'], 'dlc_df': data[f'{cam}_dlc'], 'cam': cam })) else: panels.append( (None, f'Data missing\n{cam.capitalize()} cam trace on frame')) # If trials data is not there, we cannot plot any of the trial average plots, skip all remaining panels if data['trials'] is None: panels.extend([(None, 'No trial data,\ncannot compute trial avgs') for i in range(7)]) else: # Panel D: Motion energy camera_dict = { 'left': { 'motion_energy': data['left_ROIMotionEnergy'], 'times': data['left_times'] }, 'right': { 'motion_energy': data['right_ROIMotionEnergy'], 'times': data['right_times'] }, 'body': { 'motion_energy': data['body_ROIMotionEnergy'], 'times': data['body_times'] } } for cam in [ 'left', 'right', 'body' ]: # Remove cameras where we don't have motion energy AND camera times if camera_dict[cam]['motion_energy'] is None or camera_dict[cam][ 'times'] is None: _ = camera_dict.pop(cam) if len(camera_dict) > 0: panels.append((plot_motion_energy_hist, { 'camera_dict': camera_dict, 'trials_df': data['trials'] })) else: panels.append((None, 'Data missing\nMotion energy')) # Panel E: Wheel position if data['wheel']: panels.append((plot_wheel_position, { 'wheel_position': data['wheel'].position, 'wheel_time': data['wheel'].timestamps, 'trials_df': data['trials'] })) else: panels.append((None, 'Data missing\nWheel position')) # Panel F, G: Paw speed and nose speed # Try if all data is there for left cam first, otherwise right for cam in ['left', 'right']: fail = False if (data[f'{cam}_dlc'] is not None and data[f'{cam}_times'] is not None and len(data[f'{cam}_times']) >= len(data[f'{cam}_dlc'])): break fail = True if not fail: paw = 'r' if cam == 'left' else 'l' panels.append((plot_speed_hist, { 'dlc_df': data[f'{cam}_dlc'], 'cam_times': data[f'{cam}_times'], 'trials_df': data['trials'], 'feature': f'paw_{paw}', 'cam': cam })) panels.append((plot_speed_hist, { 'dlc_df': data[f'{cam}_dlc'], 'cam_times': data[f'{cam}_times'], 'trials_df': data['trials'], 'feature': 'nose_tip', 'legend': False, 'cam': cam })) else: panels.extend([(None, 'Data missing or corrupt\nSpeed histograms') for i in range(2)]) # Panel H and I: Lick plots if data['licks'] and data['licks'].times.shape[0] > 0: panels.append((plot_lick_hist, { 'lick_times': data['licks'].times, 'trials_df': data['trials'] })) panels.append((plot_lick_raster, { 'lick_times': data['licks'].times, 'trials_df': data['trials'] })) else: panels.extend([(None, 'Data missing\nLicks plots') for i in range(2)]) # Panel J: pupil plot # Try if all data is there for left cam first, otherwise right for cam in ['left', 'right']: fail = False if (data[f'{cam}_times'] is not None and data[f'{cam}_features'] is not None and len(data[f'{cam}_times']) >= len(data[f'{cam}_features']) and not np.all( np.isnan( data[f'{cam}_features'].pupilDiameter_smooth))): break fail = True if not fail: panels.append((plot_pupil_diameter_hist, { 'pupil_diameter': data[f'{cam}_features'].pupilDiameter_smooth, 'cam_times': data[f'{cam}_times'], 'trials_df': data['trials'], 'cam': cam })) else: panels.append((None, 'Data missing or corrupt\nPupil diameter')) # Plotting plt.rcParams.update({'font.size': 10}) fig = plt.figure(figsize=(17, 10)) for i, panel in enumerate(panels): ax = plt.subplot(2, 5, i + 1) ax.text(-0.1, 1.15, ascii_uppercase[i], transform=ax.transAxes, fontsize=16, fontweight='bold') # Check if there was in issue with inputs, if yes, print the respective text if panel[0] is None: ax.text(.5, .5, panel[1], color='r', fontweight='bold', fontsize=12, horizontalalignment='center', verticalalignment='center', transform=ax.transAxes) plt.axis('off') else: try: panel[0](**panel[1]) except BaseException: logger.error(f'Error in {panel[0].__name__}\n' + traceback.format_exc()) ax.text(.5, .5, f'Error while plotting\n{panel[0].__name__}', color='r', fontweight='bold', fontsize=12, horizontalalignment='center', verticalalignment='center', transform=ax.transAxes) plt.axis('off') plt.tight_layout(rect=[0, 0.03, 1, 0.95]) return fig
from one.api import ONE import numpy as np import matplotlib.pyplot as plt from ibllib.atlas import BrainRegions one = ONE(base_url='https://openalyx.internationalbrainlab.org', silent=True) # Specify subject, date and probe we are interested in subject = 'CSHL049' date = '2020-01-08' sess_no = 1 probe_label = 'probe00' eid = one.search(subject=subject, date=date, number=sess_no)[0] cluster_chans = one.load_dataset(eid, 'clusters.channels.npy', collection=f'alf/{probe_label}') aligned_traj = one.alyx.rest('trajectories', 'list', subject=subject, session=eid, probe=probe_label, provenance='Ephys aligned histology track') if len(aligned_traj) > 0: print('Getting channels for provenance ' + aligned_traj[0]['provenance']) channels = one.alyx.rest('channels', 'list', trajectory_estimate=aligned_traj[0]['id']) chans = {'atlas_id': np.array([ch['brain_region'] for ch in channels]), 'x': np.array([ch['x'] for ch in channels]) / 1e6, 'y': np.array([ch['y'] for ch in channels]) / 1e6, 'z': np.array([ch['z'] for ch in channels]) / 1e6, 'axial_um': np.array([ch['axial'] for ch in channels]),
import matplotlib.pyplot as plt import numpy as np from one.api import ONE import brainbox.io.one as bbone import brainbox.task.passive as passive eid = '4ecb5d24-f5cc-402c-be28-9d0f7cb14b3a' probe = 'probe00' one = ONE(base_url='https://openalyx.internationalbrainlab.org', silent=True) # Load in the receptive field map data rf_map = bbone.load_passive_rfmap(eid, one=one) spike_times = one.load_dataset(eid, dataset='spikes.times.npy', collection=f'alf/{probe}') spike_depths = one.load_dataset(eid, dataset='spikes.depths.npy', collection=f'alf/{probe}') # Remove any nan depths kp_idx = np.where(~np.isnan(spike_depths))[0] spike_times = spike_times[kp_idx] spike_depths = spike_depths[kp_idx] # Pass this data into brainbox function to find out at what times each voxel on screen was # activated rf_stim_times, rf_stim_pos, rf_stim_frames = passive.get_on_off_times_and_positions( rf_map) # rf_stim_times - time of frame change # rf_stim_pos - position of each voxel on (15 x 15) screen