(example taken for nidq.cbin file, but also applicable for lf.cbin and ap.cbin files) """ # Author: Olivier, Gaelle, Mayo from pprint import pprint from ibllib.io import spikeglx from one.api import ONE one = ONE(base_url='https://openalyx.internationalbrainlab.org', silent=True) # Download a dataset of interest eid = one.search(subject='KS023', date_range='2019-12-10')[0] # Optionally list the raw ephys data for this session pprint([x for x in one.list_datasets(eid) if 'ephysData' in x]) files = one.load_object(eid, 'ephysData_g0_t0', attribute='nidq', collection='raw_ephys_data', download_only=True) # Get file path of interest efile = next(x for x in files if str(x).endswith('.cbin')) # Read the files and get the data # Enough to do analysis sr = spikeglx.Reader(efile) # Decompress the data
""" Get single probe label and directory, using the probes description dataset. """ # Author: Gaelle Chapuis, Miles Wells from one.api import ONE one = ONE() eid = 'da188f2c-553c-4e04-879b-c9ea2d1b9a93' # --- Get single probe directory filename either by # 1. getting probe description in alf # 2. using alyx rest end point # Option 1. prob_des = one.load_dataset(eid, 'probes.description.json') labels = [x['label'] for x in prob_des] # You can then use this label into dict, e.g. channels[label[0]] # -- Load single probe data with probe-level collection # List datsets for first probe collection = f'alf/{labels[0]}' datasets = one.list_datasets(eid, collection=collection)
def dlc_qc_plot(session_path, one=None): """ Creates DLC QC plot. Data is searched first locally, then on Alyx. Panels that lack required data are skipped. Required data to create all panels 'raw_video_data/_iblrig_bodyCamera.raw.mp4', 'raw_video_data/_iblrig_leftCamera.raw.mp4', 'raw_video_data/_iblrig_rightCamera.raw.mp4', 'alf/_ibl_bodyCamera.dlc.pqt', 'alf/_ibl_leftCamera.dlc.pqt', 'alf/_ibl_rightCamera.dlc.pqt', 'alf/_ibl_bodyCamera.times.npy', 'alf/_ibl_leftCamera.times.npy', 'alf/_ibl_rightCamera.times.npy', 'alf/_ibl_leftCamera.features.pqt', 'alf/_ibl_rightCamera.features.pqt', 'alf/rightROIMotionEnergy.position.npy', 'alf/leftROIMotionEnergy.position.npy', 'alf/bodyROIMotionEnergy.position.npy', 'alf/_ibl_trials.choice.npy', 'alf/_ibl_trials.feedbackType.npy', 'alf/_ibl_trials.feedback_times.npy', 'alf/_ibl_trials.stimOn_times.npy', 'alf/_ibl_wheel.position.npy', 'alf/_ibl_wheel.timestamps.npy', 'alf/licks.times.npy', :params session_path: Path to session data on disk :params one: ONE instance, if None is given, default ONE is instantiated :returns: Matplotlib figure """ one = one or ONE() # hack for running on cortexlab local server if one.alyx.base_url == 'https://alyx.cortexlab.net': one = ONE(base_url='https://alyx.internationalbrainlab.org') data = {} cams = ['left', 'right', 'body'] session_path = Path(session_path) # Load data for each camera for cam in cams: # Load a single frame for each video # Check if video data is available locally,if yes, load a single frame video_path = session_path.joinpath('raw_video_data', f'_iblrig_{cam}Camera.raw.mp4') if video_path.exists(): data[f'{cam}_frame'] = get_video_frame(video_path, frame_number=5 * 60 * SAMPLING[cam])[:, :, 0] # If not, try to stream a frame (try three times) else: try: video_url = url_from_eid(one.path2eid(session_path), one=one)[cam] for tries in range(3): try: data[f'{cam}_frame'] = get_video_frame( video_url, frame_number=5 * 60 * SAMPLING[cam])[:, :, 0] break except BaseException: if tries < 2: tries += 1 logger.info( f"Streaming {cam} video failed, retrying x{tries}" ) time.sleep(30) else: logger.warning( f"Could not load video frame for {cam} cam. Skipping trace on frame." ) data[f'{cam}_frame'] = None except KeyError: logger.warning( f"Could not load video frame for {cam} cam. Skipping trace on frame." ) data[f'{cam}_frame'] = None # Other camera associated data for feat in ['dlc', 'times', 'features', 'ROIMotionEnergy']: # Check locally first, then try to load from alyx, if nothing works, set to None if feat == 'features' and cam == 'body': # this doesn't exist for body cam continue local_file = list( session_path.joinpath('alf').glob(f'*{cam}Camera.{feat}*')) if len(local_file) > 0: data[f'{cam}_{feat}'] = alfio.load_file_content(local_file[0]) else: alyx_ds = [ ds for ds in one.list_datasets(one.path2eid(session_path)) if f'{cam}Camera.{feat}' in ds ] if len(alyx_ds) > 0: data[f'{cam}_{feat}'] = one.load_dataset( one.path2eid(session_path), alyx_ds[0]) else: logger.warning( f"Could not load _ibl_{cam}Camera.{feat} some plots have to be skipped." ) data[f'{cam}_{feat}'] = None # Sometimes there is a file but the object is empty, set to None if data[f'{cam}_{feat}'] is not None and len( data[f'{cam}_{feat}']) == 0: logger.warning( f"Object loaded from _ibl_{cam}Camera.{feat} is empty, some plots have to be skipped." ) data[f'{cam}_{feat}'] = None # If we have no frame and/or no DLC and/or no times for all cams, raise an error, something is really wrong assert any([data[f'{cam}_frame'] is not None for cam in cams]), "No camera data could be loaded, aborting." assert any([data[f'{cam}_dlc'] is not None for cam in cams]), "No DLC data could be loaded, aborting." assert any([data[f'{cam}_times'] is not None for cam in cams ]), "No camera times data could be loaded, aborting." # Load session level data for alf_object in ['trials', 'wheel', 'licks']: try: data[f'{alf_object}'] = alfio.load_object( session_path.joinpath('alf'), alf_object) # load locally continue except ALFObjectNotFound: pass try: data[f'{alf_object}'] = one.load_object( one.path2eid(session_path), alf_object) # then try from alyx except ALFObjectNotFound: logger.warning( f"Could not load {alf_object} object, some plots have to be skipped." ) data[f'{alf_object}'] = None # Simplify and clean up trials data if data['trials']: data['trials'] = pd.DataFrame({ k: data['trials'][k] for k in ['stimOn_times', 'feedback_times', 'choice', 'feedbackType'] }) # Discard nan events and too long trials data['trials'] = data['trials'].dropna() data['trials'] = data['trials'].drop( data['trials'][(data['trials']['feedback_times'] - data['trials']['stimOn_times']) > 10].index) # Make a list of panels, if inputs are missing, instead input a text to display panels = [] # Panel A, B, C: Trace on frame for cam in cams: if data[f'{cam}_frame'] is not None and data[f'{cam}_dlc'] is not None: panels.append((plot_trace_on_frame, { 'frame': data[f'{cam}_frame'], 'dlc_df': data[f'{cam}_dlc'], 'cam': cam })) else: panels.append( (None, f'Data missing\n{cam.capitalize()} cam trace on frame')) # If trials data is not there, we cannot plot any of the trial average plots, skip all remaining panels if data['trials'] is None: panels.extend([(None, 'No trial data,\ncannot compute trial avgs') for i in range(7)]) else: # Panel D: Motion energy camera_dict = { 'left': { 'motion_energy': data['left_ROIMotionEnergy'], 'times': data['left_times'] }, 'right': { 'motion_energy': data['right_ROIMotionEnergy'], 'times': data['right_times'] }, 'body': { 'motion_energy': data['body_ROIMotionEnergy'], 'times': data['body_times'] } } for cam in [ 'left', 'right', 'body' ]: # Remove cameras where we don't have motion energy AND camera times if camera_dict[cam]['motion_energy'] is None or camera_dict[cam][ 'times'] is None: _ = camera_dict.pop(cam) if len(camera_dict) > 0: panels.append((plot_motion_energy_hist, { 'camera_dict': camera_dict, 'trials_df': data['trials'] })) else: panels.append((None, 'Data missing\nMotion energy')) # Panel E: Wheel position if data['wheel']: panels.append((plot_wheel_position, { 'wheel_position': data['wheel'].position, 'wheel_time': data['wheel'].timestamps, 'trials_df': data['trials'] })) else: panels.append((None, 'Data missing\nWheel position')) # Panel F, G: Paw speed and nose speed # Try if all data is there for left cam first, otherwise right for cam in ['left', 'right']: fail = False if (data[f'{cam}_dlc'] is not None and data[f'{cam}_times'] is not None and len(data[f'{cam}_times']) >= len(data[f'{cam}_dlc'])): break fail = True if not fail: paw = 'r' if cam == 'left' else 'l' panels.append((plot_speed_hist, { 'dlc_df': data[f'{cam}_dlc'], 'cam_times': data[f'{cam}_times'], 'trials_df': data['trials'], 'feature': f'paw_{paw}', 'cam': cam })) panels.append((plot_speed_hist, { 'dlc_df': data[f'{cam}_dlc'], 'cam_times': data[f'{cam}_times'], 'trials_df': data['trials'], 'feature': 'nose_tip', 'legend': False, 'cam': cam })) else: panels.extend([(None, 'Data missing or corrupt\nSpeed histograms') for i in range(2)]) # Panel H and I: Lick plots if data['licks'] and data['licks'].times.shape[0] > 0: panels.append((plot_lick_hist, { 'lick_times': data['licks'].times, 'trials_df': data['trials'] })) panels.append((plot_lick_raster, { 'lick_times': data['licks'].times, 'trials_df': data['trials'] })) else: panels.extend([(None, 'Data missing\nLicks plots') for i in range(2)]) # Panel J: pupil plot # Try if all data is there for left cam first, otherwise right for cam in ['left', 'right']: fail = False if (data[f'{cam}_times'] is not None and data[f'{cam}_features'] is not None and len(data[f'{cam}_times']) >= len(data[f'{cam}_features']) and not np.all( np.isnan( data[f'{cam}_features'].pupilDiameter_smooth))): break fail = True if not fail: panels.append((plot_pupil_diameter_hist, { 'pupil_diameter': data[f'{cam}_features'].pupilDiameter_smooth, 'cam_times': data[f'{cam}_times'], 'trials_df': data['trials'], 'cam': cam })) else: panels.append((None, 'Data missing or corrupt\nPupil diameter')) # Plotting plt.rcParams.update({'font.size': 10}) fig = plt.figure(figsize=(17, 10)) for i, panel in enumerate(panels): ax = plt.subplot(2, 5, i + 1) ax.text(-0.1, 1.15, ascii_uppercase[i], transform=ax.transAxes, fontsize=16, fontweight='bold') # Check if there was in issue with inputs, if yes, print the respective text if panel[0] is None: ax.text(.5, .5, panel[1], color='r', fontweight='bold', fontsize=12, horizontalalignment='center', verticalalignment='center', transform=ax.transAxes) plt.axis('off') else: try: panel[0](**panel[1]) except BaseException: logger.error(f'Error in {panel[0].__name__}\n' + traceback.format_exc()) ax.text(.5, .5, f'Error while plotting\n{panel[0].__name__}', color='r', fontweight='bold', fontsize=12, horizontalalignment='center', verticalalignment='center', transform=ax.transAxes) plt.axis('off') plt.tight_layout(rect=[0, 0.03, 1, 0.95]) return fig