def load_trials(sess_path, one): """ Load trials data for session. First attempts to load from local session path, if this fails will attempt to download via ONE, if this also fails, will then attempt to re-extraxt locally :param sess_path: session path :param one: ONE instance :return: """ # try and load trials locally try: trials = alfio.load_object(sess_path.joinpath('alf'), 'trials') if 'probabilityLeft' not in trials.keys(): raise ALFObjectNotFound except ALFObjectNotFound: try: # attempt to download trials using ONE trials = one.load_object(one.path2eid(sess_path), 'trials') if 'probabilityLeft' not in trials.keys(): raise ALFObjectNotFound except Exception: try: task = get_trials_task(sess_path, one=one) task.run() trials = alfio.load_object(sess_path.joinpath('alf'), 'trials') if 'probabilityLeft' not in trials.keys(): raise ALFObjectNotFound except Exception: # TODO how can i make this more specific trials = None return trials
def test_size_outputs(self): # check the output dimensions from ibllib.io.extractors.bpod_trials import extract_all extract_all(self.training_ge5['path']) trials = alfio.load_object(self.training_ge5['path'] / 'alf', object='trials') self.assertTrue(alfio.check_dimensions(trials) == 0) extract_all(self.training_lt5['path']) trials = alfio.load_object(self.training_lt5['path'] / 'alf', object='trials') self.assertTrue(alfio.check_dimensions(trials) == 0) extract_all(self.biased_ge5['path']) trials = alfio.load_object(self.biased_ge5['path'] / 'alf', object='trials') self.assertTrue(alfio.check_dimensions(trials) == 0)
def load_data(self): self.spikes = alfio.load_object(self.probe_path, 'spikes') self.trials = alfio.load_object(self.alf_path, 'trials') self.clusters = alfio.load_object(self.probe_path, 'clusters') try: chan_locations = np.load( self.probe_path.joinpath('channels.locations.npy'), allow_pickle=True) self.clusters.locations = chan_locations[self.clusters.channels] except: print('WARNING: locations not available') self.prepare_data(self.spikes, self.clusters, self.trials)
def extract_sync(session_path, overwrite=False, ephys_files=None): """ Reads ephys binary file (s) and extract sync within the binary file folder Assumes ephys data is within a `raw_ephys_data` folder :param session_path: '/path/to/subject/yyyy-mm-dd/001' :param overwrite: Bool on re-extraction, forces overwrite instead of loading existing files :return: list of sync dictionaries """ session_path = Path(session_path) if not ephys_files: ephys_files = spikeglx.glob_ephys_files(session_path) syncs = [] outputs = [] for efi in ephys_files: bin_file = efi.get('ap', efi.get('nidq', None)) if not bin_file: continue alfname = dict(object='sync', namespace='spikeglx') if efi.label: alfname['extra'] = efi.label file_exists = alfio.exists(bin_file.parent, **alfname) if not overwrite and file_exists: _logger.warning(f'Skipping raw sync: SGLX sync found for probe {efi.label} !') sync = alfio.load_object(bin_file.parent, **alfname) out_files, _ = alfio._ls(bin_file.parent, **alfname) else: sr = spikeglx.Reader(bin_file) sync, out_files = _sync_to_alf(sr, bin_file.parent, save=True, parts=efi.label) outputs.extend(out_files) syncs.extend([sync]) return syncs, outputs
def _get_all_probes_sync(session_path, bin_exists=True): # round-up of all bin ephys files in the session, infer revision and get sync map ephys_files = spikeglx.glob_ephys_files(session_path, bin_exists=bin_exists) version = spikeglx.get_neuropixel_version_from_files(ephys_files) # attach the sync information to each binary file found for ef in ephys_files: ef['sync'] = alfio.load_object(ef.path, 'sync', namespace='spikeglx', short_keys=True) ef['sync_map'] = get_ibl_sync_map(ef, version) return ephys_files
def get_micro_manipulator_data(subject, one=None, force_extract=False): """ Looks for all ephys sessions for a given subject and get the probe micro-manipulator trajectories. If probes ALF object not on flat-iron, attempts to perform the extraction from meta-data and task settings file. """ if not one: one = ONE(cache_rest=None) eids, sessions = one.search(subject=subject, task_protocol='ephys', details=True) probes = alfio.AlfBunch({}) for ses in sessions: sess_path = Path(ses['local_path']) probe = None if not force_extract: probe = one.load_object(ses['url'], 'probes') if not probe: _logger.warning(f"Re-extraction probe info for {sess_path}") dtypes = ['_iblrig_taskSettings.raw', 'ephysData.raw.meta'] raw_files = one.load(ses['url'], dataset_types=dtypes, download_only=True) if all([rf is None for rf in raw_files]): _logger.warning( f"no raw settings files nor ephys data found for" f" {ses['local_path']}. Skip this session.") continue extract_probes(sess_path, bin_exists=False) probe = alfio.load_object(sess_path.joinpath('alf'), 'probes') one.load(ses['url'], dataset_types='channels.localCoordinates', download_only=True) # get for each insertion the sites local mapping: if not found assumes checkerboard pattern probe['sites_coordinates'] = [] for prb in probe.description: chfile = Path(ses['local_path']).joinpath( 'alf', prb['label'], 'channels.localCoordinates.npy') if chfile.exists(): probe['sites_coordinates'].append(np.load(chfile)) else: _logger.warning( f"no channel.localCoordinates found for {ses['local_path']}." f"Assumes checkerboard pattern") probe['sites_coordinates'].append(SITES_COORDINATES) # put the session information in there probe['session'] = [ses] * len(probe.description) probes = probes.append(probe) return probes
def load_data(self, download=False): """ Load wheel, trial and camera timestamp data :return: wheel, trials """ if download: self.data.wheel = self.one.load_object(self.eid, 'wheel') self.data.trials = self.one.load_object(self.eid, 'trials') cam = self.one.load(self.eid, ['camera.times'], dclass_output=True) self.data.camera_times = { vidio.label_from_path(url): ts for ts, url in zip(cam.data, cam.url) } else: alf_path = self.session_path / 'alf' self.data.wheel = alfio.load_object(alf_path, 'wheel', short_keys=True) self.data.trials = alfio.load_object(alf_path, 'trials') self.data.camera_times = { vidio.label_from_path(x): alfio.load_file_content(x) for x in alf_path.glob('*Camera.times*') } assert all(x is not None for x in self.data.values())
def load_data(self, download_data: bool = None) -> None: """Extract the data from data files Extracts all the required task data from the data files. Data keys: - camera_times (float array): camera frame timestamps extracted from frame headers - dlc_coords (dict): keys are the points traced by dlc, items are x-y coordinates of these points over time, those with likelihood <0.9 set to NaN :param download_data: if True, any missing raw data is downloaded via ONE. """ if download_data is not None: self.download_data = download_data if self.one and not self.one.offline: self._ensure_required_data() alf_path = self.session_path / 'alf' # Load times self.data['camera_times'] = alfio.load_object(alf_path, f'{self.side}Camera')['times'] # Load dlc traces dlc_df = alfio.load_object(alf_path, f'{self.side}Camera', namespace='ibl')['dlc'] targets = np.unique(['_'.join(col.split('_')[:-1]) for col in dlc_df.columns]) # Set values to nan if likelihood is too low dlc_coords = {} for t in targets: idx = dlc_df.loc[dlc_df[f'{t}_likelihood'] < 0.9].index dlc_df.loc[idx, [f'{t}_x', f'{t}_y']] = np.nan dlc_coords[t] = np.array((dlc_df[f'{t}_x'], dlc_df[f'{t}_y'])) self.data['dlc_coords'] = dlc_coords # load pupil diameters if self.side in ['left', 'right']: features = alfio.load_object(alf_path, f'{self.side}Camera', namespace='ibl')['features'] self.data['pupilDiameter_raw'] = features['pupilDiameter_raw'] self.data['pupilDiameter_smooth'] = features['pupilDiameter_smooth']
def _plot_rmsmap(outpath, typ, savefig=True): ''' TODO document this function ''' rmsmap = alfio.load_object(outpath, 'ephysQcTime' + typ.upper(), namespace='spikeglx') # hack to ensure a single key name if 'times.probe_00' in rmsmap.keys(): rmsmap['times'] = rmsmap.pop('times.probe_00') rmsmap['rms'] = rmsmap.pop('rms.probe_00') elif 'times.probe_01' in rmsmap.keys(): rmsmap['times'] = rmsmap.pop('times.probe_01') rmsmap['rms'] = rmsmap.pop('rms.probe_01') plt.figure(figsize=[12, 4.5]) axim = plt.axes([0.2, 0.1, 0.7, 0.8]) axrms = plt.axes([0.05, 0.1, 0.15, 0.8]) axcb = plt.axes([0.92, 0.1, 0.02, 0.8]) axrms.plot( np.median(rmsmap['rms'], axis=0)[:-1] * 1e6, np.arange(1, rmsmap['rms'].shape[1])) axrms.set_ylim(0, rmsmap['rms'].shape[1]) im = axim.imshow(20 * np.log10(rmsmap['rms'].T + 1e-15), aspect='auto', origin='lower', extent=[ rmsmap['times'][0], rmsmap['times'][-1], 0, rmsmap['rms'].shape[1] ]) axim.set_xlabel(r'Time (s)') axrms.set_ylabel(r'Channel Number') plt.colorbar(im, cax=axcb) if typ == 'ap': im.set_clim(-110, -90) axrms.set_xlim(100, 0) elif typ == 'lf': im.set_clim(-100, -60) axrms.set_xlim(500, 0) axim.set_xlim(0, 4000) axim.set_title(outpath) if savefig: plt.savefig(outpath / (typ + '_rms.png'), dpi=150)
def _run(self): assert self.pid output_files = [] if self.location != 'server': self.histology_status = self.get_histology_status() electrodes = self.get_channels('electrodeSites', f'alf/{self.pname}') # TODO need to figure out the clim range fig, axs = plt.subplots(1, 2, gridspec_kw={'width_ratios': [.95, .05]}, figsize=(16, 9)) ap = alfio.load_object( self.session_path.joinpath(f'raw_ephys_data/{self.pname}'), 'ephysTimeRmsAP', namespace='iblqc') _, _, _ = image_rms_plot(ap.rms, ap.timestamps, median_subtract=False, band='AP', ax=axs[0], fig_kwargs={'figsize': (8, 6)}, display=True, title=f"{self.pid_label}") set_axis_label_size(axs[0], cmap=True) if self.histology_status: plot_brain_regions(electrodes['atlas_id'], brain_regions=self.brain_regions, display=True, ax=axs[1], title=self.histology_status) set_axis_label_size(axs[1]) else: remove_axis_outline(axs[1]) save_path = Path(self.output_directory).joinpath("ap_rms.png") output_files.append(save_path) fig.savefig(save_path) plt.close(fig) return output_files
def get_sync_fronts(auxiliary_name): d = Bunch({'times': [], 'nsync': np.zeros(nprobes, )}) # auxiliary_name: frame2ttl or right_camera for ind, ephys_file in enumerate(ephys_files): sync = alfio.load_object(ephys_file.ap.parent, 'sync', namespace='spikeglx', short_keys=True) sync_map = get_ibl_sync_map(ephys_file, '3A') # exits if sync label not found for current probe if auxiliary_name not in sync_map: return isync = np.in1d(sync['channels'], np.array([sync_map[auxiliary_name]])) # only returns syncs if we get fronts for all probes if np.all(~isync): return d.nsync[ind] = len(sync.channels) d['times'].append(sync['times'][isync]) return d
def _plot_spectra(outpath, typ, savefig=True): ''' TODO document this function ''' spec = alfio.load_object(outpath, 'ephysQcFreq' + typ.upper(), namespace='spikeglx') # hack to ensure a single key name if 'power.probe_00' in spec.keys(): spec['power'] = spec.pop('power.probe_00') spec['freq'] = spec.pop('freq.probe_00') elif 'power.probe_01' in spec.keys(): spec['power'] = spec.pop('power.probe_01') spec['freq'] = spec.pop('freq.probe_01') # plot sns.set_style("whitegrid") plt.figure(figsize=[9, 4.5]) ax = plt.axes() ax.plot(spec['freq'], 20 * np.log10(spec['power'] + 1e-14), linewidth=0.5, color=[0.5, 0.5, 0.5]) ax.plot(spec['freq'], 20 * np.log10(np.median(spec['power'] + 1e-14, axis=1)), label='median') ax.set_xlabel(r'Frequency (Hz)') ax.set_ylabel(r'dB rel to $V^2.$Hz$^{-1}$') if typ == 'ap': ax.set_ylim([-275, -125]) elif typ == 'lf': ax.set_ylim([-260, -60]) ax.legend() ax.set_title(outpath) if savefig: plt.savefig(outpath / (typ + '_spec.png'), dpi=150) print('saved figure to %s' % (outpath / (typ + '_spec.png')))
def _qc_from_path(sess_path, display=True): WHEEL = False sess_path = Path(sess_path) temp_alf_folder = sess_path.joinpath('fpga_test', 'alf') temp_alf_folder.mkdir(parents=True, exist_ok=True) sync, chmap = ephys_fpga.get_main_probe_sync(sess_path, bin_exists=False) _ = ephys_fpga.extract_all(sess_path, output_path=temp_alf_folder, save=True) # check that the output is complete fpga_trials = ephys_fpga.extract_behaviour_sync(sync, chmap=chmap, display=display) # align with the bpod bpod2fpga = ephys_fpga.align_with_bpod(temp_alf_folder.parent) alf_trials = alfio.load_object(temp_alf_folder, 'trials') shutil.rmtree(temp_alf_folder) # do the QC qcs, qct = qc_fpga_task(fpga_trials, alf_trials) # do the wheel part if WHEEL: bpod_wheel = training_wheel.get_wheel_data(sess_path, save=False) fpga_wheel = ephys_fpga.extract_wheel_sync(sync, chmap=chmap, save=False) if display: import matplotlib.pyplot as plt t0 = max(np.min(bpod2fpga(bpod_wheel['re_ts'])), np.min(fpga_wheel['re_ts'])) dy = np.interp(t0, fpga_wheel['re_ts'], fpga_wheel['re_pos']) - np.interp( t0, bpod2fpga(bpod_wheel['re_ts']), bpod_wheel['re_pos']) fix, axes = plt.subplots(nrows=2, sharex='all', sharey='all') # axes[0].plot(t, pos), axes[0].title.set_text('Extracted') axes[0].plot(bpod2fpga(bpod_wheel['re_ts']), bpod_wheel['re_pos'] + dy) axes[0].plot(fpga_wheel['re_ts'], fpga_wheel['re_pos']) axes[0].title.set_text('FPGA') axes[1].plot(bpod2fpga(bpod_wheel['re_ts']), bpod_wheel['re_pos'] + dy) axes[1].title.set_text('Bpod') return alfio.dataframe({**fpga_trials, **alf_trials, **qct})
def _run(self): output_files = [] trials = alfio.load_object(self.session_path.joinpath('alf'), 'trials') title = '_'.join(list(self.session_path.parts[-3:])) fig, ax = training.plot_psychometric(trials, title=title, figsize=(8, 6)) set_axis_label_size(ax) save_path = Path( self.output_directory).joinpath("psychometric_curve.png") output_files.append(save_path) fig.savefig(save_path) plt.close(fig) fig, ax = training.plot_reaction_time(trials, title=title, figsize=(8, 6)) set_axis_label_size(ax) save_path = Path( self.output_directory).joinpath("chronometric_curve.png") output_files.append(save_path) fig.savefig(save_path) plt.close(fig) fig, ax = training.plot_reaction_time_over_trials(trials, title=title, figsize=(8, 6)) set_axis_label_size(ax) save_path = Path( self.output_directory).joinpath("reaction_time_with_trials.png") output_files.append(save_path) fig.savefig(save_path) plt.close(fig) return output_files
def load_data(self, download_data: bool = None, extract_times: bool = False, load_video: bool = True) -> None: """Extract the data from raw data files Extracts all the required task data from the raw data files. Data keys: - count (int array): the sequential frame number (n, n+1, n+2...) - pin_state (): the camera GPIO pin; records the audio TTLs; should be one per frame - audio (float array): timestamps of audio TTL fronts - fpga_times (float array): timestamps of camera TTLs recorded by FPGA - timestamps (float array): extracted video timestamps (the camera.times ALF) - bonsai_times (datetime array): system timestamps of video PC; should be one per frame - camera_times (float array): camera frame timestamps extracted from frame headers - wheel (Bunch): rotary encoder timestamps, position and period used for wheel motion - video (Bunch): video meta data, including dimensions and FPS - frame_samples (h x w x n array): array of evenly sampled frames (1 colour channel) :param download_data: if True, any missing raw data is downloaded via ONE. Missing data will raise an AssertionError :param extract_times: if True, the camera.times are re-extracted from the raw data :param load_video: if True, calls the load_video_data method """ assert self.session_path, 'no session path set' if download_data is not None: self.download_data = download_data if self.download_data and self.eid and self.one and not self.one.offline: self.ensure_required_data() _log.info('Gathering data for QC') # Get frame count and pin state self.data['count'], self.data['pin_state'] = \ raw.load_embedded_frame_data(self.session_path, self.label, raw=True) # Load the audio and raw FPGA times if self.type == 'ephys': sync, chmap = ephys_fpga.get_main_probe_sync(self.session_path) audio_ttls = ephys_fpga.get_sync_fronts(sync, chmap['audio']) self.data['audio'] = audio_ttls['times'] # Get rises # Load raw FPGA times cam_ts = extract_camera_sync(sync, chmap) self.data['fpga_times'] = cam_ts[self.label] else: bpod_data = raw.load_data(self.session_path) _, audio_ttls = raw.load_bpod_fronts(self.session_path, bpod_data) self.data['audio'] = audio_ttls['times'] # Load extracted frame times alf_path = self.session_path / 'alf' try: assert not extract_times self.data['timestamps'] = alfio.load_object( alf_path, f'{self.label}Camera', short_keys=True)['times'] except AssertionError: # Re-extract kwargs = dict(video_path=self.video_path, labels=self.label) if self.type == 'ephys': kwargs = {**kwargs, 'sync': sync, 'chmap': chmap} # noqa outputs, _ = extract_all(self.session_path, self.type, save=False, **kwargs) self.data['timestamps'] = outputs[ f'{self.label}_camera_timestamps'] except ALFObjectNotFound: _log.warning('no camera.times ALF found for session') # Get audio and wheel data wheel_keys = ('timestamps', 'position') try: self.data['wheel'] = alfio.load_object(alf_path, 'wheel', short_keys=True) except ALFObjectNotFound: # Extract from raw data if self.type == 'ephys': wheel_data = ephys_fpga.extract_wheel_sync(sync, chmap) else: wheel_data = training_wheel.get_wheel_position( self.session_path) self.data['wheel'] = Bunch(zip(wheel_keys, wheel_data)) # Find short period of wheel motion for motion correlation. if data_for_keys( wheel_keys, self.data['wheel']) and self.data['timestamps'] is not None: self.data['wheel'].period = self.get_active_wheel_period( self.data['wheel']) # Load Bonsai frame timestamps try: ssv_times = raw.load_camera_ssv_times(self.session_path, self.label) self.data['bonsai_times'], self.data['camera_times'] = ssv_times except AssertionError: _log.warning('No Bonsai video timestamps file found') # Gather information from video file if load_video: _log.info('Inspecting video file...') self.load_video_data()
def dlc_qc_plot(session_path, one=None): """ Creates DLC QC plot. Data is searched first locally, then on Alyx. Panels that lack required data are skipped. Required data to create all panels 'raw_video_data/_iblrig_bodyCamera.raw.mp4', 'raw_video_data/_iblrig_leftCamera.raw.mp4', 'raw_video_data/_iblrig_rightCamera.raw.mp4', 'alf/_ibl_bodyCamera.dlc.pqt', 'alf/_ibl_leftCamera.dlc.pqt', 'alf/_ibl_rightCamera.dlc.pqt', 'alf/_ibl_bodyCamera.times.npy', 'alf/_ibl_leftCamera.times.npy', 'alf/_ibl_rightCamera.times.npy', 'alf/_ibl_leftCamera.features.pqt', 'alf/_ibl_rightCamera.features.pqt', 'alf/rightROIMotionEnergy.position.npy', 'alf/leftROIMotionEnergy.position.npy', 'alf/bodyROIMotionEnergy.position.npy', 'alf/_ibl_trials.choice.npy', 'alf/_ibl_trials.feedbackType.npy', 'alf/_ibl_trials.feedback_times.npy', 'alf/_ibl_trials.stimOn_times.npy', 'alf/_ibl_wheel.position.npy', 'alf/_ibl_wheel.timestamps.npy', 'alf/licks.times.npy', :params session_path: Path to session data on disk :params one: ONE instance, if None is given, default ONE is instantiated :returns: Matplotlib figure """ one = one or ONE() # hack for running on cortexlab local server if one.alyx.base_url == 'https://alyx.cortexlab.net': one = ONE(base_url='https://alyx.internationalbrainlab.org') data = {} cams = ['left', 'right', 'body'] session_path = Path(session_path) # Load data for each camera for cam in cams: # Load a single frame for each video # Check if video data is available locally,if yes, load a single frame video_path = session_path.joinpath('raw_video_data', f'_iblrig_{cam}Camera.raw.mp4') if video_path.exists(): data[f'{cam}_frame'] = get_video_frame(video_path, frame_number=5 * 60 * SAMPLING[cam])[:, :, 0] # If not, try to stream a frame (try three times) else: try: video_url = url_from_eid(one.path2eid(session_path), one=one)[cam] for tries in range(3): try: data[f'{cam}_frame'] = get_video_frame( video_url, frame_number=5 * 60 * SAMPLING[cam])[:, :, 0] break except BaseException: if tries < 2: tries += 1 logger.info( f"Streaming {cam} video failed, retrying x{tries}" ) time.sleep(30) else: logger.warning( f"Could not load video frame for {cam} cam. Skipping trace on frame." ) data[f'{cam}_frame'] = None except KeyError: logger.warning( f"Could not load video frame for {cam} cam. Skipping trace on frame." ) data[f'{cam}_frame'] = None # Other camera associated data for feat in ['dlc', 'times', 'features', 'ROIMotionEnergy']: # Check locally first, then try to load from alyx, if nothing works, set to None if feat == 'features' and cam == 'body': # this doesn't exist for body cam continue local_file = list( session_path.joinpath('alf').glob(f'*{cam}Camera.{feat}*')) if len(local_file) > 0: data[f'{cam}_{feat}'] = alfio.load_file_content(local_file[0]) else: alyx_ds = [ ds for ds in one.list_datasets(one.path2eid(session_path)) if f'{cam}Camera.{feat}' in ds ] if len(alyx_ds) > 0: data[f'{cam}_{feat}'] = one.load_dataset( one.path2eid(session_path), alyx_ds[0]) else: logger.warning( f"Could not load _ibl_{cam}Camera.{feat} some plots have to be skipped." ) data[f'{cam}_{feat}'] = None # Sometimes there is a file but the object is empty, set to None if data[f'{cam}_{feat}'] is not None and len( data[f'{cam}_{feat}']) == 0: logger.warning( f"Object loaded from _ibl_{cam}Camera.{feat} is empty, some plots have to be skipped." ) data[f'{cam}_{feat}'] = None # If we have no frame and/or no DLC and/or no times for all cams, raise an error, something is really wrong assert any([data[f'{cam}_frame'] is not None for cam in cams]), "No camera data could be loaded, aborting." assert any([data[f'{cam}_dlc'] is not None for cam in cams]), "No DLC data could be loaded, aborting." assert any([data[f'{cam}_times'] is not None for cam in cams ]), "No camera times data could be loaded, aborting." # Load session level data for alf_object in ['trials', 'wheel', 'licks']: try: data[f'{alf_object}'] = alfio.load_object( session_path.joinpath('alf'), alf_object) # load locally continue except ALFObjectNotFound: pass try: data[f'{alf_object}'] = one.load_object( one.path2eid(session_path), alf_object) # then try from alyx except ALFObjectNotFound: logger.warning( f"Could not load {alf_object} object, some plots have to be skipped." ) data[f'{alf_object}'] = None # Simplify and clean up trials data if data['trials']: data['trials'] = pd.DataFrame({ k: data['trials'][k] for k in ['stimOn_times', 'feedback_times', 'choice', 'feedbackType'] }) # Discard nan events and too long trials data['trials'] = data['trials'].dropna() data['trials'] = data['trials'].drop( data['trials'][(data['trials']['feedback_times'] - data['trials']['stimOn_times']) > 10].index) # Make a list of panels, if inputs are missing, instead input a text to display panels = [] # Panel A, B, C: Trace on frame for cam in cams: if data[f'{cam}_frame'] is not None and data[f'{cam}_dlc'] is not None: panels.append((plot_trace_on_frame, { 'frame': data[f'{cam}_frame'], 'dlc_df': data[f'{cam}_dlc'], 'cam': cam })) else: panels.append( (None, f'Data missing\n{cam.capitalize()} cam trace on frame')) # If trials data is not there, we cannot plot any of the trial average plots, skip all remaining panels if data['trials'] is None: panels.extend([(None, 'No trial data,\ncannot compute trial avgs') for i in range(7)]) else: # Panel D: Motion energy camera_dict = { 'left': { 'motion_energy': data['left_ROIMotionEnergy'], 'times': data['left_times'] }, 'right': { 'motion_energy': data['right_ROIMotionEnergy'], 'times': data['right_times'] }, 'body': { 'motion_energy': data['body_ROIMotionEnergy'], 'times': data['body_times'] } } for cam in [ 'left', 'right', 'body' ]: # Remove cameras where we don't have motion energy AND camera times if camera_dict[cam]['motion_energy'] is None or camera_dict[cam][ 'times'] is None: _ = camera_dict.pop(cam) if len(camera_dict) > 0: panels.append((plot_motion_energy_hist, { 'camera_dict': camera_dict, 'trials_df': data['trials'] })) else: panels.append((None, 'Data missing\nMotion energy')) # Panel E: Wheel position if data['wheel']: panels.append((plot_wheel_position, { 'wheel_position': data['wheel'].position, 'wheel_time': data['wheel'].timestamps, 'trials_df': data['trials'] })) else: panels.append((None, 'Data missing\nWheel position')) # Panel F, G: Paw speed and nose speed # Try if all data is there for left cam first, otherwise right for cam in ['left', 'right']: fail = False if (data[f'{cam}_dlc'] is not None and data[f'{cam}_times'] is not None and len(data[f'{cam}_times']) >= len(data[f'{cam}_dlc'])): break fail = True if not fail: paw = 'r' if cam == 'left' else 'l' panels.append((plot_speed_hist, { 'dlc_df': data[f'{cam}_dlc'], 'cam_times': data[f'{cam}_times'], 'trials_df': data['trials'], 'feature': f'paw_{paw}', 'cam': cam })) panels.append((plot_speed_hist, { 'dlc_df': data[f'{cam}_dlc'], 'cam_times': data[f'{cam}_times'], 'trials_df': data['trials'], 'feature': 'nose_tip', 'legend': False, 'cam': cam })) else: panels.extend([(None, 'Data missing or corrupt\nSpeed histograms') for i in range(2)]) # Panel H and I: Lick plots if data['licks'] and data['licks'].times.shape[0] > 0: panels.append((plot_lick_hist, { 'lick_times': data['licks'].times, 'trials_df': data['trials'] })) panels.append((plot_lick_raster, { 'lick_times': data['licks'].times, 'trials_df': data['trials'] })) else: panels.extend([(None, 'Data missing\nLicks plots') for i in range(2)]) # Panel J: pupil plot # Try if all data is there for left cam first, otherwise right for cam in ['left', 'right']: fail = False if (data[f'{cam}_times'] is not None and data[f'{cam}_features'] is not None and len(data[f'{cam}_times']) >= len(data[f'{cam}_features']) and not np.all( np.isnan( data[f'{cam}_features'].pupilDiameter_smooth))): break fail = True if not fail: panels.append((plot_pupil_diameter_hist, { 'pupil_diameter': data[f'{cam}_features'].pupilDiameter_smooth, 'cam_times': data[f'{cam}_times'], 'trials_df': data['trials'], 'cam': cam })) else: panels.append((None, 'Data missing or corrupt\nPupil diameter')) # Plotting plt.rcParams.update({'font.size': 10}) fig = plt.figure(figsize=(17, 10)) for i, panel in enumerate(panels): ax = plt.subplot(2, 5, i + 1) ax.text(-0.1, 1.15, ascii_uppercase[i], transform=ax.transAxes, fontsize=16, fontweight='bold') # Check if there was in issue with inputs, if yes, print the respective text if panel[0] is None: ax.text(.5, .5, panel[1], color='r', fontweight='bold', fontsize=12, horizontalalignment='center', verticalalignment='center', transform=ax.transAxes) plt.axis('off') else: try: panel[0](**panel[1]) except BaseException: logger.error(f'Error in {panel[0].__name__}\n' + traceback.format_exc()) ax.text(.5, .5, f'Error while plotting\n{panel[0].__name__}', color='r', fontweight='bold', fontsize=12, horizontalalignment='center', verticalalignment='center', transform=ax.transAxes) plt.axis('off') plt.tight_layout(rect=[0, 0.03, 1, 0.95]) return fig
def _run(self, collection=None): """runs for initiated PID, streams data, destripe and check bad channels""" def plot_driftmap(self, spikes, clusters, channels, collection, ylim=(0, 3840)): fig, axs = plt.subplots(1, 2, gridspec_kw={'width_ratios': [.95, .05]}, figsize=(16, 9)) driftmap(spikes.times, spikes.depths, t_bin=0.007, d_bin=10, vmax=0.5, ax=axs[0]) title_str = f"{self.pid_label}, {collection}, {self.pid} \n " \ f"{spikes.clusters.size:_} spikes, {clusters.depths.size:_} clusters" axs[0].set(ylim=ylim, title=title_str) run_label = str(Path(collection).relative_to(f'alf/{self.pname}')) run_label = "ks2matlab" if run_label == '.' else run_label outfile = self.output_directory.joinpath( f"spike_sorting_raster_{run_label}.png") set_axis_label_size(axs[0]) if self.histology_status: plot_brain_regions(channels['atlas_id'], channel_depths=channels['axial_um'], brain_regions=self.brain_regions, display=True, ax=axs[1], title=self.histology_status) axs[1].set(ylim=ylim) set_axis_label_size(axs[1]) else: remove_axis_outline(axs[1]) fig.savefig(outfile) plt.close(fig) return outfile, fig, axs output_files = [] if self.location == 'server': assert collection spikes = alfio.load_object(self.session_path.joinpath(collection), 'spikes') clusters = alfio.load_object( self.session_path.joinpath(collection), 'clusters') channels = alfio.load_object( self.session_path.joinpath(collection), 'channels') out, fig, axs = plot_driftmap(self, spikes, clusters, channels, collection) output_files.append(out) else: self.histology_status = self.get_histology_status() all_here, output_files = self.assert_expected(self.output_files, silent=True) spike_sorting_runs = self.one.list_datasets( self.eid, filename='spikes.times.npy', collection=f'alf/{self.pname}*') if all_here and len(output_files) == len(spike_sorting_runs): return output_files logger.info(self.output_directory) for run in spike_sorting_runs: collection = str(Path(run).parent.as_posix()) spikes, clusters, channels = load_spike_sorting_fast( eid=self.eid, probe=self.pname, one=self.one, nested=False, collection=collection, dataset_types=['spikes.depths'], brain_regions=self.brain_regions) if 'atlas_id' not in channels.keys(): channels = self.get_channels('channels', collection) out, fig, axs = plot_driftmap(self, spikes, clusters, channels, collection) output_files.append(out) return output_files
def version3B(ses_path, display=True, type=None, tol=2.5): """ From a session path with _spikeglx_sync arrays extraccted, locate ephys files for 3A and outputs one sync.timestamps.probeN.npy file per acquired probe. By convention the reference probe is the one with the most synchronisation pulses. Assumes the _spikeglx_sync datasets are already extracted from binary data :param ses_path: :param type: linear, exact or smooth :return: None """ DEFAULT_TYPE = 'smooth' ephys_files = spikeglx.glob_ephys_files(ses_path, ext='meta', bin_exists=False) for ef in ephys_files: ef['sync'] = alfio.load_object(ef.path, 'sync', namespace='spikeglx', short_keys=True) ef['sync_map'] = get_ibl_sync_map(ef, '3B') nidq_file = [ef for ef in ephys_files if ef.get('nidq')] ephys_files = [ef for ef in ephys_files if not ef.get('nidq')] # should have at least 2 probes and only one nidq assert (len(nidq_file) == 1) nidq_file = nidq_file[0] sync_nidq = get_sync_fronts(nidq_file.sync, nidq_file.sync_map['imec_sync']) qc_all = True out_files = [] for ef in ephys_files: sync_probe = get_sync_fronts(ef.sync, ef.sync_map['imec_sync']) sr = _get_sr(ef) try: # we say that the number of pulses should be within 10 % assert (np.isclose(sync_nidq.times.size, sync_probe.times.size, rtol=0.1)) except AssertionError: raise Neuropixel3BSyncFrontsNonMatching(f"{ses_path}") # Find the indexes in case the sizes don't match if sync_nidq.times.size != sync_probe.times.size: _logger.warning( f'Sync mismatch by {np.abs(sync_nidq.times.size - sync_probe.times.size)} ' f'NIDQ sync times: {sync_nidq.times.size}, Probe sync times {sync_probe.times.size}' ) sync_idx = np.min([sync_nidq.times.size, sync_probe.times.size]) # if the qc of the diff finds anomalies, do not attempt to smooth the interp function qcdiff = _check_diff_3b(sync_probe) if not qcdiff: qc_all = False type_probe = type or 'exact' else: type_probe = type or DEFAULT_TYPE timestamps, qc = sync_probe_front_times(sync_probe.times[:sync_idx], sync_nidq.times[:sync_idx], sr, display=display, type=type_probe, tol=tol) qc_all &= qc out_files.extend(_save_timestamps_npy(ef, timestamps, sr)) return qc_all, out_files