def load_data(self, download_data: bool = None) -> None: """Extract the data from data files Extracts all the required task data from the data files. Data keys: - camera_times (float array): camera frame timestamps extracted from frame headers - dlc_coords (dict): keys are the points traced by dlc, items are x-y coordinates of these points over time, those with likelihood <0.9 set to NaN :param download_data: if True, any missing raw data is downloaded via ONE. """ if download_data is not None: self.download_data = download_data if self.one and not isinstance(self.one, OneOffline): self._ensure_required_data() _log.info('Gathering data for QC') alf_path = self.session_path / 'alf' # Load times self.data['camera_times'] = alfio.load_object( alf_path, f'{self.side}Camera')['times'] # Load dlc traces dlc_df = alfio.load_object(alf_path, f'{self.side}Camera', namespace='ibl')['dlc'] targets = np.unique( ['_'.join(col.split('_')[:-1]) for col in dlc_df.columns]) # Set values to nan if likelyhood is too low dlc_coords = {} for t in targets: idx = dlc_df.loc[dlc_df[f'{t}_likelihood'] < 0.9].index dlc_df.loc[idx, [f'{t}_x', f'{t}_y']] = np.nan dlc_coords[t] = np.array((dlc_df[f'{t}_x'], dlc_df[f'{t}_y'])) self.data['dlc_coords'] = dlc_coords
def test_wheel_motion(self, mock_cv): side = 'left' period = np.array([1730.3513333, 1734.1743333]) mock_cv().read.side_effect = self.side_effect() aln = MotionAlignment(self.dummy_id, one=self.one) aln.session_path = self.data_path / 'camera' / 'SWC_054' / '2020-10-07' / '001' cam = alfio.load_object(aln.session_path / 'alf', f'{side}Camera') aln.data.camera_times = {side: cam['times']} aln.video_paths = { side: aln.session_path / 'raw_video_data' / f'_iblrig_{side}Camera.raw.mp4' } aln.data.wheel = alfio.load_object(aln.session_path / 'alf', 'wheel') # Test value error when invalid period given with self.assertRaises(ValueError): aln.align_motion(period=[5000, 5000.01], side=side) dt_i, c, df = aln.align_motion(period=period, side=side) expected = np.array([0.90278801, 0.68067675, 0.73734772, 0.82648895, 0.80950881, 0.88054471, 0.84264046, 0.302118, 0.94302567, 0.86188695]) np.testing.assert_array_almost_equal(expected, df[:10]) self.assertEqual(dt_i, 0) self.assertEqual(round(c, 5), 19.48842) # Test saving alignment video with tempfile.TemporaryDirectory() as tdir: aln.plot_alignment(save=tdir) vid = next(Path(tdir).glob('*.mp4')) self.assertEqual(vid.name, '2018-07-13_1_flowers_l.mp4') self.assertEqual(round(vid.stat().st_size / 1e5), 18)
def test_qc_extract(self): # extract a short lf signal RMS for fbin in Path(self.init_folder).rglob('*.lf.bin'): ephysqc.extract_rmsmap(fbin, out_folder=self.alf_folder) rmsmap_lf = alf.load_object(self.alf_folder, '_ibl_ephysRmsMap_lf') spec_lf = alf.load_object(self.alf_folder, '_ibl_ephysSpectra_lf') ntimes = rmsmap_lf['times'].shape[0] nchannels = rmsmap_lf['rms'].shape[1] nfreqs = spec_lf['frequencies'].shape[0] # makes sure the dimensions are consistend self.assertTrue(rmsmap_lf['rms'].shape == (ntimes, nchannels)) self.assertTrue(spec_lf['power'].shape == (nfreqs, nchannels))
def test_training(self): with tempfile.TemporaryDirectory() as tdir: subjects_path = Path(tdir).joinpath('Subjects', *self.training_folder.parts[-3:]) session_path = shutil.copytree(self.training_folder, subjects_path) # task running part - there are no videos, should exit gracefully job = TrainingVideoCompress(session_path, one=self.one) with self.assertLogs('ibllib', level='INFO'): job.run() self.assertEqual(job.status, 0) self.assertTrue('skipping' in job.log) # Now mock the video data so that extraction and QC succeed video_path = session_path.joinpath('raw_video_data') video_path.mkdir(parents=True) video_path.joinpath('_iblrig_leftCamera.raw.mp4').touch() with mock.patch('ibllib.io.extractors.camera.cv2.VideoCapture') as mock_vc, \ mock.patch('ibllib.pipes.training_preprocessing.CameraQC') as mock_qc: def side_effect(): return True, np.random.randint(0, 255, size=(1024, 1280, 3)) mock_vc().read.side_effect = side_effect length = 68453 mock_vc().get.return_value = length job.run() self.assertEqual(job.status, 0) self.assertEqual(mock_qc.call_args.args, (session_path, 'left')) mock_qc().run.assert_called_once_with(update=True) self.assertTrue(len(job.outputs) > 0) # check the file objects ts = alfio.load_object(session_path / 'alf', 'leftCamera') self.assertTrue(ts['times'].size == length)
def load_object(self, eid: Union[str, Path, UUID], obj: str, collection: Optional[str] = 'alf', download_only: bool = False, **kwargs) -> Union[alfio.AlfBunch, List[Path]]: """ Load all attributes of an ALF object from a Session ID and an object name. :param eid: Experiment session identifier; may be a UUID, URL, experiment reference string details dict or Path :param obj: The ALF object to load. Supports asterisks as wildcards. :param collection: The collection to which the object belongs, e.g. 'alf/probe01'. Supports asterisks as wildcards. :param download_only: When true the data are downloaded and the file paths are returned :param kwargs: Optional filters for the ALF objects, including namespace and timescale :return: An ALF bunch or if download_only is True, a list of Paths objects Examples: load_object(eid, '*moves') load_object(eid, 'trials') load_object(eid, 'spikes', collection='*probe01') """ # Filter server-side by collection and dataset name search_str = 'name__regex,' + obj.replace('*', '.*') if collection and collection != 'all': search_str += ',collection__regex,' + collection.replace('*', '.*') results = self.alyx.rest('datasets', 'list', session=eid, django=search_str) pattern = re.compile(fnmatch.translate(obj)) # Further refine by matching object part of ALF datasets def match(r): return is_valid(r['name']) and pattern.match(alf_parts(r['name'])[1]) # Get filenames of returned ALF files returned_obj = {alf_parts(x['name'])[1] for x in results if match(x)} # Validate result before loading if len(returned_obj) > 1: raise ALFMultipleObjectsFound('The following matching objects were found: ' + ', '.join(returned_obj)) elif len(returned_obj) == 0: raise ALFObjectNotFound(f'ALF object "{obj}" not found on Alyx') collection_set = {x['collection'] for x in results if match(x)} if len(collection_set) > 1: raise ALFMultipleCollectionsFound('Matching object belongs to multiple collections:' + ', '.join(collection_set)) # Download and optionally load the datasets out_files = self.download_datasets(x for x in results if match(x)) assert not any(x is None for x in out_files), 'failed to download object' if download_only: return out_files else: return alfio.load_object(out_files[0].parent, obj, **kwargs)
def load_data(self, download=False): """ Load wheel, trial and camera timestamp data :return: wheel, trials """ if download: self.data.wheel = self.one.load_object(self.eid, 'wheel') self.data.trials = self.one.load_object(self.eid, 'trials') cam = self.one.load(self.eid, ['camera.times'], dclass_output=True) self.data.camera_times = { vidio.label_from_path(url): ts for ts, url in zip(cam.data, cam.url) } else: alf_path = self.session_path / 'alf' self.data.wheel = alfio.load_object(alf_path, 'wheel') self.data.trials = alfio.load_object(alf_path, 'trials') self.data.camera_times = { vidio.label_from_path(x): alfio.load_file_content(x) for x in alf_path.glob('*Camera.times*') } assert all(x is not None for x in self.data.values())
def test_ephys(self, mock_qc): # task running part job = EphysVideoCompress(self.ephys_folder, one=self.one) job.run() self.assertEqual(job.status, 0) self.assertEqual(len(mock_qc.call_args_list), 3) # Once per camera labels = ('left', 'right', 'body') self.assertCountEqual(labels, [arg.args[-1]for arg in mock_qc.call_args_list]) [self.assertEqual(call[0][0], self.ephys_folder) for call in mock_qc.call_args_list] mock_qc().run.assert_called_with(update=True) self.assertEqual(len(job.outputs), 6) # check the file objects for label in labels: ts = alfio.load_object(self.ephys_folder / 'alf', f'{label}Camera') self.assertTrue(ts['times'].size > 0)
# Get paths ses_nr = listdir( join(DATA_PATH, sessions.loc[i, 'lab'], 'Subjects', sessions.loc[i, 'subject'], sessions.loc[i, 'date']))[0] session_path = join(DATA_PATH, sessions.loc[i, 'lab'], 'Subjects', sessions.loc[i, 'subject'], sessions.loc[i, 'date'], ses_nr) alf_path = join(DATA_PATH, sessions.loc[i, 'lab'], 'Subjects', sessions.loc[i, 'subject'], sessions.loc[i, 'date'], ses_nr, 'alf') probe_path = join(DATA_PATH, sessions.loc[i, 'lab'], 'Subjects', sessions.loc[i, 'subject'], sessions.loc[i, 'date'], ses_nr, 'alf', 'probe%s' % sessions.loc[i, 'probe']) # Load in data spikes = ioalf.load_object(probe_path, 'spikes') clusters = ioalf.load_object(probe_path, 'clusters') trials = ioalf.load_object(alf_path, '_ibl_trials') # Only use single units spikes.times = spikes.times[np.isin( spikes.clusters, clusters.metrics.cluster_id[clusters.metrics.ks2_label == 'good'])] spikes.clusters = spikes.clusters[np.isin( spikes.clusters, clusters.metrics.cluster_id[clusters.metrics.ks2_label == 'good'])] # Calculate whether neuron discriminates blocks trial_times = trials.stimOn_times[((trials.probabilityLeft > 0.55) | (trials.probabilityLeft < 0.45))] trial_blocks = (trials.probabilityLeft[((
from oneibl.one import ONE import alf.io as ioalf import ibllib.plots as iblplt from brainbox.processing import bincount2D T_BIN = 0.01 # get the data from flatiron and the current folder one = ONE() eid = one.search(subject='ZM_1150', date='2019-05-07', number=1) D = one.load(eid[0], clobber=False, download_only=True) session_path = Path(D.local_path[0]).parent # load objects spikes = ioalf.load_object(session_path, 'spikes') clusters = ioalf.load_object(session_path, 'clusters') channels = ioalf.load_object(session_path, 'channels') trials = ioalf.load_object(session_path, '_ibl_trials') # compute raster map as a function of cluster number R, times, clusters = bincount2D(spikes['times'], spikes['clusters'], T_BIN) # plot raster map plt.imshow(R, aspect='auto', cmap='binary', vmax=T_BIN / 0.001 / 4, extent=np.r_[times[[0, -1]], clusters[[0, -1]]], origin='lower') # plot trial start and reward time
PCA_DIMS = 20 CCA_DIMS = PCA_DIMS N_SPLITS = 5 RNG_SEED = 0 # get the data from flatiron subject = 'KS005' date = '2019-08-30' number = 1 one = ONE() eid = one.search(subject=subject, date=date, number=number) D = one.load(eid[0], download_only=True) session_path = Path(D.local_path[0]).parent spikes = ioalf.load_object(session_path, 'spikes') clusters = ioalf.load_object(session_path, 'clusters') # channels = ioalf.load_object(session_path, 'channels') trials = ioalf.load_object(session_path, 'trials') # bin spikes and get trial IDs associated with them binned_spikes, binned_trialIDs, _ = bin_spikes_trials(spikes, trials, bin_size=0.01) # define areas brain_areas = np.unique(clusters.brainAcronyms) brain_areas = brain_areas[1:4] # [take subset for testing] # split data by brain area # (bin_spikes_trials does not return info for innactive clusters)
def gen_metrics_labels(eid, probe_name): one = ONE() ses_path = one.path_from_eid(eid) alf_probe_dir = os.path.join(ses_path, 'alf', probe_name) ks_dir = alf_probe_dir spks_b = aio.load_object(alf_probe_dir, 'spikes') units_b = bb.processing.get_units_bunch(spks_b) units = list(units_b.amps.keys()) lengths_samples = [len(v) for k, v in units_b.samples.items()] units_nonzeros = [i for i, d in enumerate(lengths_samples) if d > 0] n_units = len( units_nonzeros) #only compute metrics for units with no samples #for cases where raw data is available locally: ephys_file_dir = os.path.join(ses_path, 'raw_ephys_data', probe_name) ephys_file = os.path.join(ses_path, 'raw_ephys_data', probe_name, '_iblrig_ephysData.raw_g0_t0.imec.ap.cbin') #create params.py file params_file = os.path.join(ks_dir, 'params.py') if os.path.exists(ephys_file) and not os.path.exists(params_file): f = open(params_file, "w+") f.write('dat_path = ' + 'r"' + ephys_file + '"\n' + '''n_channels_dat = 385 dtype = 'int16' offset = 0 sample_rate = 30000 hp_filtered = False uidx=0''') f.close() # Initialize metrics cum_amp_drift = np.full((n_units, ), np.nan) cum_depth_drift = np.full((n_units, ), np.nan) cv_amp = np.full((n_units, ), np.nan) cv_fr = np.full((n_units, ), np.nan) frac_isi_viol = np.full((n_units, ), np.nan) frac_missing_spks = np.full((n_units, ), np.nan) fp_estimate = np.full((n_units, ), np.nan) presence_ratio = np.full((n_units, ), np.nan) pres_ratio_std = np.full((n_units, ), np.nan) ptp_sigma = np.full((n_units, ), np.nan) units_missing_metrics = set() label = np.empty([len(units)]) RefPViol = np.empty([len(units)]) NoiseCutoff = np.empty([len(units)]) MeanAmpTrue = np.empty([len(units)]) for idx, unit in enumerate(units_nonzeros): if unit == units_nonzeros[0]: t0 = time.perf_counter() # used for computation time estimate print('computing metrics for unit ' + str(unit) + '...') #load relevant data for unit ts = units_b['times'][str(unit)] amps = units_b['amps'][str(unit)] samples = units_b['samples'][str(unit)] depths = units_b['depths'][str(unit)] RefPViol[idx] = FP_RP(ts) NoiseCutoff[idx] = noise_cutoff(amps, quartile_length=.25) #create 'label' based on RPviol,NoiseCutoff, and MeanAmp if len( samples > 50 ): #only compute mean amplitude for units with more than 50 samples try: MeanAmpTrue[int(unit)] = peak_to_peak_amp(ephys_file, samples, nsamps=20) if (FP_RP(ts) and noise_cutoff(amps, quartile_length=.25) < 20 and MeanAmpTrue[int(unit)] > 50): label[idx] = 1 else: label[idx] = 0 except: if (FP_RP(ts) and noise_cutoff(amps, quartile_length=.25) < 20): label[idx] = 1 else: label[idx] = 0 else: #no ephys file, do not include true mean amps if (FP_RP(ts) and noise_cutoff(amps, quartile_length=.25) < 20): label[idx] = 1 else: label[idx] = 0 #now compute additional metrics that label does not depend on: # Cumulative drift of spike amplitudes, normalized by total number of spikes. try: cum_amp_drift[idx] = cum_drift(amps) except Exception as err: print( "Failed to compute 'cum_drift(amps)' for unit {}. Details: \n {}" .format(unit, err)) units_missing_metrics.add(unit) # Cumulative drift of spike depths, normalized by total number of spikes. try: cum_depth_drift[idx] = cum_drift(depths) except Exception as err: print( "Failed to compute 'cum_drift(depths)' for unit {}. Details: \n {}" .format(unit, err)) units_missing_metrics.add(unit) # Coefficient of variation of spike amplitudes. try: cv_amp[idx] = np.std(amps) / np.mean(amps) except Exception as err: print("Failed to compute 'cv_amp' for unit {}. Details: \n {}". format(unit, err)) units_missing_metrics.add(unit) # Coefficient of variation of computed instantaneous firing rate. try: fr = bb.singlecell.firing_rate(ts, hist_win=0.01, fr_win=0.25) cv_fr[idx] = np.std(fr) / np.mean(fr) except Exception as err: print( "Failed to compute 'cv_fr' for unit {}. Details: \n {}".format( unit, err)) units_missing_metrics.add(unit) # Fraction of isi violations. try: frac_isi_viol[idx], _, _ = isi_viol(ts, rp=0.002) except Exception as err: print( "Failed to compute 'frac_isi_viol' for unit {}. Details: \n {}" .format(unit, err)) units_missing_metrics.add(unit) # Estimated fraction of missing spikes. try: frac_missing_spks[idx], _, _ = feat_cutoff(amps, spks_per_bin=10, sigma=4, min_num_bins=50) except Exception as err: print( "Failed to compute 'frac_missing_spks' for unit {}. Details: \n {}" .format(unit, err)) units_missing_metrics.add(unit) # Estimated fraction of false positives. try: fp_estimate[idx] = fp_est(ts, rp=0.002) except Exception as err: print("Failed to compute 'fp_est' for unit {}. Details: \n {}". format(unit, err)) units_missing_metrics.add(unit) # Presence ratio try: presence_ratio[idx], _ = pres_ratio(ts, hist_win=10) except Exception as err: print("Failed to compute 'pres_ratio' for unit {}. Details: \n {}". format(unit, err)) units_missing_metrics.add(unit) # Presence ratio over the standard deviation of spike counts in each bin try: pr, pr_bins = pres_ratio(ts, hist_win=10) pres_ratio_std[idx] = pr / np.std(pr_bins) except Exception as err: print( "Failed to compute 'pres_ratio_std' for unit {}. Details: \n {}" .format(unit, err)) units_missing_metrics.add(unit) #append metrics to the current clusters.metrics metrics_read = pd.read_csv(Path(alf_probe_dir, 'clusters.metrics.csv')) if not 'label' in metrics_read.columns: try: label_df = pd.DataFrame(label) pd.DataFrame.insert(metrics_read, 1, 'label', label_df) except ValueError: pd.DataFrame.drop(metrics_read, columns='label') pd.DataFrame.insert(metrics_read, 1, 'label', label_df) except: print("Could not save 'label' to .csv.") try: df_cum_amp_drift = pd.DataFrame(cum_amp_drift.round(2)) metrics_read['cum_amp_drift'] = df_cum_amp_drift except Exception as err: print("Could not save 'cum_amp_drift' to .csv. Details: \n {}". format(err)) try: df_cum_depth_drift = pd.DataFrame(cum_depth_drift.round(2)) metrics_read['cum_depth_drift'] = df_cum_depth_drift except Exception as err: print("Could not save 'cum_depth_drift' to .tsv. Details: \n {}". format(err)) try: df_cv_amp = pd.DataFrame(cv_amp.round(2)) metrics_read['cv_amp'] = df_cv_amp except Exception as err: print( "Could not save 'cv_amp' to .tsv. Details: \n {}".format(err)) try: df_cv_fr = pd.DataFrame(cv_fr.round(2)) metrics_read['cv_fr'] = df_cv_fr except Exception as err: print("Could not save 'cv_fr' to .tsv. Details: \n {}".format(err)) try: df_frac_isi_viol = pd.DataFrame(frac_isi_viol.round(2)) metrics_read['frac_isi_viol'] = df_frac_isi_viol except Exception as err: print("Could not save 'frac_isi_viol' to .tsv. Details: \n {}". format(err)) try: df_frac_missing_spks = pd.DataFrame(frac_missing_spks.round(2)) metrics_read['frac_missing_spks'] = df_frac_missing_spks except Exception as err: print("Could not save 'frac_missing_spks' to .tsv. Details: \n {}". format(err)) try: df_fp_est = pd.DataFrame(fp_estimate.round(2)) metrics_read['fp_est'] = df_fp_est except Exception as err: print( "Could not save 'fp_est' to .tsv. Details: \n {}".format(err)) try: df_pres_ratio = pd.DataFrame(presence_ratio.round(2)) metrics_read['pres_ratio'] = df_pres_ratio except Exception as err: print("Could not save 'pres_ratio' to .tsv. Details: \n {}".format( err)) try: df_pres_ratio_std = pd.DataFrame(pres_ratio_std.round(2)) metrics_read['pres_ratio_std'] = df_pres_ratio_std except Exception as err: print("Could not save 'pres_ratio_std' to .tsv. Details: \n {}". format(err)) try: df_refp_viol = pd.DataFrame(RefPViol) pd.DataFrame.insert(metrics_read, 2, 'refp_viol', df_refp_viol) except ValueError: pd.DataFrame.drop(metrics_read, columns='refp_viol') pd.DataFrame.insert(metrics_read, 2, 'refp_viol', df_refp_viol) except Exception as err: print("Could not save 'RefPViol' to .tsv. Details: \n {}".format( err)) try: df_noise_cutoff = pd.DataFrame(NoiseCutoff) pd.DataFrame.insert(metrics_read, 3, 'noise_cutoff', df_noise_cutoff) except ValueError: pd.DataFrame.drop(metrics_read, columns='noise_cutoff') pd.DataFrame.insert(metrics_read, 3, 'noise_cutoff', df_noise_cutoff) except Exception as err: print( "Could not save 'NoiseCutoff' to .tsv. Details: \n {}".format( err)) try: df_mean_amp_true = pd.DataFrame(MeanAmpTrue) pd.DataFrame.insert(metrics_read, 4, 'mean_amp_true', df_mean_amp_true) except ValueError: pd.DataFrame.drop(metrics_read, columns='mean_amp_true') pd.DataFrame.insert(metrics_read, 4, 'mean_amp_true', df_mean_amp_true) except Exception as err: print("Could not save 'Mean Amp True' to .tsv. Details: \n {}". format(err)) #now add df to csv metrics_read.to_csv(Path(alf_probe_dir, 'clusters.metrics.csv')) print('Launching phy') else: print('Launching phy') try: numpass = int(sum(label)) print("\n Number of units that pass: "******"Number of units that pass RP threshold: ", numpassRP) print("Number of units that pass Amp Cutoff threshold: ", numpassAC) print("Number of total units: ", ntot) except Exception as err: print("Could not compute number of units that pass. Details \n {}". format(err)) return metrics_read
eid5 = one.search(subject='ZM_2406', date='2019-11-12', number=1)[0] eid6 = one.search(subject='CSK-scan-008', date='2019-12-09', number=8)[0] eid7 = one.search(subject='NYU-18', date='2019-10-23', number=1)[0] eids = [eid0, eid1, eid2, eid3, eid4, eid5, eid6, eid7] n_units = np.zeros((len(eids),)) n_good_units = np.zeros((len(eids),)) for i, eid in enumerate(eids): # load data to disk d_paths = one.load(eid, dataset_types=dtypes, clobber=False, download_only=True) # get paths and load data in memory alf_probe_path = os.path.split(d_paths[0])[0] spks_b = aio.load_object(alf_probe_path, 'spikes') units_b = bb.processing.get_units_bunch(spks_b) # filter units and get nubmer of good units and number of total units T = spks_b.times[-1] - spks_b.times[0] # length of recording session filt_units = bb.processing.filter_units(units_b, T, min_amp=50e-6, min_fr=1) # get num_units & num_good units n_units[i] = np.max(spks_b.clusters) n_good_units = len(filt_units) # 2 group bar plot(all units, good units) names = ['swc026_09-16_1', 'ks003_11-19_1', 'lic3_08-27_2', 'cer5_10-25_1', 'CSHL020_12-04_5', 'ZM2406_11-12_1', 'CSKscan008_12-09_1', 'NYU18_10-23_1'] units_eid0 = [n_units[0], n_good_units[0]]
def gen_metrics(alf_dir, ks_dir, ephys_file_path=None): """ Tries to generate single unit metrics for all units metric-by-metric and save the metrics as .tsv files, and displays an error if unable to create one of the metric .tsv files, before continuing to the next one. Parameters ---------- alf_dir : string Full path to alf output directory. ks_dir : string Full path to the ks2 output directory. The .tsv files will be saved here. ephys_file_path : string (optional) Full path to binary ephys file. Returns ------- units_missing_metrics : set Set of units for which not all metrics were able to be calculated. Examples ------- 1) Create an alf directory from a ks2 output directory. The alf directory cannot be the same as the ks2 output directory. Then, generate all metrics that don't require raw ephys data. >>> import ibllib.ephys.spikes as e_spks >>> from gen_phy_metrics import gen_metrics >>> e_spks.ks2_to_alf(ks_dir_full_path, alf_dir_full_path) >>> gen_metrics(alf_dir, ks_dir) 2) Generate metrics from an alf directory and metrics that require an ephys_file_path. For phy, the ephys file should be in `ks_dir`. >>> from gen_phy_metrics import gen_metrics >>> gen_metrics(alf_dir, ks_dir, ephys_file_path=ks_dir) """ # Setup # # ----- # # Extract alf objects from `alf_dir` and get units info alf_dir = Path(alf_dir) if not (Path.exists(alf_dir)): raise FileNotFoundError( 'The given alf directory {} does not exist!'.format(alf_dir)) spks_b = aio.load_object(alf_dir, 'spikes') clstrs_b = aio.load_object(alf_dir, 'clusters') units_b = bb.processing.get_units_bunch(spks_b) units = list(units_b.amps.keys()) n_units = np.max(spks_b.clusters) + 1 # Initialize metrics cum_amp_drift = np.full((n_units, ), np.nan) cum_depth_drift = np.full((n_units, ), np.nan) cv_amp = np.full((n_units, ), np.nan) cv_fr = np.full((n_units, ), np.nan) frac_isi_viol = np.full((n_units, ), np.nan) fn_est = np.full((n_units, ), np.nan) fp_est = np.full((n_units, ), np.nan) pres_ratio = np.full((n_units, ), np.nan) pres_ratio_std = np.full((n_units, ), np.nan) ptp_sigma = np.full((n_units, ), np.nan) units_missing_metrics = set() # Compute metrics # # --------------- # # Iterate over all units for unit in units: if unit == units[0]: t0 = time.perf_counter() # used for computation time estimate # Need timestamps, amps, depths ts = units_b['times'][unit] #Bug was below: amps and depths were defined as units_b['times'][unit]! amps = units_b['amps'][unit] depths = units_b['depths'][unit] # Cumulative drift of spike amplitudes, normalized by total number of spikes. try: cum_amp_drift[int(unit)] = bb.metrics.cum_drift(amps) except Exception as err: print( "Failed to compute 'cum_drift(amps)' for unit {}. Details: \n {}" .format(unit, err)) units_missing_metrics.add(unit) # Cumulative drift of spike depths, normalized by total number of spikes. try: cum_depth_drift[int(unit)] = bb.metrics.cum_drift(depths) except Exception as err: print( "Failed to compute 'cum_drift(depths)' for unit {}. Details: \n {}" .format(unit, err)) units_missing_metrics.add(unit) # Coefficient of variation of spike amplitudes. try: cv_amp[int(unit)] = np.std(amps) / np.mean(amps) except Exception as err: print("Failed to compute 'cv_amp' for unit {}. Details: \n {}". format(unit, err)) units_missing_metrics.add(unit) # Coefficient of variation of computed instantaneous firing rate. try: fr = bb.singlecell.firing_rate(ts, hist_win=0.01, fr_win=0.25) cv_fr[int(unit)] = np.std(fr) / np.mean(fr) except Exception as err: print( "Failed to compute 'cv_fr' for unit {}. Details: \n {}".format( unit, err)) units_missing_metrics.add(unit) # Fraction of isi violations. try: frac_isi_viol[int(unit)], _, _ = bb.metrics.isi_viol(ts, rp=0.002) except Exception as err: print( "Failed to compute 'frac_isi_viol' for unit {}. Details: \n {}" .format(unit, err)) units_missing_metrics.add(unit) # Estimated fraction of missing spikes. try: fn_est[int(unit)], _, _ = bb.metrics.missed_spikes_est( amps, spks_per_bin=10, sigma=4, min_num_bins=50) except Exception as err: print( "Failed to compute 'missed_spikes_est' for unit {}. Details: \n {}" .format(unit, err)) units_missing_metrics.add(unit) # Estimated fraction of false positives. try: fp_est[int(unit)] = bb.metrics.contamination(ts, rp=0.002) except Exception as err: print( "Failed to compute 'contamination' for unit {}. Details: \n {}" .format(unit, err)) units_missing_metrics.add(unit) # Presence ratio try: pres_ratio[int(unit)], _ = bb.metrics.pres_ratio(ts, hist_win=10) except Exception as err: print("Failed to compute 'pres_ratio' for unit {}. Details: \n {}". format(unit, err)) units_missing_metrics.add(unit) # Presence ratio over the standard deviation of spike counts in each bin try: pr, pr_bins = bb.metrics.pres_ratio(ts, hist_win=10) pres_ratio_std[int(unit)] = pr / np.std(pr_bins) except Exception as err: print( "Failed to compute 'pres_ratio_std' for unit {}. Details: \n {}" .format(unit, err)) units_missing_metrics.add(unit) # The mean peak-to-peak value over the background noise of the channel of max amplitude. if ephys_file_path: try: ch = clstrs_b['channels'][int( unit)] # channel of max amplitude ptp_sigma[int(unit)] = bb.metrics.ptp_over_noise( ephys_file_path, ts, ch, t=2.0, sr=30000, n_ch_probe=385, dtype='int16', car=False) except Exception as err: print( "Failed to compute 'ptp_sigma' for unit {}. Details: \n {}" .format(unit, err)) units_missing_metrics.add(unit) if unit == units[0]: # give time estimate dt = time.perf_counter() - t0 print( '\nComputing metrics. Estimated time is {:.2f} mins\n'.format( len(units) * dt / 60)) # Extract to .tsv files # # --------------------- # try: df_cum_amp_drift = pd.DataFrame(cum_amp_drift.round(2)) df_cum_amp_drift.to_csv(Path(ks_dir, 'cum_amp_drift.tsv'), sep='\t', header=['cum_amp_drift']) except Exception as err: print("Could not save 'cum_amp_drift' to .tsv. Details: \n {}".format( err)) try: df_cum_depth_drift = pd.DataFrame(cum_depth_drift.round(2)) df_cum_depth_drift.to_csv(Path(ks_dir, 'cum_depth_drift.tsv'), sep='\t', header=['cum_depth_drift']) except Exception as err: print( "Could not save 'cum_depth_drift' to .tsv. Details: \n {}".format( err)) try: df_cv_amp = pd.DataFrame(cv_amp.round(2)) df_cv_amp.to_csv(Path(ks_dir, 'cv_amp.tsv'), sep='\t', header=['cv_amp']) except Exception as err: print("Could not save 'cv_amp' to .tsv. Details: \n {}".format(err)) try: df_cv_fr = pd.DataFrame(cv_fr.round(2)) df_cv_fr.to_csv(Path(ks_dir, 'cv_fr.tsv'), sep='\t', header=['cv_fr']) except Exception as err: print("Could not save 'cv_fr' to .tsv. Details: \n {}".format(err)) try: df_frac_isi_viol = pd.DataFrame(frac_isi_viol.round(2)) df_frac_isi_viol.to_csv(Path(ks_dir, 'frac_isi_viol.tsv'), sep='\t', header=['frac_isi_viol']) except Exception as err: print("Could not save 'frac_isi_viol' to .tsv. Details: \n {}".format( err)) try: df_fn_est = pd.DataFrame(fn_est.round(2)) df_fn_est.to_csv(Path(ks_dir, ' missed_spikes_est.tsv'), sep='\t', header=['missed_spikes_est']) except Exception as err: print("Could not save 'missed_spikes_est' to .tsv. Details: \n {}". format(err)) try: df_fp_est = pd.DataFrame(fp_est.round(2)) df_fp_est.to_csv(Path(ks_dir, 'contamination.tsv'), sep='\t', header=['contamination']) except Exception as err: print("Could not save 'contamination' to .tsv. Details: \n {}".format( err)) try: df_pres_ratio = pd.DataFrame(pres_ratio.round(2)) df_pres_ratio.to_csv(Path(ks_dir, 'pres_ratio.tsv'), sep='\t', header=['pres_ratio']) except Exception as err: print( "Could not save 'pres_ratio' to .tsv. Details: \n {}".format(err)) try: df_pres_ratio_std = pd.DataFrame(pres_ratio_std.round(2)) df_pres_ratio_std.to_csv(Path(ks_dir, 'pres_ratio_std.tsv'), sep='\t', header=['pres_ratio_std']) except Exception as err: print("Could not save 'pres_ratio_std' to .tsv. Details: \n {}".format( err)) if ephys_file_path: try: df_ptp_sigma = pd.DataFrame(ptp_sigma.round(2)) df_ptp_sigma.to_csv(Path(ks_dir, 'ptp_sigma.tsv'), sep='\t', header=['ptp_sigma']) except Exception as err: print("Could not save 'cum_amp_drift' to .tsv. Details: \n {}". format(err)) return units_missing_metrics
'Ephys file not found! Some of the examples in this script require ephys file,' ) # Ensure st least alf directory can be found assert os.path.isdir(alf_probe_dir), \ 'Directories set incorrectly. {} not found'.format(alf_probe_dir) # Call brainbox functions # #-------------------------# # Change variable names to same names used in brainbox docstrings path_to_ephys_file = ephys_file_path path_to_alf_out = alf_probe_dir # Load alf objects spks_b = aio.load_object(path_to_alf_out, 'spikes') clstrs_b = aio.load_object(path_to_alf_out, 'clusters') chnls_b = aio.load_object(path_to_alf_out, 'channels') tmplts_b = aio.load_object(path_to_alf_out, 'templates') # Convert spikes bunch into a units bunch units_b = bb.processing.get_units_bunch(spks_b) # this may take a few mins unit4_amps = units_b['amps']['4'] # get amplitudes for unit 4. # Filter units according to some parameters T = spks_b['times'][-1] - spks_b['times'][0] filtered_units = bb.processing.filter_units(units_b, T, min_amp=0, min_fr=0.5, max_fpr=1,
alf_probe_dir = os.path.join(alf_dir, probe) ephys_file_dir = os.path.join(session_path, 'raw_ephys_data', probe) # Find 'ap' ephys file in `ephys_file_dir` for i in os.listdir(ephys_file_dir): if 'ap' in i and 'bin' in i: ephys_file_path = os.path.join(ephys_file_dir, i) # Ensure directories can be found assert os.path.isdir(ephys_file_dir) and os.path.isdir(alf_probe_dir) \ and os.path.isabs(ephys_file_path), 'Directories set incorrectly' else: ephys_file_path = '/Users/gaelle/Downloads/FlatIron/mainenlab/Subjects/ZM_2407/2019-11-05/003/raw_ephys_data/probe_00/_spikeglx_ephysData_probe00.raw_g0_t0.imec.ap.cbin' alf_probe_dir = '/Users/gaelle/Downloads/FlatIron/mainenlab/Subjects/ZM_2407/2019-11-05/003/alf/probe_00' # -- Get spks and clusters spks = aio.load_object(alf_probe_dir, 'spikes') clstrs = aio.load_object(alf_probe_dir, 'clusters') # Fig 1 -- ALL UNITS # Create a bar plot of the variances of the spike amplitudes for each unit. fig, var_vals, p_vals = bb.plot.feat_vars(spks) # Fig 2 -- SINGLE UNIT # Plot cutoff line indicating the fraction of spikes missing from a unit based on the recorded # unit's spike amplitudes, assuming the distribution of the unit's spike amplitudes is symmetric. fig, fraction_missing = bb.plot.missed_spikes_est(spks, 1) # Fig 3 -- SINGLE UNIT # Compare first and last 100 spike waveforms for unit1, across 20 channels around the channel # of max amplitude, and compare the first and last 50 spike waveforms for unit2, across 15 # channels around the mean.
def get_vr_clusters(session_path, clusters=None, n_selected_cl=4): ''' Gets visually responsive clusters Parameters ---------- session_path : str The path to to the appropriate 'alf/probe' directory. clusters : ndarray The clusters to use to get a subset of visually responsive clusters. (if `None`, take visually response subset from all clusters from recording session.) n_selected_cl : int The number of clusters to return in `vr_clusters_selected` Returns ------- clusters_vr : ndarray The visually responsive clusters. clusters_selected_vr : ndarray A subset of `n_selected_cl` `vr_clusters` ''' print('finding visually responsive clusters...', end='', flush=True) # ------------------------- # load required alf objects # ------------------------- spikes = ioalf.load_object(session_path, 'spikes') gratings = ioalf.load_object(session_path, '_iblcertif_.odsgratings') spontaneous = ioalf.load_object(session_path, '_iblcertif_.spontaneous') grating_times = { 'beg': gratings['odsgratings.times.00'], 'end': gratings['odsgratings.times.01']} grating_vals = { 'beg': gratings['odsgratings.stims.00'], 'end': gratings['odsgratings.stims.01']} spont_times = { 'beg': spontaneous['spontaneous.times.00'], 'end': spontaneous['spontaneous.times.01']} # --------------------------------- # find visually responsive clusters # --------------------------------- epochs = ['beg', 'end'] if clusters is None: # use all clusters # speed up downstream computations by restricting data to relevant time periods mask_times = np.full(spikes.times.shape, fill_value=False) for epoch in epochs: mask_times |= (spikes.times >= grating_times[epoch].min()) & \ (spikes.times <= grating_times[epoch].max()) clusters = np.unique(spikes.clusters[mask_times]) # only calculate responsiveness for clusters that were active during gratings mask_clust = np.isin(spikes.clusters, clusters) resp = {epoch: [] for epoch in epochs} for epoch in epochs: resp[epoch] = are_neurons_responsive( spikes.times[mask_clust], spikes.clusters[mask_clust], grating_times[epoch], grating_vals[epoch], spont_times[epoch]) resp_agg = resp['beg'] & resp['end'] # remove non-responsive clusters clusters_vr = clusters[resp_agg] print('done') if n_selected_cl < len(clusters_vr): clusters_selected_vr = np.random.choice(clusters_vr, size=n_selected_cl, replace=False) else: clusters_selected_vr = clusters_vr return clusters_vr, clusters_selected_vr
def plot_grating_figures( session_path, cluster_ids_summary, cluster_ids_selected, save_dir=None, format='png', pre_time=0.5, post_time=2.5, bin_size=0.005, smoothing=0.025, n_rand_clusters=20, plot_summary=True, plot_selected=True): """ Produces two summary figures for the oriented grating protocol; the first summary figure contains plots that compare different measures during the first and second grating protocols, such as orientation selectivity index (OSI), orientation preference, fraction of visual clusters, PSTHs, firing rate histograms, etc. The second summary figure contains plots of polar PSTHs and corresponding rasters for a random subset of visually responsive clusters. Parameters ---------- session_path : str absolute path to experimental session directory cluster_ids_summary : list the clusters for which to plot summary psths/rasters; if empty, all clusters with responses during the grating presentations are used cluster_ids_selected : list the clusters for which to plot individual psths/rasters; if empty, `n_rand_clusters` are randomly chosen save_dir : str or NoneType if NoneType, figures are displayed; else a string defining the absolute filepath to the directory in which figures will be saved format : str file format, i.e. 'png' | 'pdf' | 'jpg' pre_time : float time (sec) to plot before grating presentation onset post_time : float time (sec) to plot after grating presentation onset (should include length of stimulus) bin_size : float size of bins for raster plots/psths smoothing : float size of smoothing kernel (sec) n_rand_clusters : int the number of random clusters to choose for which to plot psths/rasters if `cluster_ids_slected` is empty plot_summary : bool a flag for plotting the summary figure plot_selected : bool a flag for plotting the selected units figure Returns ------- metrics : dict - 'osi' (dict): keys 'beg', 'end' point to arrays of osis during these epochs - 'orientation_pref' (dict): keys 'beg', 'end' point to arrays of orientation preference - 'frac_resp_by_depth' (dict): fraction of responsive clusters by depth fig_dict : dict A dict whose values are handles to one or both figures generated. """ fig_dict = {} cluster_ids = cluster_ids_summary cluster_idxs = cluster_ids_selected epochs = ['beg', 'end'] # ------------------------- # load required alf objects # ------------------------- print('loading alf objects...', end='', flush=True) spikes = ioalf.load_object(session_path, 'spikes') clusters = ioalf.load_object(session_path, 'clusters') gratings = ioalf.load_object(session_path, '_iblcertif_.odsgratings') spontaneous = ioalf.load_object(session_path, '_iblcertif_.spontaneous') grating_times = { 'beg': gratings['odsgratings.times.00'], 'end': gratings['odsgratings.times.01']} grating_vals = { 'beg': gratings['odsgratings.stims.00'], 'end': gratings['odsgratings.stims.01']} spont_times = { 'beg': spontaneous['spontaneous.times.00'], 'end': spontaneous['spontaneous.times.01']} # -------------------------- # calculate relevant metrics # -------------------------- print('calcuating mean responses to gratings...', end='', flush=True) # calculate mean responses to gratings mask_clust = np.isin(spikes.clusters, cluster_ids) # update mask for responsive clusters mask_times = np.full(spikes.times.shape, fill_value=False) for epoch in epochs: mask_times |= (spikes.times >= grating_times[epoch].min()) & \ (spikes.times <= grating_times[epoch].max()) resp = {epoch: [] for epoch in epochs} for epoch in epochs: resp[epoch] = are_neurons_responsive( spikes.times[mask_clust], spikes.clusters[mask_clust], grating_times[epoch], grating_vals[epoch], spont_times[epoch]) responses = {epoch: [] for epoch in epochs} for epoch in epochs: responses[epoch] = bin_responses( spikes.times[mask_clust], spikes.clusters[mask_clust], grating_times[epoch], grating_vals[epoch]) responses_mean = {epoch: np.mean(responses[epoch], axis=2) for epoch in epochs} # responses_se = {epoch: np.std(responses[epoch], axis=2) / np.sqrt(responses[epoch].shape[2]) # for epoch in responses.keys()} print('done') # calculate osi and orientation preference print('calcuating osi/orientation preference...', end='', flush=True) ori_pref = {epoch: [] for epoch in epochs} osi = {epoch: [] for epoch in epochs} for epoch in epochs: osi[epoch], ori_pref[epoch] = compute_selectivity( responses_mean[epoch], np.unique(grating_vals[epoch]), 'ori') print('done') # calculate depth vs osi ratio (osi_beg/osi_end) print('calcuating osi ratio as a function of depth...', end='', flush=True) depths = np.array([clusters.depths[c] for c in cluster_ids]) ratios = np.array([osi['beg'][c] / osi['end'][c] for c in range(len(cluster_ids))]) print('done') # calculate fraction of visual neurons by depth print('calcuating fraction of visual clusters by depth...', end='', flush=True) n_bins = 10 min_depth = np.min(clusters['depths']) max_depth = np.max(clusters['depths']) depth_limits = np.linspace(min_depth - 1, max_depth, n_bins + 1) depth_avg = (depth_limits[:-1] + depth_limits[1:]) / 2 # aggregate clusters clusters_binned = {epoch: [] for epoch in epochs} frac_responsive = {epoch: [] for epoch in epochs} cids = cluster_ids for epoch in epochs: # just look at responsive clusters during this epoch cids_tmp = cids[resp[epoch]] for d in range(n_bins): lo_limit = depth_limits[d] up_limit = depth_limits[d + 1] # clusters.depth index is cluster id cids_curr_depth = np.where( (lo_limit < clusters.depths) & (clusters.depths <= up_limit))[0] clusters_binned[epoch].append(cids_curr_depth) frac_responsive[epoch].append(np.sum( np.isin(cids_tmp, cids_curr_depth)) / len(cids_curr_depth)) # package for plotting responsive = {'fraction': frac_responsive, 'depth': depth_avg} print('done') # calculate PSTH averaged over all clusters/orientations print('calcuating average PSTH...', end='', flush=True) peths = {epoch: [] for epoch in epochs} peths_avg = {epoch: [] for epoch in epochs} for epoch in epochs: stim_ids = np.unique(grating_vals[epoch]) peths[epoch] = {i: None for i in range(len(stim_ids))} peths_avg_tmp = [] for i, stim_id in enumerate(stim_ids): curr_stim_idxs = np.where(grating_vals[epoch] == stim_id) align_times = grating_times[epoch][curr_stim_idxs, 0][0] peths[epoch][i], _ = calculate_peths( spikes.times[mask_times], spikes.clusters[mask_times], cluster_ids, align_times, pre_time=pre_time, post_time=post_time, bin_size=bin_size, smoothing=smoothing, return_fr=True) peths_avg_tmp.append( np.mean(peths[epoch][i]['means'], axis=0, keepdims=True)) peths_avg_tmp = np.vstack(peths_avg_tmp) peths_avg[epoch] = { 'mean': np.mean(peths_avg_tmp, axis=0), 'std': np.std(peths_avg_tmp, axis=0) / np.sqrt(peths_avg_tmp.shape[0])} peths_avg['bin_size'] = bin_size peths_avg['on_idx'] = int(pre_time / bin_size) peths_avg['off_idx'] = peths_avg['on_idx'] + int(2 / bin_size) print('done') # compute rasters for entire orientation sequence at beg/end epoch if plot_summary: print('computing rasters for example stimulus sequences...', end='', flush=True) r = {epoch: None for epoch in epochs} r_times = {epoch: None for epoch in epochs} r_clusters = {epoch: None for epoch in epochs} for epoch in epochs: # restrict activity to a single stim series; assumes each possible grating direction # is displayed before repeating n_stims = len(np.unique(grating_vals[epoch])) mask_idxs_e = (spikes.times >= grating_times[epoch][:n_stims].min()) & \ (spikes.times <= grating_times[epoch][:n_stims].max()) r_tmp, r_times[epoch], r_clusters[epoch] = bincount2D( spikes.times[mask_idxs_e], spikes.clusters[mask_idxs_e], bin_size) # order activity by anatomical depth of neurons d = dict(zip(spikes.clusters[mask_idxs_e], spikes.depths[mask_idxs_e])) y = sorted([[i, d[i]] for i in d]) isort = np.argsort([x[1] for x in y]) r[epoch] = r_tmp[isort, :] # package for plotting rasters = {'spikes': r, 'times': r_times, 'clusters': r_clusters, 'bin_size': bin_size} print('done') # ------------------------------------------------- # compute psths and rasters for individual clusters # ------------------------------------------------- if plot_selected: print('computing psths and rasters for clusters...', end='', flush=True) if len(cluster_ids_selected) == 0: if (n_rand_clusters < len(cluster_ids)): cluster_idxs = np.random.choice(cluster_ids, size=n_rand_clusters, replace=False) else: cluster_idxs = cluster_ids else: cluster_idxs = cluster_ids_selected mean_responses = {cluster: {epoch: [] for epoch in epochs} for cluster in cluster_idxs} osis = {cluster: {epoch: [] for epoch in epochs} for cluster in cluster_idxs} binned = {cluster: {epoch: [] for epoch in epochs} for cluster in cluster_idxs} for cluster_idx in cluster_idxs: cluster = np.where(cluster_ids == cluster_idx)[0] for epoch in epochs: mean_responses[cluster_idx][epoch] = responses_mean[epoch][cluster, :][0] osis[cluster_idx][epoch] = osi[epoch][cluster] stim_ids = np.unique(grating_vals[epoch]) binned[cluster_idx][epoch] = {j: None for j in range(len(stim_ids))} for j, stim_id in enumerate(stim_ids): curr_stim_idxs = np.where(grating_vals[epoch] == stim_id) align_times = grating_times[epoch][curr_stim_idxs, 0][0] _, binned[cluster_idx][epoch][j] = calculate_peths( spikes.times[mask_times], spikes.clusters[mask_times], [cluster_idx], align_times, pre_time=pre_time, post_time=post_time, bin_size=bin_size) print('done') # -------------- # output figures # -------------- print('producing figures...', end='') if plot_summary: if save_dir is None: save_file = None else: if not os.path.exists(save_dir): os.makedirs(save_dir) save_file = os.path.join(save_dir, 'grating_summary_figure.' + format) fig_gr_summary = plot_summary_figure( ratios=ratios, depths=depths, responsive=responsive, peths_avg=peths_avg, osi=osi, ori_pref=ori_pref, responses_mean=responses_mean, rasters=rasters, save_file=save_file) fig_gr_summary.suptitle('Summary Grating Responses') fig_dict['gr_summary'] = fig_gr_summary if plot_selected: if save_dir is None: save_file = None else: save_file = os.path.join(save_dir, 'grating_random_responses.' + format) fig_gr_selected = plot_psths_and_rasters( mean_responses, binned, osis, grating_vals, on_idx=peths_avg['on_idx'], off_idx=peths_avg['off_idx'], bin_size=bin_size, save_file=save_file) fig_gr_selected.suptitle('Selected Units Grating Responses') print('done') fig_dict['gr_selected'] = fig_gr_selected # ----------------------------- # package up and return metrics # ----------------------------- metrics = { 'osi': osi, 'orientation_pref': ori_pref, 'frac_resp_by_depth': responsive, } return fig_dict, metrics
maxlags = maxlags or ns - 1 return np.correlate(x, y, mode='full')[ns - 1 - maxlags:ns + maxlags] T_BIN = 0.01 # seconds CORR_LEN = 1 # seconds CORR_BINS = int(CORR_LEN / T_BIN) # bins # get the data from flatiron and the current folder one = ONE() eid = one.search(subject='ZM_1150', date='2019-05-07', number=1) D = one.load(eid[0], clobber=False, download_only=True) session_path = Path(D.local_path[0]).parent # load objects spikes = ioalf.load_object(session_path, 'spikes') # Get a Bunch instance. b = one_to_bunch(spikes) # Compute the firing rates. rates = firing_rates(b.spike_times, b.spike_clusters, T_BIN) # Note: I would rather just use spikes['times'] and spikes['clusters'] instead of going # via a Bunch or DataFrame or similar... # Compute the cross-correlation between the firing rate of two neurons. c = xcorr(rates[0], rates[1], CORR_BINS) # Plot it. lags = np.linspace(-CORR_LEN, +CORR_LEN, len(c)) plt.plot(c)
def plot_rfs_by_depth_wrapper( alf_path, axes=None, cluster_ids=[], method='corr', binsize=0.025, lags=8, n_depths=30, use_svd=False): """ Wrapper function to load spikes and rf stimulus info, aggregate clusters over depths, compute rfs, and plot spatial components as a function of linear depth on probe. Must have ibllib package in python path in order to use alf loaders. Parameters ---------- alf_path : str absolute path to experimental session directory axes : array-like object of matplotlib axes or `NoneType`, optional axes in which to plot the rfs; if `NoneType`, a figure with appropriate axes is created cluster_ids : array-like, optional clusters to use in rf calculation; if empty, all clusters are used method : str, optional method for calculating receptive fields 'sta': method used in Durand et al 2016 'corr': reverse correlation method; uses convolution and is therefor faster than `'sta'` binsize : float, optional width of bins in seconds for rf computation lags : int, optional number of bins after pixel flip to use for rf computation n_depths : int, optional number of bins to divide probe depth into for aggregating clusters use_svd : bool, optional `True` plots 1st spatial SVD component of rf; `False` plots time lag with peak response Returns ------- dict depths and associated receptive fields """ import alf.io as ioalf # load objects spikes = ioalf.load_object(alf_path, 'spikes') clusters = ioalf.load_object(alf_path, 'clusters') rfmap = ioalf.load_object(alf_path, '_iblcertif_.rfmap') rf_stim_times = rfmap['rfmap.times.00'] rf_stim = rfmap['rfmap.stims.00'].astype('float') # combine clusters across similar depths min_depth = np.min(clusters['depths']) max_depth = np.max(clusters['depths']) depth_limits = np.linspace(min_depth - 1, max_depth, n_depths + 1) if len(cluster_ids) == 0: times_agg = spikes.times depths_agg = spikes.depths else: clust_mask = np.isin(spikes.clusters, np.array(cluster_ids)) times_agg = spikes.times[clust_mask] depths_agg = spikes.depths[clust_mask] clusters_agg = np.full(times_agg.shape, fill_value=np.nan) for d in range(n_depths): lo_limit = depth_limits[d] up_limit = depth_limits[d + 1] clusters_agg[(lo_limit < depths_agg) & (depths_agg <= up_limit)] = d print('computing receptive fields...', end='') if method == 'sta': # method in Durand et al 2016 rfs = compute_rfs( times_agg, clusters_agg, rf_stim_times, rf_stim, lags=lags, binsize=binsize) elif method == 'corr': # reverse correlation method rfs = compute_rfs_corr( times_agg, clusters_agg, rf_stim_times, rf_stim, lags=lags, binsize=binsize) else: raise NotImplementedError # get single spatial footprint of rf if use_svd: rfs_spatial = compute_rf_svds(rfs, scale='spatial')['spatial'] else: rfs_spatial = find_peak_responses(rfs) rfs_interp = interpolate_rfs(rfs_spatial, bin_scale=0.5) print('done') plot_rfs_by_depth(rfs_interp, axes=axes) return {'depths': depths_agg, 'rfs': rfs_interp}
from oneibl.one import ONE import alf.io as ioalf # user options BINSIZE = 0.05 # sec LAGS = 4 # number of bins for calculating RF METHOD = 'corr' # 'corr' | 'sta' # get the data from flatiron and the current folder one = ONE() eid = one.search(subject='ZM_2104', date='2019-09-19', number=1) D = one.load(eid[0], clobber=False, download_only=True) session_path = Path(D.local_path[0]).parent # load objects spikes = ioalf.load_object(session_path, 'spikes') rfmap = ioalf.load_object(session_path, '_iblcertif_.rfmap') rf_stim_times = rfmap['rfmap.times.00'] rf_stim = rfmap['rfmap.stims.00'].astype('float') # compute receptive fields if METHOD == 'sta': # method in Durand et al 2016; ~9 min for 700 units on a single cpu core print('computing receptive fields...', end='') rfs = compute_rfs( spikes.times, spikes.clusters, rf_stim_times, rf_stim, lags=LAGS, binsize=BINSIZE) print('done') elif METHOD == 'corr': # reverse correlation method; ~3 min for 700 units on a single cpu core print('computing receptive fields...', end='') rfs = compute_rfs_corr(
def load_data(self, download_data: bool = None, extract_times: bool = False, load_video: bool = True) -> None: """Extract the data from raw data files Extracts all the required task data from the raw data files. Data keys: - count (int array): the sequential frame number (n, n+1, n+2...) - pin_state (): the camera GPIO pin; records the audio TTLs; should be one per frame - audio (float array): timestamps of audio TTL fronts - fpga_times (float array): timestamps of camera TTLs recorded by FPGA - timestamps (float array): extracted video timestamps (the camera.times ALF) - bonsai_times (datetime array): system timestamps of video PC; should be one per frame - camera_times (float array): camera frame timestamps extracted from frame headers - wheel (Bunch): rotary encoder timestamps, position and period used for wheel motion - video (Bunch): video meta data, including dimensions and FPS - frame_samples (h x w x n array): array of evenly sampled frames (1 colour channel) :param download_data: if True, any missing raw data is downloaded via ONE. :param extract_times: if True, the camera.times are re-extracted from the raw data :param load_video: if True, calls the load_video_data method """ assert self.session_path, 'no session path set' if download_data is not None: self.download_data = download_data if self.eid and self.one and not isinstance(self.one, OneOffline): self._ensure_required_data() _log.info('Gathering data for QC') # Get frame count and pin state self.data['count'], self.data['pin_state'] = \ raw.load_embedded_frame_data(self.session_path, self.side, raw=True) # Load the audio and raw FPGA times if self.type == 'ephys': sync, chmap = ephys_fpga.get_main_probe_sync(self.session_path) audio_ttls = ephys_fpga._get_sync_fronts(sync, chmap['audio']) self.data['audio'] = audio_ttls['times'] # Get rises # Load raw FPGA times cam_ts = extract_camera_sync(sync, chmap) self.data['fpga_times'] = cam_ts[self.side] else: bpod_data = raw.load_data(self.session_path) _, audio_ttls = raw.load_bpod_fronts(self.session_path, bpod_data) self.data['audio'] = audio_ttls['times'] # Load extracted frame times alf_path = self.session_path / 'alf' try: assert not extract_times self.data['timestamps'] = alfio.load_object( alf_path, f'{self.side}Camera')['times'] except AssertionError: # Re-extract kwargs = dict(video_path=self.video_path, labels=self.side) if self.type == 'ephys': kwargs = {**kwargs, 'sync': sync, 'chmap': chmap} # noqa outputs, _ = extract_all(self.session_path, self.type, save=False, **kwargs) self.data['timestamps'] = outputs[f'{self.side}_camera_timestamps'] except ALFObjectNotFound: _log.warning('no camera.times ALF found for session') # Get audio and wheel data wheel_keys = ('timestamps', 'position') try: self.data['wheel'] = alfio.load_object(alf_path, 'wheel') except ALFObjectNotFound: # Extract from raw data if self.type == 'ephys': wheel_data = ephys_fpga.extract_wheel_sync(sync, chmap) else: wheel_data = training_wheel.get_wheel_position( self.session_path) self.data['wheel'] = Bunch(zip(wheel_keys, wheel_data)) # Find short period of wheel motion for motion correlation. For speed start with the # fist 2 minutes (nearly always enough), extract wheel movements and pick one. # TODO Pick movement towards the end of the session (but not right at the end as some # are extrapolated). Make sure the movement isn't too long. if data_for_keys( wheel_keys, self.data['wheel']) and self.data['timestamps'] is not None: self.data['wheel'].period = self.get_active_wheel_period( self.data['wheel']) # Load Bonsai frame timestamps try: ssv_times = raw.load_camera_ssv_times(self.session_path, self.side) self.data['bonsai_times'], self.data['camera_times'] = ssv_times except AssertionError: _log.warning('No Bonsai video timestamps file found') # Gather information from video file if load_video: _log.info('Inspecting video file...') self.load_video_data()