def load_data(self, download_data: bool = None) -> None:
"""Extract the data from data files
Extracts all the required task data from the data files.
Data keys:
- camera_times (float array): camera frame timestamps extracted from frame headers
- dlc_coords (dict): keys are the points traced by dlc, items are x-y coordinates of
these points over time, those with likelihood <0.9 set to NaN
:param download_data: if True, any missing raw data is downloaded via ONE.
"""
if download_data is not None:
self.download_data = download_data
if self.one and not isinstance(self.one, OneOffline):
self._ensure_required_data()
_log.info('Gathering data for QC')
alf_path = self.session_path / 'alf'
# Load times
self.data['camera_times'] = alfio.load_object(
alf_path, f'{self.side}Camera')['times']
# Load dlc traces
dlc_df = alfio.load_object(alf_path,
f'{self.side}Camera',
namespace='ibl')['dlc']
targets = np.unique(
['_'.join(col.split('_')[:-1]) for col in dlc_df.columns])
# Set values to nan if likelyhood is too low
dlc_coords = {}
for t in targets:
idx = dlc_df.loc[dlc_df[f'{t}_likelihood'] < 0.9].index
dlc_df.loc[idx, [f'{t}_x', f'{t}_y']] = np.nan
dlc_coords[t] = np.array((dlc_df[f'{t}_x'], dlc_df[f'{t}_y']))
self.data['dlc_coords'] = dlc_coords
def test_wheel_motion(self, mock_cv):
side = 'left'
period = np.array([1730.3513333, 1734.1743333])
mock_cv().read.side_effect = self.side_effect()
aln = MotionAlignment(self.dummy_id, one=self.one)
aln.session_path = self.data_path / 'camera' / 'SWC_054' / '2020-10-07' / '001'
cam = alfio.load_object(aln.session_path / 'alf', f'{side}Camera')
aln.data.camera_times = {side: cam['times']}
aln.video_paths = {
side: aln.session_path / 'raw_video_data' / f'_iblrig_{side}Camera.raw.mp4'
}
aln.data.wheel = alfio.load_object(aln.session_path / 'alf', 'wheel')
# Test value error when invalid period given
with self.assertRaises(ValueError):
aln.align_motion(period=[5000, 5000.01], side=side)
dt_i, c, df = aln.align_motion(period=period, side=side)
expected = np.array([0.90278801, 0.68067675, 0.73734772, 0.82648895, 0.80950881,
0.88054471, 0.84264046, 0.302118, 0.94302567, 0.86188695])
np.testing.assert_array_almost_equal(expected, df[:10])
self.assertEqual(dt_i, 0)
self.assertEqual(round(c, 5), 19.48842)
# Test saving alignment video
with tempfile.TemporaryDirectory() as tdir:
aln.plot_alignment(save=tdir)
vid = next(Path(tdir).glob('*.mp4'))
self.assertEqual(vid.name, '2018-07-13_1_flowers_l.mp4')
self.assertEqual(round(vid.stat().st_size / 1e5), 18)
def test_qc_extract(self):
# extract a short lf signal RMS
for fbin in Path(self.init_folder).rglob('*.lf.bin'):
ephysqc.extract_rmsmap(fbin, out_folder=self.alf_folder)
rmsmap_lf = alf.load_object(self.alf_folder, '_ibl_ephysRmsMap_lf')
spec_lf = alf.load_object(self.alf_folder, '_ibl_ephysSpectra_lf')
ntimes = rmsmap_lf['times'].shape[0]
nchannels = rmsmap_lf['rms'].shape[1]
nfreqs = spec_lf['frequencies'].shape[0]
# makes sure the dimensions are consistend
self.assertTrue(rmsmap_lf['rms'].shape == (ntimes, nchannels))
self.assertTrue(spec_lf['power'].shape == (nfreqs, nchannels))
def test_training(self):
with tempfile.TemporaryDirectory() as tdir:
subjects_path = Path(tdir).joinpath('Subjects', *self.training_folder.parts[-3:])
session_path = shutil.copytree(self.training_folder, subjects_path)
# task running part - there are no videos, should exit gracefully
job = TrainingVideoCompress(session_path, one=self.one)
with self.assertLogs('ibllib', level='INFO'):
job.run()
self.assertEqual(job.status, 0)
self.assertTrue('skipping' in job.log)
# Now mock the video data so that extraction and QC succeed
video_path = session_path.joinpath('raw_video_data')
video_path.mkdir(parents=True)
video_path.joinpath('_iblrig_leftCamera.raw.mp4').touch()
with mock.patch('ibllib.io.extractors.camera.cv2.VideoCapture') as mock_vc, \
mock.patch('ibllib.pipes.training_preprocessing.CameraQC') as mock_qc:
def side_effect():
return True, np.random.randint(0, 255, size=(1024, 1280, 3))
mock_vc().read.side_effect = side_effect
length = 68453
mock_vc().get.return_value = length
job.run()
self.assertEqual(job.status, 0)
self.assertEqual(mock_qc.call_args.args, (session_path, 'left'))
mock_qc().run.assert_called_once_with(update=True)
self.assertTrue(len(job.outputs) > 0)
# check the file objects
ts = alfio.load_object(session_path / 'alf', 'leftCamera')
self.assertTrue(ts['times'].size == length)
def load_object(self,
eid: Union[str, Path, UUID],
obj: str,
collection: Optional[str] = 'alf',
download_only: bool = False,
**kwargs) -> Union[alfio.AlfBunch, List[Path]]:
"""
Load all attributes of an ALF object from a Session ID and an object name.
:param eid: Experiment session identifier; may be a UUID, URL, experiment reference string
details dict or Path
:param obj: The ALF object to load. Supports asterisks as wildcards.
:param collection: The collection to which the object belongs, e.g. 'alf/probe01'.
Supports asterisks as wildcards.
:param download_only: When true the data are downloaded and the file paths are returned
:param kwargs: Optional filters for the ALF objects, including namespace and timescale
:return: An ALF bunch or if download_only is True, a list of Paths objects
Examples:
load_object(eid, '*moves')
load_object(eid, 'trials')
load_object(eid, 'spikes', collection='*probe01')
"""
# Filter server-side by collection and dataset name
search_str = 'name__regex,' + obj.replace('*', '.*')
if collection and collection != 'all':
search_str += ',collection__regex,' + collection.replace('*', '.*')
results = self.alyx.rest('datasets', 'list', session=eid, django=search_str)
pattern = re.compile(fnmatch.translate(obj))
# Further refine by matching object part of ALF datasets
def match(r):
return is_valid(r['name']) and pattern.match(alf_parts(r['name'])[1])
# Get filenames of returned ALF files
returned_obj = {alf_parts(x['name'])[1] for x in results if match(x)}
# Validate result before loading
if len(returned_obj) > 1:
raise ALFMultipleObjectsFound('The following matching objects were found: ' +
', '.join(returned_obj))
elif len(returned_obj) == 0:
raise ALFObjectNotFound(f'ALF object "{obj}" not found on Alyx')
collection_set = {x['collection'] for x in results if match(x)}
if len(collection_set) > 1:
raise ALFMultipleCollectionsFound('Matching object belongs to multiple collections:' +
', '.join(collection_set))
# Download and optionally load the datasets
out_files = self.download_datasets(x for x in results if match(x))
assert not any(x is None for x in out_files), 'failed to download object'
if download_only:
return out_files
else:
return alfio.load_object(out_files[0].parent, obj, **kwargs)
def load_data(self, download=False):
"""
Load wheel, trial and camera timestamp data
:return: wheel, trials
"""
if download:
self.data.wheel = self.one.load_object(self.eid, 'wheel')
self.data.trials = self.one.load_object(self.eid, 'trials')
cam = self.one.load(self.eid, ['camera.times'], dclass_output=True)
self.data.camera_times = {
vidio.label_from_path(url): ts
for ts, url in zip(cam.data, cam.url)
}
else:
alf_path = self.session_path / 'alf'
self.data.wheel = alfio.load_object(alf_path, 'wheel')
self.data.trials = alfio.load_object(alf_path, 'trials')
self.data.camera_times = {
vidio.label_from_path(x): alfio.load_file_content(x)
for x in alf_path.glob('*Camera.times*')
}
assert all(x is not None for x in self.data.values())
def test_ephys(self, mock_qc):
# task running part
job = EphysVideoCompress(self.ephys_folder, one=self.one)
job.run()
self.assertEqual(job.status, 0)
self.assertEqual(len(mock_qc.call_args_list), 3) # Once per camera
labels = ('left', 'right', 'body')
self.assertCountEqual(labels, [arg.args[-1]for arg in mock_qc.call_args_list])
[self.assertEqual(call[0][0], self.ephys_folder) for call in mock_qc.call_args_list]
mock_qc().run.assert_called_with(update=True)
self.assertEqual(len(job.outputs), 6)
# check the file objects
for label in labels:
ts = alfio.load_object(self.ephys_folder / 'alf', f'{label}Camera')
self.assertTrue(ts['times'].size > 0)
# Get paths
ses_nr = listdir(
join(DATA_PATH, sessions.loc[i, 'lab'], 'Subjects',
sessions.loc[i, 'subject'], sessions.loc[i, 'date']))[0]
session_path = join(DATA_PATH, sessions.loc[i, 'lab'], 'Subjects',
sessions.loc[i, 'subject'], sessions.loc[i, 'date'],
ses_nr)
alf_path = join(DATA_PATH, sessions.loc[i, 'lab'], 'Subjects',
sessions.loc[i, 'subject'], sessions.loc[i, 'date'],
ses_nr, 'alf')
probe_path = join(DATA_PATH, sessions.loc[i, 'lab'], 'Subjects',
sessions.loc[i, 'subject'], sessions.loc[i, 'date'],
ses_nr, 'alf', 'probe%s' % sessions.loc[i, 'probe'])
# Load in data
spikes = ioalf.load_object(probe_path, 'spikes')
clusters = ioalf.load_object(probe_path, 'clusters')
trials = ioalf.load_object(alf_path, '_ibl_trials')
# Only use single units
spikes.times = spikes.times[np.isin(
spikes.clusters,
clusters.metrics.cluster_id[clusters.metrics.ks2_label == 'good'])]
spikes.clusters = spikes.clusters[np.isin(
spikes.clusters,
clusters.metrics.cluster_id[clusters.metrics.ks2_label == 'good'])]
# Calculate whether neuron discriminates blocks
trial_times = trials.stimOn_times[((trials.probabilityLeft > 0.55)
| (trials.probabilityLeft < 0.45))]
trial_blocks = (trials.probabilityLeft[((
from oneibl.one import ONE
import alf.io as ioalf
import ibllib.plots as iblplt
from brainbox.processing import bincount2D
T_BIN = 0.01
# get the data from flatiron and the current folder
one = ONE()
eid = one.search(subject='ZM_1150', date='2019-05-07', number=1)
D = one.load(eid[0], clobber=False, download_only=True)
session_path = Path(D.local_path[0]).parent
# load objects
spikes = ioalf.load_object(session_path, 'spikes')
clusters = ioalf.load_object(session_path, 'clusters')
channels = ioalf.load_object(session_path, 'channels')
trials = ioalf.load_object(session_path, '_ibl_trials')
# compute raster map as a function of cluster number
R, times, clusters = bincount2D(spikes['times'], spikes['clusters'], T_BIN)
# plot raster map
plt.imshow(R,
aspect='auto',
cmap='binary',
vmax=T_BIN / 0.001 / 4,
extent=np.r_[times[[0, -1]], clusters[[0, -1]]],
origin='lower')
# plot trial start and reward time
PCA_DIMS = 20
CCA_DIMS = PCA_DIMS
N_SPLITS = 5
RNG_SEED = 0
# get the data from flatiron
subject = 'KS005'
date = '2019-08-30'
number = 1
one = ONE()
eid = one.search(subject=subject, date=date, number=number)
D = one.load(eid[0], download_only=True)
session_path = Path(D.local_path[0]).parent
spikes = ioalf.load_object(session_path, 'spikes')
clusters = ioalf.load_object(session_path, 'clusters')
# channels = ioalf.load_object(session_path, 'channels')
trials = ioalf.load_object(session_path, 'trials')
# bin spikes and get trial IDs associated with them
binned_spikes, binned_trialIDs, _ = bin_spikes_trials(spikes,
trials,
bin_size=0.01)
# define areas
brain_areas = np.unique(clusters.brainAcronyms)
brain_areas = brain_areas[1:4] # [take subset for testing]
# split data by brain area
# (bin_spikes_trials does not return info for innactive clusters)
def gen_metrics_labels(eid, probe_name):
one = ONE()
ses_path = one.path_from_eid(eid)
alf_probe_dir = os.path.join(ses_path, 'alf', probe_name)
ks_dir = alf_probe_dir
spks_b = aio.load_object(alf_probe_dir, 'spikes')
units_b = bb.processing.get_units_bunch(spks_b)
units = list(units_b.amps.keys())
lengths_samples = [len(v) for k, v in units_b.samples.items()]
units_nonzeros = [i for i, d in enumerate(lengths_samples) if d > 0]
n_units = len(
units_nonzeros) #only compute metrics for units with no samples
#for cases where raw data is available locally:
ephys_file_dir = os.path.join(ses_path, 'raw_ephys_data', probe_name)
ephys_file = os.path.join(ses_path, 'raw_ephys_data', probe_name,
'_iblrig_ephysData.raw_g0_t0.imec.ap.cbin')
#create params.py file
params_file = os.path.join(ks_dir, 'params.py')
if os.path.exists(ephys_file) and not os.path.exists(params_file):
f = open(params_file, "w+")
f.write('dat_path = ' + 'r"' + ephys_file + '"\n' +
'''n_channels_dat = 385
dtype = 'int16'
offset = 0
sample_rate = 30000
hp_filtered = False
uidx=0''')
f.close()
# Initialize metrics
cum_amp_drift = np.full((n_units, ), np.nan)
cum_depth_drift = np.full((n_units, ), np.nan)
cv_amp = np.full((n_units, ), np.nan)
cv_fr = np.full((n_units, ), np.nan)
frac_isi_viol = np.full((n_units, ), np.nan)
frac_missing_spks = np.full((n_units, ), np.nan)
fp_estimate = np.full((n_units, ), np.nan)
presence_ratio = np.full((n_units, ), np.nan)
pres_ratio_std = np.full((n_units, ), np.nan)
ptp_sigma = np.full((n_units, ), np.nan)
units_missing_metrics = set()
label = np.empty([len(units)])
RefPViol = np.empty([len(units)])
NoiseCutoff = np.empty([len(units)])
MeanAmpTrue = np.empty([len(units)])
for idx, unit in enumerate(units_nonzeros):
if unit == units_nonzeros[0]:
t0 = time.perf_counter() # used for computation time estimate
print('computing metrics for unit ' + str(unit) + '...')
#load relevant data for unit
ts = units_b['times'][str(unit)]
amps = units_b['amps'][str(unit)]
samples = units_b['samples'][str(unit)]
depths = units_b['depths'][str(unit)]
RefPViol[idx] = FP_RP(ts)
NoiseCutoff[idx] = noise_cutoff(amps, quartile_length=.25)
#create 'label' based on RPviol,NoiseCutoff, and MeanAmp
if len(
samples > 50
): #only compute mean amplitude for units with more than 50 samples
try:
MeanAmpTrue[int(unit)] = peak_to_peak_amp(ephys_file,
samples,
nsamps=20)
if (FP_RP(ts) and noise_cutoff(amps, quartile_length=.25) < 20
and MeanAmpTrue[int(unit)] > 50):
label[idx] = 1
else:
label[idx] = 0
except:
if (FP_RP(ts)
and noise_cutoff(amps, quartile_length=.25) < 20):
label[idx] = 1
else:
label[idx] = 0
else: #no ephys file, do not include true mean amps
if (FP_RP(ts) and noise_cutoff(amps, quartile_length=.25) < 20):
label[idx] = 1
else:
label[idx] = 0
#now compute additional metrics that label does not depend on:
# Cumulative drift of spike amplitudes, normalized by total number of spikes.
try:
cum_amp_drift[idx] = cum_drift(amps)
except Exception as err:
print(
"Failed to compute 'cum_drift(amps)' for unit {}. Details: \n {}"
.format(unit, err))
units_missing_metrics.add(unit)
# Cumulative drift of spike depths, normalized by total number of spikes.
try:
cum_depth_drift[idx] = cum_drift(depths)
except Exception as err:
print(
"Failed to compute 'cum_drift(depths)' for unit {}. Details: \n {}"
.format(unit, err))
units_missing_metrics.add(unit)
# Coefficient of variation of spike amplitudes.
try:
cv_amp[idx] = np.std(amps) / np.mean(amps)
except Exception as err:
print("Failed to compute 'cv_amp' for unit {}. Details: \n {}".
format(unit, err))
units_missing_metrics.add(unit)
# Coefficient of variation of computed instantaneous firing rate.
try:
fr = bb.singlecell.firing_rate(ts, hist_win=0.01, fr_win=0.25)
cv_fr[idx] = np.std(fr) / np.mean(fr)
except Exception as err:
print(
"Failed to compute 'cv_fr' for unit {}. Details: \n {}".format(
unit, err))
units_missing_metrics.add(unit)
# Fraction of isi violations.
try:
frac_isi_viol[idx], _, _ = isi_viol(ts, rp=0.002)
except Exception as err:
print(
"Failed to compute 'frac_isi_viol' for unit {}. Details: \n {}"
.format(unit, err))
units_missing_metrics.add(unit)
# Estimated fraction of missing spikes.
try:
frac_missing_spks[idx], _, _ = feat_cutoff(amps,
spks_per_bin=10,
sigma=4,
min_num_bins=50)
except Exception as err:
print(
"Failed to compute 'frac_missing_spks' for unit {}. Details: \n {}"
.format(unit, err))
units_missing_metrics.add(unit)
# Estimated fraction of false positives.
try:
fp_estimate[idx] = fp_est(ts, rp=0.002)
except Exception as err:
print("Failed to compute 'fp_est' for unit {}. Details: \n {}".
format(unit, err))
units_missing_metrics.add(unit)
# Presence ratio
try:
presence_ratio[idx], _ = pres_ratio(ts, hist_win=10)
except Exception as err:
print("Failed to compute 'pres_ratio' for unit {}. Details: \n {}".
format(unit, err))
units_missing_metrics.add(unit)
# Presence ratio over the standard deviation of spike counts in each bin
try:
pr, pr_bins = pres_ratio(ts, hist_win=10)
pres_ratio_std[idx] = pr / np.std(pr_bins)
except Exception as err:
print(
"Failed to compute 'pres_ratio_std' for unit {}. Details: \n {}"
.format(unit, err))
units_missing_metrics.add(unit)
#append metrics to the current clusters.metrics
metrics_read = pd.read_csv(Path(alf_probe_dir, 'clusters.metrics.csv'))
if not 'label' in metrics_read.columns:
try:
label_df = pd.DataFrame(label)
pd.DataFrame.insert(metrics_read, 1, 'label', label_df)
except ValueError:
pd.DataFrame.drop(metrics_read, columns='label')
pd.DataFrame.insert(metrics_read, 1, 'label', label_df)
except:
print("Could not save 'label' to .csv.")
try:
df_cum_amp_drift = pd.DataFrame(cum_amp_drift.round(2))
metrics_read['cum_amp_drift'] = df_cum_amp_drift
except Exception as err:
print("Could not save 'cum_amp_drift' to .csv. Details: \n {}".
format(err))
try:
df_cum_depth_drift = pd.DataFrame(cum_depth_drift.round(2))
metrics_read['cum_depth_drift'] = df_cum_depth_drift
except Exception as err:
print("Could not save 'cum_depth_drift' to .tsv. Details: \n {}".
format(err))
try:
df_cv_amp = pd.DataFrame(cv_amp.round(2))
metrics_read['cv_amp'] = df_cv_amp
except Exception as err:
print(
"Could not save 'cv_amp' to .tsv. Details: \n {}".format(err))
try:
df_cv_fr = pd.DataFrame(cv_fr.round(2))
metrics_read['cv_fr'] = df_cv_fr
except Exception as err:
print("Could not save 'cv_fr' to .tsv. Details: \n {}".format(err))
try:
df_frac_isi_viol = pd.DataFrame(frac_isi_viol.round(2))
metrics_read['frac_isi_viol'] = df_frac_isi_viol
except Exception as err:
print("Could not save 'frac_isi_viol' to .tsv. Details: \n {}".
format(err))
try:
df_frac_missing_spks = pd.DataFrame(frac_missing_spks.round(2))
metrics_read['frac_missing_spks'] = df_frac_missing_spks
except Exception as err:
print("Could not save 'frac_missing_spks' to .tsv. Details: \n {}".
format(err))
try:
df_fp_est = pd.DataFrame(fp_estimate.round(2))
metrics_read['fp_est'] = df_fp_est
except Exception as err:
print(
"Could not save 'fp_est' to .tsv. Details: \n {}".format(err))
try:
df_pres_ratio = pd.DataFrame(presence_ratio.round(2))
metrics_read['pres_ratio'] = df_pres_ratio
except Exception as err:
print("Could not save 'pres_ratio' to .tsv. Details: \n {}".format(
err))
try:
df_pres_ratio_std = pd.DataFrame(pres_ratio_std.round(2))
metrics_read['pres_ratio_std'] = df_pres_ratio_std
except Exception as err:
print("Could not save 'pres_ratio_std' to .tsv. Details: \n {}".
format(err))
try:
df_refp_viol = pd.DataFrame(RefPViol)
pd.DataFrame.insert(metrics_read, 2, 'refp_viol', df_refp_viol)
except ValueError:
pd.DataFrame.drop(metrics_read, columns='refp_viol')
pd.DataFrame.insert(metrics_read, 2, 'refp_viol', df_refp_viol)
except Exception as err:
print("Could not save 'RefPViol' to .tsv. Details: \n {}".format(
err))
try:
df_noise_cutoff = pd.DataFrame(NoiseCutoff)
pd.DataFrame.insert(metrics_read, 3, 'noise_cutoff',
df_noise_cutoff)
except ValueError:
pd.DataFrame.drop(metrics_read, columns='noise_cutoff')
pd.DataFrame.insert(metrics_read, 3, 'noise_cutoff',
df_noise_cutoff)
except Exception as err:
print(
"Could not save 'NoiseCutoff' to .tsv. Details: \n {}".format(
err))
try:
df_mean_amp_true = pd.DataFrame(MeanAmpTrue)
pd.DataFrame.insert(metrics_read, 4, 'mean_amp_true',
df_mean_amp_true)
except ValueError:
pd.DataFrame.drop(metrics_read, columns='mean_amp_true')
pd.DataFrame.insert(metrics_read, 4, 'mean_amp_true',
df_mean_amp_true)
except Exception as err:
print("Could not save 'Mean Amp True' to .tsv. Details: \n {}".
format(err))
#now add df to csv
metrics_read.to_csv(Path(alf_probe_dir, 'clusters.metrics.csv'))
print('Launching phy')
else:
print('Launching phy')
try:
numpass = int(sum(label))
print("\n Number of units that pass: "******"Number of units that pass RP threshold: ", numpassRP)
print("Number of units that pass Amp Cutoff threshold: ", numpassAC)
print("Number of total units: ", ntot)
except Exception as err:
print("Could not compute number of units that pass. Details \n {}".
format(err))
return metrics_read
eid5 = one.search(subject='ZM_2406', date='2019-11-12', number=1)[0]
eid6 = one.search(subject='CSK-scan-008', date='2019-12-09', number=8)[0]
eid7 = one.search(subject='NYU-18', date='2019-10-23', number=1)[0]
eids = [eid0, eid1, eid2, eid3, eid4, eid5, eid6, eid7]
n_units = np.zeros((len(eids),))
n_good_units = np.zeros((len(eids),))
for i, eid in enumerate(eids):
# load data to disk
d_paths = one.load(eid, dataset_types=dtypes, clobber=False, download_only=True)
# get paths and load data in memory
alf_probe_path = os.path.split(d_paths[0])[0]
spks_b = aio.load_object(alf_probe_path, 'spikes')
units_b = bb.processing.get_units_bunch(spks_b)
# filter units and get nubmer of good units and number of total units
T = spks_b.times[-1] - spks_b.times[0] # length of recording session
filt_units = bb.processing.filter_units(units_b, T, min_amp=50e-6, min_fr=1)
# get num_units & num_good units
n_units[i] = np.max(spks_b.clusters)
n_good_units = len(filt_units)
# 2 group bar plot(all units, good units)
names = ['swc026_09-16_1', 'ks003_11-19_1', 'lic3_08-27_2', 'cer5_10-25_1', 'CSHL020_12-04_5',
'ZM2406_11-12_1', 'CSKscan008_12-09_1', 'NYU18_10-23_1']
units_eid0 = [n_units[0], n_good_units[0]]
def gen_metrics(alf_dir, ks_dir, ephys_file_path=None):
"""
Tries to generate single unit metrics for all units metric-by-metric and save the metrics
as .tsv files, and displays an error if unable to create one of the metric .tsv files,
before continuing to the next one.
Parameters
----------
alf_dir : string
Full path to alf output directory.
ks_dir : string
Full path to the ks2 output directory. The .tsv files will be saved here.
ephys_file_path : string (optional)
Full path to binary ephys file.
Returns
-------
units_missing_metrics : set
Set of units for which not all metrics were able to be calculated.
Examples
-------
1) Create an alf directory from a ks2 output directory. The alf directory cannot be the same as
the ks2 output directory. Then, generate all metrics that don't require raw ephys data.
>>> import ibllib.ephys.spikes as e_spks
>>> from gen_phy_metrics import gen_metrics
>>> e_spks.ks2_to_alf(ks_dir_full_path, alf_dir_full_path)
>>> gen_metrics(alf_dir, ks_dir)
2) Generate metrics from an alf directory and metrics that require an ephys_file_path. For phy,
the ephys file should be in `ks_dir`.
>>> from gen_phy_metrics import gen_metrics
>>> gen_metrics(alf_dir, ks_dir, ephys_file_path=ks_dir)
"""
# Setup #
# ----- #
# Extract alf objects from `alf_dir` and get units info
alf_dir = Path(alf_dir)
if not (Path.exists(alf_dir)):
raise FileNotFoundError(
'The given alf directory {} does not exist!'.format(alf_dir))
spks_b = aio.load_object(alf_dir, 'spikes')
clstrs_b = aio.load_object(alf_dir, 'clusters')
units_b = bb.processing.get_units_bunch(spks_b)
units = list(units_b.amps.keys())
n_units = np.max(spks_b.clusters) + 1
# Initialize metrics
cum_amp_drift = np.full((n_units, ), np.nan)
cum_depth_drift = np.full((n_units, ), np.nan)
cv_amp = np.full((n_units, ), np.nan)
cv_fr = np.full((n_units, ), np.nan)
frac_isi_viol = np.full((n_units, ), np.nan)
fn_est = np.full((n_units, ), np.nan)
fp_est = np.full((n_units, ), np.nan)
pres_ratio = np.full((n_units, ), np.nan)
pres_ratio_std = np.full((n_units, ), np.nan)
ptp_sigma = np.full((n_units, ), np.nan)
units_missing_metrics = set()
# Compute metrics #
# --------------- #
# Iterate over all units
for unit in units:
if unit == units[0]:
t0 = time.perf_counter() # used for computation time estimate
# Need timestamps, amps, depths
ts = units_b['times'][unit]
#Bug was below: amps and depths were defined as units_b['times'][unit]!
amps = units_b['amps'][unit]
depths = units_b['depths'][unit]
# Cumulative drift of spike amplitudes, normalized by total number of spikes.
try:
cum_amp_drift[int(unit)] = bb.metrics.cum_drift(amps)
except Exception as err:
print(
"Failed to compute 'cum_drift(amps)' for unit {}. Details: \n {}"
.format(unit, err))
units_missing_metrics.add(unit)
# Cumulative drift of spike depths, normalized by total number of spikes.
try:
cum_depth_drift[int(unit)] = bb.metrics.cum_drift(depths)
except Exception as err:
print(
"Failed to compute 'cum_drift(depths)' for unit {}. Details: \n {}"
.format(unit, err))
units_missing_metrics.add(unit)
# Coefficient of variation of spike amplitudes.
try:
cv_amp[int(unit)] = np.std(amps) / np.mean(amps)
except Exception as err:
print("Failed to compute 'cv_amp' for unit {}. Details: \n {}".
format(unit, err))
units_missing_metrics.add(unit)
# Coefficient of variation of computed instantaneous firing rate.
try:
fr = bb.singlecell.firing_rate(ts, hist_win=0.01, fr_win=0.25)
cv_fr[int(unit)] = np.std(fr) / np.mean(fr)
except Exception as err:
print(
"Failed to compute 'cv_fr' for unit {}. Details: \n {}".format(
unit, err))
units_missing_metrics.add(unit)
# Fraction of isi violations.
try:
frac_isi_viol[int(unit)], _, _ = bb.metrics.isi_viol(ts, rp=0.002)
except Exception as err:
print(
"Failed to compute 'frac_isi_viol' for unit {}. Details: \n {}"
.format(unit, err))
units_missing_metrics.add(unit)
# Estimated fraction of missing spikes.
try:
fn_est[int(unit)], _, _ = bb.metrics.missed_spikes_est(
amps, spks_per_bin=10, sigma=4, min_num_bins=50)
except Exception as err:
print(
"Failed to compute 'missed_spikes_est' for unit {}. Details: \n {}"
.format(unit, err))
units_missing_metrics.add(unit)
# Estimated fraction of false positives.
try:
fp_est[int(unit)] = bb.metrics.contamination(ts, rp=0.002)
except Exception as err:
print(
"Failed to compute 'contamination' for unit {}. Details: \n {}"
.format(unit, err))
units_missing_metrics.add(unit)
# Presence ratio
try:
pres_ratio[int(unit)], _ = bb.metrics.pres_ratio(ts, hist_win=10)
except Exception as err:
print("Failed to compute 'pres_ratio' for unit {}. Details: \n {}".
format(unit, err))
units_missing_metrics.add(unit)
# Presence ratio over the standard deviation of spike counts in each bin
try:
pr, pr_bins = bb.metrics.pres_ratio(ts, hist_win=10)
pres_ratio_std[int(unit)] = pr / np.std(pr_bins)
except Exception as err:
print(
"Failed to compute 'pres_ratio_std' for unit {}. Details: \n {}"
.format(unit, err))
units_missing_metrics.add(unit)
# The mean peak-to-peak value over the background noise of the channel of max amplitude.
if ephys_file_path:
try:
ch = clstrs_b['channels'][int(
unit)] # channel of max amplitude
ptp_sigma[int(unit)] = bb.metrics.ptp_over_noise(
ephys_file_path,
ts,
ch,
t=2.0,
sr=30000,
n_ch_probe=385,
dtype='int16',
car=False)
except Exception as err:
print(
"Failed to compute 'ptp_sigma' for unit {}. Details: \n {}"
.format(unit, err))
units_missing_metrics.add(unit)
if unit == units[0]: # give time estimate
dt = time.perf_counter() - t0
print(
'\nComputing metrics. Estimated time is {:.2f} mins\n'.format(
len(units) * dt / 60))
# Extract to .tsv files #
# --------------------- #
try:
df_cum_amp_drift = pd.DataFrame(cum_amp_drift.round(2))
df_cum_amp_drift.to_csv(Path(ks_dir, 'cum_amp_drift.tsv'),
sep='\t',
header=['cum_amp_drift'])
except Exception as err:
print("Could not save 'cum_amp_drift' to .tsv. Details: \n {}".format(
err))
try:
df_cum_depth_drift = pd.DataFrame(cum_depth_drift.round(2))
df_cum_depth_drift.to_csv(Path(ks_dir, 'cum_depth_drift.tsv'),
sep='\t',
header=['cum_depth_drift'])
except Exception as err:
print(
"Could not save 'cum_depth_drift' to .tsv. Details: \n {}".format(
err))
try:
df_cv_amp = pd.DataFrame(cv_amp.round(2))
df_cv_amp.to_csv(Path(ks_dir, 'cv_amp.tsv'),
sep='\t',
header=['cv_amp'])
except Exception as err:
print("Could not save 'cv_amp' to .tsv. Details: \n {}".format(err))
try:
df_cv_fr = pd.DataFrame(cv_fr.round(2))
df_cv_fr.to_csv(Path(ks_dir, 'cv_fr.tsv'), sep='\t', header=['cv_fr'])
except Exception as err:
print("Could not save 'cv_fr' to .tsv. Details: \n {}".format(err))
try:
df_frac_isi_viol = pd.DataFrame(frac_isi_viol.round(2))
df_frac_isi_viol.to_csv(Path(ks_dir, 'frac_isi_viol.tsv'),
sep='\t',
header=['frac_isi_viol'])
except Exception as err:
print("Could not save 'frac_isi_viol' to .tsv. Details: \n {}".format(
err))
try:
df_fn_est = pd.DataFrame(fn_est.round(2))
df_fn_est.to_csv(Path(ks_dir, ' missed_spikes_est.tsv'),
sep='\t',
header=['missed_spikes_est'])
except Exception as err:
print("Could not save 'missed_spikes_est' to .tsv. Details: \n {}".
format(err))
try:
df_fp_est = pd.DataFrame(fp_est.round(2))
df_fp_est.to_csv(Path(ks_dir, 'contamination.tsv'),
sep='\t',
header=['contamination'])
except Exception as err:
print("Could not save 'contamination' to .tsv. Details: \n {}".format(
err))
try:
df_pres_ratio = pd.DataFrame(pres_ratio.round(2))
df_pres_ratio.to_csv(Path(ks_dir, 'pres_ratio.tsv'),
sep='\t',
header=['pres_ratio'])
except Exception as err:
print(
"Could not save 'pres_ratio' to .tsv. Details: \n {}".format(err))
try:
df_pres_ratio_std = pd.DataFrame(pres_ratio_std.round(2))
df_pres_ratio_std.to_csv(Path(ks_dir, 'pres_ratio_std.tsv'),
sep='\t',
header=['pres_ratio_std'])
except Exception as err:
print("Could not save 'pres_ratio_std' to .tsv. Details: \n {}".format(
err))
if ephys_file_path:
try:
df_ptp_sigma = pd.DataFrame(ptp_sigma.round(2))
df_ptp_sigma.to_csv(Path(ks_dir, 'ptp_sigma.tsv'),
sep='\t',
header=['ptp_sigma'])
except Exception as err:
print("Could not save 'cum_amp_drift' to .tsv. Details: \n {}".
format(err))
return units_missing_metrics
'Ephys file not found! Some of the examples in this script require ephys file,'
)
# Ensure st least alf directory can be found
assert os.path.isdir(alf_probe_dir), \
'Directories set incorrectly. {} not found'.format(alf_probe_dir)
# Call brainbox functions #
#-------------------------#
# Change variable names to same names used in brainbox docstrings
path_to_ephys_file = ephys_file_path
path_to_alf_out = alf_probe_dir
# Load alf objects
spks_b = aio.load_object(path_to_alf_out, 'spikes')
clstrs_b = aio.load_object(path_to_alf_out, 'clusters')
chnls_b = aio.load_object(path_to_alf_out, 'channels')
tmplts_b = aio.load_object(path_to_alf_out, 'templates')
# Convert spikes bunch into a units bunch
units_b = bb.processing.get_units_bunch(spks_b) # this may take a few mins
unit4_amps = units_b['amps']['4'] # get amplitudes for unit 4.
# Filter units according to some parameters
T = spks_b['times'][-1] - spks_b['times'][0]
filtered_units = bb.processing.filter_units(units_b,
T,
min_amp=0,
min_fr=0.5,
max_fpr=1,
alf_probe_dir = os.path.join(alf_dir, probe)
ephys_file_dir = os.path.join(session_path, 'raw_ephys_data', probe)
# Find 'ap' ephys file in `ephys_file_dir`
for i in os.listdir(ephys_file_dir):
if 'ap' in i and 'bin' in i:
ephys_file_path = os.path.join(ephys_file_dir, i)
# Ensure directories can be found
assert os.path.isdir(ephys_file_dir) and os.path.isdir(alf_probe_dir) \
and os.path.isabs(ephys_file_path), 'Directories set incorrectly'
else:
ephys_file_path = '/Users/gaelle/Downloads/FlatIron/mainenlab/Subjects/ZM_2407/2019-11-05/003/raw_ephys_data/probe_00/_spikeglx_ephysData_probe00.raw_g0_t0.imec.ap.cbin'
alf_probe_dir = '/Users/gaelle/Downloads/FlatIron/mainenlab/Subjects/ZM_2407/2019-11-05/003/alf/probe_00'
# -- Get spks and clusters
spks = aio.load_object(alf_probe_dir, 'spikes')
clstrs = aio.load_object(alf_probe_dir, 'clusters')
# Fig 1 -- ALL UNITS
# Create a bar plot of the variances of the spike amplitudes for each unit.
fig, var_vals, p_vals = bb.plot.feat_vars(spks)
# Fig 2 -- SINGLE UNIT
# Plot cutoff line indicating the fraction of spikes missing from a unit based on the recorded
# unit's spike amplitudes, assuming the distribution of the unit's spike amplitudes is symmetric.
fig, fraction_missing = bb.plot.missed_spikes_est(spks, 1)
# Fig 3 -- SINGLE UNIT
# Compare first and last 100 spike waveforms for unit1, across 20 channels around the channel
# of max amplitude, and compare the first and last 50 spike waveforms for unit2, across 15
# channels around the mean.
def get_vr_clusters(session_path, clusters=None, n_selected_cl=4):
'''
Gets visually responsive clusters
Parameters
----------
session_path : str
The path to to the appropriate 'alf/probe' directory.
clusters : ndarray
The clusters to use to get a subset of visually responsive clusters. (if `None`, take
visually response subset from all clusters from recording session.)
n_selected_cl : int
The number of clusters to return in `vr_clusters_selected`
Returns
-------
clusters_vr : ndarray
The visually responsive clusters.
clusters_selected_vr : ndarray
A subset of `n_selected_cl` `vr_clusters`
'''
print('finding visually responsive clusters...', end='', flush=True)
# -------------------------
# load required alf objects
# -------------------------
spikes = ioalf.load_object(session_path, 'spikes')
gratings = ioalf.load_object(session_path, '_iblcertif_.odsgratings')
spontaneous = ioalf.load_object(session_path, '_iblcertif_.spontaneous')
grating_times = {
'beg': gratings['odsgratings.times.00'],
'end': gratings['odsgratings.times.01']}
grating_vals = {
'beg': gratings['odsgratings.stims.00'],
'end': gratings['odsgratings.stims.01']}
spont_times = {
'beg': spontaneous['spontaneous.times.00'],
'end': spontaneous['spontaneous.times.01']}
# ---------------------------------
# find visually responsive clusters
# ---------------------------------
epochs = ['beg', 'end']
if clusters is None: # use all clusters
# speed up downstream computations by restricting data to relevant time periods
mask_times = np.full(spikes.times.shape, fill_value=False)
for epoch in epochs:
mask_times |= (spikes.times >= grating_times[epoch].min()) & \
(spikes.times <= grating_times[epoch].max())
clusters = np.unique(spikes.clusters[mask_times])
# only calculate responsiveness for clusters that were active during gratings
mask_clust = np.isin(spikes.clusters, clusters)
resp = {epoch: [] for epoch in epochs}
for epoch in epochs:
resp[epoch] = are_neurons_responsive(
spikes.times[mask_clust], spikes.clusters[mask_clust], grating_times[epoch],
grating_vals[epoch], spont_times[epoch])
resp_agg = resp['beg'] & resp['end']
# remove non-responsive clusters
clusters_vr = clusters[resp_agg]
print('done')
if n_selected_cl < len(clusters_vr):
clusters_selected_vr = np.random.choice(clusters_vr, size=n_selected_cl, replace=False)
else:
clusters_selected_vr = clusters_vr
return clusters_vr, clusters_selected_vr
def plot_grating_figures(
session_path, cluster_ids_summary, cluster_ids_selected, save_dir=None, format='png',
pre_time=0.5, post_time=2.5, bin_size=0.005, smoothing=0.025, n_rand_clusters=20,
plot_summary=True, plot_selected=True):
"""
Produces two summary figures for the oriented grating protocol; the first summary figure
contains plots that compare different measures during the first and second grating protocols,
such as orientation selectivity index (OSI), orientation preference, fraction of visual
clusters, PSTHs, firing rate histograms, etc. The second summary figure contains plots of polar
PSTHs and corresponding rasters for a random subset of visually responsive clusters.
Parameters
----------
session_path : str
absolute path to experimental session directory
cluster_ids_summary : list
the clusters for which to plot summary psths/rasters; if empty, all clusters with responses
during the grating presentations are used
cluster_ids_selected : list
the clusters for which to plot individual psths/rasters; if empty, `n_rand_clusters` are
randomly chosen
save_dir : str or NoneType
if NoneType, figures are displayed; else a string defining the absolute filepath to the
directory in which figures will be saved
format : str
file format, i.e. 'png' | 'pdf' | 'jpg'
pre_time : float
time (sec) to plot before grating presentation onset
post_time : float
time (sec) to plot after grating presentation onset (should include length of stimulus)
bin_size : float
size of bins for raster plots/psths
smoothing : float
size of smoothing kernel (sec)
n_rand_clusters : int
the number of random clusters to choose for which to plot psths/rasters if
`cluster_ids_slected` is empty
plot_summary : bool
a flag for plotting the summary figure
plot_selected : bool
a flag for plotting the selected units figure
Returns
-------
metrics : dict
- 'osi' (dict): keys 'beg', 'end' point to arrays of osis during these epochs
- 'orientation_pref' (dict): keys 'beg', 'end' point to arrays of orientation preference
- 'frac_resp_by_depth' (dict): fraction of responsive clusters by depth
fig_dict : dict
A dict whose values are handles to one or both figures generated.
"""
fig_dict = {}
cluster_ids = cluster_ids_summary
cluster_idxs = cluster_ids_selected
epochs = ['beg', 'end']
# -------------------------
# load required alf objects
# -------------------------
print('loading alf objects...', end='', flush=True)
spikes = ioalf.load_object(session_path, 'spikes')
clusters = ioalf.load_object(session_path, 'clusters')
gratings = ioalf.load_object(session_path, '_iblcertif_.odsgratings')
spontaneous = ioalf.load_object(session_path, '_iblcertif_.spontaneous')
grating_times = {
'beg': gratings['odsgratings.times.00'],
'end': gratings['odsgratings.times.01']}
grating_vals = {
'beg': gratings['odsgratings.stims.00'],
'end': gratings['odsgratings.stims.01']}
spont_times = {
'beg': spontaneous['spontaneous.times.00'],
'end': spontaneous['spontaneous.times.01']}
# --------------------------
# calculate relevant metrics
# --------------------------
print('calcuating mean responses to gratings...', end='', flush=True)
# calculate mean responses to gratings
mask_clust = np.isin(spikes.clusters, cluster_ids) # update mask for responsive clusters
mask_times = np.full(spikes.times.shape, fill_value=False)
for epoch in epochs:
mask_times |= (spikes.times >= grating_times[epoch].min()) & \
(spikes.times <= grating_times[epoch].max())
resp = {epoch: [] for epoch in epochs}
for epoch in epochs:
resp[epoch] = are_neurons_responsive(
spikes.times[mask_clust], spikes.clusters[mask_clust], grating_times[epoch],
grating_vals[epoch], spont_times[epoch])
responses = {epoch: [] for epoch in epochs}
for epoch in epochs:
responses[epoch] = bin_responses(
spikes.times[mask_clust], spikes.clusters[mask_clust], grating_times[epoch],
grating_vals[epoch])
responses_mean = {epoch: np.mean(responses[epoch], axis=2) for epoch in epochs}
# responses_se = {epoch: np.std(responses[epoch], axis=2) / np.sqrt(responses[epoch].shape[2])
# for epoch in responses.keys()}
print('done')
# calculate osi and orientation preference
print('calcuating osi/orientation preference...', end='', flush=True)
ori_pref = {epoch: [] for epoch in epochs}
osi = {epoch: [] for epoch in epochs}
for epoch in epochs:
osi[epoch], ori_pref[epoch] = compute_selectivity(
responses_mean[epoch], np.unique(grating_vals[epoch]), 'ori')
print('done')
# calculate depth vs osi ratio (osi_beg/osi_end)
print('calcuating osi ratio as a function of depth...', end='', flush=True)
depths = np.array([clusters.depths[c] for c in cluster_ids])
ratios = np.array([osi['beg'][c] / osi['end'][c] for c in range(len(cluster_ids))])
print('done')
# calculate fraction of visual neurons by depth
print('calcuating fraction of visual clusters by depth...', end='', flush=True)
n_bins = 10
min_depth = np.min(clusters['depths'])
max_depth = np.max(clusters['depths'])
depth_limits = np.linspace(min_depth - 1, max_depth, n_bins + 1)
depth_avg = (depth_limits[:-1] + depth_limits[1:]) / 2
# aggregate clusters
clusters_binned = {epoch: [] for epoch in epochs}
frac_responsive = {epoch: [] for epoch in epochs}
cids = cluster_ids
for epoch in epochs:
# just look at responsive clusters during this epoch
cids_tmp = cids[resp[epoch]]
for d in range(n_bins):
lo_limit = depth_limits[d]
up_limit = depth_limits[d + 1]
# clusters.depth index is cluster id
cids_curr_depth = np.where(
(lo_limit < clusters.depths) & (clusters.depths <= up_limit))[0]
clusters_binned[epoch].append(cids_curr_depth)
frac_responsive[epoch].append(np.sum(
np.isin(cids_tmp, cids_curr_depth)) / len(cids_curr_depth))
# package for plotting
responsive = {'fraction': frac_responsive, 'depth': depth_avg}
print('done')
# calculate PSTH averaged over all clusters/orientations
print('calcuating average PSTH...', end='', flush=True)
peths = {epoch: [] for epoch in epochs}
peths_avg = {epoch: [] for epoch in epochs}
for epoch in epochs:
stim_ids = np.unique(grating_vals[epoch])
peths[epoch] = {i: None for i in range(len(stim_ids))}
peths_avg_tmp = []
for i, stim_id in enumerate(stim_ids):
curr_stim_idxs = np.where(grating_vals[epoch] == stim_id)
align_times = grating_times[epoch][curr_stim_idxs, 0][0]
peths[epoch][i], _ = calculate_peths(
spikes.times[mask_times], spikes.clusters[mask_times], cluster_ids,
align_times, pre_time=pre_time, post_time=post_time, bin_size=bin_size,
smoothing=smoothing, return_fr=True)
peths_avg_tmp.append(
np.mean(peths[epoch][i]['means'], axis=0, keepdims=True))
peths_avg_tmp = np.vstack(peths_avg_tmp)
peths_avg[epoch] = {
'mean': np.mean(peths_avg_tmp, axis=0),
'std': np.std(peths_avg_tmp, axis=0) / np.sqrt(peths_avg_tmp.shape[0])}
peths_avg['bin_size'] = bin_size
peths_avg['on_idx'] = int(pre_time / bin_size)
peths_avg['off_idx'] = peths_avg['on_idx'] + int(2 / bin_size)
print('done')
# compute rasters for entire orientation sequence at beg/end epoch
if plot_summary:
print('computing rasters for example stimulus sequences...', end='', flush=True)
r = {epoch: None for epoch in epochs}
r_times = {epoch: None for epoch in epochs}
r_clusters = {epoch: None for epoch in epochs}
for epoch in epochs:
# restrict activity to a single stim series; assumes each possible grating direction
# is displayed before repeating
n_stims = len(np.unique(grating_vals[epoch]))
mask_idxs_e = (spikes.times >= grating_times[epoch][:n_stims].min()) & \
(spikes.times <= grating_times[epoch][:n_stims].max())
r_tmp, r_times[epoch], r_clusters[epoch] = bincount2D(
spikes.times[mask_idxs_e], spikes.clusters[mask_idxs_e], bin_size)
# order activity by anatomical depth of neurons
d = dict(zip(spikes.clusters[mask_idxs_e], spikes.depths[mask_idxs_e]))
y = sorted([[i, d[i]] for i in d])
isort = np.argsort([x[1] for x in y])
r[epoch] = r_tmp[isort, :]
# package for plotting
rasters = {'spikes': r, 'times': r_times, 'clusters': r_clusters, 'bin_size': bin_size}
print('done')
# -------------------------------------------------
# compute psths and rasters for individual clusters
# -------------------------------------------------
if plot_selected:
print('computing psths and rasters for clusters...', end='', flush=True)
if len(cluster_ids_selected) == 0:
if (n_rand_clusters < len(cluster_ids)):
cluster_idxs = np.random.choice(cluster_ids, size=n_rand_clusters, replace=False)
else:
cluster_idxs = cluster_ids
else:
cluster_idxs = cluster_ids_selected
mean_responses = {cluster: {epoch: [] for epoch in epochs} for cluster in cluster_idxs}
osis = {cluster: {epoch: [] for epoch in epochs} for cluster in cluster_idxs}
binned = {cluster: {epoch: [] for epoch in epochs} for cluster in cluster_idxs}
for cluster_idx in cluster_idxs:
cluster = np.where(cluster_ids == cluster_idx)[0]
for epoch in epochs:
mean_responses[cluster_idx][epoch] = responses_mean[epoch][cluster, :][0]
osis[cluster_idx][epoch] = osi[epoch][cluster]
stim_ids = np.unique(grating_vals[epoch])
binned[cluster_idx][epoch] = {j: None for j in range(len(stim_ids))}
for j, stim_id in enumerate(stim_ids):
curr_stim_idxs = np.where(grating_vals[epoch] == stim_id)
align_times = grating_times[epoch][curr_stim_idxs, 0][0]
_, binned[cluster_idx][epoch][j] = calculate_peths(
spikes.times[mask_times], spikes.clusters[mask_times], [cluster_idx],
align_times, pre_time=pre_time, post_time=post_time, bin_size=bin_size)
print('done')
# --------------
# output figures
# --------------
print('producing figures...', end='')
if plot_summary:
if save_dir is None:
save_file = None
else:
if not os.path.exists(save_dir):
os.makedirs(save_dir)
save_file = os.path.join(save_dir, 'grating_summary_figure.' + format)
fig_gr_summary = plot_summary_figure(
ratios=ratios, depths=depths, responsive=responsive, peths_avg=peths_avg, osi=osi,
ori_pref=ori_pref, responses_mean=responses_mean, rasters=rasters, save_file=save_file)
fig_gr_summary.suptitle('Summary Grating Responses')
fig_dict['gr_summary'] = fig_gr_summary
if plot_selected:
if save_dir is None:
save_file = None
else:
save_file = os.path.join(save_dir, 'grating_random_responses.' + format)
fig_gr_selected = plot_psths_and_rasters(
mean_responses, binned, osis, grating_vals, on_idx=peths_avg['on_idx'],
off_idx=peths_avg['off_idx'], bin_size=bin_size, save_file=save_file)
fig_gr_selected.suptitle('Selected Units Grating Responses')
print('done')
fig_dict['gr_selected'] = fig_gr_selected
# -----------------------------
# package up and return metrics
# -----------------------------
metrics = {
'osi': osi,
'orientation_pref': ori_pref,
'frac_resp_by_depth': responsive,
}
return fig_dict, metrics
maxlags = maxlags or ns - 1
return np.correlate(x, y, mode='full')[ns - 1 - maxlags:ns + maxlags]
T_BIN = 0.01 # seconds
CORR_LEN = 1 # seconds
CORR_BINS = int(CORR_LEN / T_BIN) # bins
# get the data from flatiron and the current folder
one = ONE()
eid = one.search(subject='ZM_1150', date='2019-05-07', number=1)
D = one.load(eid[0], clobber=False, download_only=True)
session_path = Path(D.local_path[0]).parent
# load objects
spikes = ioalf.load_object(session_path, 'spikes')
# Get a Bunch instance.
b = one_to_bunch(spikes)
# Compute the firing rates.
rates = firing_rates(b.spike_times, b.spike_clusters, T_BIN)
# Note: I would rather just use spikes['times'] and spikes['clusters'] instead of going
# via a Bunch or DataFrame or similar...
# Compute the cross-correlation between the firing rate of two neurons.
c = xcorr(rates[0], rates[1], CORR_BINS)
# Plot it.
lags = np.linspace(-CORR_LEN, +CORR_LEN, len(c))
plt.plot(c)
def plot_rfs_by_depth_wrapper(
alf_path, axes=None, cluster_ids=[], method='corr', binsize=0.025, lags=8, n_depths=30,
use_svd=False):
"""
Wrapper function to load spikes and rf stimulus info, aggregate clusters over depths, compute
rfs, and plot spatial components as a function of linear depth on probe. Must have ibllib
package in python path in order to use alf loaders.
Parameters
----------
alf_path : str
absolute path to experimental session directory
axes : array-like object of matplotlib axes or `NoneType`, optional
axes in which to plot the rfs; if `NoneType`, a figure with appropriate axes is created
cluster_ids : array-like, optional
clusters to use in rf calculation; if empty, all clusters are used
method : str, optional
method for calculating receptive fields
'sta': method used in Durand et al 2016
'corr': reverse correlation method; uses convolution and is therefor faster than `'sta'`
binsize : float, optional
width of bins in seconds for rf computation
lags : int, optional
number of bins after pixel flip to use for rf computation
n_depths : int, optional
number of bins to divide probe depth into for aggregating clusters
use_svd : bool, optional
`True` plots 1st spatial SVD component of rf; `False` plots time lag with peak response
Returns
-------
dict
depths and associated receptive fields
"""
import alf.io as ioalf
# load objects
spikes = ioalf.load_object(alf_path, 'spikes')
clusters = ioalf.load_object(alf_path, 'clusters')
rfmap = ioalf.load_object(alf_path, '_iblcertif_.rfmap')
rf_stim_times = rfmap['rfmap.times.00']
rf_stim = rfmap['rfmap.stims.00'].astype('float')
# combine clusters across similar depths
min_depth = np.min(clusters['depths'])
max_depth = np.max(clusters['depths'])
depth_limits = np.linspace(min_depth - 1, max_depth, n_depths + 1)
if len(cluster_ids) == 0:
times_agg = spikes.times
depths_agg = spikes.depths
else:
clust_mask = np.isin(spikes.clusters, np.array(cluster_ids))
times_agg = spikes.times[clust_mask]
depths_agg = spikes.depths[clust_mask]
clusters_agg = np.full(times_agg.shape, fill_value=np.nan)
for d in range(n_depths):
lo_limit = depth_limits[d]
up_limit = depth_limits[d + 1]
clusters_agg[(lo_limit < depths_agg) & (depths_agg <= up_limit)] = d
print('computing receptive fields...', end='')
if method == 'sta':
# method in Durand et al 2016
rfs = compute_rfs(
times_agg, clusters_agg, rf_stim_times, rf_stim, lags=lags, binsize=binsize)
elif method == 'corr':
# reverse correlation method
rfs = compute_rfs_corr(
times_agg, clusters_agg, rf_stim_times, rf_stim, lags=lags, binsize=binsize)
else:
raise NotImplementedError
# get single spatial footprint of rf
if use_svd:
rfs_spatial = compute_rf_svds(rfs, scale='spatial')['spatial']
else:
rfs_spatial = find_peak_responses(rfs)
rfs_interp = interpolate_rfs(rfs_spatial, bin_scale=0.5)
print('done')
plot_rfs_by_depth(rfs_interp, axes=axes)
return {'depths': depths_agg, 'rfs': rfs_interp}
from oneibl.one import ONE
import alf.io as ioalf
# user options
BINSIZE = 0.05 # sec
LAGS = 4 # number of bins for calculating RF
METHOD = 'corr' # 'corr' | 'sta'
# get the data from flatiron and the current folder
one = ONE()
eid = one.search(subject='ZM_2104', date='2019-09-19', number=1)
D = one.load(eid[0], clobber=False, download_only=True)
session_path = Path(D.local_path[0]).parent
# load objects
spikes = ioalf.load_object(session_path, 'spikes')
rfmap = ioalf.load_object(session_path, '_iblcertif_.rfmap')
rf_stim_times = rfmap['rfmap.times.00']
rf_stim = rfmap['rfmap.stims.00'].astype('float')
# compute receptive fields
if METHOD == 'sta':
# method in Durand et al 2016; ~9 min for 700 units on a single cpu core
print('computing receptive fields...', end='')
rfs = compute_rfs(
spikes.times, spikes.clusters, rf_stim_times, rf_stim, lags=LAGS, binsize=BINSIZE)
print('done')
elif METHOD == 'corr':
# reverse correlation method; ~3 min for 700 units on a single cpu core
print('computing receptive fields...', end='')
rfs = compute_rfs_corr(
def load_data(self,
download_data: bool = None,
extract_times: bool = False,
load_video: bool = True) -> None:
"""Extract the data from raw data files
Extracts all the required task data from the raw data files.
Data keys:
- count (int array): the sequential frame number (n, n+1, n+2...)
- pin_state (): the camera GPIO pin; records the audio TTLs; should be one per frame
- audio (float array): timestamps of audio TTL fronts
- fpga_times (float array): timestamps of camera TTLs recorded by FPGA
- timestamps (float array): extracted video timestamps (the camera.times ALF)
- bonsai_times (datetime array): system timestamps of video PC; should be one per frame
- camera_times (float array): camera frame timestamps extracted from frame headers
- wheel (Bunch): rotary encoder timestamps, position and period used for wheel motion
- video (Bunch): video meta data, including dimensions and FPS
- frame_samples (h x w x n array): array of evenly sampled frames (1 colour channel)
:param download_data: if True, any missing raw data is downloaded via ONE.
:param extract_times: if True, the camera.times are re-extracted from the raw data
:param load_video: if True, calls the load_video_data method
"""
assert self.session_path, 'no session path set'
if download_data is not None:
self.download_data = download_data
if self.eid and self.one and not isinstance(self.one, OneOffline):
self._ensure_required_data()
_log.info('Gathering data for QC')
# Get frame count and pin state
self.data['count'], self.data['pin_state'] = \
raw.load_embedded_frame_data(self.session_path, self.side, raw=True)
# Load the audio and raw FPGA times
if self.type == 'ephys':
sync, chmap = ephys_fpga.get_main_probe_sync(self.session_path)
audio_ttls = ephys_fpga._get_sync_fronts(sync, chmap['audio'])
self.data['audio'] = audio_ttls['times'] # Get rises
# Load raw FPGA times
cam_ts = extract_camera_sync(sync, chmap)
self.data['fpga_times'] = cam_ts[self.side]
else:
bpod_data = raw.load_data(self.session_path)
_, audio_ttls = raw.load_bpod_fronts(self.session_path, bpod_data)
self.data['audio'] = audio_ttls['times']
# Load extracted frame times
alf_path = self.session_path / 'alf'
try:
assert not extract_times
self.data['timestamps'] = alfio.load_object(
alf_path, f'{self.side}Camera')['times']
except AssertionError: # Re-extract
kwargs = dict(video_path=self.video_path, labels=self.side)
if self.type == 'ephys':
kwargs = {**kwargs, 'sync': sync, 'chmap': chmap} # noqa
outputs, _ = extract_all(self.session_path,
self.type,
save=False,
**kwargs)
self.data['timestamps'] = outputs[f'{self.side}_camera_timestamps']
except ALFObjectNotFound:
_log.warning('no camera.times ALF found for session')
# Get audio and wheel data
wheel_keys = ('timestamps', 'position')
try:
self.data['wheel'] = alfio.load_object(alf_path, 'wheel')
except ALFObjectNotFound:
# Extract from raw data
if self.type == 'ephys':
wheel_data = ephys_fpga.extract_wheel_sync(sync, chmap)
else:
wheel_data = training_wheel.get_wheel_position(
self.session_path)
self.data['wheel'] = Bunch(zip(wheel_keys, wheel_data))
# Find short period of wheel motion for motion correlation. For speed start with the
# fist 2 minutes (nearly always enough), extract wheel movements and pick one.
# TODO Pick movement towards the end of the session (but not right at the end as some
# are extrapolated). Make sure the movement isn't too long.
if data_for_keys(
wheel_keys,
self.data['wheel']) and self.data['timestamps'] is not None:
self.data['wheel'].period = self.get_active_wheel_period(
self.data['wheel'])
# Load Bonsai frame timestamps
try:
ssv_times = raw.load_camera_ssv_times(self.session_path, self.side)
self.data['bonsai_times'], self.data['camera_times'] = ssv_times
except AssertionError:
_log.warning('No Bonsai video timestamps file found')
# Gather information from video file
if load_video:
_log.info('Inspecting video file...')
self.load_video_data()
