def test_drifting_gratings(dg, nwb_a, analysis_a_new):
logging.debug("reading outputs")
dg_new = DriftingGratings.from_analysis_file(BODS(nwb_a), analysis_a_new)
#assert np.allclose(dg.sweep_response, dg_new.sweep_response)
assert np.allclose(dg.mean_sweep_response, dg_new.mean_sweep_response, equal_nan=True)
assert np.allclose(dg.response, dg_new.response, equal_nan=True)
assert np.allclose(dg.noise_correlation, dg_new.noise_correlation, equal_nan=True)
assert np.allclose(dg.signal_correlation, dg_new.signal_correlation, equal_nan=True)
assert np.allclose(dg.representational_similarity, dg_new.representational_similarity, equal_nan=True)
def test_drifting_gratings(dg, nwb_a, analysis_a_new):
logging.debug("reading outputs")
dg_new = DriftingGratings.from_analysis_file(BODS(nwb_a), analysis_a_new)
#assert np.allclose(dg.sweep_response, dg_new.sweep_response)
assert np.allclose(dg.mean_sweep_response, dg_new.mean_sweep_response, equal_nan=True)
assert np.allclose(dg.response, dg_new.response, equal_nan=True)
assert np.allclose(dg.noise_correlation, dg_new.noise_correlation, equal_nan=True)
assert np.allclose(dg.signal_correlation, dg_new.signal_correlation, equal_nan=True)
assert np.allclose(dg.representational_similarity, dg_new.representational_similarity, equal_nan=True)
def build_type(nwb_file, data_file, configs, output_dir, type_name):
data_set = BrainObservatoryNwbDataSet(nwb_file)
try:
if type_name == "dg":
dga = DriftingGratings.from_analysis_file(data_set, data_file)
build_drifting_gratings(dga, configs, output_dir)
elif type_name == "sg":
sga = StaticGratings.from_analysis_file(data_set, data_file)
build_static_gratings(sga, configs, output_dir)
elif type_name == "nm1":
nma = NaturalMovie.from_analysis_file(data_set, data_file, si.NATURAL_MOVIE_ONE)
build_natural_movie(nma, configs, output_dir, si.NATURAL_MOVIE_ONE)
elif type_name == "nm2":
nma = NaturalMovie.from_analysis_file(data_set, data_file, si.NATURAL_MOVIE_TWO)
build_natural_movie(nma, configs, output_dir, si.NATURAL_MOVIE_TWO)
elif type_name == "nm3":
nma = NaturalMovie.from_analysis_file(data_set, data_file, si.NATURAL_MOVIE_THREE)
build_natural_movie(nma, configs, output_dir, si.NATURAL_MOVIE_THREE)
elif type_name == "ns":
nsa = NaturalScenes.from_analysis_file(data_set, data_file)
build_natural_scenes(nsa, configs, output_dir)
elif type_name == "sp":
nma = NaturalMovie.from_analysis_file(data_set, data_file, si.NATURAL_MOVIE_ONE)
build_speed_tuning(nma, configs, output_dir)
elif type_name == "lsn_on":
lsna = lsna_check_hvas(data_set, data_file)
build_locally_sparse_noise(lsna, configs, output_dir, True)
elif type_name == "lsn_off":
lsna = lsna_check_hvas(data_set, data_file)
build_locally_sparse_noise(lsna, configs, output_dir, False)
elif type_name == "rf":
lsna = lsna_check_hvas(data_set, data_file)
build_receptive_field(lsna, configs, output_dir)
elif type_name == "corr":
build_correlation_plots(data_set, data_file, configs, output_dir)
elif type_name == "eye":
build_eye_tracking_plots(data_set, configs, output_dir)
except MissingStimulusException as e:
logging.warning("could not load stimulus (%s)", type_name)
except Exception as e:
traceback.print_exc()
logging.critical("error running stimulus (%s)", type_name)
raise e
def session_a(self, plot_flag=False, save_flag=True):
nm1 = NaturalMovie(self.nwb, 'natural_movie_one', speed_tuning=True)
nm3 = NaturalMovie(self.nwb, 'natural_movie_three')
dg = DriftingGratings(self.nwb)
SessionAnalysis._log.info("Session A analyzed")
peak = multi_dataframe_merge(
[nm1.peak_run, dg.peak, nm1.peak, nm3.peak])
self.append_metrics_drifting_grating(self.metrics_a, dg)
self.metrics_a["roi_id"] = dg.roi_id
self.append_metadata(peak)
if save_flag:
self.save_session_a(dg, nm1, nm3, peak)
if plot_flag:
cp._plot_3sa(dg, nm1, nm3, self.save_dir)
cp.plot_drifting_grating_traces(dg, self.save_dir)
def test_brain_observatory_trace_analysis_notebook(boc):
# Drifting Gratings
data_set = boc.get_ophys_experiment_data(502376461)
dg = DriftingGratings(data_set)
specimen_id = 517425074
specimen_ids = data_set.get_cell_specimen_ids()
cell_loc = np.argwhere(specimen_ids == specimen_id)[0][0]
assert cell_loc == 97
# temporal frequency plot
response = dg.response[:, 1:, cell_loc, 0]
tfvals = dg.tfvals[1:]
orivals = dg.orivals
# peak
pk = dg.peak.loc[cell_loc]
# trials for cell's preferred condition
pref_ori = dg.orivals[dg.peak.ori_dg[cell_loc]]
pref_tf = dg.tfvals[dg.peak.tf_dg[cell_loc]]
assert pref_ori == 180
assert pref_tf == 2
pref_trials = dg.stim_table[(dg.stim_table.orientation == pref_ori) &
(dg.stim_table.temporal_frequency == pref_tf)]
assert pref_trials['start'][1] == 837
assert pref_trials['end'][1] == 897
# mean sweep response
subset = dg.sweep_response[(dg.stim_table.orientation == pref_ori)
& (dg.stim_table.temporal_frequency == pref_tf)]
subset_mean = dg.mean_sweep_response[
(dg.stim_table.orientation == pref_ori)
& (dg.stim_table.temporal_frequency == pref_tf)]
assert np.isclose(subset_mean['dx'][1], 0.920868)
# response to each trial
trial_timestamps = np.arange(-1 * dg.interlength, dg.interlength +
dg.sweeplength, 1.) / dg.acquisition_rate
boc = BrainObservatoryCache()
# Download a list of all targeted areas
targeted_structures = boc.get_all_targeted_structures()
# Download cells for a set of experiments and convert to DataFrame
cells = boc.get_cell_specimens()
cells = pd.DataFrame.from_records(cells)
dsi_cells = cells.query('area == "VISp" & g_dsi_dg >= .2 & g_dsi_dg < .9')
# find experiment containers for those cells
dsi_ec_ids = dsi_cells['experiment_container_id'].unique()
# Download the ophys experiments containing the static gratings stimulus for VISp experiment containers
dsi_exps = boc.get_ophys_experiments(experiment_container_ids=dsi_ec_ids, stimuli=[stim_info.DRIFTING_GRATINGS])
exp_id = dsi_exps[1]['id']
data_set = boc.get_ophys_experiment_data(exp_id)
dg = DriftingGratings(data_set)
mean_sweeps = dg.mean_sweep_response.values
d = xr.DataArray(
mean_sweeps,
dims=("stim", "cell"),
coords = {'cell' : [str(x) for x in dg.cell_id] + ['dx']})
d.to_dataframe(name='value').reset_index().to_feather('cells_dg1.feather')
dg.stim_table.to_feather('stim_table_dg1.feather')
def dg(nwb_a, analysis_a):
return DriftingGratings.from_analysis_file(BODS(nwb_a), analysis_a)
def test_harness(dataset, trigger):
dg = DriftingGratings(dataset)
assert dg._stim_table is StimulusAnalysis._PRELOAD
assert dg._orivals is StimulusAnalysis._PRELOAD
assert dg._tfvals is StimulusAnalysis._PRELOAD
assert dg._number_ori is StimulusAnalysis._PRELOAD
assert dg._number_tf is StimulusAnalysis._PRELOAD
assert dg._sweep_response is StimulusAnalysis._PRELOAD
assert dg._mean_sweep_response is StimulusAnalysis._PRELOAD
assert dg._pval is StimulusAnalysis._PRELOAD
assert dg._response is StimulusAnalysis._PRELOAD
assert dg._peak is StimulusAnalysis._PRELOAD
if trigger == 1:
print(dg.stim_table)
print(dg.sweep_response)
print(dg.response)
print(dg.peak)
elif trigger == 2:
print(dg.orivals)
print(dg.mean_sweep_response)
print(dg.response)
print(dg.peak)
elif trigger == 3:
print(dg.tfvals)
print(dg.pval)
print(dg.response)
print(dg.peak)
elif trigger == 4:
print(dg.number_ori)
print(dg.sweep_response)
print(dg.response)
print(dg.peak)
elif trigger == 5:
print(dg.number_tf)
print(dg.sweep_response)
print(dg.response)
print(dg.peak)
assert dg._stim_table is not StimulusAnalysis._PRELOAD
assert dg._orivals is not StimulusAnalysis._PRELOAD
assert dg._tfvals is not StimulusAnalysis._PRELOAD
assert dg._number_ori is not StimulusAnalysis._PRELOAD
assert dg._number_tf is not StimulusAnalysis._PRELOAD
assert dg._sweep_response is not StimulusAnalysis._PRELOAD
assert dg._mean_sweep_response is not StimulusAnalysis._PRELOAD
assert dg._pval is not StimulusAnalysis._PRELOAD
assert dg._response is not StimulusAnalysis._PRELOAD
assert dg._peak is not StimulusAnalysis._PRELOAD
# check super properties
dataset.get_corrected_fluorescence_traces.assert_called_once_with()
assert dg._timestamps != DriftingGratings._PRELOAD
assert dg._celltraces != DriftingGratings._PRELOAD
assert dg._numbercells != DriftingGratings._PRELOAD
assert not dataset.get_roi_ids.called
assert dg._roi_id is DriftingGratings._PRELOAD
assert dataset.get_cell_specimen_ids.called
assert dg._cell_id is DriftingGratings._PRELOAD
assert not dataset.get_dff_traces.called
assert dg._dfftraces is DriftingGratings._PRELOAD
assert dg._dxcm is DriftingGratings._PRELOAD
assert dg._dxtime is DriftingGratings._PRELOAD
def test_brain_observatory_experiment_containers_notebook(boc):
targeted_structures = boc.get_all_targeted_structures()
visp_ecs = boc.get_experiment_containers(targeted_structures=['VISp'])
depths = boc.get_all_imaging_depths()
stims = boc.get_all_stimuli()
cre_lines = boc.get_all_cre_lines()
cux2_ecs = boc.get_experiment_containers(cre_lines=['Cux2-CreERT2'])
cux2_ec_id = cux2_ecs[-1]['id']
exps = boc.get_ophys_experiments(experiment_container_ids=[cux2_ec_id])
exp = boc.get_ophys_experiments(experiment_container_ids=[cux2_ec_id],
stimuli=[stim_info.STATIC_GRATINGS])[0]
exp = boc.get_ophys_experiment_data(exp['id'])
assert set(depths) == set([
175, 185, 195, 200, 205, 225, 250, 265, 275, 276, 285, 300, 320, 325,
335, 350, 365, 375, 390, 400, 550, 570, 625
])
expected_stimuli = [
'drifting_gratings', 'locally_sparse_noise',
'locally_sparse_noise_4deg', 'locally_sparse_noise_8deg',
'natural_movie_one', 'natural_movie_three', 'natural_movie_two',
'natural_scenes', 'spontaneous', 'static_gratings'
]
assert set(stims) == set(expected_stimuli)
expected_cre_lines = [
u'Cux2-CreERT2', u'Emx1-IRES-Cre', u'Fezf2-CreER', u'Nr5a1-Cre',
u'Ntsr1-Cre_GN220', u'Pvalb-IRES-Cre', u'Rbp4-Cre_KL100',
u'Rorb-IRES2-Cre', u'Scnn1a-Tg3-Cre', u'Slc17a7-IRES2-Cre',
u'Sst-IRES-Cre', u'Tlx3-Cre_PL56', u'Vip-IRES-Cre'
]
assert set(cre_lines) == set(expected_cre_lines)
cells = boc.get_cell_specimens()
cells = pd.DataFrame.from_records(cells)
# find direction selective cells in VISp
visp_ec_ids = [ec['id'] for ec in visp_ecs]
visp_cells = cells[cells['experiment_container_id'].isin(visp_ec_ids)]
# significant response to drifting gratings stimulus
sig_cells = visp_cells[visp_cells['p_dg'] < 0.05]
# direction selective cells
dsi_cells = sig_cells[(sig_cells['dsi_dg'] > 0.5)
& (sig_cells['dsi_dg'] < 1.5)]
#assert len(cells) == 27124
assert len(cells) > 0
#assert len(visp_cells) == 16031
assert len(visp_cells) > 0
#assert len(sig_cells) == 8669
assert len(sig_cells) > 0
#assert len(dsi_cells) == 4943
assert len(dsi_cells) > 0
# find experiment containers for those cells
dsi_ec_ids = dsi_cells['experiment_container_id'].unique()
# Download the ophys experiments containing the drifting gratings stimulus for VISp experiment containers
dsi_exps = boc.get_ophys_experiments(experiment_container_ids=dsi_ec_ids,
stimuli=[stim_info.DRIFTING_GRATINGS])
# pick a direction-selective cell and find its NWB file
dsi_cell = dsi_cells.iloc[0]
# figure out which ophys experiment has the drifting gratings stimulus for the cell's experiment container
cell_exp = boc.get_ophys_experiments(
experiment_container_ids=[dsi_cell['experiment_container_id']],
stimuli=[stim_info.DRIFTING_GRATINGS])[0]
data_set = boc.get_ophys_experiment_data(cell_exp['id'])
# Fluorescence
dsi_cell_id = dsi_cell['cell_specimen_id']
time, raw_traces = data_set.get_fluorescence_traces(
cell_specimen_ids=[dsi_cell_id])
_, demixed_traces = data_set.get_demixed_traces(
cell_specimen_ids=[dsi_cell_id])
_, neuropil_traces = data_set.get_neuropil_traces(
cell_specimen_ids=[dsi_cell_id])
_, corrected_traces = data_set.get_corrected_fluorescence_traces(
cell_specimen_ids=[dsi_cell_id])
_, dff_traces = data_set.get_dff_traces(cell_specimen_ids=[dsi_cell_id])
# ROI Masks
data_set = boc.get_ophys_experiment_data(510221121)
# get the specimen IDs for a few cells
cids = data_set.get_cell_specimen_ids()[:15:5]
# get masks for specific cells
roi_mask_list = data_set.get_roi_mask(cell_specimen_ids=cids)
# make a mask of all ROIs in the experiment
all_roi_masks = data_set.get_roi_mask_array()
combined_mask = all_roi_masks.max(axis=0)
max_projection = data_set.get_max_projection()
# ROI Analysis
# example loading drifing grating data
data_set = boc.get_ophys_experiment_data(512326618)
dg = DriftingGratings(data_set)
# filter for visually responding, selective cells
vis_cells = (dg.peak.ptest_dg < 0.05) & (dg.peak.peak_dff_dg > 3)
osi_cells = vis_cells & (dg.peak.osi_dg > 0.5) & (dg.peak.osi_dg <= 1.5)
dsi_cells = vis_cells & (dg.peak.dsi_dg > 0.5) & (dg.peak.dsi_dg <= 1.5)
# 2-d tf vs. ori histogram
# tfval = 0 is used for the blank sweep, so we are ignoring it here
os = np.zeros((len(dg.orivals), len(dg.tfvals) - 1))
ds = np.zeros((len(dg.orivals), len(dg.tfvals) - 1))
for i, trial in dg.peak[osi_cells].iterrows():
os[trial.ori_dg, trial.tf_dg - 1] += 1
for i, trial in dg.peak[dsi_cells].iterrows():
ds[trial.ori_dg, trial.tf_dg - 1] += 1
max_count = max(os.max(), ds.max())
# Neuropil correction
data_set = boc.get_ophys_experiment_data(569407590)
csid = data_set.get_cell_specimen_ids()[0]
time, demixed_traces = data_set.get_demixed_traces(
cell_specimen_ids=[csid])
_, neuropil_traces = data_set.get_neuropil_traces(cell_specimen_ids=[csid])
results = estimate_contamination_ratios(demixed_traces[0],
neuropil_traces[0])
correction = demixed_traces[0] - results['r'] * neuropil_traces[0]
_, corrected_traces = data_set.get_corrected_fluorescence_traces(
cell_specimen_ids=[csid])
# Running Speed and Motion Correction
data_set = boc.get_ophys_experiment_data(512326618)
dxcm, dxtime = data_set.get_running_speed()
mc = data_set.get_motion_correction()
assert True
def build_correlation_plots(data_set, analysis_file, configs, output_dir):
sig_corrs = []
noise_corrs = []
avail_stims = si.stimuli_in_session(data_set.get_session_type())
ans = []
labels = []
colors = []
if si.DRIFTING_GRATINGS in avail_stims:
dg = DriftingGratings.from_analysis_file(data_set, analysis_file)
if hasattr(dg, 'representational_similarity'):
ans.append(dg)
labels.append(si.DRIFTING_GRATINGS_SHORT)
colors.append(si.DRIFTING_GRATINGS_COLOR)
setups = [ ( [configs['large']], True ), ( [configs['small']], False )]
for cfgs, show_labels in setups:
for fn in build_plots("drifting_gratings_representational_similarity", 1.0, cfgs, output_dir):
oplots.plot_representational_similarity(dg.representational_similarity,
dims=[dg.orivals, dg.tfvals[1:]],
dim_labels=["dir", "tf"],
dim_order=[1,0],
colors=['r','b'],
labels=show_labels)
if si.STATIC_GRATINGS in avail_stims:
sg = StaticGratings.from_analysis_file(data_set, analysis_file)
if hasattr(sg, 'representational_similarity'):
ans.append(sg)
labels.append(si.STATIC_GRATINGS_SHORT)
colors.append(si.STATIC_GRATINGS_COLOR)
setups = [ ( [configs['large']], True ), ( [configs['small']], False )]
for cfgs, show_labels in setups:
for fn in build_plots("static_gratings_representational_similarity", 1.0, cfgs, output_dir):
oplots.plot_representational_similarity(sg.representational_similarity,
dims=[sg.orivals, sg.sfvals[1:], sg.phasevals],
dim_labels=["ori", "sf", "ph"],
dim_order=[1,0,2],
colors=['r','g','b'],
labels=show_labels)
if si.NATURAL_SCENES in avail_stims:
ns = NaturalScenes.from_analysis_file(data_set, analysis_file)
if hasattr(ns, 'representational_similarity'):
ans.append(ns)
labels.append(si.NATURAL_SCENES_SHORT)
colors.append(si.NATURAL_SCENES_COLOR)
setups = [ ( [configs['large']], True ), ( [configs['small']], False )]
for cfgs, show_labels in setups:
for fn in build_plots("natural_scenes_representational_similarity", 1.0, cfgs, output_dir):
oplots.plot_representational_similarity(ns.representational_similarity, labels=show_labels)
if len(ans):
for an in ans:
sig_corrs.append(an.signal_correlation)
extra_dims = range(2,len(an.noise_correlation.shape))
noise_corrs.append(an.noise_correlation.mean(axis=tuple(extra_dims)))
for fn in build_plots("correlation", 1.0, [configs['large'], configs['svg']], output_dir):
oplots.population_correlation_scatter(sig_corrs, noise_corrs, labels, colors, scale=16.0)
oplots.finalize_with_axes()
for fn in build_plots("correlation", 1.0, [configs['small']], output_dir):
oplots.population_correlation_scatter(sig_corrs, noise_corrs, labels, colors, scale=4.0)
oplots.finalize_no_labels()
csids = ans[0].data_set.get_cell_specimen_ids()
for fn, csid, i in build_cell_plots(csids, "signal_correlation", 1.0, [configs['large']], output_dir):
row = ans[0].row_from_cell_id(csid, i)
oplots.plot_cell_correlation([ np.delete(sig_corr[row],i) for sig_corr in sig_corrs ],
labels, colors)
oplots.finalize_with_axes()
for fn, csid, i in build_cell_plots(csids, "signal_correlation", 1.0, [configs['small']], output_dir):
row = ans[0].row_from_cell_id(csid, i)
oplots.plot_cell_correlation([ np.delete(sig_corr[row],i) for sig_corr in sig_corrs ],
labels, colors)
oplots.finalize_no_labels()
def dg(nwb_a, analysis_a):
return DriftingGratings.from_analysis_file(BODS(nwb_a), analysis_a)