def get_neuropil_subtracted_traces(roi_traces, neuropil_traces):
"""
get_neuropil_subtracted_traces(roi_traces, neuropil_traces)
Returns ROI traces with neuropil subtracted, as well as the contamination
ratio for each ROI.
Required args:
- roi_traces (2D array) : ROI traces, structured as ROI x frame
- neuropil_traces (2D array): neuropil traces, structured as ROI x frame
Returns:
- neuropilsub_traces (2D array): ROI traces with neuropil subtracted,
structured as ROI x frame
- r (1D array) : contamination ratio (0-1) for each ROI
"""
r = np.full(len(roi_traces), 0.)
for i, (roi_trace, neuropil_trace) in enumerate(zip(
roi_traces, neuropil_traces)):
if np.isfinite(roi_trace).all():
r[i] = r_neuropil.estimate_contamination_ratios(
roi_trace, neuropil_trace, iterations=3)["r"]
neuropilsub_traces = roi_traces - neuropil_traces * r[:, np.newaxis]
return neuropilsub_traces, r
def main():
module = PipelineModule()
args = module.args
jin = module.input_data()
########################################################################
# prelude -- get processing metadata
trace_file = jin["roi_trace_file"]
neuropil_file = jin["neuropil_trace_file"]
storage_dir = jin["storage_directory"]
plot_dir = os.path.join(storage_dir, "neuropil_subtraction_plots")
if os.path.exists(plot_dir):
shutil.rmtree(plot_dir)
try:
os.makedirs(plot_dir)
except:
pass
logging.info("Neuropil correcting '%s'", trace_file)
########################################################################
# process data
try:
roi_traces = h5py.File(trace_file, "r")
except:
logging.error("Error: unable to open ROI trace file '%s'", trace_file)
raise
try:
neuropil_traces = h5py.File(neuropil_file, "r")
except:
logging.error("Error: unable to open neuropil trace file '%s'",
neuropil_file)
raise
'''
get number of traces, length, etc.
'''
num_traces, T = roi_traces['data'].shape
T_orig = T
T_cross_val = int(T / 2)
if (T - T_cross_val > T_cross_val):
T = T - 1
# make sure that ROI and neuropil trace files are organized the same
n_id = neuropil_traces["roi_names"][:].astype(str)
r_id = roi_traces["roi_names"][:].astype(str)
logging.info("Processing %d traces", len(n_id))
assert len(n_id) == len(
r_id), "Input trace files are not aligned (ROI count)"
for i in range(len(n_id)):
assert n_id[i] == r_id[
i], "Input trace files are not aligned (ROI IDs)"
'''
initialize storage variables and analysis routine
'''
r_list = [None] * num_traces
RMSE_list = [-1] * num_traces
roi_names = n_id
corrected = np.zeros((num_traces, T_orig))
r_vals = [None] * num_traces
for n in range(num_traces):
roi = roi_traces['data'][n]
neuropil = neuropil_traces['data'][n]
if np.any(np.isnan(neuropil)):
logging.warning(
"neuropil trace for roi %d contains NaNs, skipping", n)
continue
if np.any(np.isnan(roi)):
logging.warning("roi trace for roi %d contains NaNs, skipping", n)
continue
r = None
logging.info("Correcting trace %d (roi %s)", n, str(n_id[n]))
results = estimate_contamination_ratios(roi, neuropil)
logging.info("r=%f err=%f it=%d", results["r"], results["err"],
results["it"])
r = results["r"]
fc = roi - r * neuropil
RMSE_list[n] = results["err"]
r_vals[n] = results["r_vals"]
debug_plot(os.path.join(plot_dir, "initial_%04d.png" % n), roi,
neuropil, fc, r, results["r_vals"], results["err_vals"])
# mean of the corrected trace must be positive
if fc.mean() > 0:
r_list[n] = r
corrected[n, :] = fc
else:
logging.warning("fc has negative baseline, skipping this r value")
# compute mean valid r value
r_mean = np.array([r for r in r_list if r is not None]).mean()
# fill in empty r values
for n in range(num_traces):
roi = roi_traces['data'][n]
neuropil = neuropil_traces['data'][n]
if r_list[n] is None:
logging.warning("Error estimated r for trace %d. Setting to zero.",
n)
r_list[n] = 0
corrected[n, :] = roi
# save a debug plot
debug_plot(os.path.join(plot_dir, "final_%04d.png" % n), roi, neuropil,
corrected[n, :], r_list[n])
# one last sanity check
eps = -0.0001
if np.mean(corrected[n, :]) < eps:
raise Exception(
"Trace %d baseline is still negative value after correction" %
n)
if r_list[n] < 0.0:
raise Exception("Trace %d ended with negative r" % n)
########################################################################
# write out processed data
try:
savefile = os.path.join(storage_dir, "neuropil_correction.h5")
hf = h5py.File(savefile, 'w')
hf.create_dataset("r", data=r_list)
hf.create_dataset("RMSE", data=RMSE_list)
hf.create_dataset("FC", data=corrected, compression="gzip")
hf.create_dataset("roi_names", data=roi_names.astype(np.string_))
for n in range(num_traces):
r = r_vals[n]
if r is not None:
hf.create_dataset("r_vals/%d" % n, data=r)
hf.close()
except:
logging.error("Error creating output h5 file")
raise
roi_traces.close()
neuropil_traces.close()
jout = copy.copy(jin)
jout["neuropil_correction"] = savefile
module.write_output_data(jout)
logging.info("finished")
def test_brain_observatory_experiment_containers_notebook(boc):
targeted_structures = boc.get_all_targeted_structures()
visp_ecs = boc.get_experiment_containers(targeted_structures=['VISp'])
depths = boc.get_all_imaging_depths()
stims = boc.get_all_stimuli()
cre_lines = boc.get_all_cre_lines()
cux2_ecs = boc.get_experiment_containers(cre_lines=['Cux2-CreERT2'])
cux2_ec_id = cux2_ecs[-1]['id']
exps = boc.get_ophys_experiments(experiment_container_ids=[cux2_ec_id])
exp = boc.get_ophys_experiments(experiment_container_ids=[cux2_ec_id],
stimuli=[stim_info.STATIC_GRATINGS])[0]
exp = boc.get_ophys_experiment_data(exp['id'])
assert set(depths) == set([
175, 185, 195, 200, 205, 225, 250, 265, 275, 276, 285, 300, 320, 325,
335, 350, 365, 375, 390, 400, 550, 570, 625
])
expected_stimuli = [
'drifting_gratings', 'locally_sparse_noise',
'locally_sparse_noise_4deg', 'locally_sparse_noise_8deg',
'natural_movie_one', 'natural_movie_three', 'natural_movie_two',
'natural_scenes', 'spontaneous', 'static_gratings'
]
assert set(stims) == set(expected_stimuli)
expected_cre_lines = [
u'Cux2-CreERT2', u'Emx1-IRES-Cre', u'Fezf2-CreER', u'Nr5a1-Cre',
u'Ntsr1-Cre_GN220', u'Pvalb-IRES-Cre', u'Rbp4-Cre_KL100',
u'Rorb-IRES2-Cre', u'Scnn1a-Tg3-Cre', u'Slc17a7-IRES2-Cre',
u'Sst-IRES-Cre', u'Tlx3-Cre_PL56', u'Vip-IRES-Cre'
]
assert set(cre_lines) == set(expected_cre_lines)
cells = boc.get_cell_specimens()
cells = pd.DataFrame.from_records(cells)
# find direction selective cells in VISp
visp_ec_ids = [ec['id'] for ec in visp_ecs]
visp_cells = cells[cells['experiment_container_id'].isin(visp_ec_ids)]
# significant response to drifting gratings stimulus
sig_cells = visp_cells[visp_cells['p_dg'] < 0.05]
# direction selective cells
dsi_cells = sig_cells[(sig_cells['dsi_dg'] > 0.5)
& (sig_cells['dsi_dg'] < 1.5)]
#assert len(cells) == 27124
assert len(cells) > 0
#assert len(visp_cells) == 16031
assert len(visp_cells) > 0
#assert len(sig_cells) == 8669
assert len(sig_cells) > 0
#assert len(dsi_cells) == 4943
assert len(dsi_cells) > 0
# find experiment containers for those cells
dsi_ec_ids = dsi_cells['experiment_container_id'].unique()
# Download the ophys experiments containing the drifting gratings stimulus for VISp experiment containers
dsi_exps = boc.get_ophys_experiments(experiment_container_ids=dsi_ec_ids,
stimuli=[stim_info.DRIFTING_GRATINGS])
# pick a direction-selective cell and find its NWB file
dsi_cell = dsi_cells.iloc[0]
# figure out which ophys experiment has the drifting gratings stimulus for the cell's experiment container
cell_exp = boc.get_ophys_experiments(
experiment_container_ids=[dsi_cell['experiment_container_id']],
stimuli=[stim_info.DRIFTING_GRATINGS])[0]
data_set = boc.get_ophys_experiment_data(cell_exp['id'])
# Fluorescence
dsi_cell_id = dsi_cell['cell_specimen_id']
time, raw_traces = data_set.get_fluorescence_traces(
cell_specimen_ids=[dsi_cell_id])
_, demixed_traces = data_set.get_demixed_traces(
cell_specimen_ids=[dsi_cell_id])
_, neuropil_traces = data_set.get_neuropil_traces(
cell_specimen_ids=[dsi_cell_id])
_, corrected_traces = data_set.get_corrected_fluorescence_traces(
cell_specimen_ids=[dsi_cell_id])
_, dff_traces = data_set.get_dff_traces(cell_specimen_ids=[dsi_cell_id])
# ROI Masks
data_set = boc.get_ophys_experiment_data(510221121)
# get the specimen IDs for a few cells
cids = data_set.get_cell_specimen_ids()[:15:5]
# get masks for specific cells
roi_mask_list = data_set.get_roi_mask(cell_specimen_ids=cids)
# make a mask of all ROIs in the experiment
all_roi_masks = data_set.get_roi_mask_array()
combined_mask = all_roi_masks.max(axis=0)
max_projection = data_set.get_max_projection()
# ROI Analysis
# example loading drifing grating data
data_set = boc.get_ophys_experiment_data(512326618)
dg = DriftingGratings(data_set)
# filter for visually responding, selective cells
vis_cells = (dg.peak.ptest_dg < 0.05) & (dg.peak.peak_dff_dg > 3)
osi_cells = vis_cells & (dg.peak.osi_dg > 0.5) & (dg.peak.osi_dg <= 1.5)
dsi_cells = vis_cells & (dg.peak.dsi_dg > 0.5) & (dg.peak.dsi_dg <= 1.5)
# 2-d tf vs. ori histogram
# tfval = 0 is used for the blank sweep, so we are ignoring it here
os = np.zeros((len(dg.orivals), len(dg.tfvals) - 1))
ds = np.zeros((len(dg.orivals), len(dg.tfvals) - 1))
for i, trial in dg.peak[osi_cells].iterrows():
os[trial.ori_dg, trial.tf_dg - 1] += 1
for i, trial in dg.peak[dsi_cells].iterrows():
ds[trial.ori_dg, trial.tf_dg - 1] += 1
max_count = max(os.max(), ds.max())
# Neuropil correction
data_set = boc.get_ophys_experiment_data(569407590)
csid = data_set.get_cell_specimen_ids()[0]
time, demixed_traces = data_set.get_demixed_traces(
cell_specimen_ids=[csid])
_, neuropil_traces = data_set.get_neuropil_traces(cell_specimen_ids=[csid])
results = estimate_contamination_ratios(demixed_traces[0],
neuropil_traces[0])
correction = demixed_traces[0] - results['r'] * neuropil_traces[0]
_, corrected_traces = data_set.get_corrected_fluorescence_traces(
cell_specimen_ids=[csid])
# Running Speed and Motion Correction
data_set = boc.get_ophys_experiment_data(512326618)
dxcm, dxtime = data_set.get_running_speed()
mc = data_set.get_motion_correction()
assert True