def summarize_responses(experiment_id):
dataset = loading.get_ophys_dataset(experiment_id)
analysis = ResponseAnalysis(dataset,
overwrite_analysis_files=False,
dataframe_format='tidy',
use_extended_stimulus_presentations=True)
valid_cells = list(
dataset.cell_specimen_table.query('valid_roi==True').index.values)
summaries = []
for df_type in ['omission', 'stimulus', 'trials']:
summary = (getattr(analysis, '{}_response_df'.format(df_type)).query(
'cell_specimen_id in @valid_cells').drop_duplicates([
c for c in
['cell_specimen_id', 'stimulus_presentations_id', 'trials_id']
if c in getattr(analysis, '{}_response_df'.format(df_type))
]).groupby(['cell_specimen_id', 'engagement_state'
])[['mean_response',
'mean_baseline']].mean().reset_index())
for key in analysis.dataset.metadata.keys():
val = analysis.dataset.metadata[key]
if type(val) is not list:
summary[key] = val
summary['event_type'] = df_type
summaries.append(summary)
return pd.concat(summaries)
def get_matched_specimen_id(oeids):
for i, oeid in enumerate(oeids):
dataset = loading.get_ophys_dataset(oeid)
df = dataset.dff_traces.reset_index()
if i == 0:
same_ids = dataset.events.index.values
else:
index = df.cell_specimen_id.isin(same_ids).values
same_ids = df.cell_specimen_id[index].values
return same_ids
def generate_save_plots(experiment_id, split_by):
dataset = loading.get_ophys_dataset(experiment_id)
analysis = ResponseAnalysis(dataset,
overwrite_analysis_files=False,
dataframe_format='tidy',
use_extended_stimulus_presentations=True)
for cell_specimen_id in dataset.cell_specimen_table.query(
'valid_roi==True').index.values:
sf.make_cell_response_summary_plot(analysis,
cell_specimen_id,
split_by,
save=True,
show=False,
errorbar_bootstrap_iterations=1000)
str(cell_specimen_id),
ylabel='dF/F')
fig.tight_layout()
title = title + '_' + str(cell_specimen_id)
plt.suptitle(title, x=0.5, y=1.01, fontsize=18)
save_dir = r'/allen/programs/braintv/workgroups/nc-ophys/visual_behavior/qc_plots/roi_filtering_validation'
utils.save_figure(fig, figsize, save_dir, 'single_cell_metrics', title)
if __name__ == '__main__':
import sys
experiment_id = sys.argv[1]
dataset = loading.get_ophys_dataset(experiment_id,
include_invalid_rois=True)
max_projection = dataset.max_projection.data
ct = dataset.cell_specimen_table.copy()
title = dataset.metadata_string
metrics_df = get_metrics_df(experiment_id)
# metrics_df = metrics_df[metrics_df.valid_roi==True] #only filter valid ROIs
for cell_specimen_id in dataset.cell_specimen_ids:
plot_roi_metrics_for_cell(dataset, metrics_df, cell_specimen_id, title)
metric = 'area'
thresholds = [50, 75, 100, 150, 200, 250]
plot_metric_range_dataset(dataset,
ct,
max_projection,
metrics_df,
def get_multi_session_mean_df(experiment_ids,
analysis_cache_dir,
df_name,
conditions=[
'cell_specimen_id', 'change_image_name',
'behavioral_response_type'
],
flashes=False,
use_events=False,
omitted=False,
get_reliability=False,
get_pref_stim=True,
exclude_omitted_from_pref_stim=True,
use_sdk_dataset=True):
experiments_table = loading.get_filtered_ophys_experiment_table()
mega_mdf = pd.DataFrame()
for experiment_id in experiment_ids:
print(experiment_id)
# try:
if use_sdk_dataset:
dataset = loading.get_ophys_dataset(experiment_id)
else:
dataset = VisualBehaviorOphysDataset(experiment_id,
analysis_cache_dir)
analysis = ResponseAnalysis(dataset,
use_events=use_events,
overwrite_analysis_files=False,
use_extended_stimulus_presentations=True)
df = analysis.get_response_df(df_name)
df['ophys_experiment_id'] = dataset.ophys_experiment_id
df['project_code'] = experiments_table.loc[experiment_id].project_code
df['session_type'] = experiments_table.loc[experiment_id].session_type
# if 'engaged' in conditions:
# df['engaged'] = [True if reward_rate > 2 else False for reward_rate in df.reward_rate.values]
if 'running' in conditions:
df['running'] = [
True if window_running_speed > 5 else False
for window_running_speed in df.window_running_speed.values
]
# if 'large_pupil' in conditions:
# if 'mean_pupil_area' in df.keys():
# df = df[df.mean_pupil_area.isnull() == False]
# if len(df) > 100:
# median_pupil_area = df.mean_pupil_area.median()
# df['large_pupil'] = [True if mean_pupil_area > median_pupil_area else False for mean_pupil_area in
# df.mean_pupil_area.values]
mdf = ut.get_mean_df(
df,
analysis,
conditions=conditions,
get_pref_stim=get_pref_stim,
flashes=flashes,
omitted=omitted,
get_reliability=get_reliability,
exclude_omitted_from_pref_stim=exclude_omitted_from_pref_stim)
mdf['ophys_experiment_id'] = dataset.ophys_experiment_id
dataset.metadata['reporter_line'] = dataset.metadata['reporter_line'][
0]
dataset.metadata['driver_line'] = dataset.metadata['driver_line'][0]
metadata = pd.DataFrame(dataset.metadata, index=[experiment_id])
mdf = ut.add_metadata_to_mean_df(mdf, metadata)
mega_mdf = pd.concat([mega_mdf, mdf])
# except Exception as e: # flake8: noqa: E722
# print(e)
# print('problem for', experiment_id)
if use_events:
suffix = '_events'
else:
suffix = ''
if 'level_0' in mega_mdf.keys():
mega_mdf = mega_mdf.drop(columns='level_0')
if 'index' in mega_mdf.keys():
mega_mdf = mega_mdf.drop(columns='index')
mega_mdf_write_dir = os.path.join(analysis_cache_dir,
'multi_session_summary_dfs')
if not os.path.exists(mega_mdf_write_dir):
os.makedirs(mega_mdf_write_dir)
if len(conditions) == 5:
filename = 'mean_' + df_name + '_' + conditions[1] + '_' + conditions[2] + '_' + conditions[3] + '_' + \
conditions[4] + suffix + '.h5'
elif len(conditions) == 4:
filename = 'mean_' + df_name + '_' + conditions[1] + '_' + conditions[
2] + '_' + conditions[3] + suffix + '.h5'
elif len(conditions) == 3:
filename = 'mean_' + df_name + '_' + conditions[1] + '_' + conditions[
2] + suffix + '.h5'
elif len(conditions) == 2:
filename = 'mean_' + df_name + '_' + conditions[1] + suffix + '.h5'
elif len(conditions) == 1:
filename = 'mean_' + df_name + '_' + conditions[0] + suffix + '.h5'
print('saving multi session mean df to ', filename)
mega_mdf.to_hdf(os.path.join(mega_mdf_write_dir, filename), key='df')
print('saved')
return mega_mdf
def plot_experiment_summary_figure(experiment_id, save_figure=True):
dataset = loading.get_ophys_dataset(experiment_id)
analysis = ra.ResponseAnalysis(dataset, use_events=True)
fig, ax, figsize = make_fig_ax()
ax['0_0'] = ep.plot_max_intensity_projection_for_experiment(experiment_id,
ax=ax['0_0'])
ax['0_0'].set_title('max projection')
ax['0_1'] = ep.plot_valid_segmentation_mask_outlines_per_cell_for_experiment(
experiment_id, ax=ax['0_1'])
# ax['0_0'].set_title('max projection')
ax['0_2'] = ep.plot_valid_and_invalid_segmentation_mask_overlay_per_cell_for_experiment(
experiment_id, ax=ax['0_2'])
ax['0_2'].set_title('red = valid ROIs, blue = invalid ROIs')
ax['0_3:'] = ep.plot_motion_correction_and_population_average(
experiment_id, ax=ax['0_3:'])
ax['1_0'] = ep.plot_average_image_for_experiment(experiment_id,
ax=ax['1_0'])
ax['1_1'] = ep.plot_average_image_for_experiment(experiment_id,
ax=ax['1_1'])
try:
ax['1_2'] = ep.plot_remaining_decrosstalk_masks_for_experiment(
experiment_id, ax=ax['1_2'])
except BaseException:
print('no decrosstalk for experiment', experiment_id)
ax['1_3:'] = ep.plot_behavior_timeseries_for_experiment(experiment_id,
ax=ax['1_3:'])
# ax['2_0'] = population_image_selectivity(experiment_id, ax=ax['2_0'])
# ax['2_0'] = ep.plot_average_image_for_experiment(experiment_id, ax=ax['2_1'])
ax['2_2'] = ep.plot_cell_snr_distribution_for_experiment(experiment_id,
ax=ax['2_2'])
ax['2_3:'] = ep.plot_traces_heatmap_for_experiment(experiment_id,
ax=ax['2_3:'])
df_name = 'trials_response_df'
df = analysis.get_response_df(df_name)
mean_df = ut.get_mean_df(df,
analysis=analysis,
conditions=['cell_specimen_id', 'go'],
flashes=False,
omitted=False,
get_reliability=False,
get_pref_stim=False,
exclude_omitted_from_pref_stim=True)
ax['3_0'] = ep.plot_population_average_for_experiment(experiment_id,
df,
mean_df,
df_name,
color=None,
label=None,
ax=ax['3_0'])
ax['3_0'].set_xlim(-2.5, 2.8)
df_name = 'omission_response_df'
df = analysis.get_response_df(df_name)
mean_df = ut.get_mean_df(df,
analysis=analysis,
conditions=['cell_specimen_id'],
flashes=False,
omitted=True,
get_reliability=False,
get_pref_stim=False,
exclude_omitted_from_pref_stim=False)
ax['3_1'] = ep.plot_population_average_for_experiment(experiment_id,
df,
mean_df,
df_name,
color=None,
label=None,
ax=ax['3_1'])
ax['3_1'].set_xlim(-2.5, 2.8)
df_name = 'trials_run_speed_df'
df = analysis.get_response_df(df_name)
df['condition'] = True
mean_df = ut.get_mean_df(df,
analysis=analysis,
conditions=['condition', 'go'],
flashes=False,
omitted=True,
get_reliability=False,
get_pref_stim=False,
exclude_omitted_from_pref_stim=False)
ax['4_0'] = ep.plot_population_average_for_experiment(
experiment_id,
df,
df,
df_name,
trace_type='trace',
color=sns.color_palette()[4],
label=None,
ax=ax['4_0'])
ax['4_0'].set_ylabel('run speed (cm/s)')
ax['4_0'].set_xlim(-2.5, 2.8)
df_name = 'omission_run_speed_df'
df = analysis.get_response_df(df_name)
df['condition'] = True
mean_df = ut.get_mean_df(df,
analysis=analysis,
conditions=['condition'],
flashes=False,
omitted=False,
get_reliability=False,
get_pref_stim=True,
exclude_omitted_from_pref_stim=True)
ax['4_1'] = ep.plot_population_average_for_experiment(
experiment_id,
df,
df,
df_name,
trace_type='trace',
color=sns.color_palette()[4],
label=None,
ax=ax['4_1'])
ax['4_1'].set_ylabel('run speed (cm/s)')
ax['4_1'].set_xlim(-2.5, 2.8)
xlim_seconds = [int(10 * 60), int(15 * 60)]
ax['3_3:'] = ep.plot_high_low_snr_trace_examples(experiment_id,
xlim_seconds=xlim_seconds,
ax=ax['3_3:'])
ax['4_3:'] = ep.plot_behavior_timeseries_for_experiment(
experiment_id, xlim_seconds=xlim_seconds, ax=ax['4_3:'])
fig.tight_layout()
title = dataset.metadata_string
plt.suptitle(title, x=0.5, y=.91, fontsize=20)
if save_figure:
# save_dir = r'\\allen\programs\braintv\workgroups\nc-ophys\visual_behavior\qc_plots\experiment_plots'
save_dir = loading.get_experiment_plots_dir()
utils.save_figure(fig, figsize, save_dir, 'experiment_summary_figure',
title)
analysis_dir = os.path.join(cache_dir, analysis_folder)
else:
print('no analysis folder for', experiment_id)
# These files contain the output of SDK session object attribute dff_traces,
# saved separately for production output currently in lims (without decrosstalk) and Wayne's dev lims environment (with decrosstalk)
dff_traces_pre_decrosstalk = pd.read_hdf(os.path.join(
analysis_dir, 'dff_traces_without_decrosstalk.h5'),
key='df')
dff_traces_post_decrosstalk = pd.read_hdf(os.path.join(
analysis_dir, 'dff_traces_with_decrosstalk.h5'),
key='df')
# get metadata for plot title
dataset = loading.get_ophys_dataset(
experiment_id, include_invalid_rois=True
) # set to False to limit to valid ROIs (post filtering)
for cell_roi_id in dff_traces_pre_decrosstalk.cell_roi_id.values:
figsize = (20, 15)
fig, ax = plt.subplots(3, 1, figsize=figsize)
ax[0].plot(
dff_traces_post_decrosstalk[dff_traces_post_decrosstalk.cell_roi_id
== cell_roi_id].dff.values[0],
color='blue',
label='dev - with decrosstalk')
ax[0].plot(
dff_traces_pre_decrosstalk[dff_traces_pre_decrosstalk.cell_roi_id
== cell_roi_id].dff.values[0],
color='black',
def plot_matched_roi_and_traces_example(cell_metadata, include_omissions=True,
use_events=False, filter_events=False, save_dir=None, folder=None):
"""
Plots the ROI masks and cell traces for a cell matched across sessions
Cell_metadata is a subset of the ophys_cells_table limited to the cell_specimen_id of interest
Masks and traces will be plotted for all ophys_experiment_ids in the cell_metadata table
To limit to a single session of each type, set last_familiar_second_novel to True
ROI mask for each ophys_experiment_id in cell_metadata is plotted on its own axis
Average cell traces across all experiments are plotted on a single axis with each trace colored by its experience_level
if include_omissions is True, there will be one axis for the change response and one axis for the omission response across sessions
if include_omissions is False, only change responses will be plotted
Only plots data for ophys_experiment_ids where the cell_specimen_id is present, does not plot max projections without an ROI mask for expts in a container where the cell was not detected
To generate plots showing max projections from experiments in a container where a cell was not detected, use plot_matched_roi_and_trace
"""
if len(cell_metadata.cell_specimen_id.unique()) > 1:
print('There is more than one cell_specimen_id in the provided cell_metadata table')
print('Please limit input to a single cell_specimen_id')
# get relevant info for this cell
cell_metadata = cell_metadata.sort_values(by='experience_level')
cell_specimen_id = cell_metadata.cell_specimen_id.unique()[0]
ophys_container_id = cell_metadata.ophys_container_id.unique()[0]
ophys_experiment_ids = cell_metadata.ophys_experiment_id.unique()
n_expts = len(ophys_experiment_ids)
# set up labels for different trace types
if use_events:
if filter_events:
suffix = 'filtered_events'
else:
suffix = 'events'
ylabel = 'response'
else:
suffix = 'dff'
ylabel = 'dF/F'
# number of columns is one for each experiments ROI mask, plus additional columns for stimulus and omission traces
if include_omissions:
n_cols = n_expts + 2
else:
n_cols = n_expts + 1
experience_levels = ['Familiar', 'Novel 1', 'Novel >1']
colors = utils.get_experience_level_colors()
figsize = (3 * n_cols, 3)
fig, ax = plt.subplots(1, n_cols, figsize=figsize)
print('cell_specimen_id:', cell_specimen_id)
print('ophys_container_id:', ophys_container_id)
for i, ophys_experiment_id in enumerate(ophys_experiment_ids):
print('ophys_experiment_id:', ophys_experiment_id)
experience_level = \
cell_metadata[cell_metadata.ophys_experiment_id == ophys_experiment_id].experience_level.values[0]
ind = experience_levels.index(experience_level)
color = colors[ind]
try:
dataset = loading.get_ophys_dataset(ophys_experiment_id, get_extended_stimulus_presentations=False)
if cell_specimen_id in dataset.dff_traces.index:
ct = dataset.cell_specimen_table.copy()
cell_roi_id = ct.loc[cell_specimen_id].cell_roi_id
ax[i] = sf.plot_cell_zoom(dataset.roi_masks, dataset.max_projection, cell_roi_id,
spacex=50, spacey=50, show_mask=True, ax=ax[i])
ax[i].set_title(experience_level)
# get change responses and plot on second to last axis
window = [-1, 1.5] # window around event
sdf = loading.get_stimulus_response_df(dataset, time_window=window, interpolate=True,
output_sampling_rate=30,
data_type='events', event_type='changes',
load_from_file=True)
cell_data = sdf[(sdf.cell_specimen_id == cell_specimen_id) & (sdf.is_change == True)]
ax[n_expts] = utils.plot_mean_trace(cell_data.trace.values, cell_data.trace_timestamps.values[0],
ylabel=ylabel, legend_label=None, color=color, interval_sec=1,
xlim_seconds=window, plot_sem=True, ax=ax[n_expts])
ax[n_expts] = utils.plot_flashes_on_trace(ax[n_expts], cell_data.trace_timestamps.values[0],
change=True, omitted=False)
ax[n_expts].set_title('changes')
# get omission responses and plot on last axis
if include_omissions:
sdf = loading.get_stimulus_response_df(dataset, time_window=window, interpolate=True,
output_sampling_rate=30,
data_type='events', event_type='omissions',
load_from_file=True)
cell_data = sdf[(sdf.cell_specimen_id == cell_specimen_id) & (sdf.omitted == True)]
ax[n_expts + 1] = utils.plot_mean_trace(cell_data.trace.values,
cell_data.trace_timestamps.values[0],
ylabel=ylabel, legend_label=None, color=color,
interval_sec=1,
xlim_seconds=window, plot_sem=True, ax=ax[n_expts + 1])
ax[n_expts + 1] = utils.plot_flashes_on_trace(ax[n_expts + 1],
cell_data.trace_timestamps.values[0],
change=False, omitted=True)
ax[n_expts + 1].set_title('omissions')
metadata_string = utils.get_metadata_string(dataset.metadata)
fig.tight_layout()
fig.suptitle(str(cell_specimen_id) + '_' + metadata_string, x=0.53, y=1.02,
horizontalalignment='center', fontsize=16)
except Exception as e:
print('problem for cell_specimen_id:', cell_specimen_id, ', ophys_experiment_id:', ophys_experiment_id)
print(e)
if save_dir:
utils.save_figure(fig, figsize, save_dir, folder,
str(cell_specimen_id) + '_' + metadata_string + '_' + suffix)
plt.close()
def plot_matched_roi_and_trace(ophys_container_id, cell_specimen_id, limit_to_last_familiar_second_novel=True,
use_events=False, filter_events=False, save_figure=True):
"""
Generates plots characterizing single cell activity in response to stimulus, omissions, and changes.
Compares across all sessions in a container for each cell, including the ROI mask across days.
Useful to validate cell matching as well as examine changes in activity profiles over days.
"""
experiments_table = loading.get_platform_paper_experiment_table()
if limit_to_last_familiar_second_novel: # this ensures only one session per experience level
experiments_table = utilities.limit_to_last_familiar_second_novel_active(experiments_table)
experiments_table = utilities.limit_to_containers_with_all_experience_levels(experiments_table)
container_expts = experiments_table[experiments_table.ophys_container_id == ophys_container_id]
container_expts = container_expts.sort_values(by=['experience_level'])
expts = np.sort(container_expts.index.values)
if use_events:
if filter_events:
suffix = 'filtered_events'
else:
suffix = 'events'
ylabel = 'response'
else:
suffix = 'dff'
ylabel = 'dF/F'
n = len(expts)
if limit_to_last_familiar_second_novel:
figsize = (9, 6)
folder = 'matched_cells_exp_levels'
else:
figsize = (20, 6)
folder = 'matched_cells_all_sessions'
fig, ax = plt.subplots(2, n, figsize=figsize, sharey='row')
ax = ax.ravel()
print('ophys_container_id:', ophys_container_id)
for i, ophys_experiment_id in enumerate(expts):
print('ophys_experiment_id:', ophys_experiment_id)
try:
dataset = loading.get_ophys_dataset(ophys_experiment_id, get_extended_stimulus_presentations=False)
if cell_specimen_id in dataset.dff_traces.index:
ct = dataset.cell_specimen_table.copy()
cell_roi_id = ct.loc[cell_specimen_id].cell_roi_id
roi_masks = dataset.roi_masks.copy() # save this to get approx ROI position if subsequent session is missing the ROI (fails if the first session is the one missing the ROI)
ax[i] = sf.plot_cell_zoom(dataset.roi_masks, dataset.max_projection, cell_roi_id,
spacex=50, spacey=50, show_mask=True, ax=ax[i])
ax[i].set_title(container_expts.loc[ophys_experiment_id].experience_level)
analysis = ResponseAnalysis(dataset, use_events=use_events, filter_events=filter_events,
use_extended_stimulus_presentations=False)
sdf = analysis.get_response_df(df_name='stimulus_response_df')
cell_data = sdf[(sdf.cell_specimen_id == cell_specimen_id) & (sdf.is_change == True)]
window = rp.get_default_stimulus_response_params()["window_around_timepoint_seconds"]
ax[i + n] = utils.plot_mean_trace(cell_data.trace.values, cell_data.trace_timestamps.values[0],
ylabel=ylabel, legend_label=None, color='gray', interval_sec=0.5,
xlim_seconds=window, plot_sem=True, ax=ax[i + n])
ax[i + n] = utils.plot_flashes_on_trace(ax[i + n], cell_data.trace_timestamps.values[0], change=True, omitted=False,
alpha=0.15, facecolor='gray')
ax[i + n].set_title('')
if i != 0:
ax[i + n].set_ylabel('')
else:
# plot the max projection image with the xy location of the previous ROI
# this will fail if the familiar session is the one without the cell matched
ax[i] = sf.plot_cell_zoom(roi_masks, dataset.max_projection, cell_roi_id,
spacex=50, spacey=50, show_mask=False, ax=ax[i])
ax[i].set_title(container_expts.loc[ophys_experiment_id].experience_level)
metadata_string = utils.get_metadata_string(dataset.metadata)
fig.tight_layout()
fig.suptitle(str(cell_specimen_id) + '_' + metadata_string, x=0.53, y=1.02,
horizontalalignment='center', fontsize=16)
except Exception as e:
print('problem for cell_specimen_id:', cell_specimen_id, ', ophys_experiment_id:', ophys_experiment_id)
print(e)
if save_figure:
save_dir = r'//allen/programs/braintv/workgroups/nc-ophys/visual_behavior/platform_paper_plots/cell_matching'
utils.save_figure(fig, figsize, save_dir, folder, str(cell_specimen_id) + '_' + metadata_string + '_' + suffix)
plt.close()
