def summarize_responses(experiment_id): dataset = loading.get_ophys_dataset(experiment_id) analysis = ResponseAnalysis(dataset, overwrite_analysis_files=False, dataframe_format='tidy', use_extended_stimulus_presentations=True) valid_cells = list( dataset.cell_specimen_table.query('valid_roi==True').index.values) summaries = [] for df_type in ['omission', 'stimulus', 'trials']: summary = (getattr(analysis, '{}_response_df'.format(df_type)).query( 'cell_specimen_id in @valid_cells').drop_duplicates([ c for c in ['cell_specimen_id', 'stimulus_presentations_id', 'trials_id'] if c in getattr(analysis, '{}_response_df'.format(df_type)) ]).groupby(['cell_specimen_id', 'engagement_state' ])[['mean_response', 'mean_baseline']].mean().reset_index()) for key in analysis.dataset.metadata.keys(): val = analysis.dataset.metadata[key] if type(val) is not list: summary[key] = val summary['event_type'] = df_type summaries.append(summary) return pd.concat(summaries)
def get_matched_specimen_id(oeids): for i, oeid in enumerate(oeids): dataset = loading.get_ophys_dataset(oeid) df = dataset.dff_traces.reset_index() if i == 0: same_ids = dataset.events.index.values else: index = df.cell_specimen_id.isin(same_ids).values same_ids = df.cell_specimen_id[index].values return same_ids
def generate_save_plots(experiment_id, split_by): dataset = loading.get_ophys_dataset(experiment_id) analysis = ResponseAnalysis(dataset, overwrite_analysis_files=False, dataframe_format='tidy', use_extended_stimulus_presentations=True) for cell_specimen_id in dataset.cell_specimen_table.query( 'valid_roi==True').index.values: sf.make_cell_response_summary_plot(analysis, cell_specimen_id, split_by, save=True, show=False, errorbar_bootstrap_iterations=1000)
str(cell_specimen_id), ylabel='dF/F') fig.tight_layout() title = title + '_' + str(cell_specimen_id) plt.suptitle(title, x=0.5, y=1.01, fontsize=18) save_dir = r'/allen/programs/braintv/workgroups/nc-ophys/visual_behavior/qc_plots/roi_filtering_validation' utils.save_figure(fig, figsize, save_dir, 'single_cell_metrics', title) if __name__ == '__main__': import sys experiment_id = sys.argv[1] dataset = loading.get_ophys_dataset(experiment_id, include_invalid_rois=True) max_projection = dataset.max_projection.data ct = dataset.cell_specimen_table.copy() title = dataset.metadata_string metrics_df = get_metrics_df(experiment_id) # metrics_df = metrics_df[metrics_df.valid_roi==True] #only filter valid ROIs for cell_specimen_id in dataset.cell_specimen_ids: plot_roi_metrics_for_cell(dataset, metrics_df, cell_specimen_id, title) metric = 'area' thresholds = [50, 75, 100, 150, 200, 250] plot_metric_range_dataset(dataset, ct, max_projection, metrics_df,
def get_multi_session_mean_df(experiment_ids, analysis_cache_dir, df_name, conditions=[ 'cell_specimen_id', 'change_image_name', 'behavioral_response_type' ], flashes=False, use_events=False, omitted=False, get_reliability=False, get_pref_stim=True, exclude_omitted_from_pref_stim=True, use_sdk_dataset=True): experiments_table = loading.get_filtered_ophys_experiment_table() mega_mdf = pd.DataFrame() for experiment_id in experiment_ids: print(experiment_id) # try: if use_sdk_dataset: dataset = loading.get_ophys_dataset(experiment_id) else: dataset = VisualBehaviorOphysDataset(experiment_id, analysis_cache_dir) analysis = ResponseAnalysis(dataset, use_events=use_events, overwrite_analysis_files=False, use_extended_stimulus_presentations=True) df = analysis.get_response_df(df_name) df['ophys_experiment_id'] = dataset.ophys_experiment_id df['project_code'] = experiments_table.loc[experiment_id].project_code df['session_type'] = experiments_table.loc[experiment_id].session_type # if 'engaged' in conditions: # df['engaged'] = [True if reward_rate > 2 else False for reward_rate in df.reward_rate.values] if 'running' in conditions: df['running'] = [ True if window_running_speed > 5 else False for window_running_speed in df.window_running_speed.values ] # if 'large_pupil' in conditions: # if 'mean_pupil_area' in df.keys(): # df = df[df.mean_pupil_area.isnull() == False] # if len(df) > 100: # median_pupil_area = df.mean_pupil_area.median() # df['large_pupil'] = [True if mean_pupil_area > median_pupil_area else False for mean_pupil_area in # df.mean_pupil_area.values] mdf = ut.get_mean_df( df, analysis, conditions=conditions, get_pref_stim=get_pref_stim, flashes=flashes, omitted=omitted, get_reliability=get_reliability, exclude_omitted_from_pref_stim=exclude_omitted_from_pref_stim) mdf['ophys_experiment_id'] = dataset.ophys_experiment_id dataset.metadata['reporter_line'] = dataset.metadata['reporter_line'][ 0] dataset.metadata['driver_line'] = dataset.metadata['driver_line'][0] metadata = pd.DataFrame(dataset.metadata, index=[experiment_id]) mdf = ut.add_metadata_to_mean_df(mdf, metadata) mega_mdf = pd.concat([mega_mdf, mdf]) # except Exception as e: # flake8: noqa: E722 # print(e) # print('problem for', experiment_id) if use_events: suffix = '_events' else: suffix = '' if 'level_0' in mega_mdf.keys(): mega_mdf = mega_mdf.drop(columns='level_0') if 'index' in mega_mdf.keys(): mega_mdf = mega_mdf.drop(columns='index') mega_mdf_write_dir = os.path.join(analysis_cache_dir, 'multi_session_summary_dfs') if not os.path.exists(mega_mdf_write_dir): os.makedirs(mega_mdf_write_dir) if len(conditions) == 5: filename = 'mean_' + df_name + '_' + conditions[1] + '_' + conditions[2] + '_' + conditions[3] + '_' + \ conditions[4] + suffix + '.h5' elif len(conditions) == 4: filename = 'mean_' + df_name + '_' + conditions[1] + '_' + conditions[ 2] + '_' + conditions[3] + suffix + '.h5' elif len(conditions) == 3: filename = 'mean_' + df_name + '_' + conditions[1] + '_' + conditions[ 2] + suffix + '.h5' elif len(conditions) == 2: filename = 'mean_' + df_name + '_' + conditions[1] + suffix + '.h5' elif len(conditions) == 1: filename = 'mean_' + df_name + '_' + conditions[0] + suffix + '.h5' print('saving multi session mean df to ', filename) mega_mdf.to_hdf(os.path.join(mega_mdf_write_dir, filename), key='df') print('saved') return mega_mdf
def plot_experiment_summary_figure(experiment_id, save_figure=True): dataset = loading.get_ophys_dataset(experiment_id) analysis = ra.ResponseAnalysis(dataset, use_events=True) fig, ax, figsize = make_fig_ax() ax['0_0'] = ep.plot_max_intensity_projection_for_experiment(experiment_id, ax=ax['0_0']) ax['0_0'].set_title('max projection') ax['0_1'] = ep.plot_valid_segmentation_mask_outlines_per_cell_for_experiment( experiment_id, ax=ax['0_1']) # ax['0_0'].set_title('max projection') ax['0_2'] = ep.plot_valid_and_invalid_segmentation_mask_overlay_per_cell_for_experiment( experiment_id, ax=ax['0_2']) ax['0_2'].set_title('red = valid ROIs, blue = invalid ROIs') ax['0_3:'] = ep.plot_motion_correction_and_population_average( experiment_id, ax=ax['0_3:']) ax['1_0'] = ep.plot_average_image_for_experiment(experiment_id, ax=ax['1_0']) ax['1_1'] = ep.plot_average_image_for_experiment(experiment_id, ax=ax['1_1']) try: ax['1_2'] = ep.plot_remaining_decrosstalk_masks_for_experiment( experiment_id, ax=ax['1_2']) except BaseException: print('no decrosstalk for experiment', experiment_id) ax['1_3:'] = ep.plot_behavior_timeseries_for_experiment(experiment_id, ax=ax['1_3:']) # ax['2_0'] = population_image_selectivity(experiment_id, ax=ax['2_0']) # ax['2_0'] = ep.plot_average_image_for_experiment(experiment_id, ax=ax['2_1']) ax['2_2'] = ep.plot_cell_snr_distribution_for_experiment(experiment_id, ax=ax['2_2']) ax['2_3:'] = ep.plot_traces_heatmap_for_experiment(experiment_id, ax=ax['2_3:']) df_name = 'trials_response_df' df = analysis.get_response_df(df_name) mean_df = ut.get_mean_df(df, analysis=analysis, conditions=['cell_specimen_id', 'go'], flashes=False, omitted=False, get_reliability=False, get_pref_stim=False, exclude_omitted_from_pref_stim=True) ax['3_0'] = ep.plot_population_average_for_experiment(experiment_id, df, mean_df, df_name, color=None, label=None, ax=ax['3_0']) ax['3_0'].set_xlim(-2.5, 2.8) df_name = 'omission_response_df' df = analysis.get_response_df(df_name) mean_df = ut.get_mean_df(df, analysis=analysis, conditions=['cell_specimen_id'], flashes=False, omitted=True, get_reliability=False, get_pref_stim=False, exclude_omitted_from_pref_stim=False) ax['3_1'] = ep.plot_population_average_for_experiment(experiment_id, df, mean_df, df_name, color=None, label=None, ax=ax['3_1']) ax['3_1'].set_xlim(-2.5, 2.8) df_name = 'trials_run_speed_df' df = analysis.get_response_df(df_name) df['condition'] = True mean_df = ut.get_mean_df(df, analysis=analysis, conditions=['condition', 'go'], flashes=False, omitted=True, get_reliability=False, get_pref_stim=False, exclude_omitted_from_pref_stim=False) ax['4_0'] = ep.plot_population_average_for_experiment( experiment_id, df, df, df_name, trace_type='trace', color=sns.color_palette()[4], label=None, ax=ax['4_0']) ax['4_0'].set_ylabel('run speed (cm/s)') ax['4_0'].set_xlim(-2.5, 2.8) df_name = 'omission_run_speed_df' df = analysis.get_response_df(df_name) df['condition'] = True mean_df = ut.get_mean_df(df, analysis=analysis, conditions=['condition'], flashes=False, omitted=False, get_reliability=False, get_pref_stim=True, exclude_omitted_from_pref_stim=True) ax['4_1'] = ep.plot_population_average_for_experiment( experiment_id, df, df, df_name, trace_type='trace', color=sns.color_palette()[4], label=None, ax=ax['4_1']) ax['4_1'].set_ylabel('run speed (cm/s)') ax['4_1'].set_xlim(-2.5, 2.8) xlim_seconds = [int(10 * 60), int(15 * 60)] ax['3_3:'] = ep.plot_high_low_snr_trace_examples(experiment_id, xlim_seconds=xlim_seconds, ax=ax['3_3:']) ax['4_3:'] = ep.plot_behavior_timeseries_for_experiment( experiment_id, xlim_seconds=xlim_seconds, ax=ax['4_3:']) fig.tight_layout() title = dataset.metadata_string plt.suptitle(title, x=0.5, y=.91, fontsize=20) if save_figure: # save_dir = r'\\allen\programs\braintv\workgroups\nc-ophys\visual_behavior\qc_plots\experiment_plots' save_dir = loading.get_experiment_plots_dir() utils.save_figure(fig, figsize, save_dir, 'experiment_summary_figure', title)
analysis_dir = os.path.join(cache_dir, analysis_folder) else: print('no analysis folder for', experiment_id) # These files contain the output of SDK session object attribute dff_traces, # saved separately for production output currently in lims (without decrosstalk) and Wayne's dev lims environment (with decrosstalk) dff_traces_pre_decrosstalk = pd.read_hdf(os.path.join( analysis_dir, 'dff_traces_without_decrosstalk.h5'), key='df') dff_traces_post_decrosstalk = pd.read_hdf(os.path.join( analysis_dir, 'dff_traces_with_decrosstalk.h5'), key='df') # get metadata for plot title dataset = loading.get_ophys_dataset( experiment_id, include_invalid_rois=True ) # set to False to limit to valid ROIs (post filtering) for cell_roi_id in dff_traces_pre_decrosstalk.cell_roi_id.values: figsize = (20, 15) fig, ax = plt.subplots(3, 1, figsize=figsize) ax[0].plot( dff_traces_post_decrosstalk[dff_traces_post_decrosstalk.cell_roi_id == cell_roi_id].dff.values[0], color='blue', label='dev - with decrosstalk') ax[0].plot( dff_traces_pre_decrosstalk[dff_traces_pre_decrosstalk.cell_roi_id == cell_roi_id].dff.values[0], color='black',
def plot_matched_roi_and_traces_example(cell_metadata, include_omissions=True, use_events=False, filter_events=False, save_dir=None, folder=None): """ Plots the ROI masks and cell traces for a cell matched across sessions Cell_metadata is a subset of the ophys_cells_table limited to the cell_specimen_id of interest Masks and traces will be plotted for all ophys_experiment_ids in the cell_metadata table To limit to a single session of each type, set last_familiar_second_novel to True ROI mask for each ophys_experiment_id in cell_metadata is plotted on its own axis Average cell traces across all experiments are plotted on a single axis with each trace colored by its experience_level if include_omissions is True, there will be one axis for the change response and one axis for the omission response across sessions if include_omissions is False, only change responses will be plotted Only plots data for ophys_experiment_ids where the cell_specimen_id is present, does not plot max projections without an ROI mask for expts in a container where the cell was not detected To generate plots showing max projections from experiments in a container where a cell was not detected, use plot_matched_roi_and_trace """ if len(cell_metadata.cell_specimen_id.unique()) > 1: print('There is more than one cell_specimen_id in the provided cell_metadata table') print('Please limit input to a single cell_specimen_id') # get relevant info for this cell cell_metadata = cell_metadata.sort_values(by='experience_level') cell_specimen_id = cell_metadata.cell_specimen_id.unique()[0] ophys_container_id = cell_metadata.ophys_container_id.unique()[0] ophys_experiment_ids = cell_metadata.ophys_experiment_id.unique() n_expts = len(ophys_experiment_ids) # set up labels for different trace types if use_events: if filter_events: suffix = 'filtered_events' else: suffix = 'events' ylabel = 'response' else: suffix = 'dff' ylabel = 'dF/F' # number of columns is one for each experiments ROI mask, plus additional columns for stimulus and omission traces if include_omissions: n_cols = n_expts + 2 else: n_cols = n_expts + 1 experience_levels = ['Familiar', 'Novel 1', 'Novel >1'] colors = utils.get_experience_level_colors() figsize = (3 * n_cols, 3) fig, ax = plt.subplots(1, n_cols, figsize=figsize) print('cell_specimen_id:', cell_specimen_id) print('ophys_container_id:', ophys_container_id) for i, ophys_experiment_id in enumerate(ophys_experiment_ids): print('ophys_experiment_id:', ophys_experiment_id) experience_level = \ cell_metadata[cell_metadata.ophys_experiment_id == ophys_experiment_id].experience_level.values[0] ind = experience_levels.index(experience_level) color = colors[ind] try: dataset = loading.get_ophys_dataset(ophys_experiment_id, get_extended_stimulus_presentations=False) if cell_specimen_id in dataset.dff_traces.index: ct = dataset.cell_specimen_table.copy() cell_roi_id = ct.loc[cell_specimen_id].cell_roi_id ax[i] = sf.plot_cell_zoom(dataset.roi_masks, dataset.max_projection, cell_roi_id, spacex=50, spacey=50, show_mask=True, ax=ax[i]) ax[i].set_title(experience_level) # get change responses and plot on second to last axis window = [-1, 1.5] # window around event sdf = loading.get_stimulus_response_df(dataset, time_window=window, interpolate=True, output_sampling_rate=30, data_type='events', event_type='changes', load_from_file=True) cell_data = sdf[(sdf.cell_specimen_id == cell_specimen_id) & (sdf.is_change == True)] ax[n_expts] = utils.plot_mean_trace(cell_data.trace.values, cell_data.trace_timestamps.values[0], ylabel=ylabel, legend_label=None, color=color, interval_sec=1, xlim_seconds=window, plot_sem=True, ax=ax[n_expts]) ax[n_expts] = utils.plot_flashes_on_trace(ax[n_expts], cell_data.trace_timestamps.values[0], change=True, omitted=False) ax[n_expts].set_title('changes') # get omission responses and plot on last axis if include_omissions: sdf = loading.get_stimulus_response_df(dataset, time_window=window, interpolate=True, output_sampling_rate=30, data_type='events', event_type='omissions', load_from_file=True) cell_data = sdf[(sdf.cell_specimen_id == cell_specimen_id) & (sdf.omitted == True)] ax[n_expts + 1] = utils.plot_mean_trace(cell_data.trace.values, cell_data.trace_timestamps.values[0], ylabel=ylabel, legend_label=None, color=color, interval_sec=1, xlim_seconds=window, plot_sem=True, ax=ax[n_expts + 1]) ax[n_expts + 1] = utils.plot_flashes_on_trace(ax[n_expts + 1], cell_data.trace_timestamps.values[0], change=False, omitted=True) ax[n_expts + 1].set_title('omissions') metadata_string = utils.get_metadata_string(dataset.metadata) fig.tight_layout() fig.suptitle(str(cell_specimen_id) + '_' + metadata_string, x=0.53, y=1.02, horizontalalignment='center', fontsize=16) except Exception as e: print('problem for cell_specimen_id:', cell_specimen_id, ', ophys_experiment_id:', ophys_experiment_id) print(e) if save_dir: utils.save_figure(fig, figsize, save_dir, folder, str(cell_specimen_id) + '_' + metadata_string + '_' + suffix) plt.close()
def plot_matched_roi_and_trace(ophys_container_id, cell_specimen_id, limit_to_last_familiar_second_novel=True, use_events=False, filter_events=False, save_figure=True): """ Generates plots characterizing single cell activity in response to stimulus, omissions, and changes. Compares across all sessions in a container for each cell, including the ROI mask across days. Useful to validate cell matching as well as examine changes in activity profiles over days. """ experiments_table = loading.get_platform_paper_experiment_table() if limit_to_last_familiar_second_novel: # this ensures only one session per experience level experiments_table = utilities.limit_to_last_familiar_second_novel_active(experiments_table) experiments_table = utilities.limit_to_containers_with_all_experience_levels(experiments_table) container_expts = experiments_table[experiments_table.ophys_container_id == ophys_container_id] container_expts = container_expts.sort_values(by=['experience_level']) expts = np.sort(container_expts.index.values) if use_events: if filter_events: suffix = 'filtered_events' else: suffix = 'events' ylabel = 'response' else: suffix = 'dff' ylabel = 'dF/F' n = len(expts) if limit_to_last_familiar_second_novel: figsize = (9, 6) folder = 'matched_cells_exp_levels' else: figsize = (20, 6) folder = 'matched_cells_all_sessions' fig, ax = plt.subplots(2, n, figsize=figsize, sharey='row') ax = ax.ravel() print('ophys_container_id:', ophys_container_id) for i, ophys_experiment_id in enumerate(expts): print('ophys_experiment_id:', ophys_experiment_id) try: dataset = loading.get_ophys_dataset(ophys_experiment_id, get_extended_stimulus_presentations=False) if cell_specimen_id in dataset.dff_traces.index: ct = dataset.cell_specimen_table.copy() cell_roi_id = ct.loc[cell_specimen_id].cell_roi_id roi_masks = dataset.roi_masks.copy() # save this to get approx ROI position if subsequent session is missing the ROI (fails if the first session is the one missing the ROI) ax[i] = sf.plot_cell_zoom(dataset.roi_masks, dataset.max_projection, cell_roi_id, spacex=50, spacey=50, show_mask=True, ax=ax[i]) ax[i].set_title(container_expts.loc[ophys_experiment_id].experience_level) analysis = ResponseAnalysis(dataset, use_events=use_events, filter_events=filter_events, use_extended_stimulus_presentations=False) sdf = analysis.get_response_df(df_name='stimulus_response_df') cell_data = sdf[(sdf.cell_specimen_id == cell_specimen_id) & (sdf.is_change == True)] window = rp.get_default_stimulus_response_params()["window_around_timepoint_seconds"] ax[i + n] = utils.plot_mean_trace(cell_data.trace.values, cell_data.trace_timestamps.values[0], ylabel=ylabel, legend_label=None, color='gray', interval_sec=0.5, xlim_seconds=window, plot_sem=True, ax=ax[i + n]) ax[i + n] = utils.plot_flashes_on_trace(ax[i + n], cell_data.trace_timestamps.values[0], change=True, omitted=False, alpha=0.15, facecolor='gray') ax[i + n].set_title('') if i != 0: ax[i + n].set_ylabel('') else: # plot the max projection image with the xy location of the previous ROI # this will fail if the familiar session is the one without the cell matched ax[i] = sf.plot_cell_zoom(roi_masks, dataset.max_projection, cell_roi_id, spacex=50, spacey=50, show_mask=False, ax=ax[i]) ax[i].set_title(container_expts.loc[ophys_experiment_id].experience_level) metadata_string = utils.get_metadata_string(dataset.metadata) fig.tight_layout() fig.suptitle(str(cell_specimen_id) + '_' + metadata_string, x=0.53, y=1.02, horizontalalignment='center', fontsize=16) except Exception as e: print('problem for cell_specimen_id:', cell_specimen_id, ', ophys_experiment_id:', ophys_experiment_id) print(e) if save_figure: save_dir = r'//allen/programs/braintv/workgroups/nc-ophys/visual_behavior/platform_paper_plots/cell_matching' utils.save_figure(fig, figsize, save_dir, folder, str(cell_specimen_id) + '_' + metadata_string + '_' + suffix) plt.close()