parser.add_argument('--env', type=str, default='visual_behavior_sdk', metavar='name of conda environment to use') parser.add_argument('--scriptname', type=str, default='create_cell_metrics_table.py', metavar='name of script to run (must be in same folder)') # from allensdk.brain_observatory.behavior.behavior_project_cache import VisualBehaviorOphysProjectCache as bpc # cache_dir = loading.get_platform_analysis_cache_dir() # cache = bpc.from_s3_cache(cache_dir=cache_dir) # experiments_table = cache.get_ophys_experiment_table() # print(cache_dir) experiments_table = loading.get_platform_paper_experiment_table() ophys_experiment_ids = experiments_table.index.values if __name__ == "__main__": args = parser.parse_args() python_executable = "{}/anaconda2/envs/{}/bin/python".format( os.path.expanduser('~'), args.env) python_file = os.path.join(os.getcwd(), args.scriptname) # define the job record output folder stdout_location = r"/allen/programs/braintv/workgroups/nc-ophys/visual_behavior/cluster_jobs/cell_metrics" # instantiate a Slurm object slurm = Slurm( mem='60g', # '24g' cpus_per_task=1,
data_type = 'events' # set base_dir to load and save results base_dir = os.path.join(base_dir, glm_version) # folder in save_dir where you want to load GLM results from glm_output_folder = '220527_across_session_norm' glm_output_dir = os.path.join(base_dir, glm_output_folder) print(glm_output_dir) # directory to save plots to, in a subfolder called 'matched_cell_examples plot_save_dir = glm_output_dir # get experiments and cells tables limited to the data you want to plot # whatever filtering is applied here will be applied to GLM results experiments_table = loading.get_platform_paper_experiment_table( add_extra_columns=True, limit_to_closest_active=True) print(len(experiments_table), 'expts in expts table') cells_table = loading.get_cell_table(platform_paper_only=True, limit_to_closest_active=True, limit_to_matched_cells=True, add_extra_columns=True) print(len(cells_table.ophys_experiment_id.unique()), 'expts in cells table') print( 'should only be 402 experiments if limited to matched platform dataset' ) print(len(cells_table.cell_specimen_id.unique()), 'unique cell_specimen_ids in cells table') ### get GLM output ### # whatever pre-processing has been applied to results_pivoted saved to glm_output_dir will be applied here
def plot_cell_rois_and_GLM_weights(cell_specimen_id, cells_table, experiments_table, dropout_features, results_pivoted, weights_df, weights_features, kernels, save_dir=None, folder=None, data_type='dff'): """ This function limits inputs just to the provided cell_specimen_id to hand off to the function plot_matched_roi_and_traces_example_GLM That function will plot the following panels: cell ROI masks matched across sessions for a given cell_specimen_id, change and omission triggered average respones across sessions, image locked running and pupil if included 'running' and 'pupil' in included weights_features dropout scores across dropout_features and sessions as a heatmap, kernels weights across sessions for the kernels in weights_features :param cell_specimen_id: cell_specimen_id for cell to plot :param cells_table: must only include a max of one one experiment per experience level for a given container. ok if less than 1. :param experiments_table: must only include one experiment per experience level for a given container :param results_pivoted: must only include one experiment per experience level for a given container must be limited to features for plotting, plus cell_specimen_id + experiment_id :param weights_df: must only include one experiment per experience level for a given container :param weights_features: columns in weights_df to use for plotting :param save_dir: top level directory where files exist for this run of analysis code will create a folder within save_dir called 'matched_cell_examples' :param subfolder: name of subfolder to create within os.path.join(save_dir, 'matched_cell_examples') to save plots, ex: 'cluster_0' or 'without_exp_var_full_model' or 'with_running_and_pupil' :param data_type: can be 'dff', 'events', or 'filtered_events' - to be used for cell response plots :return: """ # make sure weights and dropouts are limited to matched experiments / cells cells_table = loading.get_cell_table(platform_paper_only=True, limit_to_closest_active=True, limit_to_matched_cells=True, add_extra_columns=True) experiments_table = loading.get_platform_paper_experiment_table( add_extra_columns=True, limit_to_closest_active=True) matched_cells = cells_table.cell_specimen_id.unique() matched_experiments = cells_table.ophys_experiment_id.unique() weights_df = weights_df[weights_df.ophys_experiment_id.isin( matched_experiments)] weights_df = weights_df[weights_df.cell_specimen_id.isin(matched_cells)] results_pivoted = results_pivoted.reset_index() # reset just in case results_pivoted = results_pivoted[results_pivoted.cell_specimen_id.isin( matched_cells)] # get cell info cell_metadata = cells_table[cells_table.cell_specimen_id == cell_specimen_id] # get metadata for this cell cell_metadata = cells_table[cells_table.cell_specimen_id == cell_specimen_id] # get weights for example cell cell_weights = weights_df[weights_df.cell_specimen_id == cell_specimen_id] # if exp var full model is in features (must be first feature), scale it by 10x so its on similar scale as dropouts if 'variance_explained_full' in results_pivoted.keys(): results_pivoted['variance_explained_full'] = results_pivoted[ 'variance_explained_full'] * 10 # get dropouts just for one cell cell_dropouts = results_pivoted[results_pivoted.cell_specimen_id == cell_specimen_id] plot_matched_roi_and_traces_example_GLM(cell_metadata, cell_dropouts, cell_weights, weights_features, kernels, dropout_features, experiments_table, data_type, save_dir, folder)
def plot_matched_roi_and_trace(ophys_container_id, cell_specimen_id, limit_to_last_familiar_second_novel=True, use_events=False, filter_events=False, save_figure=True): """ Generates plots characterizing single cell activity in response to stimulus, omissions, and changes. Compares across all sessions in a container for each cell, including the ROI mask across days. Useful to validate cell matching as well as examine changes in activity profiles over days. """ experiments_table = loading.get_platform_paper_experiment_table() if limit_to_last_familiar_second_novel: # this ensures only one session per experience level experiments_table = utilities.limit_to_last_familiar_second_novel_active(experiments_table) experiments_table = utilities.limit_to_containers_with_all_experience_levels(experiments_table) container_expts = experiments_table[experiments_table.ophys_container_id == ophys_container_id] container_expts = container_expts.sort_values(by=['experience_level']) expts = np.sort(container_expts.index.values) if use_events: if filter_events: suffix = 'filtered_events' else: suffix = 'events' ylabel = 'response' else: suffix = 'dff' ylabel = 'dF/F' n = len(expts) if limit_to_last_familiar_second_novel: figsize = (9, 6) folder = 'matched_cells_exp_levels' else: figsize = (20, 6) folder = 'matched_cells_all_sessions' fig, ax = plt.subplots(2, n, figsize=figsize, sharey='row') ax = ax.ravel() print('ophys_container_id:', ophys_container_id) for i, ophys_experiment_id in enumerate(expts): print('ophys_experiment_id:', ophys_experiment_id) try: dataset = loading.get_ophys_dataset(ophys_experiment_id, get_extended_stimulus_presentations=False) if cell_specimen_id in dataset.dff_traces.index: ct = dataset.cell_specimen_table.copy() cell_roi_id = ct.loc[cell_specimen_id].cell_roi_id roi_masks = dataset.roi_masks.copy() # save this to get approx ROI position if subsequent session is missing the ROI (fails if the first session is the one missing the ROI) ax[i] = sf.plot_cell_zoom(dataset.roi_masks, dataset.max_projection, cell_roi_id, spacex=50, spacey=50, show_mask=True, ax=ax[i]) ax[i].set_title(container_expts.loc[ophys_experiment_id].experience_level) analysis = ResponseAnalysis(dataset, use_events=use_events, filter_events=filter_events, use_extended_stimulus_presentations=False) sdf = analysis.get_response_df(df_name='stimulus_response_df') cell_data = sdf[(sdf.cell_specimen_id == cell_specimen_id) & (sdf.is_change == True)] window = rp.get_default_stimulus_response_params()["window_around_timepoint_seconds"] ax[i + n] = utils.plot_mean_trace(cell_data.trace.values, cell_data.trace_timestamps.values[0], ylabel=ylabel, legend_label=None, color='gray', interval_sec=0.5, xlim_seconds=window, plot_sem=True, ax=ax[i + n]) ax[i + n] = utils.plot_flashes_on_trace(ax[i + n], cell_data.trace_timestamps.values[0], change=True, omitted=False, alpha=0.15, facecolor='gray') ax[i + n].set_title('') if i != 0: ax[i + n].set_ylabel('') else: # plot the max projection image with the xy location of the previous ROI # this will fail if the familiar session is the one without the cell matched ax[i] = sf.plot_cell_zoom(roi_masks, dataset.max_projection, cell_roi_id, spacex=50, spacey=50, show_mask=False, ax=ax[i]) ax[i].set_title(container_expts.loc[ophys_experiment_id].experience_level) metadata_string = utils.get_metadata_string(dataset.metadata) fig.tight_layout() fig.suptitle(str(cell_specimen_id) + '_' + metadata_string, x=0.53, y=1.02, horizontalalignment='center', fontsize=16) except Exception as e: print('problem for cell_specimen_id:', cell_specimen_id, ', ophys_experiment_id:', ophys_experiment_id) print(e) if save_figure: save_dir = r'//allen/programs/braintv/workgroups/nc-ophys/visual_behavior/platform_paper_plots/cell_matching' utils.save_figure(fig, figsize, save_dir, folder, str(cell_specimen_id) + '_' + metadata_string + '_' + suffix) plt.close()