parser.add_argument('--env',
type=str,
default='visual_behavior_sdk',
metavar='name of conda environment to use')
parser.add_argument('--scriptname',
type=str,
default='create_cell_metrics_table.py',
metavar='name of script to run (must be in same folder)')
# from allensdk.brain_observatory.behavior.behavior_project_cache import VisualBehaviorOphysProjectCache as bpc
# cache_dir = loading.get_platform_analysis_cache_dir()
# cache = bpc.from_s3_cache(cache_dir=cache_dir)
# experiments_table = cache.get_ophys_experiment_table()
# print(cache_dir)
experiments_table = loading.get_platform_paper_experiment_table()
ophys_experiment_ids = experiments_table.index.values
if __name__ == "__main__":
args = parser.parse_args()
python_executable = "{}/anaconda2/envs/{}/bin/python".format(
os.path.expanduser('~'), args.env)
python_file = os.path.join(os.getcwd(), args.scriptname)
# define the job record output folder
stdout_location = r"/allen/programs/braintv/workgroups/nc-ophys/visual_behavior/cluster_jobs/cell_metrics"
# instantiate a Slurm object
slurm = Slurm(
mem='60g', # '24g'
cpus_per_task=1,
data_type = 'events'
# set base_dir to load and save results
base_dir = os.path.join(base_dir, glm_version)
# folder in save_dir where you want to load GLM results from
glm_output_folder = '220527_across_session_norm'
glm_output_dir = os.path.join(base_dir, glm_output_folder)
print(glm_output_dir)
# directory to save plots to, in a subfolder called 'matched_cell_examples
plot_save_dir = glm_output_dir
# get experiments and cells tables limited to the data you want to plot
# whatever filtering is applied here will be applied to GLM results
experiments_table = loading.get_platform_paper_experiment_table(
add_extra_columns=True, limit_to_closest_active=True)
print(len(experiments_table), 'expts in expts table')
cells_table = loading.get_cell_table(platform_paper_only=True,
limit_to_closest_active=True,
limit_to_matched_cells=True,
add_extra_columns=True)
print(len(cells_table.ophys_experiment_id.unique()),
'expts in cells table')
print(
'should only be 402 experiments if limited to matched platform dataset'
)
print(len(cells_table.cell_specimen_id.unique()),
'unique cell_specimen_ids in cells table')
### get GLM output ###
# whatever pre-processing has been applied to results_pivoted saved to glm_output_dir will be applied here
def plot_cell_rois_and_GLM_weights(cell_specimen_id,
cells_table,
experiments_table,
dropout_features,
results_pivoted,
weights_df,
weights_features,
kernels,
save_dir=None,
folder=None,
data_type='dff'):
"""
This function limits inputs just to the provided cell_specimen_id to hand off to the function plot_matched_roi_and_traces_example_GLM
That function will plot the following panels:
cell ROI masks matched across sessions for a given cell_specimen_id,
change and omission triggered average respones across sessions,
image locked running and pupil if included 'running' and 'pupil' in included weights_features
dropout scores across dropout_features and sessions as a heatmap,
kernels weights across sessions for the kernels in weights_features
:param cell_specimen_id: cell_specimen_id for cell to plot
:param cells_table: must only include a max of one one experiment per experience level for a given container. ok if less than 1.
:param experiments_table: must only include one experiment per experience level for a given container
:param results_pivoted: must only include one experiment per experience level for a given container
must be limited to features for plotting, plus cell_specimen_id + experiment_id
:param weights_df: must only include one experiment per experience level for a given container
:param weights_features: columns in weights_df to use for plotting
:param save_dir: top level directory where files exist for this run of analysis
code will create a folder within save_dir called 'matched_cell_examples'
:param subfolder: name of subfolder to create within os.path.join(save_dir, 'matched_cell_examples') to save plots, ex: 'cluster_0' or 'without_exp_var_full_model' or 'with_running_and_pupil'
:param data_type: can be 'dff', 'events', or 'filtered_events' - to be used for cell response plots
:return:
"""
# make sure weights and dropouts are limited to matched experiments / cells
cells_table = loading.get_cell_table(platform_paper_only=True,
limit_to_closest_active=True,
limit_to_matched_cells=True,
add_extra_columns=True)
experiments_table = loading.get_platform_paper_experiment_table(
add_extra_columns=True, limit_to_closest_active=True)
matched_cells = cells_table.cell_specimen_id.unique()
matched_experiments = cells_table.ophys_experiment_id.unique()
weights_df = weights_df[weights_df.ophys_experiment_id.isin(
matched_experiments)]
weights_df = weights_df[weights_df.cell_specimen_id.isin(matched_cells)]
results_pivoted = results_pivoted.reset_index() # reset just in case
results_pivoted = results_pivoted[results_pivoted.cell_specimen_id.isin(
matched_cells)]
# get cell info
cell_metadata = cells_table[cells_table.cell_specimen_id ==
cell_specimen_id]
# get metadata for this cell
cell_metadata = cells_table[cells_table.cell_specimen_id ==
cell_specimen_id]
# get weights for example cell
cell_weights = weights_df[weights_df.cell_specimen_id == cell_specimen_id]
# if exp var full model is in features (must be first feature), scale it by 10x so its on similar scale as dropouts
if 'variance_explained_full' in results_pivoted.keys():
results_pivoted['variance_explained_full'] = results_pivoted[
'variance_explained_full'] * 10
# get dropouts just for one cell
cell_dropouts = results_pivoted[results_pivoted.cell_specimen_id ==
cell_specimen_id]
plot_matched_roi_and_traces_example_GLM(cell_metadata, cell_dropouts,
cell_weights, weights_features,
kernels, dropout_features,
experiments_table, data_type,
save_dir, folder)
def plot_matched_roi_and_trace(ophys_container_id, cell_specimen_id, limit_to_last_familiar_second_novel=True,
use_events=False, filter_events=False, save_figure=True):
"""
Generates plots characterizing single cell activity in response to stimulus, omissions, and changes.
Compares across all sessions in a container for each cell, including the ROI mask across days.
Useful to validate cell matching as well as examine changes in activity profiles over days.
"""
experiments_table = loading.get_platform_paper_experiment_table()
if limit_to_last_familiar_second_novel: # this ensures only one session per experience level
experiments_table = utilities.limit_to_last_familiar_second_novel_active(experiments_table)
experiments_table = utilities.limit_to_containers_with_all_experience_levels(experiments_table)
container_expts = experiments_table[experiments_table.ophys_container_id == ophys_container_id]
container_expts = container_expts.sort_values(by=['experience_level'])
expts = np.sort(container_expts.index.values)
if use_events:
if filter_events:
suffix = 'filtered_events'
else:
suffix = 'events'
ylabel = 'response'
else:
suffix = 'dff'
ylabel = 'dF/F'
n = len(expts)
if limit_to_last_familiar_second_novel:
figsize = (9, 6)
folder = 'matched_cells_exp_levels'
else:
figsize = (20, 6)
folder = 'matched_cells_all_sessions'
fig, ax = plt.subplots(2, n, figsize=figsize, sharey='row')
ax = ax.ravel()
print('ophys_container_id:', ophys_container_id)
for i, ophys_experiment_id in enumerate(expts):
print('ophys_experiment_id:', ophys_experiment_id)
try:
dataset = loading.get_ophys_dataset(ophys_experiment_id, get_extended_stimulus_presentations=False)
if cell_specimen_id in dataset.dff_traces.index:
ct = dataset.cell_specimen_table.copy()
cell_roi_id = ct.loc[cell_specimen_id].cell_roi_id
roi_masks = dataset.roi_masks.copy() # save this to get approx ROI position if subsequent session is missing the ROI (fails if the first session is the one missing the ROI)
ax[i] = sf.plot_cell_zoom(dataset.roi_masks, dataset.max_projection, cell_roi_id,
spacex=50, spacey=50, show_mask=True, ax=ax[i])
ax[i].set_title(container_expts.loc[ophys_experiment_id].experience_level)
analysis = ResponseAnalysis(dataset, use_events=use_events, filter_events=filter_events,
use_extended_stimulus_presentations=False)
sdf = analysis.get_response_df(df_name='stimulus_response_df')
cell_data = sdf[(sdf.cell_specimen_id == cell_specimen_id) & (sdf.is_change == True)]
window = rp.get_default_stimulus_response_params()["window_around_timepoint_seconds"]
ax[i + n] = utils.plot_mean_trace(cell_data.trace.values, cell_data.trace_timestamps.values[0],
ylabel=ylabel, legend_label=None, color='gray', interval_sec=0.5,
xlim_seconds=window, plot_sem=True, ax=ax[i + n])
ax[i + n] = utils.plot_flashes_on_trace(ax[i + n], cell_data.trace_timestamps.values[0], change=True, omitted=False,
alpha=0.15, facecolor='gray')
ax[i + n].set_title('')
if i != 0:
ax[i + n].set_ylabel('')
else:
# plot the max projection image with the xy location of the previous ROI
# this will fail if the familiar session is the one without the cell matched
ax[i] = sf.plot_cell_zoom(roi_masks, dataset.max_projection, cell_roi_id,
spacex=50, spacey=50, show_mask=False, ax=ax[i])
ax[i].set_title(container_expts.loc[ophys_experiment_id].experience_level)
metadata_string = utils.get_metadata_string(dataset.metadata)
fig.tight_layout()
fig.suptitle(str(cell_specimen_id) + '_' + metadata_string, x=0.53, y=1.02,
horizontalalignment='center', fontsize=16)
except Exception as e:
print('problem for cell_specimen_id:', cell_specimen_id, ', ophys_experiment_id:', ophys_experiment_id)
print(e)
if save_figure:
save_dir = r'//allen/programs/braintv/workgroups/nc-ophys/visual_behavior/platform_paper_plots/cell_matching'
utils.save_figure(fig, figsize, save_dir, folder, str(cell_specimen_id) + '_' + metadata_string + '_' + suffix)
plt.close()