def get_motion_corrected_movie_paths(ophys_container_id):
et = loading.get_filtered_ophys_experiment_table().reset_index()
paths = []
for oeid in et.query('ophys_container_id == @ophys_container_id').sort_values(by='ophys_experiment_id')['ophys_experiment_id']:
paths.append(
'/' + get_motion_corrected_movie_path(oeid)
)
return paths
def print_motion_corrected_movie_paths(ophys_container_id):
et = loading.get_filtered_ophys_experiment_table().reset_index()
lines = []
for oeid in et.query('ophys_container_id == @ophys_container_id')['ophys_experiment_id']:
movie_path = loading.get_motion_corrected_movie_h5_location(oeid).replace('motion_corrected_video.h5', 'motion_preview.10x.mp4')
lines.append('ophys experiment ID = {}\n'.format(oeid))
lines.append("LINUX PATH:")
lines.append('\t<a href="url">{}</a>'.format(movie_path))
lines.append('WINDOWS PATH')
lines.append('\t{}'.format(movie_path.replace('/', '\\')))
lines.append('')
return '\n'.join(lines)
def plot_lick_triggered_average_for_container(container_id, save_figure=True):
experiments = loading.get_filtered_ophys_experiment_table()
container_data = experiments[experiments.container_id == container_id]
figsize = (6, 4)
fig, ax = plt.subplots(figsize=figsize)
for session_number in container_data.session_number.unique():
experiment_ids = container_data[container_data.session_number ==
session_number].index.values
for experiment_id in experiment_ids:
dataset = loading.get_ophys_dataset(experiment_id)
analysis = ResponseAnalysis(
dataset,
use_events=True,
use_extended_stimulus_presentations=False)
ldf = analysis.get_response_df(
df_name='lick_triggered_response_df')
if len(ldf.cell_specimen_id.unique()) > 5:
colors = utils.get_colors_for_session_numbers()
ax = plot_lick_triggered_average(
ldf,
color=colors[session_number - 1],
ylabel='pop. avg. \nevent magnitude',
legend_label=session_number,
ax=ax)
ax.legend(loc='upper left', fontsize='x-small')
fig.tight_layout()
m = dataset.metadata.copy()
title = str(m['mouse_id']) + '_' + m['full_genotype'].split(
'/')[0] + '_' + m['targeted_structure'] + '_' + str(
m['imaging_depth']) + '_' + m['equipment_name'] + '_' + str(
m['experiment_container_id'])
fig.suptitle(title, x=0.53, y=1.02, fontsize=16)
if save_figure:
save_dir = loading.get_container_plots_dir()
utils.save_figure(fig, figsize, save_dir, 'lick_triggered_average',
title)
app = dash.Dash(external_stylesheets=[dbc.themes.SPACELAB])
app.title = 'Visual Behavior Data QC'
# app.config['suppress_callback_exceptions'] = True
# FUNCTION CALLS
print('setting up table')
t0 = time.time()
container_table = functions.load_container_data().sort_values(
'first_acquistion_date')
container_plot_options = functions.load_container_plot_options()
session_plot_options = functions.load_session_plot_options()
container_overview_plot_options = functions.load_container_overview_plot_options(
)
plot_inventory = functions.generate_plot_inventory()
plot_inventory_fig = functions.make_plot_inventory_heatmap(plot_inventory)
experiment_table = loading.get_filtered_ophys_experiment_table().reset_index()
session_table = functions.load_session_data()
print('done setting up table, it took {} seconds'.format(time.time() - t0))
# COMPONENT SETUP
print('setting up components')
t0 = time.time()
components.plot_selection_dropdown.options = container_plot_options
components.container_overview_dropdown.options = container_overview_plot_options
components.container_overview_iframe.src = app.get_asset_url(
'qc_plots/overview_plots/d_prime_container_overview.html')
components.plot_inventory_iframe.src = 'https://dougollerenshaw.github.io/figures_to_share/container_plot_inventory.html' # app.get_asset_url('qc_plots/container_plot_inventory.html')
components.data_table.columns = [{
"name": i.replace('_', ' '),
"id": i
} for i in container_table.columns]
def plot_across_session_responses(ophys_container_id, cell_specimen_id, use_events=False, save_figure=True):
"""
Generates plots characterizing single cell activity in response to stimulus, omissions, and changes.
Compares across all sessions in a container for each cell, including the ROI mask across days.
Useful to validate cell matching as well as examine changes in activity profiles over days.
"""
experiments_table = data_loading.get_filtered_ophys_experiment_table(release_data_only=True)
container_expts = experiments_table[experiments_table.ophys_container_id == ophys_container_id]
expts = np.sort(container_expts.index.values)
if use_events:
ylabel = 'response'
suffix = '_events'
else:
ylabel = 'dF/F'
suffix = ''
n = len(expts)
figsize = (25, 20)
fig, ax = plt.subplots(6, n, figsize=figsize)
ax = ax.ravel()
print('ophys_container_id:', ophys_container_id)
for i, ophys_experiment_id in enumerate(expts):
print('ophys_experiment_id:', ophys_experiment_id)
try:
dataset = data_loading.get_ophys_dataset(ophys_experiment_id, include_invalid_rois=False)
if cell_specimen_id in dataset.dff_traces.index:
analysis = ResponseAnalysis(dataset, use_events=use_events, use_extended_stimulus_presentations=False)
sdf = ut.get_mean_df(analysis.get_response_df(df_name='stimulus_response_df'), analysis=analysis,
conditions=['cell_specimen_id', 'is_change', 'image_name'], flashes=True, omitted=False,
get_reliability=False, get_pref_stim=True, exclude_omitted_from_pref_stim=True)
odf = ut.get_mean_df(analysis.get_response_df(df_name='omission_response_df'), analysis=analysis,
conditions=['cell_specimen_id'], flashes=False, omitted=True,
get_reliability=False, get_pref_stim=False, exclude_omitted_from_pref_stim=False)
tdf = ut.get_mean_df(analysis.get_response_df(df_name='trials_response_df'), analysis=analysis,
conditions=['cell_specimen_id', 'go', 'hit', 'change_image_name'], flashes=False, omitted=False,
get_reliability=False, get_pref_stim=True, exclude_omitted_from_pref_stim=True)
ct = dataset.cell_specimen_table.copy()
cell_roi_id = ct.loc[cell_specimen_id].cell_roi_id
ax[i] = sf.plot_cell_zoom(dataset.roi_masks, dataset.max_projection, cell_roi_id,
spacex=20, spacey=20, show_mask=True, ax=ax[i])
ax[i].set_title(container_expts.loc[ophys_experiment_id].session_type[6:])
colors = sns.color_palette('hls', 8) + [(0.5, 0.5, 0.5)]
window = rp.get_default_stimulus_response_params()["window_around_timepoint_seconds"]
cell_data = sdf[(sdf.cell_specimen_id == cell_specimen_id) & (sdf.is_change == False)]
for c, image_name in enumerate(np.sort(cell_data.image_name.unique())):
ax[i + n] = sf.plot_mean_trace_from_mean_df(cell_data[cell_data.image_name == image_name],
frame_rate=analysis.ophys_frame_rate, ylabel=ylabel,
legend_label=image_name, color=colors[c], interval_sec=0.5,
xlims=window, ax=ax[i + n])
ax[i + n] = sf.plot_flashes_on_trace(ax[i + n], analysis, window=window, trial_type=None, omitted=False, alpha=0.15, facecolor='gray')
ax[i + n].set_title(container_expts.loc[ophys_experiment_id].session_type[6:] + '\n image response')
analysis = ResponseAnalysis(dataset, use_events=False, use_extended_stimulus_presentations=True)
tmp = analysis.get_response_df(df_name='stimulus_response_df')
tmp['running'] = [True if run_speed > 2 else False for run_speed in tmp.mean_running_speed.values]
sdf = ut.get_mean_df(tmp, analysis=analysis,
conditions=['cell_specimen_id', 'is_change', 'image_name', 'running'], flashes=True, omitted=False,
get_reliability=False, get_pref_stim=True, exclude_omitted_from_pref_stim=False)
cell_data = sdf[(sdf.cell_specimen_id == cell_specimen_id) & (sdf.is_change == False) & (sdf.pref_stim == True)]
run_colors = [sns.color_palette()[3], sns.color_palette()[2]]
for c, running in enumerate(np.sort(cell_data.running.unique())):
if len(cell_data[cell_data.running == running]) > 0:
ax[i + (n * 2)] = sf.plot_mean_trace_from_mean_df(cell_data[cell_data.running == running],
frame_rate=analysis.ophys_frame_rate, ylabel=ylabel,
legend_label=running, color=run_colors[c], interval_sec=0.5,
xlims=window, ax=ax[i + (n * 2)])
ax[i + (n * 2)].legend(fontsize='xx-small', title='running', title_fontsize='xx-small')
ax[i + (n * 2)] = sf.plot_flashes_on_trace(ax[i + (n * 2)], analysis, window=window, trial_type=None, omitted=False, alpha=0.15, facecolor='gray')
ax[i + (n * 2)].set_title(container_expts.loc[ophys_experiment_id].session_type[6:] + '\n image response')
window = rp.get_default_omission_response_params()["window_around_timepoint_seconds"]
cell_data = odf[(odf.cell_specimen_id == cell_specimen_id)]
ax[i + (n * 3)] = sf.plot_mean_trace_from_mean_df(cell_data,
frame_rate=analysis.ophys_frame_rate, ylabel=ylabel,
legend_label=image_name, color='gray', interval_sec=1,
xlims=window, ax=ax[i + (n * 3)])
ax[i + (n * 3)] = sf.plot_flashes_on_trace(ax[i + (n * 3)], analysis, window=window, trial_type=None, omitted=True, alpha=0.15, facecolor='gray')
ax[i + (n * 3)].set_title(container_expts.loc[ophys_experiment_id].session_type[6:] + '\n omission response')
window = rp.get_default_trial_response_params()["window_around_timepoint_seconds"]
cell_data = tdf[(tdf.cell_specimen_id == cell_specimen_id) & (tdf.go == True) & (tdf.pref_stim == True)]
hit_colors = [sns.color_palette()[2], sns.color_palette()[3]]
for c, hit in enumerate([True, False]):
if len(cell_data[cell_data.hit == hit]) > 0:
ax[i + (n * 4)] = sf.plot_mean_trace_from_mean_df(cell_data[cell_data.hit == hit],
frame_rate=analysis.ophys_frame_rate, ylabel=ylabel,
legend_label=hit, color=hit_colors[c], interval_sec=1,
xlims=window, ax=ax[i + (n * 4)])
ax[i + (n * 4)].legend(fontsize='xx-small', title='hit', title_fontsize='xx-small')
ax[i + (n * 4)] = sf.plot_flashes_on_trace(ax[i + (n * 4)], analysis, window=window, trial_type='go', omitted=False, alpha=0.15, facecolor='gray')
ax[i + (n * 4)].set_title(container_expts.loc[ophys_experiment_id].session_type[6:] + '\n change response')
fig.tight_layout()
fig.suptitle(str(cell_specimen_id) + '_' + dataset.metadata_string, x=0.5, y=1.01,
horizontalalignment='center')
except Exception as e:
print('problem for cell_specimen_id:', cell_specimen_id, ', ophys_experiment_id:', ophys_experiment_id)
print(e)
if save_figure:
save_dir = utils.get_single_cell_plots_dir()
utils.save_figure(fig, figsize, save_dir, 'across_session_responses', str(
cell_specimen_id) + '_' + dataset.metadata_string + '_across_session_responses' + suffix)
plt.close()
def test_get_filtered_ophys_experiment_table():
import visual_behavior.data_access.loading as loading
experiment_table = loading.get_filtered_ophys_experiment_table()
problems_list.append(experiment_id)
print('problem for', experiment_id)
columns = ['experiment_id', 'basic_snr_depth_image', 'basic_snr_average_image', 'basic_snr_max_image',
'std_avg_image', 'std_max_image', 'mean_avg_image', 'mean_max_image',
# 'basic_snr_fingerprint_avg_image', 'basic_snr_fingerprint_max_image',
# 'contrast_std_fingerprint_avg_image', 'contrast_std_fingerprint_max_image', 'basic_snr_movie_frame',
'median_robust_snr_traces', 'max_robust_snr_traces', 'peak_over_std_max_traces', ]
metrics_df = pd.DataFrame(metrics_list, columns=columns)
return metrics_df, problems_list
if __name__ == '__main__':
try:
experiments_table = loading.get_filtered_ophys_experiment_table(include_failed_data=True)
experiments = experiments_table[(experiments_table.cre_line == 'Slc17a7-IRES2-Cre') &
(experiments_table.imaging_depth < 250) &
(experiments_table.imaging_depth > 150)]
experiment_ids = experiments.index.values
metrics_df, problems_list = get_snr_metrics_df_for_experiments(experiment_ids)
save_dir = r'/allen/programs/braintv/workgroups/nc-ophys/visual_behavior/qc_plots/snr_metrics'
# save_dir = r'\\allen\programs\braintv\workgroups\nc-ophys\visual_behavior\qc_plots\snr_metrics'
metrics_df.to_csv(os.path.join(save_dir, 'snr_metrics_Slc.csv'))
problems_list = pd.DataFrame(problems_list)
problems_list.to_csv(os.path.join(save_dir, 'problems_list_Slc.csv'))
except Exception as e:
print(e)
import os
import sys
import platform
if platform.system() == 'Linux':
# sys.path.append('/allen/programs/braintv/workgroups/nc-ophys/Doug/pbstools')
sys.path.append('/allen/programs/braintv/workgroups/nc-ophys/nick.ponvert/src/pbstools')
from pbstools import PythonJob # flake8: noqa: E999
import visual_behavior.data_access.loading as loading
experiments_table = loading.get_filtered_ophys_experiment_table()
experiments = experiments_table[experiments_table.project_code.isin(['VisualBehavior','VisualBehaviorTask1B','VisualBehaviorMultiscope'])]
experiment_ids = experiments.index.values
python_file = r"/home/marinag/visual_behavior_analysis/scripts/create_analysis_files.py"
jobdir = '/allen/programs/braintv/workgroups/nc-ophys/Marina/ClusterJobs/JobRecords2'
job_settings = {'queue': 'braintv',
'mem': '100g',
'walltime': '10:00:00',
'ppn': 1,
'jobdir': jobdir,
}
for experiment_id in experiment_ids:
PythonJob(
python_file,
python_executable='/home/marinag/anaconda2/envs/visual_behavior_sdk/bin/python',
python_args=experiment_id,
def plot_behavior_and_pop_avg_mesoscope(ophys_session_id,
xlim_seconds=None,
save_figure=True):
if xlim_seconds is None:
suffix = ''
elif xlim_seconds[1] < 2000:
suffix = 'early'
elif xlim_seconds[1] > -2000:
suffix = 'late'
experiments_table = loading.get_filtered_ophys_experiment_table()
experiment_ids = experiments_table[
experiments_table.ophys_session_id == ophys_session_id].sort_values(
by='date_of_acquisition').index.values
experiment_id = experiment_ids[0]
dataset = loading.get_ophys_dataset(experiment_id)
colors = sns.color_palette()
figsize = (15, 8)
fig, ax = plt.subplots(3, 1, figsize=figsize, sharex=True)
try:
ax[0] = plot_behavior_model_weights(dataset,
xlim_seconds=xlim_seconds,
plot_stimuli=True,
ax=ax[0])
except BaseException:
print('no behavior model output for', dataset.ophys_experiment_id)
ax[1] = plot_behavior(dataset.ophys_experiment_id,
xlim_seconds=xlim_seconds,
plot_stimuli=True,
ax=ax[1])
label = dataset.metadata['targeted_structure'] + ', ' + str(
dataset.metadata['imaging_depth'])
ax[2] = plot_traces(dataset,
include_cell_traces=False,
plot_stimuli=True,
xlim_seconds=xlim_seconds,
label=label,
color=colors[0],
ax=ax[2])
for i, experiment_id in enumerate(experiment_ids[1:]):
dataset = loading.get_ophys_dataset(experiment_id)
label = dataset.metadata['targeted_structure'] + ', ' + str(
dataset.metadata['imaging_depth'])
ax[2] = plot_traces(dataset,
include_cell_traces=False,
plot_stimuli=False,
xlim_seconds=xlim_seconds,
label=label,
color=colors[i + 1],
ax=ax[2])
ax[2].legend(fontsize='x-small', loc='upper left')
plt.subplots_adjust(wspace=0, hspace=0.1)
ax[0].set_title(dataset.metadata_string)
if save_figure:
save_dir = os.path.abspath(
os.path.join(loading.get_qc_plots_dir(), 'timeseries_plots'))
utils.save_figure(fig, figsize, save_dir,
'behavior_traces_population_average_mesoscope',
dataset.metadata_string + '_' + suffix)
def get_multi_session_mean_df(experiment_ids,
analysis_cache_dir,
df_name,
conditions=[
'cell_specimen_id', 'change_image_name',
'behavioral_response_type'
],
flashes=False,
use_events=False,
omitted=False,
get_reliability=False,
get_pref_stim=True,
exclude_omitted_from_pref_stim=True,
use_sdk_dataset=True):
experiments_table = loading.get_filtered_ophys_experiment_table()
mega_mdf = pd.DataFrame()
for experiment_id in experiment_ids:
print(experiment_id)
# try:
if use_sdk_dataset:
dataset = loading.get_ophys_dataset(experiment_id)
else:
dataset = VisualBehaviorOphysDataset(experiment_id,
analysis_cache_dir)
analysis = ResponseAnalysis(dataset,
use_events=use_events,
overwrite_analysis_files=False,
use_extended_stimulus_presentations=True)
df = analysis.get_response_df(df_name)
df['ophys_experiment_id'] = dataset.ophys_experiment_id
df['project_code'] = experiments_table.loc[experiment_id].project_code
df['session_type'] = experiments_table.loc[experiment_id].session_type
# if 'engaged' in conditions:
# df['engaged'] = [True if reward_rate > 2 else False for reward_rate in df.reward_rate.values]
if 'running' in conditions:
df['running'] = [
True if window_running_speed > 5 else False
for window_running_speed in df.window_running_speed.values
]
# if 'large_pupil' in conditions:
# if 'mean_pupil_area' in df.keys():
# df = df[df.mean_pupil_area.isnull() == False]
# if len(df) > 100:
# median_pupil_area = df.mean_pupil_area.median()
# df['large_pupil'] = [True if mean_pupil_area > median_pupil_area else False for mean_pupil_area in
# df.mean_pupil_area.values]
mdf = ut.get_mean_df(
df,
analysis,
conditions=conditions,
get_pref_stim=get_pref_stim,
flashes=flashes,
omitted=omitted,
get_reliability=get_reliability,
exclude_omitted_from_pref_stim=exclude_omitted_from_pref_stim)
mdf['ophys_experiment_id'] = dataset.ophys_experiment_id
dataset.metadata['reporter_line'] = dataset.metadata['reporter_line'][
0]
dataset.metadata['driver_line'] = dataset.metadata['driver_line'][0]
metadata = pd.DataFrame(dataset.metadata, index=[experiment_id])
mdf = ut.add_metadata_to_mean_df(mdf, metadata)
mega_mdf = pd.concat([mega_mdf, mdf])
# except Exception as e: # flake8: noqa: E722
# print(e)
# print('problem for', experiment_id)
if use_events:
suffix = '_events'
else:
suffix = ''
if 'level_0' in mega_mdf.keys():
mega_mdf = mega_mdf.drop(columns='level_0')
if 'index' in mega_mdf.keys():
mega_mdf = mega_mdf.drop(columns='index')
mega_mdf_write_dir = os.path.join(analysis_cache_dir,
'multi_session_summary_dfs')
if not os.path.exists(mega_mdf_write_dir):
os.makedirs(mega_mdf_write_dir)
if len(conditions) == 5:
filename = 'mean_' + df_name + '_' + conditions[1] + '_' + conditions[2] + '_' + conditions[3] + '_' + \
conditions[4] + suffix + '.h5'
elif len(conditions) == 4:
filename = 'mean_' + df_name + '_' + conditions[1] + '_' + conditions[
2] + '_' + conditions[3] + suffix + '.h5'
elif len(conditions) == 3:
filename = 'mean_' + df_name + '_' + conditions[1] + '_' + conditions[
2] + suffix + '.h5'
elif len(conditions) == 2:
filename = 'mean_' + df_name + '_' + conditions[1] + suffix + '.h5'
elif len(conditions) == 1:
filename = 'mean_' + df_name + '_' + conditions[0] + suffix + '.h5'
print('saving multi session mean df to ', filename)
mega_mdf.to_hdf(os.path.join(mega_mdf_write_dir, filename), key='df')
print('saved')
return mega_mdf
def get_behavior_session_table():
'''
returns a table of every behavior session that meets the following criteria:
* is an ophys session and has at least one passing experiment associated with it
* is a behavior only session performed by a mouse with at least one passing ophys experiment
'''
cache = loading.get_visual_behavior_cache()
behavior_session_table = cache.get_behavior_session_table().reset_index()
filtered_ophys_experiment_table = loading.get_filtered_ophys_experiment_table(
)
# get donor id for experiment_table
filtered_ophys_experiment_table[
'donor_id'] = filtered_ophys_experiment_table['specimen_id'].map(
lambda sid: get_donor_from_specimen_id(sid))
# get behavior donor dataframe - all mice in behavior table
behavior_donors = pd.DataFrame(
{'donor_id': behavior_session_table['donor_id'].unique()})
# add a flag identifying which donors have associated ophys sessions
behavior_donors['donor_in_ophys'] = behavior_donors['donor_id'].map(
lambda did: did in list(filtered_ophys_experiment_table['donor_id'].
unique()))
# merge back in behavior donors to determine which behavior sessions have associated ophys
behavior_session_table = behavior_session_table.merge(
behavior_donors,
left_on='donor_id',
right_on='donor_id',
how='left',
)
# get project table
project_table = db.lims_query("select id,code from projects")
query = '''SELECT behavior_sessions.id, specimens.project_id FROM specimens
JOIN donors ON specimens.donor_id=donors.id
JOIN behavior_sessions ON donors.id=behavior_sessions.donor_id'''
behavior_id_project_id_map = db.lims_query(query).rename(columns={
'id':
'behavior_session_id'
}).merge(
project_table,
left_on='project_id',
right_on='id',
how='left',
).drop(columns=['id']).rename(columns={
'code': 'project_code'
}).drop_duplicates('behavior_session_id').set_index('behavior_session_id')
# merge project table with behavior sessions
behavior_session_table = behavior_session_table.merge(
behavior_id_project_id_map.reset_index(),
left_on='behavior_session_id',
right_on='behavior_session_id',
how='left')
# add a boolean for whether or not a session is in the filtered experiment table
def osid_in_filtered_experiments(osid):
if pd.notnull(osid):
return osid in filtered_ophys_experiment_table[
'ophys_session_id'].unique()
else:
return True
behavior_session_table['in_filtered_experiments'] = behavior_session_table[
'ophys_session_id'].apply(osid_in_filtered_experiments)
# add missing session types (I have no idea why some are missing!)
def get_session_type(osid):
if osid in filtered_ophys_experiment_table['ophys_session_id'].unique(
).tolist():
return filtered_ophys_experiment_table.query(
'ophys_session_id == {}'.format(osid)).iloc[0]['session_type']
else:
return None
# drop sessions for mice with no ophys AND sessions that aren't in the filtered list
behavior_session_table = behavior_session_table.set_index(
'behavior_session_id').query(
'donor_in_ophys and in_filtered_experiments').copy()
for idx, row in behavior_session_table.iterrows():
if pd.isnull(row['session_type']):
behavior_session_table.at[idx, 'session_type'] = get_session_type(
row['ophys_session_id'])
validation_results = get_validation_results().sort_index()
behavior_session_table = behavior_session_table.merge(validation_results,
left_index=True,
right_index=True,
how='left')
return behavior_session_table
