def get_motion_corrected_movie_paths(ophys_container_id): et = loading.get_filtered_ophys_experiment_table().reset_index() paths = [] for oeid in et.query('ophys_container_id == @ophys_container_id').sort_values(by='ophys_experiment_id')['ophys_experiment_id']: paths.append( '/' + get_motion_corrected_movie_path(oeid) ) return paths
def print_motion_corrected_movie_paths(ophys_container_id): et = loading.get_filtered_ophys_experiment_table().reset_index() lines = [] for oeid in et.query('ophys_container_id == @ophys_container_id')['ophys_experiment_id']: movie_path = loading.get_motion_corrected_movie_h5_location(oeid).replace('motion_corrected_video.h5', 'motion_preview.10x.mp4') lines.append('ophys experiment ID = {}\n'.format(oeid)) lines.append("LINUX PATH:") lines.append('\t<a href="url">{}</a>'.format(movie_path)) lines.append('WINDOWS PATH') lines.append('\t{}'.format(movie_path.replace('/', '\\'))) lines.append('') return '\n'.join(lines)
def plot_lick_triggered_average_for_container(container_id, save_figure=True): experiments = loading.get_filtered_ophys_experiment_table() container_data = experiments[experiments.container_id == container_id] figsize = (6, 4) fig, ax = plt.subplots(figsize=figsize) for session_number in container_data.session_number.unique(): experiment_ids = container_data[container_data.session_number == session_number].index.values for experiment_id in experiment_ids: dataset = loading.get_ophys_dataset(experiment_id) analysis = ResponseAnalysis( dataset, use_events=True, use_extended_stimulus_presentations=False) ldf = analysis.get_response_df( df_name='lick_triggered_response_df') if len(ldf.cell_specimen_id.unique()) > 5: colors = utils.get_colors_for_session_numbers() ax = plot_lick_triggered_average( ldf, color=colors[session_number - 1], ylabel='pop. avg. \nevent magnitude', legend_label=session_number, ax=ax) ax.legend(loc='upper left', fontsize='x-small') fig.tight_layout() m = dataset.metadata.copy() title = str(m['mouse_id']) + '_' + m['full_genotype'].split( '/')[0] + '_' + m['targeted_structure'] + '_' + str( m['imaging_depth']) + '_' + m['equipment_name'] + '_' + str( m['experiment_container_id']) fig.suptitle(title, x=0.53, y=1.02, fontsize=16) if save_figure: save_dir = loading.get_container_plots_dir() utils.save_figure(fig, figsize, save_dir, 'lick_triggered_average', title)
app = dash.Dash(external_stylesheets=[dbc.themes.SPACELAB]) app.title = 'Visual Behavior Data QC' # app.config['suppress_callback_exceptions'] = True # FUNCTION CALLS print('setting up table') t0 = time.time() container_table = functions.load_container_data().sort_values( 'first_acquistion_date') container_plot_options = functions.load_container_plot_options() session_plot_options = functions.load_session_plot_options() container_overview_plot_options = functions.load_container_overview_plot_options( ) plot_inventory = functions.generate_plot_inventory() plot_inventory_fig = functions.make_plot_inventory_heatmap(plot_inventory) experiment_table = loading.get_filtered_ophys_experiment_table().reset_index() session_table = functions.load_session_data() print('done setting up table, it took {} seconds'.format(time.time() - t0)) # COMPONENT SETUP print('setting up components') t0 = time.time() components.plot_selection_dropdown.options = container_plot_options components.container_overview_dropdown.options = container_overview_plot_options components.container_overview_iframe.src = app.get_asset_url( 'qc_plots/overview_plots/d_prime_container_overview.html') components.plot_inventory_iframe.src = 'https://dougollerenshaw.github.io/figures_to_share/container_plot_inventory.html' # app.get_asset_url('qc_plots/container_plot_inventory.html') components.data_table.columns = [{ "name": i.replace('_', ' '), "id": i } for i in container_table.columns]
def plot_across_session_responses(ophys_container_id, cell_specimen_id, use_events=False, save_figure=True): """ Generates plots characterizing single cell activity in response to stimulus, omissions, and changes. Compares across all sessions in a container for each cell, including the ROI mask across days. Useful to validate cell matching as well as examine changes in activity profiles over days. """ experiments_table = data_loading.get_filtered_ophys_experiment_table(release_data_only=True) container_expts = experiments_table[experiments_table.ophys_container_id == ophys_container_id] expts = np.sort(container_expts.index.values) if use_events: ylabel = 'response' suffix = '_events' else: ylabel = 'dF/F' suffix = '' n = len(expts) figsize = (25, 20) fig, ax = plt.subplots(6, n, figsize=figsize) ax = ax.ravel() print('ophys_container_id:', ophys_container_id) for i, ophys_experiment_id in enumerate(expts): print('ophys_experiment_id:', ophys_experiment_id) try: dataset = data_loading.get_ophys_dataset(ophys_experiment_id, include_invalid_rois=False) if cell_specimen_id in dataset.dff_traces.index: analysis = ResponseAnalysis(dataset, use_events=use_events, use_extended_stimulus_presentations=False) sdf = ut.get_mean_df(analysis.get_response_df(df_name='stimulus_response_df'), analysis=analysis, conditions=['cell_specimen_id', 'is_change', 'image_name'], flashes=True, omitted=False, get_reliability=False, get_pref_stim=True, exclude_omitted_from_pref_stim=True) odf = ut.get_mean_df(analysis.get_response_df(df_name='omission_response_df'), analysis=analysis, conditions=['cell_specimen_id'], flashes=False, omitted=True, get_reliability=False, get_pref_stim=False, exclude_omitted_from_pref_stim=False) tdf = ut.get_mean_df(analysis.get_response_df(df_name='trials_response_df'), analysis=analysis, conditions=['cell_specimen_id', 'go', 'hit', 'change_image_name'], flashes=False, omitted=False, get_reliability=False, get_pref_stim=True, exclude_omitted_from_pref_stim=True) ct = dataset.cell_specimen_table.copy() cell_roi_id = ct.loc[cell_specimen_id].cell_roi_id ax[i] = sf.plot_cell_zoom(dataset.roi_masks, dataset.max_projection, cell_roi_id, spacex=20, spacey=20, show_mask=True, ax=ax[i]) ax[i].set_title(container_expts.loc[ophys_experiment_id].session_type[6:]) colors = sns.color_palette('hls', 8) + [(0.5, 0.5, 0.5)] window = rp.get_default_stimulus_response_params()["window_around_timepoint_seconds"] cell_data = sdf[(sdf.cell_specimen_id == cell_specimen_id) & (sdf.is_change == False)] for c, image_name in enumerate(np.sort(cell_data.image_name.unique())): ax[i + n] = sf.plot_mean_trace_from_mean_df(cell_data[cell_data.image_name == image_name], frame_rate=analysis.ophys_frame_rate, ylabel=ylabel, legend_label=image_name, color=colors[c], interval_sec=0.5, xlims=window, ax=ax[i + n]) ax[i + n] = sf.plot_flashes_on_trace(ax[i + n], analysis, window=window, trial_type=None, omitted=False, alpha=0.15, facecolor='gray') ax[i + n].set_title(container_expts.loc[ophys_experiment_id].session_type[6:] + '\n image response') analysis = ResponseAnalysis(dataset, use_events=False, use_extended_stimulus_presentations=True) tmp = analysis.get_response_df(df_name='stimulus_response_df') tmp['running'] = [True if run_speed > 2 else False for run_speed in tmp.mean_running_speed.values] sdf = ut.get_mean_df(tmp, analysis=analysis, conditions=['cell_specimen_id', 'is_change', 'image_name', 'running'], flashes=True, omitted=False, get_reliability=False, get_pref_stim=True, exclude_omitted_from_pref_stim=False) cell_data = sdf[(sdf.cell_specimen_id == cell_specimen_id) & (sdf.is_change == False) & (sdf.pref_stim == True)] run_colors = [sns.color_palette()[3], sns.color_palette()[2]] for c, running in enumerate(np.sort(cell_data.running.unique())): if len(cell_data[cell_data.running == running]) > 0: ax[i + (n * 2)] = sf.plot_mean_trace_from_mean_df(cell_data[cell_data.running == running], frame_rate=analysis.ophys_frame_rate, ylabel=ylabel, legend_label=running, color=run_colors[c], interval_sec=0.5, xlims=window, ax=ax[i + (n * 2)]) ax[i + (n * 2)].legend(fontsize='xx-small', title='running', title_fontsize='xx-small') ax[i + (n * 2)] = sf.plot_flashes_on_trace(ax[i + (n * 2)], analysis, window=window, trial_type=None, omitted=False, alpha=0.15, facecolor='gray') ax[i + (n * 2)].set_title(container_expts.loc[ophys_experiment_id].session_type[6:] + '\n image response') window = rp.get_default_omission_response_params()["window_around_timepoint_seconds"] cell_data = odf[(odf.cell_specimen_id == cell_specimen_id)] ax[i + (n * 3)] = sf.plot_mean_trace_from_mean_df(cell_data, frame_rate=analysis.ophys_frame_rate, ylabel=ylabel, legend_label=image_name, color='gray', interval_sec=1, xlims=window, ax=ax[i + (n * 3)]) ax[i + (n * 3)] = sf.plot_flashes_on_trace(ax[i + (n * 3)], analysis, window=window, trial_type=None, omitted=True, alpha=0.15, facecolor='gray') ax[i + (n * 3)].set_title(container_expts.loc[ophys_experiment_id].session_type[6:] + '\n omission response') window = rp.get_default_trial_response_params()["window_around_timepoint_seconds"] cell_data = tdf[(tdf.cell_specimen_id == cell_specimen_id) & (tdf.go == True) & (tdf.pref_stim == True)] hit_colors = [sns.color_palette()[2], sns.color_palette()[3]] for c, hit in enumerate([True, False]): if len(cell_data[cell_data.hit == hit]) > 0: ax[i + (n * 4)] = sf.plot_mean_trace_from_mean_df(cell_data[cell_data.hit == hit], frame_rate=analysis.ophys_frame_rate, ylabel=ylabel, legend_label=hit, color=hit_colors[c], interval_sec=1, xlims=window, ax=ax[i + (n * 4)]) ax[i + (n * 4)].legend(fontsize='xx-small', title='hit', title_fontsize='xx-small') ax[i + (n * 4)] = sf.plot_flashes_on_trace(ax[i + (n * 4)], analysis, window=window, trial_type='go', omitted=False, alpha=0.15, facecolor='gray') ax[i + (n * 4)].set_title(container_expts.loc[ophys_experiment_id].session_type[6:] + '\n change response') fig.tight_layout() fig.suptitle(str(cell_specimen_id) + '_' + dataset.metadata_string, x=0.5, y=1.01, horizontalalignment='center') except Exception as e: print('problem for cell_specimen_id:', cell_specimen_id, ', ophys_experiment_id:', ophys_experiment_id) print(e) if save_figure: save_dir = utils.get_single_cell_plots_dir() utils.save_figure(fig, figsize, save_dir, 'across_session_responses', str( cell_specimen_id) + '_' + dataset.metadata_string + '_across_session_responses' + suffix) plt.close()
def test_get_filtered_ophys_experiment_table(): import visual_behavior.data_access.loading as loading experiment_table = loading.get_filtered_ophys_experiment_table()
problems_list.append(experiment_id) print('problem for', experiment_id) columns = ['experiment_id', 'basic_snr_depth_image', 'basic_snr_average_image', 'basic_snr_max_image', 'std_avg_image', 'std_max_image', 'mean_avg_image', 'mean_max_image', # 'basic_snr_fingerprint_avg_image', 'basic_snr_fingerprint_max_image', # 'contrast_std_fingerprint_avg_image', 'contrast_std_fingerprint_max_image', 'basic_snr_movie_frame', 'median_robust_snr_traces', 'max_robust_snr_traces', 'peak_over_std_max_traces', ] metrics_df = pd.DataFrame(metrics_list, columns=columns) return metrics_df, problems_list if __name__ == '__main__': try: experiments_table = loading.get_filtered_ophys_experiment_table(include_failed_data=True) experiments = experiments_table[(experiments_table.cre_line == 'Slc17a7-IRES2-Cre') & (experiments_table.imaging_depth < 250) & (experiments_table.imaging_depth > 150)] experiment_ids = experiments.index.values metrics_df, problems_list = get_snr_metrics_df_for_experiments(experiment_ids) save_dir = r'/allen/programs/braintv/workgroups/nc-ophys/visual_behavior/qc_plots/snr_metrics' # save_dir = r'\\allen\programs\braintv\workgroups\nc-ophys\visual_behavior\qc_plots\snr_metrics' metrics_df.to_csv(os.path.join(save_dir, 'snr_metrics_Slc.csv')) problems_list = pd.DataFrame(problems_list) problems_list.to_csv(os.path.join(save_dir, 'problems_list_Slc.csv')) except Exception as e: print(e)
import os import sys import platform if platform.system() == 'Linux': # sys.path.append('/allen/programs/braintv/workgroups/nc-ophys/Doug/pbstools') sys.path.append('/allen/programs/braintv/workgroups/nc-ophys/nick.ponvert/src/pbstools') from pbstools import PythonJob # flake8: noqa: E999 import visual_behavior.data_access.loading as loading experiments_table = loading.get_filtered_ophys_experiment_table() experiments = experiments_table[experiments_table.project_code.isin(['VisualBehavior','VisualBehaviorTask1B','VisualBehaviorMultiscope'])] experiment_ids = experiments.index.values python_file = r"/home/marinag/visual_behavior_analysis/scripts/create_analysis_files.py" jobdir = '/allen/programs/braintv/workgroups/nc-ophys/Marina/ClusterJobs/JobRecords2' job_settings = {'queue': 'braintv', 'mem': '100g', 'walltime': '10:00:00', 'ppn': 1, 'jobdir': jobdir, } for experiment_id in experiment_ids: PythonJob( python_file, python_executable='/home/marinag/anaconda2/envs/visual_behavior_sdk/bin/python', python_args=experiment_id,
def plot_behavior_and_pop_avg_mesoscope(ophys_session_id, xlim_seconds=None, save_figure=True): if xlim_seconds is None: suffix = '' elif xlim_seconds[1] < 2000: suffix = 'early' elif xlim_seconds[1] > -2000: suffix = 'late' experiments_table = loading.get_filtered_ophys_experiment_table() experiment_ids = experiments_table[ experiments_table.ophys_session_id == ophys_session_id].sort_values( by='date_of_acquisition').index.values experiment_id = experiment_ids[0] dataset = loading.get_ophys_dataset(experiment_id) colors = sns.color_palette() figsize = (15, 8) fig, ax = plt.subplots(3, 1, figsize=figsize, sharex=True) try: ax[0] = plot_behavior_model_weights(dataset, xlim_seconds=xlim_seconds, plot_stimuli=True, ax=ax[0]) except BaseException: print('no behavior model output for', dataset.ophys_experiment_id) ax[1] = plot_behavior(dataset.ophys_experiment_id, xlim_seconds=xlim_seconds, plot_stimuli=True, ax=ax[1]) label = dataset.metadata['targeted_structure'] + ', ' + str( dataset.metadata['imaging_depth']) ax[2] = plot_traces(dataset, include_cell_traces=False, plot_stimuli=True, xlim_seconds=xlim_seconds, label=label, color=colors[0], ax=ax[2]) for i, experiment_id in enumerate(experiment_ids[1:]): dataset = loading.get_ophys_dataset(experiment_id) label = dataset.metadata['targeted_structure'] + ', ' + str( dataset.metadata['imaging_depth']) ax[2] = plot_traces(dataset, include_cell_traces=False, plot_stimuli=False, xlim_seconds=xlim_seconds, label=label, color=colors[i + 1], ax=ax[2]) ax[2].legend(fontsize='x-small', loc='upper left') plt.subplots_adjust(wspace=0, hspace=0.1) ax[0].set_title(dataset.metadata_string) if save_figure: save_dir = os.path.abspath( os.path.join(loading.get_qc_plots_dir(), 'timeseries_plots')) utils.save_figure(fig, figsize, save_dir, 'behavior_traces_population_average_mesoscope', dataset.metadata_string + '_' + suffix)
def get_multi_session_mean_df(experiment_ids, analysis_cache_dir, df_name, conditions=[ 'cell_specimen_id', 'change_image_name', 'behavioral_response_type' ], flashes=False, use_events=False, omitted=False, get_reliability=False, get_pref_stim=True, exclude_omitted_from_pref_stim=True, use_sdk_dataset=True): experiments_table = loading.get_filtered_ophys_experiment_table() mega_mdf = pd.DataFrame() for experiment_id in experiment_ids: print(experiment_id) # try: if use_sdk_dataset: dataset = loading.get_ophys_dataset(experiment_id) else: dataset = VisualBehaviorOphysDataset(experiment_id, analysis_cache_dir) analysis = ResponseAnalysis(dataset, use_events=use_events, overwrite_analysis_files=False, use_extended_stimulus_presentations=True) df = analysis.get_response_df(df_name) df['ophys_experiment_id'] = dataset.ophys_experiment_id df['project_code'] = experiments_table.loc[experiment_id].project_code df['session_type'] = experiments_table.loc[experiment_id].session_type # if 'engaged' in conditions: # df['engaged'] = [True if reward_rate > 2 else False for reward_rate in df.reward_rate.values] if 'running' in conditions: df['running'] = [ True if window_running_speed > 5 else False for window_running_speed in df.window_running_speed.values ] # if 'large_pupil' in conditions: # if 'mean_pupil_area' in df.keys(): # df = df[df.mean_pupil_area.isnull() == False] # if len(df) > 100: # median_pupil_area = df.mean_pupil_area.median() # df['large_pupil'] = [True if mean_pupil_area > median_pupil_area else False for mean_pupil_area in # df.mean_pupil_area.values] mdf = ut.get_mean_df( df, analysis, conditions=conditions, get_pref_stim=get_pref_stim, flashes=flashes, omitted=omitted, get_reliability=get_reliability, exclude_omitted_from_pref_stim=exclude_omitted_from_pref_stim) mdf['ophys_experiment_id'] = dataset.ophys_experiment_id dataset.metadata['reporter_line'] = dataset.metadata['reporter_line'][ 0] dataset.metadata['driver_line'] = dataset.metadata['driver_line'][0] metadata = pd.DataFrame(dataset.metadata, index=[experiment_id]) mdf = ut.add_metadata_to_mean_df(mdf, metadata) mega_mdf = pd.concat([mega_mdf, mdf]) # except Exception as e: # flake8: noqa: E722 # print(e) # print('problem for', experiment_id) if use_events: suffix = '_events' else: suffix = '' if 'level_0' in mega_mdf.keys(): mega_mdf = mega_mdf.drop(columns='level_0') if 'index' in mega_mdf.keys(): mega_mdf = mega_mdf.drop(columns='index') mega_mdf_write_dir = os.path.join(analysis_cache_dir, 'multi_session_summary_dfs') if not os.path.exists(mega_mdf_write_dir): os.makedirs(mega_mdf_write_dir) if len(conditions) == 5: filename = 'mean_' + df_name + '_' + conditions[1] + '_' + conditions[2] + '_' + conditions[3] + '_' + \ conditions[4] + suffix + '.h5' elif len(conditions) == 4: filename = 'mean_' + df_name + '_' + conditions[1] + '_' + conditions[ 2] + '_' + conditions[3] + suffix + '.h5' elif len(conditions) == 3: filename = 'mean_' + df_name + '_' + conditions[1] + '_' + conditions[ 2] + suffix + '.h5' elif len(conditions) == 2: filename = 'mean_' + df_name + '_' + conditions[1] + suffix + '.h5' elif len(conditions) == 1: filename = 'mean_' + df_name + '_' + conditions[0] + suffix + '.h5' print('saving multi session mean df to ', filename) mega_mdf.to_hdf(os.path.join(mega_mdf_write_dir, filename), key='df') print('saved') return mega_mdf
def get_behavior_session_table(): ''' returns a table of every behavior session that meets the following criteria: * is an ophys session and has at least one passing experiment associated with it * is a behavior only session performed by a mouse with at least one passing ophys experiment ''' cache = loading.get_visual_behavior_cache() behavior_session_table = cache.get_behavior_session_table().reset_index() filtered_ophys_experiment_table = loading.get_filtered_ophys_experiment_table( ) # get donor id for experiment_table filtered_ophys_experiment_table[ 'donor_id'] = filtered_ophys_experiment_table['specimen_id'].map( lambda sid: get_donor_from_specimen_id(sid)) # get behavior donor dataframe - all mice in behavior table behavior_donors = pd.DataFrame( {'donor_id': behavior_session_table['donor_id'].unique()}) # add a flag identifying which donors have associated ophys sessions behavior_donors['donor_in_ophys'] = behavior_donors['donor_id'].map( lambda did: did in list(filtered_ophys_experiment_table['donor_id']. unique())) # merge back in behavior donors to determine which behavior sessions have associated ophys behavior_session_table = behavior_session_table.merge( behavior_donors, left_on='donor_id', right_on='donor_id', how='left', ) # get project table project_table = db.lims_query("select id,code from projects") query = '''SELECT behavior_sessions.id, specimens.project_id FROM specimens JOIN donors ON specimens.donor_id=donors.id JOIN behavior_sessions ON donors.id=behavior_sessions.donor_id''' behavior_id_project_id_map = db.lims_query(query).rename(columns={ 'id': 'behavior_session_id' }).merge( project_table, left_on='project_id', right_on='id', how='left', ).drop(columns=['id']).rename(columns={ 'code': 'project_code' }).drop_duplicates('behavior_session_id').set_index('behavior_session_id') # merge project table with behavior sessions behavior_session_table = behavior_session_table.merge( behavior_id_project_id_map.reset_index(), left_on='behavior_session_id', right_on='behavior_session_id', how='left') # add a boolean for whether or not a session is in the filtered experiment table def osid_in_filtered_experiments(osid): if pd.notnull(osid): return osid in filtered_ophys_experiment_table[ 'ophys_session_id'].unique() else: return True behavior_session_table['in_filtered_experiments'] = behavior_session_table[ 'ophys_session_id'].apply(osid_in_filtered_experiments) # add missing session types (I have no idea why some are missing!) def get_session_type(osid): if osid in filtered_ophys_experiment_table['ophys_session_id'].unique( ).tolist(): return filtered_ophys_experiment_table.query( 'ophys_session_id == {}'.format(osid)).iloc[0]['session_type'] else: return None # drop sessions for mice with no ophys AND sessions that aren't in the filtered list behavior_session_table = behavior_session_table.set_index( 'behavior_session_id').query( 'donor_in_ophys and in_filtered_experiments').copy() for idx, row in behavior_session_table.iterrows(): if pd.isnull(row['session_type']): behavior_session_table.at[idx, 'session_type'] = get_session_type( row['ophys_session_id']) validation_results = get_validation_results().sort_index() behavior_session_table = behavior_session_table.merge(validation_results, left_index=True, right_index=True, how='left') return behavior_session_table