def test_get_roi_masks_by_cell_roi_id(roi_ids, expected, cell_specimen_table_api): ssn = BehaviorOphysSession(api=cell_specimen_table_api) obtained = ssn._get_roi_masks_by_cell_roi_id(roi_ids) assert np.allclose(expected, obtained.values) assert np.allclose(obtained.coords['row'], [0.5, 1.5, 2.5, 3.5, 4.5]) assert np.allclose(obtained.coords['column'], [0.25, 0.75, 1.25, 1.75, 2.25])
def test_nwb_end_to_end(tmpdir_factory): oeid = 789359614 nwb_filepath = os.path.join(str(tmpdir_factory.mktemp('test_nwb_end_to_end')), 'nwbfile.nwb') d1 = BehaviorOphysSession.from_lims(oeid) BehaviorOphysNwbApi(nwb_filepath).save(d1) d2 = BehaviorOphysSession(api=BehaviorOphysNwbApi(nwb_filepath)) equals(d1, d2, reraise=True)
def test_legacy_dff_api(): ophys_experiment_id = 792813858 api = BehaviorOphysLimsApi(ophys_experiment_id) session = BehaviorOphysSession(api) _, dff_array = session.get_dff_traces() for csid in session.dff_traces.index.values: dff_trace = session.dff_traces.loc[csid]['dff'] ind = session.get_cell_specimen_indices([csid])[0] np.testing.assert_array_almost_equal(dff_trace, dff_array[ind, :]) assert dff_array.shape[0] == session.dff_traces.shape[0]
def test_nwb_end_to_end(tmpdir_factory): # NOTE: old test oeid 789359614 had no cell specimen ids due to not being # part of the 2021 Visual Behavior release set which broke a ton # of things... oeid = 795073741 tmpdir = 'test_nwb_end_to_end' nwb_filepath = os.path.join(str(tmpdir_factory.mktemp(tmpdir)), 'nwbfile.nwb') d1 = BehaviorOphysSession.from_lims(oeid) BehaviorOphysNwbApi(nwb_filepath).save(d1) d2 = BehaviorOphysSession(api=BehaviorOphysNwbApi(nwb_filepath)) assert sessions_are_equal(d1, d2, reraise=True)
def plot_running_speed(oeid, width=11, height=2, ax=None): """ plot running speed for a given session will create a new fig/ax if no axis is passed Arguments: oeid {int} -- ophys experiment ID Keyword Arguments: width {int} -- figure width if creating new (default: {11}) height {int} -- figure height if creating new (default: {2}) ax {matplotlib figure axis} -- axis to plot on. Will create new if none is passed (default: {None}) Returns: matplotlib figure axis -- ax """ session = BehaviorOphysSession.from_lims(oeid) if ax is None: fig, ax = plt.subplots(figsize=(width, height)) ax.plot(session.running_data_df['speed']) ax.set_xlabel('time (s)') ax.set_ylabel('speed (cm/s)') ax.set_title('running speed for ophys_session_id {}'.format(oeid)) return ax
def test_session_from_json(tmpdir_factory, session_data, get_expected, get_from_session): session = BehaviorOphysSession(api=BehaviorOphysJsonApi(session_data)) expected = get_expected(session_data) obtained = get_from_session(session) compare_fields(expected, obtained)
def get_session_data(self, ophys_session_id: int) -> BehaviorOphysSession: """Returns a BehaviorOphysSession object that contains methods to analyze a single behavior+ophys session. :param ophys_session_id: id that corresponds to a behavior session :type ophys_session_id: int :rtype: BehaviorOphysSession """ return BehaviorOphysSession(BehaviorOphysLimsApi(ophys_session_id))
def test_eye_tracking(dilation_frames, z_threshold, eye_tracking_start_value): mock = MagicMock() mock.get_eye_tracking.return_value = pd.DataFrame([1, 2, 3]) session = BehaviorOphysSession(api=mock, eye_tracking_z_threshold=z_threshold, eye_tracking_dilation_frames=dilation_frames) if eye_tracking_start_value is not None: session.eye_tracking = eye_tracking_start_value obtained = session.eye_tracking assert not session.api.get_eye_tracking.called assert obtained.equals(eye_tracking_start_value) else: obtained = session.eye_tracking assert obtained.equals(pd.DataFrame([1, 2, 3])) assert session.api.get_eye_tracking.called_with(z_threshold=z_threshold, dilation_frames=dilation_frames)
def test_visbeh_ophys_data_set(): ophys_experiment_id = 789359614 data_set = BehaviorOphysSession.from_lims(ophys_experiment_id) # TODO: need to improve testing here: # for _, row in data_set.roi_metrics.iterrows(): # print(np.array(row.to_dict()['mask']).sum()) # print # for _, row in data_set.roi_masks.iterrows(): # print(np.array(row.to_dict()['mask']).sum()) # All sorts of assert relationships: assert data_set.api.get_foraging_id() == str(data_set.api.get_behavior_session_uuid()) assert list(data_set.stimulus_templates.values())[0].shape == (8, 918, 1174) assert len(data_set.licks) == 2432 and list(data_set.licks.columns) == ['time'] assert len(data_set.rewards) == 85 and list(data_set.rewards.columns) == ['volume', 'autorewarded'] assert len(data_set.corrected_fluorescence_traces) == 269 and sorted(data_set.corrected_fluorescence_traces.columns) == ['cell_roi_id', 'corrected_fluorescence'] np.testing.assert_array_almost_equal(data_set.running_speed.timestamps, data_set.stimulus_timestamps) assert len(data_set.cell_specimen_table) == len(data_set.dff_traces) assert data_set.average_projection.data.shape == data_set.max_projection.data.shape assert list(data_set.motion_correction.columns) == ['x', 'y'] assert len(data_set.trials) == 602 assert data_set.metadata == {'stimulus_frame_rate': 60.0, 'full_genotype': 'Slc17a7-IRES2-Cre/wt;Camk2a-tTA/wt;Ai93(TITL-GCaMP6f)/wt', 'ophys_experiment_id': 789359614, 'session_type': 'OPHYS_6_images_B', 'driver_line': ['Camk2a-tTA', 'Slc17a7-IRES2-Cre'], 'behavior_session_uuid': uuid.UUID('69cdbe09-e62b-4b42-aab1-54b5773dfe78'), 'experiment_datetime': pytz.utc.localize(datetime.datetime(2018, 11, 30, 23, 28, 37)), 'ophys_frame_rate': 31.0, 'imaging_depth': 375, 'LabTracks_ID': 416369, 'experiment_container_id': 814796558, 'targeted_structure': 'VISp', 'reporter_line': ['Ai93(TITL-GCaMP6f)'], 'emission_lambda': 520.0, 'excitation_lambda': 910.0, 'field_of_view_height': 512, 'field_of_view_width': 447, 'indicator': 'GCAMP6f', 'rig_name': 'CAM2P.5', 'age': 'P139', 'sex': 'F'} assert math.isnan(data_set.task_parameters.pop('omitted_flash_fraction')) assert data_set.task_parameters == {'reward_volume': 0.007, 'stimulus_distribution': u'geometric', 'stimulus_duration_sec': 6.0, 'stimulus': 'images', 'blank_duration_sec': [0.5, 0.5], 'n_stimulus_frames': 69882, 'task': 'DoC_untranslated', 'response_window_sec': [0.15, 0.75], 'stage': u'OPHYS_6_images_B'}
def write_behavior_ophys_nwb(session_data: dict, nwb_filepath: str, skip_eye_tracking: bool): nwb_filepath_inprogress = nwb_filepath+'.inprogress' nwb_filepath_error = nwb_filepath+'.error' # Clean out files from previous runs: for filename in [nwb_filepath_inprogress, nwb_filepath_error, nwb_filepath]: if os.path.exists(filename): os.remove(filename) try: json_api = BehaviorOphysJsonApi(data=session_data, skip_eye_tracking=skip_eye_tracking) json_session = BehaviorOphysSession(api=json_api) lims_api = BehaviorOphysLimsApi( ophys_experiment_id=session_data['ophys_experiment_id'], skip_eye_tracking=skip_eye_tracking) lims_session = BehaviorOphysSession(api=lims_api) logging.info("Comparing a BehaviorOphysSession created from JSON " "with a BehaviorOphysSession created from LIMS") assert sessions_are_equal(json_session, lims_session, reraise=True) BehaviorOphysNwbApi(nwb_filepath_inprogress).save(json_session) logging.info("Comparing a BehaviorOphysSession created from JSON " "with a BehaviorOphysSession created from NWB") nwb_api = BehaviorOphysNwbApi(nwb_filepath_inprogress) nwb_session = BehaviorOphysSession(api=nwb_api) assert sessions_are_equal(json_session, nwb_session, reraise=True) os.rename(nwb_filepath_inprogress, nwb_filepath) return {'output_path': nwb_filepath} except Exception as e: os.rename(nwb_filepath_inprogress, nwb_filepath_error) raise e
def write_behavior_ophys_nwb(session_data, nwb_filepath): nwb_filepath_inprogress = nwb_filepath + '.inprogress' nwb_filepath_error = nwb_filepath + '.error' # Clean out files from previous runs: for filename in [ nwb_filepath_inprogress, nwb_filepath_error, nwb_filepath ]: if os.path.exists(filename): os.remove(filename) try: session = BehaviorOphysSession(api=BehaviorOphysJsonApi(session_data)) BehaviorOphysNwbApi(nwb_filepath_inprogress).save(session) api = BehaviorOphysNwbApi(nwb_filepath_inprogress) assert sessions_are_equal(session, BehaviorOphysSession(api=api)) os.rename(nwb_filepath_inprogress, nwb_filepath) return {'output_path': nwb_filepath} except Exception as e: os.rename(nwb_filepath_inprogress, nwb_filepath_error) raise e
def validate_last_trial_ends_adjacent_to_flash(ophys_experiment_id, api=None, verbose=False): # ensure that the last trial ends sometime on the last flash/blank cycle # i.e, if this is the last flash/blank iteration (high = stimulus present): # ------- ------- ------- ------- # | | | | | | | | # --------- --------- --------- --------- ------------------------------------------- # ^ ^ # The last trial has to have ended somewhere between the two carrots where: # the first carrot represents the time of the last recorded stimulus flash # the second carrot represents the time at which another flash should have started, after accounting for the possibility of the session ending on an omitted flash api = BehaviorOphysLimsApi() if api is None else api session = BehaviorOphysSession(api) # get the flash/blank parameters max_flash_duration = session.stimulus_presentations['duration'].max() max_blank_duration = session.task_parameters['blank_duration_sec'][1] # count number of omitted flashes at the very end of the session N_final_omitted_flashes = session.stimulus_presentations.index.max( ) - session.stimulus_presentations.query('omitted == False').index.max() # get the start/end time of the last valid (non-omitted) flash last_flash_start = session.stimulus_presentations.query( 'omitted == False')['start_time'].iloc[-1] last_flash_end = session.stimulus_presentations.query( 'omitted == False')['stop_time'].iloc[-1] # calculate when the next stimulus should have flashed, after accounting for any omitted flashes at the end of the session next_flash_would_have_started = last_flash_end + max_blank_duration + N_final_omitted_flashes * ( max_flash_duration + max_blank_duration) # get the end time of the last trial last_trial_end = session.trials.iloc[-1]['stop_time'] if verbose: print('last flash ended at {}'.format(last_flash_end)) print('another flash should have started by {}'.format( next_flash_would_have_started)) print('last trial ended at {}'.format(last_trial_end)) if not last_flash_start <= last_trial_end <= next_flash_would_have_started: raise ValidationError( 'The last trial does not end between the start of the last flash and the expected start time of the next flash' )
def test_trial_response_window_bounds_reward(ophys_experiment_id): api = BehaviorOphysLimsApi(ophys_experiment_id) session = BehaviorOphysSession(api) response_window = session.task_parameters['response_window_sec'] for _, row in session.trials.iterrows(): lick_times = [(t - row.change_time) for t in row.lick_times] if not np.isnan(row.reward_time): # monitor delay is incorporated into the trials table change time # TODO: where is this set in the session object? camstim_change_time = row.change_time - 0.0351 reward_time = (row.reward_time - camstim_change_time) assert response_window[0] < reward_time + 1 / 60 assert reward_time < response_window[1] + 1 / 60 if len(session.licks) > 0: assert lick_times[0] < reward_time
def test_event_detection(): ophys_experiment_id = 789359614 session = BehaviorOphysSession.from_lims( ophys_experiment_id=ophys_experiment_id) events = session.events assert len(events) > 0 expected_columns = [ 'events', 'filtered_events', 'lambda', 'noise_std', 'cell_roi_id' ] assert len(events.columns) == len(expected_columns) # Assert they contain the same columns assert len(set(expected_columns).intersection( events.columns)) == len(expected_columns) assert events.index.name == 'cell_specimen_id' # All events are the same length event_length = len(set([len(x) for x in events['events']])) assert event_length == 1
# Define the cache cache_json = { 'manifest_path': '/allen/programs/braintv/workgroups/nc-ophys/visual_behavior/SWDB_2019/visual_behavior_data_manifest.csv', 'nwb_base_dir': '/allen/programs/braintv/workgroups/nc-ophys/visual_behavior/SWDB_2019/nwb_files', 'analysis_files_base_dir': '/allen/programs/braintv/workgroups/nc-ophys/visual_behavior/SWDB_2019/extra_files' } # load the session cache = bpc.BehaviorProjectCache(cache_json) nwb_path = cache.get_nwb_filepath(experiment_id) api = BehaviorOphysNwbApi(nwb_path, filter_invalid_rois=True) session = BehaviorOphysSession(api) # Where to save the results output_path = '/allen/programs/braintv/workgroups/nc-ophys/visual_behavior/SWDB_2019/flash_response_500msec_response' # Define parameters for dff_trace, and response_window response_analysis_params = { 'window_around_timepoint_seconds': [-.5, .75], # -500ms, 750ms 'response_window_duration_seconds': 0.5, 'baseline_window_duration_seconds': 0.5 } # compute the base flash_response_df flash_response_df = get_flash_response_df(session, response_analysis_params)
def test_stimulus_presentations_omitted(ophys_experiment_id, number_omitted): session = BehaviorOphysSession.from_lims(ophys_experiment_id) df = session.stimulus_presentations assert df['omitted'].sum() == number_omitted
def test_equal(oeid1, oeid2, expected): d1 = BehaviorOphysSession.from_lims(oeid1) d2 = BehaviorOphysSession.from_lims(oeid2) assert equals(d1, d2) == expected
class BehaviorOphysAnalysis(LazyPropertyMixin): def __init__(self, session, api=None): self.session = session self.api = self if api is None else api # self.active_cell_roi_ids = LazyProperty(self.api.get_active_cell_roi_ids, ophys_experiment_id=self.ophys_experiment_id) def plot_example_traces_and_behavior(self, N=10): dff_traces_df = self.session.dff_traces dff_traces_df['mean'] = dff_traces_df['dff'].apply(np.mean) dff_traces_df['std'] = dff_traces_df['dff'].apply(np.std) dff_traces_df['snr'] = dff_traces_df['mean'] / dff_traces_df['std'] active_cell_roi_ids = dff_traces_df.sort_values( 'snr', ascending=False)['cell_roi_id'].values[:N] length_mins = 1 for xmin_seconds in np.arange(0, 5000, length_mins * 60): plot_example_traces_and_behavior(self.session, active_cell_roi_ids, xmin_seconds, length_mins, cell_label=False, include_running=True) if __name__ == "__main__": session = BehaviorOphysSession(789359614) analysis = BehaviorOphysAnalysis(session) analysis.plot_example_traces_and_behavior()
def test_BehaviorOphysSession_property_data(): ophys_experiment_id = 960410026 dataset = BehaviorOphysSession.from_lims(ophys_experiment_id) assert dataset.ophys_session_id == 959458018 assert dataset.ophys_experiment_id == 960410026
def test_get_roi_masks(cell_specimen_ids, expected, cell_specimen_table_api): ssn = BehaviorOphysSession(api=cell_specimen_table_api) obtained = ssn.get_roi_masks(cell_specimen_ids) assert np.allclose(expected, obtained.values)
def test_visbeh_ophys_data_set(): ophys_experiment_id = 789359614 data_set = BehaviorOphysSession.from_lims(ophys_experiment_id) # TODO: need to improve testing here: # for _, row in data_set.roi_metrics.iterrows(): # print(np.array(row.to_dict()['mask']).sum()) # print # for _, row in data_set.roi_masks.iterrows(): # print(np.array(row.to_dict()['mask']).sum()) # All sorts of assert relationships: assert data_set.api.extractor.get_foraging_id() == \ str(data_set.api.get_metadata().behavior_session_uuid) stimulus_templates = data_set._stimulus_templates assert len(stimulus_templates) == 8 assert stimulus_templates['im000'].warped.shape == MONITOR_DIMENSIONS assert stimulus_templates['im000'].unwarped.shape == MONITOR_DIMENSIONS assert len(data_set.licks) == 2421 and set(data_set.licks.columns) \ == set(['timestamps', 'frame']) assert len(data_set.rewards) == 85 and set(data_set.rewards.columns) == \ set(['timestamps', 'volume', 'autorewarded']) assert len(data_set.corrected_fluorescence_traces) == 258 and \ set(data_set.corrected_fluorescence_traces.columns) == \ set(['cell_roi_id', 'corrected_fluorescence']) np.testing.assert_array_almost_equal(data_set.running_speed.timestamps, data_set.stimulus_timestamps) assert len(data_set.cell_specimen_table) == len(data_set.dff_traces) assert data_set.average_projection.data.shape == \ data_set.max_projection.data.shape assert set(data_set.motion_correction.columns) == set(['x', 'y']) assert len(data_set.trials) == 602 expected_metadata = { 'stimulus_frame_rate': 60.0, 'full_genotype': 'Slc17a7-IRES2-Cre/wt;Camk2a-tTA/wt;Ai93(' 'TITL-GCaMP6f)/wt', 'ophys_experiment_id': 789359614, 'behavior_session_id': 789295700, 'imaging_plane_group_count': 0, 'ophys_session_id': 789220000, 'session_type': 'OPHYS_6_images_B', 'driver_line': ['Camk2a-tTA', 'Slc17a7-IRES2-Cre'], 'cre_line': 'Slc17a7-IRES2-Cre', 'behavior_session_uuid': uuid.UUID('69cdbe09-e62b-4b42-aab1-54b5773dfe78'), 'date_of_acquisition': pytz.utc.localize(datetime.datetime(2018, 11, 30, 23, 28, 37)), 'ophys_frame_rate': 31.0, 'imaging_depth': 375, 'mouse_id': 416369, 'experiment_container_id': 814796558, 'targeted_structure': 'VISp', 'reporter_line': 'Ai93(TITL-GCaMP6f)', 'emission_lambda': 520.0, 'excitation_lambda': 910.0, 'field_of_view_height': 512, 'field_of_view_width': 447, 'indicator': 'GCaMP6f', 'equipment_name': 'CAM2P.5', 'age_in_days': 139, 'sex': 'F', 'imaging_plane_group': None, 'project_code': 'VisualBehavior' } assert data_set.metadata == expected_metadata assert data_set.task_parameters == { 'reward_volume': 0.007, 'stimulus_distribution': u'geometric', 'stimulus_duration_sec': 0.25, 'stimulus': 'images', 'omitted_flash_fraction': 0.05, 'blank_duration_sec': [0.5, 0.5], 'n_stimulus_frames': 69882, 'task': 'change detection', 'response_window_sec': [0.15, 0.75], 'session_type': u'OPHYS_6_images_B', 'auto_reward_volume': 0.005 }
import os from allensdk.brain_observatory.behavior.behavior_ophys_api.behavior_ophys_nwb_api import BehaviorOphysNwbApi from allensdk.brain_observatory.behavior.behavior_ophys_session import BehaviorOphysSession basedir = '/allen/aibs/technology/nicholasc/behavior_ophys' rel_filepath_list = [ 'behavior_ophys_session_805784331.nwb', 'behavior_ophys_session_789359614.nwb', 'behavior_ophys_session_803736273.nwb', 'behavior_ophys_session_808621958.nwb', 'behavior_ophys_session_795948257.nwb' ] for rel_filepath in rel_filepath_list: full_filepath = os.path.join(basedir, rel_filepath) session = BehaviorOphysSession(api=BehaviorOphysNwbApi(full_filepath)) print(session.metadata['ophys_experiment_id'])