def test_get_roi_masks_by_cell_roi_id(roi_ids, expected,
cell_specimen_table_api):
ssn = BehaviorOphysSession(api=cell_specimen_table_api)
obtained = ssn._get_roi_masks_by_cell_roi_id(roi_ids)
assert np.allclose(expected, obtained.values)
assert np.allclose(obtained.coords['row'], [0.5, 1.5, 2.5, 3.5, 4.5])
assert np.allclose(obtained.coords['column'],
[0.25, 0.75, 1.25, 1.75, 2.25])
def test_nwb_end_to_end(tmpdir_factory):
oeid = 789359614
nwb_filepath = os.path.join(str(tmpdir_factory.mktemp('test_nwb_end_to_end')), 'nwbfile.nwb')
d1 = BehaviorOphysSession.from_lims(oeid)
BehaviorOphysNwbApi(nwb_filepath).save(d1)
d2 = BehaviorOphysSession(api=BehaviorOphysNwbApi(nwb_filepath))
equals(d1, d2, reraise=True)
def test_legacy_dff_api():
ophys_experiment_id = 792813858
api = BehaviorOphysLimsApi(ophys_experiment_id)
session = BehaviorOphysSession(api)
_, dff_array = session.get_dff_traces()
for csid in session.dff_traces.index.values:
dff_trace = session.dff_traces.loc[csid]['dff']
ind = session.get_cell_specimen_indices([csid])[0]
np.testing.assert_array_almost_equal(dff_trace, dff_array[ind, :])
assert dff_array.shape[0] == session.dff_traces.shape[0]
def test_nwb_end_to_end(tmpdir_factory):
# NOTE: old test oeid 789359614 had no cell specimen ids due to not being
# part of the 2021 Visual Behavior release set which broke a ton
# of things...
oeid = 795073741
tmpdir = 'test_nwb_end_to_end'
nwb_filepath = os.path.join(str(tmpdir_factory.mktemp(tmpdir)),
'nwbfile.nwb')
d1 = BehaviorOphysSession.from_lims(oeid)
BehaviorOphysNwbApi(nwb_filepath).save(d1)
d2 = BehaviorOphysSession(api=BehaviorOphysNwbApi(nwb_filepath))
assert sessions_are_equal(d1, d2, reraise=True)
def plot_running_speed(oeid, width=11, height=2, ax=None):
"""
plot running speed for a given session
will create a new fig/ax if no axis is passed
Arguments:
oeid {int} -- ophys experiment ID
Keyword Arguments:
width {int} -- figure width if creating new (default: {11})
height {int} -- figure height if creating new (default: {2})
ax {matplotlib figure axis} -- axis to plot on. Will create new if none is passed (default: {None})
Returns:
matplotlib figure axis -- ax
"""
session = BehaviorOphysSession.from_lims(oeid)
if ax is None:
fig, ax = plt.subplots(figsize=(width, height))
ax.plot(session.running_data_df['speed'])
ax.set_xlabel('time (s)')
ax.set_ylabel('speed (cm/s)')
ax.set_title('running speed for ophys_session_id {}'.format(oeid))
return ax
def test_session_from_json(tmpdir_factory, session_data, get_expected, get_from_session):
session = BehaviorOphysSession(api=BehaviorOphysJsonApi(session_data))
expected = get_expected(session_data)
obtained = get_from_session(session)
compare_fields(expected, obtained)
def get_session_data(self, ophys_session_id: int) -> BehaviorOphysSession:
"""Returns a BehaviorOphysSession object that contains methods
to analyze a single behavior+ophys session.
:param ophys_session_id: id that corresponds to a behavior session
:type ophys_session_id: int
:rtype: BehaviorOphysSession
"""
return BehaviorOphysSession(BehaviorOphysLimsApi(ophys_session_id))
def test_eye_tracking(dilation_frames, z_threshold, eye_tracking_start_value):
mock = MagicMock()
mock.get_eye_tracking.return_value = pd.DataFrame([1, 2, 3])
session = BehaviorOphysSession(api=mock,
eye_tracking_z_threshold=z_threshold,
eye_tracking_dilation_frames=dilation_frames)
if eye_tracking_start_value is not None:
session.eye_tracking = eye_tracking_start_value
obtained = session.eye_tracking
assert not session.api.get_eye_tracking.called
assert obtained.equals(eye_tracking_start_value)
else:
obtained = session.eye_tracking
assert obtained.equals(pd.DataFrame([1, 2, 3]))
assert session.api.get_eye_tracking.called_with(z_threshold=z_threshold,
dilation_frames=dilation_frames)
def test_visbeh_ophys_data_set():
ophys_experiment_id = 789359614
data_set = BehaviorOphysSession.from_lims(ophys_experiment_id)
# TODO: need to improve testing here:
# for _, row in data_set.roi_metrics.iterrows():
# print(np.array(row.to_dict()['mask']).sum())
# print
# for _, row in data_set.roi_masks.iterrows():
# print(np.array(row.to_dict()['mask']).sum())
# All sorts of assert relationships:
assert data_set.api.get_foraging_id() == str(data_set.api.get_behavior_session_uuid())
assert list(data_set.stimulus_templates.values())[0].shape == (8, 918, 1174)
assert len(data_set.licks) == 2432 and list(data_set.licks.columns) == ['time']
assert len(data_set.rewards) == 85 and list(data_set.rewards.columns) == ['volume', 'autorewarded']
assert len(data_set.corrected_fluorescence_traces) == 269 and sorted(data_set.corrected_fluorescence_traces.columns) == ['cell_roi_id', 'corrected_fluorescence']
np.testing.assert_array_almost_equal(data_set.running_speed.timestamps, data_set.stimulus_timestamps)
assert len(data_set.cell_specimen_table) == len(data_set.dff_traces)
assert data_set.average_projection.data.shape == data_set.max_projection.data.shape
assert list(data_set.motion_correction.columns) == ['x', 'y']
assert len(data_set.trials) == 602
assert data_set.metadata == {'stimulus_frame_rate': 60.0,
'full_genotype': 'Slc17a7-IRES2-Cre/wt;Camk2a-tTA/wt;Ai93(TITL-GCaMP6f)/wt',
'ophys_experiment_id': 789359614,
'session_type': 'OPHYS_6_images_B',
'driver_line': ['Camk2a-tTA', 'Slc17a7-IRES2-Cre'],
'behavior_session_uuid': uuid.UUID('69cdbe09-e62b-4b42-aab1-54b5773dfe78'),
'experiment_datetime': pytz.utc.localize(datetime.datetime(2018, 11, 30, 23, 28, 37)),
'ophys_frame_rate': 31.0,
'imaging_depth': 375,
'LabTracks_ID': 416369,
'experiment_container_id': 814796558,
'targeted_structure': 'VISp',
'reporter_line': ['Ai93(TITL-GCaMP6f)'],
'emission_lambda': 520.0,
'excitation_lambda': 910.0,
'field_of_view_height': 512,
'field_of_view_width': 447,
'indicator': 'GCAMP6f',
'rig_name': 'CAM2P.5',
'age': 'P139',
'sex': 'F'}
assert math.isnan(data_set.task_parameters.pop('omitted_flash_fraction'))
assert data_set.task_parameters == {'reward_volume': 0.007,
'stimulus_distribution': u'geometric',
'stimulus_duration_sec': 6.0,
'stimulus': 'images',
'blank_duration_sec': [0.5, 0.5],
'n_stimulus_frames': 69882,
'task': 'DoC_untranslated',
'response_window_sec': [0.15, 0.75],
'stage': u'OPHYS_6_images_B'}
def write_behavior_ophys_nwb(session_data: dict,
nwb_filepath: str,
skip_eye_tracking: bool):
nwb_filepath_inprogress = nwb_filepath+'.inprogress'
nwb_filepath_error = nwb_filepath+'.error'
# Clean out files from previous runs:
for filename in [nwb_filepath_inprogress,
nwb_filepath_error,
nwb_filepath]:
if os.path.exists(filename):
os.remove(filename)
try:
json_api = BehaviorOphysJsonApi(data=session_data,
skip_eye_tracking=skip_eye_tracking)
json_session = BehaviorOphysSession(api=json_api)
lims_api = BehaviorOphysLimsApi(
ophys_experiment_id=session_data['ophys_experiment_id'],
skip_eye_tracking=skip_eye_tracking)
lims_session = BehaviorOphysSession(api=lims_api)
logging.info("Comparing a BehaviorOphysSession created from JSON "
"with a BehaviorOphysSession created from LIMS")
assert sessions_are_equal(json_session, lims_session, reraise=True)
BehaviorOphysNwbApi(nwb_filepath_inprogress).save(json_session)
logging.info("Comparing a BehaviorOphysSession created from JSON "
"with a BehaviorOphysSession created from NWB")
nwb_api = BehaviorOphysNwbApi(nwb_filepath_inprogress)
nwb_session = BehaviorOphysSession(api=nwb_api)
assert sessions_are_equal(json_session, nwb_session, reraise=True)
os.rename(nwb_filepath_inprogress, nwb_filepath)
return {'output_path': nwb_filepath}
except Exception as e:
os.rename(nwb_filepath_inprogress, nwb_filepath_error)
raise e
def write_behavior_ophys_nwb(session_data, nwb_filepath):
nwb_filepath_inprogress = nwb_filepath + '.inprogress'
nwb_filepath_error = nwb_filepath + '.error'
# Clean out files from previous runs:
for filename in [
nwb_filepath_inprogress, nwb_filepath_error, nwb_filepath
]:
if os.path.exists(filename):
os.remove(filename)
try:
session = BehaviorOphysSession(api=BehaviorOphysJsonApi(session_data))
BehaviorOphysNwbApi(nwb_filepath_inprogress).save(session)
api = BehaviorOphysNwbApi(nwb_filepath_inprogress)
assert sessions_are_equal(session, BehaviorOphysSession(api=api))
os.rename(nwb_filepath_inprogress, nwb_filepath)
return {'output_path': nwb_filepath}
except Exception as e:
os.rename(nwb_filepath_inprogress, nwb_filepath_error)
raise e
def validate_last_trial_ends_adjacent_to_flash(ophys_experiment_id,
api=None,
verbose=False):
# ensure that the last trial ends sometime on the last flash/blank cycle
# i.e, if this is the last flash/blank iteration (high = stimulus present):
# ------- ------- ------- -------
# | | | | | | | |
# --------- --------- --------- --------- -------------------------------------------
# ^ ^
# The last trial has to have ended somewhere between the two carrots where:
# the first carrot represents the time of the last recorded stimulus flash
# the second carrot represents the time at which another flash should have started, after accounting for the possibility of the session ending on an omitted flash
api = BehaviorOphysLimsApi() if api is None else api
session = BehaviorOphysSession(api)
# get the flash/blank parameters
max_flash_duration = session.stimulus_presentations['duration'].max()
max_blank_duration = session.task_parameters['blank_duration_sec'][1]
# count number of omitted flashes at the very end of the session
N_final_omitted_flashes = session.stimulus_presentations.index.max(
) - session.stimulus_presentations.query('omitted == False').index.max()
# get the start/end time of the last valid (non-omitted) flash
last_flash_start = session.stimulus_presentations.query(
'omitted == False')['start_time'].iloc[-1]
last_flash_end = session.stimulus_presentations.query(
'omitted == False')['stop_time'].iloc[-1]
# calculate when the next stimulus should have flashed, after accounting for any omitted flashes at the end of the session
next_flash_would_have_started = last_flash_end + max_blank_duration + N_final_omitted_flashes * (
max_flash_duration + max_blank_duration)
# get the end time of the last trial
last_trial_end = session.trials.iloc[-1]['stop_time']
if verbose:
print('last flash ended at {}'.format(last_flash_end))
print('another flash should have started by {}'.format(
next_flash_would_have_started))
print('last trial ended at {}'.format(last_trial_end))
if not last_flash_start <= last_trial_end <= next_flash_would_have_started:
raise ValidationError(
'The last trial does not end between the start of the last flash and the expected start time of the next flash'
)
def test_trial_response_window_bounds_reward(ophys_experiment_id):
api = BehaviorOphysLimsApi(ophys_experiment_id)
session = BehaviorOphysSession(api)
response_window = session.task_parameters['response_window_sec']
for _, row in session.trials.iterrows():
lick_times = [(t - row.change_time) for t in row.lick_times]
if not np.isnan(row.reward_time):
# monitor delay is incorporated into the trials table change time
# TODO: where is this set in the session object?
camstim_change_time = row.change_time - 0.0351
reward_time = (row.reward_time - camstim_change_time)
assert response_window[0] < reward_time + 1 / 60
assert reward_time < response_window[1] + 1 / 60
if len(session.licks) > 0:
assert lick_times[0] < reward_time
def test_event_detection():
ophys_experiment_id = 789359614
session = BehaviorOphysSession.from_lims(
ophys_experiment_id=ophys_experiment_id)
events = session.events
assert len(events) > 0
expected_columns = [
'events', 'filtered_events', 'lambda', 'noise_std', 'cell_roi_id'
]
assert len(events.columns) == len(expected_columns)
# Assert they contain the same columns
assert len(set(expected_columns).intersection(
events.columns)) == len(expected_columns)
assert events.index.name == 'cell_specimen_id'
# All events are the same length
event_length = len(set([len(x) for x in events['events']]))
assert event_length == 1
# Define the cache
cache_json = {
'manifest_path':
'/allen/programs/braintv/workgroups/nc-ophys/visual_behavior/SWDB_2019/visual_behavior_data_manifest.csv',
'nwb_base_dir':
'/allen/programs/braintv/workgroups/nc-ophys/visual_behavior/SWDB_2019/nwb_files',
'analysis_files_base_dir':
'/allen/programs/braintv/workgroups/nc-ophys/visual_behavior/SWDB_2019/extra_files'
}
# load the session
cache = bpc.BehaviorProjectCache(cache_json)
nwb_path = cache.get_nwb_filepath(experiment_id)
api = BehaviorOphysNwbApi(nwb_path, filter_invalid_rois=True)
session = BehaviorOphysSession(api)
# Where to save the results
output_path = '/allen/programs/braintv/workgroups/nc-ophys/visual_behavior/SWDB_2019/flash_response_500msec_response'
# Define parameters for dff_trace, and response_window
response_analysis_params = {
'window_around_timepoint_seconds': [-.5, .75], # -500ms, 750ms
'response_window_duration_seconds': 0.5,
'baseline_window_duration_seconds': 0.5
}
# compute the base flash_response_df
flash_response_df = get_flash_response_df(session,
response_analysis_params)
def test_stimulus_presentations_omitted(ophys_experiment_id, number_omitted):
session = BehaviorOphysSession.from_lims(ophys_experiment_id)
df = session.stimulus_presentations
assert df['omitted'].sum() == number_omitted
def test_equal(oeid1, oeid2, expected):
d1 = BehaviorOphysSession.from_lims(oeid1)
d2 = BehaviorOphysSession.from_lims(oeid2)
assert equals(d1, d2) == expected
class BehaviorOphysAnalysis(LazyPropertyMixin):
def __init__(self, session, api=None):
self.session = session
self.api = self if api is None else api
# self.active_cell_roi_ids = LazyProperty(self.api.get_active_cell_roi_ids, ophys_experiment_id=self.ophys_experiment_id)
def plot_example_traces_and_behavior(self, N=10):
dff_traces_df = self.session.dff_traces
dff_traces_df['mean'] = dff_traces_df['dff'].apply(np.mean)
dff_traces_df['std'] = dff_traces_df['dff'].apply(np.std)
dff_traces_df['snr'] = dff_traces_df['mean'] / dff_traces_df['std']
active_cell_roi_ids = dff_traces_df.sort_values(
'snr', ascending=False)['cell_roi_id'].values[:N]
length_mins = 1
for xmin_seconds in np.arange(0, 5000, length_mins * 60):
plot_example_traces_and_behavior(self.session,
active_cell_roi_ids,
xmin_seconds,
length_mins,
cell_label=False,
include_running=True)
if __name__ == "__main__":
session = BehaviorOphysSession(789359614)
analysis = BehaviorOphysAnalysis(session)
analysis.plot_example_traces_and_behavior()
def test_BehaviorOphysSession_property_data():
ophys_experiment_id = 960410026
dataset = BehaviorOphysSession.from_lims(ophys_experiment_id)
assert dataset.ophys_session_id == 959458018
assert dataset.ophys_experiment_id == 960410026
def test_get_roi_masks(cell_specimen_ids, expected, cell_specimen_table_api):
ssn = BehaviorOphysSession(api=cell_specimen_table_api)
obtained = ssn.get_roi_masks(cell_specimen_ids)
assert np.allclose(expected, obtained.values)
def test_visbeh_ophys_data_set():
ophys_experiment_id = 789359614
data_set = BehaviorOphysSession.from_lims(ophys_experiment_id)
# TODO: need to improve testing here:
# for _, row in data_set.roi_metrics.iterrows():
# print(np.array(row.to_dict()['mask']).sum())
# print
# for _, row in data_set.roi_masks.iterrows():
# print(np.array(row.to_dict()['mask']).sum())
# All sorts of assert relationships:
assert data_set.api.extractor.get_foraging_id() == \
str(data_set.api.get_metadata().behavior_session_uuid)
stimulus_templates = data_set._stimulus_templates
assert len(stimulus_templates) == 8
assert stimulus_templates['im000'].warped.shape == MONITOR_DIMENSIONS
assert stimulus_templates['im000'].unwarped.shape == MONITOR_DIMENSIONS
assert len(data_set.licks) == 2421 and set(data_set.licks.columns) \
== set(['timestamps', 'frame'])
assert len(data_set.rewards) == 85 and set(data_set.rewards.columns) == \
set(['timestamps', 'volume', 'autorewarded'])
assert len(data_set.corrected_fluorescence_traces) == 258 and \
set(data_set.corrected_fluorescence_traces.columns) == \
set(['cell_roi_id', 'corrected_fluorescence'])
np.testing.assert_array_almost_equal(data_set.running_speed.timestamps,
data_set.stimulus_timestamps)
assert len(data_set.cell_specimen_table) == len(data_set.dff_traces)
assert data_set.average_projection.data.shape == \
data_set.max_projection.data.shape
assert set(data_set.motion_correction.columns) == set(['x', 'y'])
assert len(data_set.trials) == 602
expected_metadata = {
'stimulus_frame_rate':
60.0,
'full_genotype':
'Slc17a7-IRES2-Cre/wt;Camk2a-tTA/wt;Ai93('
'TITL-GCaMP6f)/wt',
'ophys_experiment_id':
789359614,
'behavior_session_id':
789295700,
'imaging_plane_group_count':
0,
'ophys_session_id':
789220000,
'session_type':
'OPHYS_6_images_B',
'driver_line': ['Camk2a-tTA', 'Slc17a7-IRES2-Cre'],
'cre_line':
'Slc17a7-IRES2-Cre',
'behavior_session_uuid':
uuid.UUID('69cdbe09-e62b-4b42-aab1-54b5773dfe78'),
'date_of_acquisition':
pytz.utc.localize(datetime.datetime(2018, 11, 30, 23, 28, 37)),
'ophys_frame_rate':
31.0,
'imaging_depth':
375,
'mouse_id':
416369,
'experiment_container_id':
814796558,
'targeted_structure':
'VISp',
'reporter_line':
'Ai93(TITL-GCaMP6f)',
'emission_lambda':
520.0,
'excitation_lambda':
910.0,
'field_of_view_height':
512,
'field_of_view_width':
447,
'indicator':
'GCaMP6f',
'equipment_name':
'CAM2P.5',
'age_in_days':
139,
'sex':
'F',
'imaging_plane_group':
None,
'project_code':
'VisualBehavior'
}
assert data_set.metadata == expected_metadata
assert data_set.task_parameters == {
'reward_volume': 0.007,
'stimulus_distribution': u'geometric',
'stimulus_duration_sec': 0.25,
'stimulus': 'images',
'omitted_flash_fraction': 0.05,
'blank_duration_sec': [0.5, 0.5],
'n_stimulus_frames': 69882,
'task': 'change detection',
'response_window_sec': [0.15, 0.75],
'session_type': u'OPHYS_6_images_B',
'auto_reward_volume': 0.005
}
import os
from allensdk.brain_observatory.behavior.behavior_ophys_api.behavior_ophys_nwb_api import BehaviorOphysNwbApi
from allensdk.brain_observatory.behavior.behavior_ophys_session import BehaviorOphysSession
basedir = '/allen/aibs/technology/nicholasc/behavior_ophys'
rel_filepath_list = [
'behavior_ophys_session_805784331.nwb',
'behavior_ophys_session_789359614.nwb',
'behavior_ophys_session_803736273.nwb',
'behavior_ophys_session_808621958.nwb',
'behavior_ophys_session_795948257.nwb'
]
for rel_filepath in rel_filepath_list:
full_filepath = os.path.join(basedir, rel_filepath)
session = BehaviorOphysSession(api=BehaviorOphysNwbApi(full_filepath))
print(session.metadata['ophys_experiment_id'])
