def test_add_events(tmp_path, nwbfile, roundtripper, roundtrip,
cell_specimen_table, metadata_fixture, dff_traces,
ophys_timestamps):
# Need to add metadata, cell specimen table, dff traces first
nwb.add_metadata(nwbfile, metadata_fixture, behavior_only=False)
nwb.add_cell_specimen_table(nwbfile, cell_specimen_table, metadata_fixture)
nwb.add_dff_traces(nwbfile, dff_traces, ophys_timestamps)
events = pd.DataFrame({
'events': [np.array([0., 0., .69]), np.array([.3, 0.0, .2])],
'filtered_events': [
np.array([0.0, 0.0, 0.22949295]),
np.array([0.09977954, 0.08805513, 0.127039049])
],
'lambda': [0., 1.0],
'noise_std': [.25, .3],
'cell_roi_id': [123, 321]
}, index=pd.Index([42, 84], name='cell_specimen_id'))
api = BehaviorOphysNwbApi.from_nwbfile(nwbfile)
nwbfile = api.add_events(
nwbfile=nwbfile,
events=events
)
if roundtrip:
obt = roundtripper(nwbfile, BehaviorOphysNwbApi)
else:
obt = BehaviorOphysNwbApi.from_nwbfile(nwbfile)
obtained = obt.get_events()
pd.testing.assert_frame_equal(obtained, events, check_like=True)
def test_nwb_end_to_end(tmpdir_factory):
oeid = 789359614
nwb_filepath = os.path.join(str(tmpdir_factory.mktemp('test_nwb_end_to_end')), 'nwbfile.nwb')
d1 = BehaviorOphysSession.from_lims(oeid)
BehaviorOphysNwbApi(nwb_filepath).save(d1)
d2 = BehaviorOphysSession(api=BehaviorOphysNwbApi(nwb_filepath))
assert sessions_are_equal(d1, d2, reraise=True)
def test_add_eye_tracking_rig_geometry_data_to_nwbfile(nwbfile, roundtripper,
roundtrip,
rig_geometry,
expected):
api = BehaviorOphysNwbApi.from_nwbfile(nwbfile)
nwbfile = api.add_eye_tracking_rig_geometry_data_to_nwbfile(nwbfile,
rig_geometry)
if roundtrip:
obt = roundtripper(nwbfile, BehaviorOphysNwbApi)
else:
obt = BehaviorOphysNwbApi.from_nwbfile(nwbfile)
obtained_eye_rig_geometry = obt.get_eye_tracking_rig_geometry()
assert obtained_eye_rig_geometry == expected
def test_add_partial_metadata(test_partial_metadata, roundtrip, roundtripper,
cell_specimen_table,
metadata_fixture, partial_metadata_fixture):
if test_partial_metadata:
meta = partial_metadata_fixture
else:
meta = metadata_fixture
nwbfile = pynwb.NWBFile(
session_description='asession',
identifier='afile',
session_start_time=meta['date_of_acquisition']
)
nwb.add_metadata(nwbfile, meta, behavior_only=False)
if not test_partial_metadata:
nwb.add_cell_specimen_table(nwbfile, cell_specimen_table, meta)
if roundtrip:
obt = roundtripper(nwbfile, BehaviorOphysNwbApi)
else:
obt = BehaviorOphysNwbApi.from_nwbfile(nwbfile)
if not test_partial_metadata:
metadata_obt = obt.get_metadata()
else:
with warnings.catch_warnings(record=True) as record:
metadata_obt = obt.get_metadata()
exp_warn_msg = "Could not locate 'ophys' module in NWB"
print(record)
assert record[0].message.args[0].startswith(exp_warn_msg)
assert len(metadata_obt) == len(meta)
for key, val in meta.items():
assert val == metadata_obt[key]
def test_get_corrected_fluorescence_traces(
nwbfile, roundtrip, filter_invalid_rois, valid_roi_ids, roundtripper,
dff_traces, corrected_fluorescence_traces, cell_specimen_table,
metadata_fixture, ophys_timestamps):
nwb.add_metadata(nwbfile, metadata_fixture, behavior_only=False)
nwb.add_cell_specimen_table(nwbfile, cell_specimen_table, metadata_fixture)
nwb.add_dff_traces(nwbfile, dff_traces, ophys_timestamps)
nwb.add_corrected_fluorescence_traces(nwbfile,
corrected_fluorescence_traces)
if roundtrip:
obt = roundtripper(nwbfile, BehaviorOphysNwbApi,
filter_invalid_rois=filter_invalid_rois)
else:
obt = BehaviorOphysNwbApi.from_nwbfile(
nwbfile, filter_invalid_rois=filter_invalid_rois)
if filter_invalid_rois:
corrected_fluorescence_traces = corrected_fluorescence_traces[
corrected_fluorescence_traces["cell_roi_id"].isin(valid_roi_ids)]
print(corrected_fluorescence_traces)
print(obt.get_corrected_fluorescence_traces())
pd.testing.assert_frame_equal(
corrected_fluorescence_traces,
obt.get_corrected_fluorescence_traces(), check_dtype=False)
def test_nwb_end_to_end(tmpdir_factory):
# NOTE: old test oeid 789359614 had no cell specimen ids due to not being
# part of the 2021 Visual Behavior release set which broke a ton
# of things...
oeid = 795073741
tmpdir = 'test_nwb_end_to_end'
nwb_filepath = os.path.join(str(tmpdir_factory.mktemp(tmpdir)),
'nwbfile.nwb')
d1 = BehaviorOphysSession.from_lims(oeid)
BehaviorOphysNwbApi(nwb_filepath).save(d1)
d2 = BehaviorOphysSession(api=BehaviorOphysNwbApi(nwb_filepath))
assert sessions_are_equal(d1, d2, reraise=True)
def from_nwb_path(
cls, nwb_path: str,
**api_kwargs: Any) -> "BehaviorOphysExperiment": # noqa: E501
api_kwargs["filter_invalid_rois"] = api_kwargs.get(
"filter_invalid_rois", True)
return cls(
api=BehaviorOphysNwbApi.from_path(path=nwb_path, **api_kwargs))
def test_add_licks(nwbfile, roundtrip, roundtripper, licks):
nwb.add_licks(nwbfile, licks)
if roundtrip:
obt = roundtripper(nwbfile, BehaviorOphysNwbApi)
else:
obt = BehaviorOphysNwbApi.from_nwbfile(nwbfile)
pd.testing.assert_frame_equal(licks, obt.get_licks(), check_dtype=False)
def test_add_eye_tracking_data_to_nwbfile(
tmp_path, nwbfile, eye_tracking_data,
rig_geometry, roundtripper, roundtrip):
api = BehaviorOphysNwbApi.from_nwbfile(nwbfile)
nwbfile = api.add_eye_tracking_data_to_nwb(
nwbfile=nwbfile,
eye_tracking_df=eye_tracking_data,
eye_tracking_rig_geometry=rig_geometry
)
if roundtrip:
obt = roundtripper(nwbfile, BehaviorOphysNwbApi)
else:
obt = BehaviorOphysNwbApi.from_nwbfile(nwbfile)
obtained = obt.get_eye_tracking()
pd.testing.assert_frame_equal(obtained,
eye_tracking_data, check_like=True)
def test_segmentation_mask_image(nwbfile, roundtrip, roundtripper,
segmentation_mask_image, image_api):
nwb.add_segmentation_mask_image(nwbfile, segmentation_mask_image)
if roundtrip:
obt = roundtripper(nwbfile, BehaviorOphysNwbApi)
else:
obt = BehaviorOphysNwbApi.from_nwbfile(nwbfile)
assert image_api.deserialize(segmentation_mask_image) == \
image_api.deserialize(obt.get_segmentation_mask_image())
def test_add_average_image(nwbfile, roundtrip, roundtripper, average_image,
image_api):
nwb.add_average_image(nwbfile, average_image)
if roundtrip:
obt = roundtripper(nwbfile, BehaviorOphysNwbApi)
else:
obt = BehaviorOphysNwbApi.from_nwbfile(nwbfile)
assert image_api.deserialize(average_image) == \
image_api.deserialize(obt.get_average_projection())
def test_add_max_projection(nwbfile, roundtrip, roundtripper,
max_projection, image_api):
nwb.add_max_projection(nwbfile, max_projection)
if roundtrip:
obt = roundtripper(nwbfile, BehaviorOphysNwbApi)
else:
obt = BehaviorOphysNwbApi.from_nwbfile(nwbfile)
assert image_api.deserialize(max_projection) == \
image_api.deserialize(obt.get_max_projection())
def test_add_stimulus_timestamps(nwbfile, stimulus_timestamps,
roundtrip, roundtripper):
nwb.add_stimulus_timestamps(nwbfile, stimulus_timestamps)
if roundtrip:
obt = roundtripper(nwbfile, BehaviorOphysNwbApi)
else:
obt = BehaviorOphysNwbApi.from_nwbfile(nwbfile)
np.testing.assert_array_almost_equal(stimulus_timestamps,
obt.get_stimulus_timestamps())
def write_behavior_ophys_nwb(session_data: dict, nwb_filepath: str,
skip_eye_tracking: bool):
nwb_filepath_inprogress = nwb_filepath + '.inprogress'
nwb_filepath_error = nwb_filepath + '.error'
# Clean out files from previous runs:
for filename in [
nwb_filepath_inprogress, nwb_filepath_error, nwb_filepath
]:
if os.path.exists(filename):
os.remove(filename)
try:
json_api = BehaviorOphysJsonApi(data=session_data,
skip_eye_tracking=skip_eye_tracking)
json_session = BehaviorOphysExperiment(api=json_api)
lims_api = BehaviorOphysLimsApi(
ophys_experiment_id=session_data['ophys_experiment_id'],
skip_eye_tracking=skip_eye_tracking)
lims_session = BehaviorOphysExperiment(api=lims_api)
logging.info("Comparing a BehaviorOphysExperiment created from JSON "
"with a BehaviorOphysExperiment created from LIMS")
assert sessions_are_equal(json_session, lims_session, reraise=True)
BehaviorOphysNwbApi(nwb_filepath_inprogress).save(json_session)
logging.info("Comparing a BehaviorOphysExperiment created from JSON "
"with a BehaviorOphysExperiment created from NWB")
nwb_api = BehaviorOphysNwbApi(nwb_filepath_inprogress)
nwb_session = BehaviorOphysExperiment(api=nwb_api)
assert sessions_are_equal(json_session, nwb_session, reraise=True)
os.rename(nwb_filepath_inprogress, nwb_filepath)
return {'output_path': nwb_filepath}
except Exception as e:
if os.path.isfile(nwb_filepath_inprogress):
os.rename(nwb_filepath_inprogress, nwb_filepath_error)
raise e
def test_add_running_speed_to_nwbfile(nwbfile, running_speed, roundtrip,
roundtripper):
nwbfile = nwb.add_running_speed_to_nwbfile(nwbfile, running_speed)
if roundtrip:
obt = roundtripper(nwbfile, BehaviorOphysNwbApi)
else:
obt = BehaviorOphysNwbApi.from_nwbfile(nwbfile)
running_speed_obt = obt.get_running_speed()
assert np.allclose(running_speed.timestamps, running_speed_obt.timestamps)
assert np.allclose(running_speed.values, running_speed_obt.values)
def write_behavior_ophys_nwb(session_data, nwb_filepath):
nwb_filepath_inprogress = nwb_filepath + '.inprogress'
nwb_filepath_error = nwb_filepath + '.error'
# Clean out files from previous runs:
for filename in [
nwb_filepath_inprogress, nwb_filepath_error, nwb_filepath
]:
if os.path.exists(filename):
os.remove(filename)
try:
session = BehaviorOphysSession(api=BehaviorOphysJsonApi(session_data))
BehaviorOphysNwbApi(nwb_filepath_inprogress).save(session)
api = BehaviorOphysNwbApi(nwb_filepath_inprogress)
assert sessions_are_equal(session, BehaviorOphysSession(api=api))
os.rename(nwb_filepath_inprogress, nwb_filepath)
return {'output_path': nwb_filepath}
except Exception as e:
os.rename(nwb_filepath_inprogress, nwb_filepath_error)
raise e
def test_add_stimulus_templates(nwbfile, stimulus_templates, roundtrip,
roundtripper):
for key, val in stimulus_templates.items():
nwb.add_stimulus_template(nwbfile, val, key)
if roundtrip:
obt = roundtripper(nwbfile, BehaviorOphysNwbApi)
else:
obt = BehaviorOphysNwbApi.from_nwbfile(nwbfile)
stimulus_templates_obt = obt.get_stimulus_templates()
for key in set(stimulus_templates.keys()).union(
set(stimulus_templates_obt.keys())):
np.testing.assert_array_almost_equal(stimulus_templates[key],
stimulus_templates_obt[key])
def test_get_motion_correction(nwbfile, roundtrip, roundtripper,
motion_correction, ophys_timestamps, metadata,
cell_specimen_table, dff_traces):
nwb.add_metadata(nwbfile, metadata)
nwb.add_cell_specimen_table(nwbfile, cell_specimen_table, metadata)
nwb.add_dff_traces(nwbfile, dff_traces, ophys_timestamps)
nwb.add_motion_correction(nwbfile, motion_correction)
if roundtrip:
obt = roundtripper(nwbfile, BehaviorOphysNwbApi)
else:
obt = BehaviorOphysNwbApi.from_nwbfile(nwbfile)
pd.testing.assert_frame_equal(motion_correction,
obt.get_motion_correction(),
check_dtype=False)
def test_add_running_data_df_to_nwbfile(nwbfile, running_data_df, roundtrip,
roundtripper):
unit_dict = {
'v_sig': 'V',
'v_in': 'V',
'speed': 'cm/s',
'timestamps': 's',
'dx': 'cm'
}
nwbfile = nwb.add_running_data_df_to_nwbfile(nwbfile, running_data_df,
unit_dict)
if roundtrip:
obt = roundtripper(nwbfile, BehaviorOphysNwbApi)
else:
obt = BehaviorOphysNwbApi.from_nwbfile(nwbfile)
pd.testing.assert_frame_equal(running_data_df, obt.get_running_data_df())
def test_add_task_parameters(nwbfile, roundtrip,
roundtripper, task_parameters):
nwb.add_task_parameters(nwbfile, task_parameters)
if roundtrip:
obt = roundtripper(nwbfile, BehaviorOphysNwbApi)
else:
obt = BehaviorOphysNwbApi.from_nwbfile(nwbfile)
task_parameters_obt = obt.get_task_parameters()
assert len(task_parameters_obt) == len(task_parameters)
for key, val in task_parameters.items():
if key == 'omitted_flash_fraction':
if math.isnan(val):
assert math.isnan(task_parameters_obt[key])
if math.isnan(task_parameters_obt[key]):
assert math.isnan(val)
else:
assert val == task_parameters_obt[key]
def test_get_dff_traces(nwbfile, roundtrip, filter_invalid_rois, valid_roi_ids,
roundtripper, dff_traces, cell_specimen_table,
metadata, ophys_timestamps):
nwb.add_metadata(nwbfile, metadata)
nwb.add_cell_specimen_table(nwbfile, cell_specimen_table, metadata)
nwb.add_dff_traces(nwbfile, dff_traces, ophys_timestamps)
if roundtrip:
obt = roundtripper(nwbfile,
BehaviorOphysNwbApi,
filter_invalid_rois=filter_invalid_rois)
else:
obt = BehaviorOphysNwbApi.from_nwbfile(
nwbfile, filter_invalid_rois=filter_invalid_rois)
if filter_invalid_rois:
dff_traces = dff_traces[dff_traces["cell_roi_id"].isin(valid_roi_ids)]
pd.testing.assert_frame_equal(dff_traces,
obt.get_dff_traces(),
check_dtype=False)
def test_add_stimulus_presentations(nwbfile, stimulus_presentations_behavior,
stimulus_timestamps, roundtrip,
roundtripper, stimulus_templates):
nwb.add_stimulus_timestamps(nwbfile, stimulus_timestamps)
nwb.add_stimulus_presentations(nwbfile, stimulus_presentations_behavior)
for key, val in stimulus_templates.items():
nwb.add_stimulus_template(nwbfile, val, key)
# Add index for this template to NWB in-memory object:
nwb_template = nwbfile.stimulus_template[key]
curr_stimulus_index = stimulus_presentations_behavior[
stimulus_presentations_behavior['image_set'] == nwb_template.name]
nwb.add_stimulus_index(nwbfile, curr_stimulus_index, nwb_template)
if roundtrip:
obt = roundtripper(nwbfile, BehaviorOphysNwbApi)
else:
obt = BehaviorOphysNwbApi.from_nwbfile(nwbfile)
pd.testing.assert_frame_equal(stimulus_presentations_behavior,
obt.get_stimulus_presentations(),
check_dtype=False)
def test_get_cell_specimen_table(nwbfile, roundtrip, filter_invalid_rois,
valid_roi_ids, roundtripper,
cell_specimen_table, metadata_fixture,
ophys_timestamps):
nwb.add_metadata(nwbfile, metadata_fixture, behavior_only=False)
nwb.add_cell_specimen_table(nwbfile, cell_specimen_table, metadata_fixture)
if roundtrip:
obt = roundtripper(nwbfile, BehaviorOphysNwbApi,
filter_invalid_rois=filter_invalid_rois)
else:
obt = BehaviorOphysNwbApi.from_nwbfile(
nwbfile, filter_invalid_rois=filter_invalid_rois)
if filter_invalid_rois:
cell_specimen_table = \
cell_specimen_table[
cell_specimen_table["cell_roi_id"].isin(
valid_roi_ids)]
pd.testing.assert_frame_equal(
cell_specimen_table,
obt.get_cell_specimen_table(),
check_dtype=False)
experiment_id = sys.argv[1]
# Define the cache
cache_json = {
'manifest_path':
'/allen/programs/braintv/workgroups/nc-ophys/visual_behavior/SWDB_2019/visual_behavior_data_manifest.csv',
'nwb_base_dir':
'/allen/programs/braintv/workgroups/nc-ophys/visual_behavior/SWDB_2019/nwb_files',
'analysis_files_base_dir':
'/allen/programs/braintv/workgroups/nc-ophys/visual_behavior/SWDB_2019/extra_files'
}
# load the session
cache = bpc.BehaviorProjectCache(cache_json)
nwb_path = cache.get_nwb_filepath(experiment_id)
api = BehaviorOphysNwbApi(nwb_path, filter_invalid_rois=True)
session = BehaviorOphysExperiment(api)
# Where to save the results
output_path = '/allen/programs/braintv/workgroups/nc-ophys/visual_behavior/SWDB_2019/flash_response_500msec_response'
# Define parameters for dff_trace, and response_window
response_analysis_params = {
'window_around_timepoint_seconds': [-.5, .75], # -500ms, 750ms
'response_window_duration_seconds': 0.5,
'baseline_window_duration_seconds': 0.5
}
# compute the base flash_response_df
flash_response_df = get_flash_response_df(session,
response_analysis_params)
def from_nwb_path(cls, nwb_path: str,
**api_kwargs: Any) -> "BehaviorOphysSession":
api_kwargs["filter_invalid_rois"] = api_kwargs.get(
"filter_invalid_rois", True)
return cls(
api=BehaviorOphysNwbApi.from_path(path=nwb_path, **api_kwargs))