def test_too_long_single_plane(self):
"""test that timestamps are truncated for single plane data"""
self.sync_file._data = {'ophys_frames': np.array([.1, .2, .3])}
ts = OphysTimestamps.from_sync_file(sync_file=self.sync_file)\
.validate(number_of_frames=2)
expected = np.array([.1, .2])
np.testing.assert_equal(ts.value, expected)
def from_nwb(cls, nwbfile: NWBFile,
segmentation_mask_image_spacing: Tuple,
exclude_invalid_rois=True,
events_params: Optional[EventsParams] = None) \
-> "CellSpecimens":
# NOTE: ROI masks are stored in full frame width and height arrays
ophys_module = nwbfile.processing['ophys']
image_seg = ophys_module.data_interfaces['image_segmentation']
plane_segmentations = image_seg.plane_segmentations
cell_specimen_table = plane_segmentations['cell_specimen_table']
def _read_table(cell_specimen_table):
df = cell_specimen_table.to_dataframe()
# Ensure int64 used instead of int32
df = df.astype(
{col: 'int64'
for col in df.select_dtypes('int32').columns})
# Because pynwb stores this field as "image_mask", it is renamed
# here
df = df.rename(columns={'image_mask': 'roi_mask'})
df.index.rename('cell_roi_id', inplace=True)
df['cell_specimen_id'] = [
None if id_ == -1 else id_
for id_ in df['cell_specimen_id'].values
]
df.reset_index(inplace=True)
df.set_index('cell_specimen_id', inplace=True)
return df
df = _read_table(cell_specimen_table=cell_specimen_table)
meta = CellSpecimenMeta.from_nwb(nwbfile=nwbfile)
dff_traces = DFFTraces.from_nwb(nwbfile=nwbfile)
corrected_fluorescence_traces = CorrectedFluorescenceTraces.from_nwb(
nwbfile=nwbfile)
def _get_events():
ep = EventsParams() if events_params is None else events_params
return Events.from_nwb(nwbfile=nwbfile,
filter_scale=ep.filter_scale,
filter_n_time_steps=ep.filter_n_time_steps)
events = _get_events()
ophys_timestamps = OphysTimestamps.from_nwb(nwbfile=nwbfile)
return CellSpecimens(
cell_specimen_table=df,
meta=meta,
dff_traces=dff_traces,
corrected_fluorescence_traces=corrected_fluorescence_traces,
events=events,
ophys_timestamps=ophys_timestamps,
segmentation_mask_image_spacing=segmentation_mask_image_spacing,
exclude_invalid_rois=exclude_invalid_rois)
def test_from_lims(self):
number_of_frames = 140296
ots = OphysTimestamps(timestamps=np.linspace(
start=.1, stop=.1 * number_of_frames, num=number_of_frames))
csp = CellSpecimens.from_lims(
ophys_experiment_id=self.ophys_experiment_id,
lims_db=self.dbconn,
ophys_timestamps=ots,
segmentation_mask_image_spacing=(.78125e-3, .78125e-3))
assert not csp.table.empty
assert not csp.events.empty
assert not csp.dff_traces.empty
assert not csp.corrected_fluorescence_traces.empty
assert csp.meta == self.expected_meta
def test_process_ophys_plane_timestamps(self, timestamps, plane_group,
group_count, expected):
"""Various test cases"""
self.sync_file._data = {'ophys_frames': timestamps}
number_of_frames = len(timestamps) if group_count == 0 else \
len(timestamps) / group_count
if group_count == 0:
ts = OphysTimestamps.from_sync_file(sync_file=self.sync_file)
else:
ts = OphysTimestampsMultiplane.from_sync_file(
sync_file=self.sync_file,
group_count=group_count,
plane_group=plane_group)
ts = ts.validate(number_of_frames=number_of_frames)
np.testing.assert_array_equal(expected, ts.value)
def setup_class(cls):
dir = Path(__file__).parent.resolve()
test_data_dir = dir / 'test_data'
with open(test_data_dir / 'test_input.json') as f:
dict_repr = json.load(f)
dict_repr = dict_repr['session_data']
dict_repr['sync_file'] = str(test_data_dir / 'sync.h5')
dict_repr['behavior_stimulus_file'] = str(test_data_dir /
'behavior_stimulus_file.pkl')
dict_repr['dff_file'] = str(test_data_dir / 'demix_file.h5')
dict_repr['demix_file'] = str(test_data_dir / 'demix_file.h5')
dict_repr['events_file'] = str(test_data_dir / 'events.h5')
cls.dict_repr = dict_repr
cls.expected_meta = CellSpecimenMeta(emission_lambda=520.0,
imaging_plane=ImagingPlane(
excitation_lambda=910.0,
indicator='GCaMP6f',
ophys_frame_rate=10.0,
targeted_structure='VISp'))
cls.ophys_timestamps = OphysTimestamps(
timestamps=np.array([.1, .2, .3]))
def setup_class(cls):
cls.ophys_timestamps = OphysTimestamps(
timestamps=np.array([.1, .2, .3]))
def from_json(cls,
session_data: dict,
eye_tracking_z_threshold: float = 3.0,
eye_tracking_dilation_frames: int = 2,
events_filter_scale: float = 2.0,
events_filter_n_time_steps: int = 20,
exclude_invalid_rois=True,
skip_eye_tracking=False) -> \
"BehaviorOphysExperiment":
"""
Parameters
----------
session_data
eye_tracking_z_threshold
See `BehaviorOphysExperiment.from_nwb`
eye_tracking_dilation_frames
See `BehaviorOphysExperiment.from_nwb`
events_filter_scale
See `BehaviorOphysExperiment.from_nwb`
events_filter_n_time_steps
See `BehaviorOphysExperiment.from_nwb`
exclude_invalid_rois
Whether to exclude invalid rois
skip_eye_tracking
Used to skip returning eye tracking data
"""
def _is_multi_plane_session():
imaging_plane_group_meta = ImagingPlaneGroup.from_json(
dict_repr=session_data)
return cls._is_multi_plane_session(
imaging_plane_group_meta=imaging_plane_group_meta)
def _get_motion_correction():
rigid_motion_transform_file = RigidMotionTransformFile.from_json(
dict_repr=session_data)
return MotionCorrection.from_data_file(
rigid_motion_transform_file=rigid_motion_transform_file)
def _get_eye_tracking_table(sync_file: SyncFile):
eye_tracking_file = EyeTrackingFile.from_json(
dict_repr=session_data)
eye_tracking_table = EyeTrackingTable.from_data_file(
data_file=eye_tracking_file,
sync_file=sync_file,
z_threshold=eye_tracking_z_threshold,
dilation_frames=eye_tracking_dilation_frames)
return eye_tracking_table
sync_file = SyncFile.from_json(dict_repr=session_data)
is_multiplane_session = _is_multi_plane_session()
meta = BehaviorOphysMetadata.from_json(
dict_repr=session_data, is_multiplane=is_multiplane_session)
monitor_delay = calculate_monitor_delay(
sync_file=sync_file, equipment=meta.behavior_metadata.equipment)
behavior_session = BehaviorSession.from_json(
session_data=session_data, monitor_delay=monitor_delay)
if is_multiplane_session:
ophys_timestamps = OphysTimestampsMultiplane.from_sync_file(
sync_file=sync_file,
group_count=meta.ophys_metadata.imaging_plane_group_count,
plane_group=meta.ophys_metadata.imaging_plane_group)
else:
ophys_timestamps = OphysTimestamps.from_sync_file(
sync_file=sync_file)
projections = Projections.from_json(dict_repr=session_data)
cell_specimens = CellSpecimens.from_json(
dict_repr=session_data,
ophys_timestamps=ophys_timestamps,
segmentation_mask_image_spacing=projections.max_projection.spacing,
events_params=EventsParams(
filter_scale=events_filter_scale,
filter_n_time_steps=events_filter_n_time_steps),
exclude_invalid_rois=exclude_invalid_rois)
motion_correction = _get_motion_correction()
if skip_eye_tracking:
eye_tracking_table = None
eye_tracking_rig_geometry = None
else:
eye_tracking_table = _get_eye_tracking_table(sync_file=sync_file)
eye_tracking_rig_geometry = EyeTrackingRigGeometry.from_json(
dict_repr=session_data)
return BehaviorOphysExperiment(
behavior_session=behavior_session,
cell_specimens=cell_specimens,
ophys_timestamps=ophys_timestamps,
metadata=meta,
projections=projections,
motion_correction=motion_correction,
eye_tracking_table=eye_tracking_table,
eye_tracking_rig_geometry=eye_tracking_rig_geometry,
date_of_acquisition=behavior_session._date_of_acquisition)
def from_nwb(cls,
nwbfile: NWBFile,
eye_tracking_z_threshold: float = 3.0,
eye_tracking_dilation_frames: int = 2,
events_filter_scale: float = 2.0,
events_filter_n_time_steps: int = 20,
exclude_invalid_rois=True) -> "BehaviorOphysExperiment":
"""
Parameters
----------
nwbfile
eye_tracking_z_threshold : float, optional
The z-threshold when determining which frames likely contain
outliers for eye or pupil areas. Influences which frames
are considered 'likely blinks'. By default 3.0
eye_tracking_dilation_frames : int, optional
Determines the number of adjacent frames that will be marked
as 'likely_blink' when performing blink detection for
`eye_tracking` data, by default 2
events_filter_scale : float, optional
Stdev of halfnorm distribution used to convolve ophys events with
a 1d causal half-gaussian filter to smooth it for visualization,
by default 2.0
events_filter_n_time_steps : int, optional
Number of time steps to use for convolution of ophys events
exclude_invalid_rois
Whether to exclude invalid rois
"""
def _is_multi_plane_session():
imaging_plane_group_meta = ImagingPlaneGroup.from_nwb(
nwbfile=nwbfile)
return cls._is_multi_plane_session(
imaging_plane_group_meta=imaging_plane_group_meta)
behavior_session = BehaviorSession.from_nwb(nwbfile=nwbfile)
projections = Projections.from_nwb(nwbfile=nwbfile)
cell_specimens = CellSpecimens.from_nwb(
nwbfile=nwbfile,
segmentation_mask_image_spacing=projections.max_projection.spacing,
events_params=EventsParams(
filter_scale=events_filter_scale,
filter_n_time_steps=events_filter_n_time_steps),
exclude_invalid_rois=exclude_invalid_rois)
eye_tracking_rig_geometry = EyeTrackingRigGeometry.from_nwb(
nwbfile=nwbfile)
eye_tracking_table = EyeTrackingTable.from_nwb(
nwbfile=nwbfile,
z_threshold=eye_tracking_z_threshold,
dilation_frames=eye_tracking_dilation_frames)
motion_correction = MotionCorrection.from_nwb(nwbfile=nwbfile)
is_multiplane_session = _is_multi_plane_session()
metadata = BehaviorOphysMetadata.from_nwb(
nwbfile=nwbfile, is_multiplane=is_multiplane_session)
if is_multiplane_session:
ophys_timestamps = OphysTimestampsMultiplane.from_nwb(
nwbfile=nwbfile)
else:
ophys_timestamps = OphysTimestamps.from_nwb(nwbfile=nwbfile)
date_of_acquisition = DateOfAcquisitionOphys.from_nwb(nwbfile=nwbfile)
return BehaviorOphysExperiment(
behavior_session=behavior_session,
cell_specimens=cell_specimens,
eye_tracking_rig_geometry=eye_tracking_rig_geometry,
eye_tracking_table=eye_tracking_table,
motion_correction=motion_correction,
metadata=metadata,
ophys_timestamps=ophys_timestamps,
projections=projections,
date_of_acquisition=date_of_acquisition)
def from_lims(cls,
ophys_experiment_id: int,
eye_tracking_z_threshold: float = 3.0,
eye_tracking_dilation_frames: int = 2,
events_filter_scale: float = 2.0,
events_filter_n_time_steps: int = 20,
exclude_invalid_rois=True,
skip_eye_tracking=False) -> \
"BehaviorOphysExperiment":
"""
Parameters
----------
ophys_experiment_id
eye_tracking_z_threshold
See `BehaviorOphysExperiment.from_nwb`
eye_tracking_dilation_frames
See `BehaviorOphysExperiment.from_nwb`
events_filter_scale
See `BehaviorOphysExperiment.from_nwb`
events_filter_n_time_steps
See `BehaviorOphysExperiment.from_nwb`
exclude_invalid_rois
Whether to exclude invalid rois
skip_eye_tracking
Used to skip returning eye tracking data
"""
def _is_multi_plane_session():
imaging_plane_group_meta = ImagingPlaneGroup.from_lims(
ophys_experiment_id=ophys_experiment_id, lims_db=lims_db)
return cls._is_multi_plane_session(
imaging_plane_group_meta=imaging_plane_group_meta)
def _get_motion_correction():
rigid_motion_transform_file = RigidMotionTransformFile.from_lims(
ophys_experiment_id=ophys_experiment_id, db=lims_db)
return MotionCorrection.from_data_file(
rigid_motion_transform_file=rigid_motion_transform_file)
def _get_eye_tracking_table(sync_file: SyncFile):
eye_tracking_file = EyeTrackingFile.from_lims(
db=lims_db, ophys_experiment_id=ophys_experiment_id)
eye_tracking_table = EyeTrackingTable.from_data_file(
data_file=eye_tracking_file,
sync_file=sync_file,
z_threshold=eye_tracking_z_threshold,
dilation_frames=eye_tracking_dilation_frames)
return eye_tracking_table
lims_db = db_connection_creator(
fallback_credentials=LIMS_DB_CREDENTIAL_MAP)
sync_file = SyncFile.from_lims(db=lims_db,
ophys_experiment_id=ophys_experiment_id)
stimulus_timestamps = StimulusTimestamps.from_sync_file(
sync_file=sync_file)
behavior_session_id = BehaviorSessionId.from_lims(
db=lims_db, ophys_experiment_id=ophys_experiment_id)
is_multiplane_session = _is_multi_plane_session()
meta = BehaviorOphysMetadata.from_lims(
ophys_experiment_id=ophys_experiment_id,
lims_db=lims_db,
is_multiplane=is_multiplane_session)
monitor_delay = calculate_monitor_delay(
sync_file=sync_file, equipment=meta.behavior_metadata.equipment)
date_of_acquisition = DateOfAcquisitionOphys.from_lims(
ophys_experiment_id=ophys_experiment_id, lims_db=lims_db)
behavior_session = BehaviorSession.from_lims(
lims_db=lims_db,
behavior_session_id=behavior_session_id.value,
stimulus_timestamps=stimulus_timestamps,
monitor_delay=monitor_delay,
date_of_acquisition=date_of_acquisition)
if is_multiplane_session:
ophys_timestamps = OphysTimestampsMultiplane.from_sync_file(
sync_file=sync_file,
group_count=meta.ophys_metadata.imaging_plane_group_count,
plane_group=meta.ophys_metadata.imaging_plane_group)
else:
ophys_timestamps = OphysTimestamps.from_sync_file(
sync_file=sync_file)
projections = Projections.from_lims(
ophys_experiment_id=ophys_experiment_id, lims_db=lims_db)
cell_specimens = CellSpecimens.from_lims(
ophys_experiment_id=ophys_experiment_id,
lims_db=lims_db,
ophys_timestamps=ophys_timestamps,
segmentation_mask_image_spacing=projections.max_projection.spacing,
events_params=EventsParams(
filter_scale=events_filter_scale,
filter_n_time_steps=events_filter_n_time_steps),
exclude_invalid_rois=exclude_invalid_rois)
motion_correction = _get_motion_correction()
if skip_eye_tracking:
eye_tracking_table = None
eye_tracking_rig_geometry = None
else:
eye_tracking_table = _get_eye_tracking_table(sync_file=sync_file)
eye_tracking_rig_geometry = EyeTrackingRigGeometry.from_lims(
ophys_experiment_id=ophys_experiment_id, lims_db=lims_db)
return BehaviorOphysExperiment(
behavior_session=behavior_session,
cell_specimens=cell_specimens,
ophys_timestamps=ophys_timestamps,
metadata=meta,
projections=projections,
motion_correction=motion_correction,
eye_tracking_table=eye_tracking_table,
eye_tracking_rig_geometry=eye_tracking_rig_geometry,
date_of_acquisition=date_of_acquisition)
def test_too_short(self):
"""test when timestamps shorter than # frames"""
self.sync_file._data = {'ophys_frames': np.array([.1, .2, .3])}
with pytest.raises(RuntimeError):
OphysTimestamps.from_sync_file(sync_file=self.sync_file)\
.validate(number_of_frames=4)
def test_from_sync_file(self):
self.sync_file._data = {'ophys_frames': np.array([.1, .2, .3])}
ts = OphysTimestamps.from_sync_file(sync_file=self.sync_file)\
.validate(number_of_frames=3)
expected = np.array([.1, .2, .3])
np.testing.assert_equal(ts.value, expected)
def __init__(self,
cell_specimen_table: pd.DataFrame,
meta: CellSpecimenMeta,
dff_traces: DFFTraces,
corrected_fluorescence_traces: CorrectedFluorescenceTraces,
events: Events,
ophys_timestamps: OphysTimestamps,
segmentation_mask_image_spacing: Tuple,
exclude_invalid_rois=True):
"""
A container for cell specimens including traces, events, metadata, etc.
Parameters
----------
cell_specimen_table
index cell_specimen_id
columns:
- cell_roi_id
- height
- mask_image_plane
- max_correction_down
- max_correction_left
- max_correction_right
- max_correction_up
- roi_mask
- valid_roi
- width
- x
- y
meta
dff_traces
corrected_fluorescence_traces
events
ophys_timestamps
segmentation_mask_image_spacing
Spacing to pass to sitk when constructing segmentation mask image
exclude_invalid_rois
Whether to exclude invalid rois
"""
super().__init__(name='cell_specimen_table', value=self)
# Validate ophys timestamps, traces
ophys_timestamps = ophys_timestamps.validate(
number_of_frames=dff_traces.get_number_of_frames())
self._validate_traces(
ophys_timestamps=ophys_timestamps,
dff_traces=dff_traces,
corrected_fluorescence_traces=corrected_fluorescence_traces,
cell_roi_ids=cell_specimen_table['cell_roi_id'].values)
if exclude_invalid_rois:
cell_specimen_table = cell_specimen_table[
cell_specimen_table['valid_roi']]
# Filter/reorder rois according to cell_specimen_table
dff_traces.filter_and_reorder(
roi_ids=cell_specimen_table['cell_roi_id'].values)
corrected_fluorescence_traces.filter_and_reorder(
roi_ids=cell_specimen_table['cell_roi_id'].values)
# Note: setting raise_if_rois_missing to False for events, since
# there seem to be cases where cell_specimen_table contains rois not in
# events
# See ie https://app.zenhub.com/workspaces/allensdk-10-5c17f74db59cfb36f158db8c/issues/alleninstitute/allensdk/2139 # noqa
events.filter_and_reorder(
roi_ids=cell_specimen_table['cell_roi_id'].values,
raise_if_rois_missing=False)
self._meta = meta
self._cell_specimen_table = cell_specimen_table
self._dff_traces = dff_traces
self._corrected_fluorescence_traces = corrected_fluorescence_traces
self._events = events
self._segmentation_mask_image = self._get_segmentation_mask_image(
spacing=segmentation_mask_image_spacing)