def from_foraging_id(
cls,
foraging_id: Union[str, uuid.UUID, int],
lims_credentials: Optional[DbCredentials] = None
) -> "BehaviorLimsApi":
"""Create a BehaviorLimsAPI instance from a foraging_id instead of
a behavior_session_id.
NOTE: 'foraging_id' in the LIMS behavior_session table should be
the same as the 'behavior_session_uuid' in mtrain which should
also be the same as the 'session_uuid' field in the .pkl
returned by 'get_behavior_stimulus_file()'.
"""
lims_db = db_connection_creator(
credentials=lims_credentials,
fallback_credentials=LIMS_DB_CREDENTIAL_MAP)
if isinstance(foraging_id, uuid.UUID):
foraging_id = str(foraging_id)
elif isinstance(foraging_id, int):
foraging_id = str(uuid.UUID(int=foraging_id))
query = f"""
SELECT id
FROM behavior_sessions
WHERE foraging_id = '{foraging_id}';
"""
session_id = lims_db.fetchone(query, strict=True)
return cls(session_id, lims_credentials=lims_credentials)
def __init__(self, ophys_experiment_id: int,
lims_credentials: Optional[DbCredentials] = None):
self.ophys_experiment_id = ophys_experiment_id
self.lims_db = db_connection_creator(
credentials=lims_credentials,
fallback_credentials=LIMS_DB_CREDENTIAL_MAP)
def default(
cls,
lims_credentials: Optional[DbCredentials] = None,
mtrain_credentials: Optional[DbCredentials] = None,
app_kwargs: Optional[Dict[str, Any]] = None,
data_release_date: Optional[str] = None) -> \
"BehaviorProjectLimsApi":
"""Construct a BehaviorProjectLimsApi instance with default
postgres and app engines.
Parameters
----------
lims_credentials: Optional[DbCredentials]
Credentials to pass to the postgres connector to the lims database.
If left unspecified, will check environment variables for the
appropriate values.
mtrain_credentials: Optional[DbCredentials]
Credentials to pass to the postgres connector to the mtrain
database. If left unspecified, will check environment variables
for the appropriate values.
data_release_date: Optional[str]
Filters tables to include only data released on date
ie 2021-03-25
app_kwargs: Dict
Dict of arguments to pass to the app engine. Currently unused.
Returns
-------
BehaviorProjectLimsApi
"""
_app_kwargs = {"scheme": "http", "host": "lims2"}
if app_kwargs:
_app_kwargs.update(app_kwargs)
lims_engine = db_connection_creator(
credentials=lims_credentials,
fallback_credentials=LIMS_DB_CREDENTIAL_MAP)
mtrain_engine = db_connection_creator(
credentials=mtrain_credentials,
fallback_credentials=MTRAIN_DB_CREDENTIAL_MAP)
app_engine = HttpEngine(**_app_kwargs)
return cls(lims_engine,
mtrain_engine,
app_engine,
data_release_date=data_release_date)
def __init__(self,
ophys_experiment_id: int,
lims_credentials: Optional[DbCredentials] = None,
mtrain_credentials: Optional[DbCredentials] = None):
self.logger = logging.getLogger(self.__class__.__name__)
self.lims_db = db_connection_creator(
credentials=lims_credentials,
fallback_credentials=LIMS_DB_CREDENTIAL_MAP)
self.mtrain_db = db_connection_creator(
credentials=mtrain_credentials,
fallback_credentials=MTRAIN_DB_CREDENTIAL_MAP)
self.ophys_experiment_id = ophys_experiment_id
self.behavior_session_id = self.get_behavior_session_id()
def setup_method(self, method):
marks = getattr(method, 'pytestmark', None)
if marks:
marks = [m.name for m in marks]
# Will only create a dbconn if the test requires_bamboo
if 'requires_bamboo' in marks:
self.dbconn = db_connection_creator(
fallback_credentials=LIMS_DB_CREDENTIAL_MAP)
def __init__(self,
behavior_session_id: int,
lims_credentials: Optional[DbCredentials] = None,
mtrain_credentials: Optional[DbCredentials] = None):
self.logger = logging.getLogger(self.__class__.__name__)
self.mtrain_db = db_connection_creator(
credentials=mtrain_credentials,
fallback_credentials=MTRAIN_DB_CREDENTIAL_MAP)
self.lims_db = db_connection_creator(
credentials=lims_credentials,
fallback_credentials=LIMS_DB_CREDENTIAL_MAP)
self.behavior_session_id = behavior_session_id
ids = self._get_ids()
self.ophys_experiment_ids = ids.get("ophys_experiment_ids")
self.ophys_session_id = ids.get("ophys_session_id")
self.behavior_training_id = ids.get("behavior_training_id")
self.foraging_id = ids.get("foraging_id")
self.ophys_container_id = ids.get("ophys_container_id")
def _get_stimulus_data():
lims_db = db_connection_creator(
fallback_credentials=LIMS_DB_CREDENTIAL_MAP)
stimulus_file = StimulusFile.from_lims(
db=lims_db, behavior_session_id=behavior_experiment_id)
return stimulus_file.data
def test_visbeh_ophys_data_set():
ophys_experiment_id = 789359614
data_set = BehaviorOphysExperiment.from_lims(ophys_experiment_id,
exclude_invalid_rois=False)
# TODO: need to improve testing here:
# for _, row in data_set.roi_metrics.iterrows():
# print(np.array(row.to_dict()['mask']).sum())
# print
# for _, row in data_set.roi_masks.iterrows():
# print(np.array(row.to_dict()['mask']).sum())
lims_db = db_connection_creator(
fallback_credentials=LIMS_DB_CREDENTIAL_MAP)
behavior_session_id = BehaviorSessionId.from_lims(
db=lims_db, ophys_experiment_id=ophys_experiment_id)
# All sorts of assert relationships:
assert ForagingId.from_lims(behavior_session_id=behavior_session_id.value,
lims_db=lims_db).value == \
data_set.metadata['behavior_session_uuid']
stimulus_templates = data_set.stimulus_templates
assert len(stimulus_templates) == 8
assert stimulus_templates.loc['im000'].warped.shape == MONITOR_DIMENSIONS
assert stimulus_templates.loc['im000'].unwarped.shape == MONITOR_DIMENSIONS
assert len(data_set.licks) == 2421 and set(data_set.licks.columns) \
== set(['timestamps', 'frame'])
assert len(data_set.rewards) == 85 and set(data_set.rewards.columns) == \
set(['timestamps', 'volume', 'autorewarded'])
assert len(data_set.corrected_fluorescence_traces) == 258 and \
set(data_set.corrected_fluorescence_traces.columns) == \
set(['cell_roi_id', 'corrected_fluorescence'])
np.testing.assert_array_almost_equal(data_set.running_speed.timestamps,
data_set.stimulus_timestamps)
assert len(data_set.cell_specimen_table) == len(data_set.dff_traces)
assert data_set.average_projection.data.shape == \
data_set.max_projection.data.shape
assert set(data_set.motion_correction.columns) == set(['x', 'y'])
assert len(data_set.trials) == 602
expected_metadata = {
'stimulus_frame_rate':
60.0,
'full_genotype':
'Slc17a7-IRES2-Cre/wt;Camk2a-tTA/wt;Ai93('
'TITL-GCaMP6f)/wt',
'ophys_experiment_id':
789359614,
'behavior_session_id':
789295700,
'imaging_plane_group_count':
0,
'ophys_session_id':
789220000,
'session_type':
'OPHYS_6_images_B',
'driver_line': ['Camk2a-tTA', 'Slc17a7-IRES2-Cre'],
'cre_line':
'Slc17a7-IRES2-Cre',
'behavior_session_uuid':
uuid.UUID('69cdbe09-e62b-4b42-aab1-54b5773dfe78'),
'date_of_acquisition':
pytz.utc.localize(datetime.datetime(2018, 11, 30, 23, 28, 37)),
'ophys_frame_rate':
31.0,
'imaging_depth':
375,
'mouse_id':
416369,
'experiment_container_id':
814796558,
'targeted_structure':
'VISp',
'reporter_line':
'Ai93(TITL-GCaMP6f)',
'emission_lambda':
520.0,
'excitation_lambda':
910.0,
'field_of_view_height':
512,
'field_of_view_width':
447,
'indicator':
'GCaMP6f',
'equipment_name':
'CAM2P.5',
'age_in_days':
139,
'sex':
'F',
'imaging_plane_group':
None,
'project_code':
'VisualBehavior'
}
assert data_set.metadata == expected_metadata
assert data_set.task_parameters == {
'reward_volume': 0.007,
'stimulus_distribution': u'geometric',
'stimulus_duration_sec': 0.25,
'stimulus': 'images',
'omitted_flash_fraction': 0.05,
'blank_duration_sec': [0.5, 0.5],
'n_stimulus_frames': 69882,
'task': 'change detection',
'response_window_sec': [0.15, 0.75],
'session_type': u'OPHYS_6_images_B',
'auto_reward_volume': 0.005
}
def from_lims(cls,
ophys_experiment_id: int,
eye_tracking_z_threshold: float = 3.0,
eye_tracking_dilation_frames: int = 2,
events_filter_scale: float = 2.0,
events_filter_n_time_steps: int = 20,
exclude_invalid_rois=True,
skip_eye_tracking=False) -> \
"BehaviorOphysExperiment":
"""
Parameters
----------
ophys_experiment_id
eye_tracking_z_threshold
See `BehaviorOphysExperiment.from_nwb`
eye_tracking_dilation_frames
See `BehaviorOphysExperiment.from_nwb`
events_filter_scale
See `BehaviorOphysExperiment.from_nwb`
events_filter_n_time_steps
See `BehaviorOphysExperiment.from_nwb`
exclude_invalid_rois
Whether to exclude invalid rois
skip_eye_tracking
Used to skip returning eye tracking data
"""
def _is_multi_plane_session():
imaging_plane_group_meta = ImagingPlaneGroup.from_lims(
ophys_experiment_id=ophys_experiment_id, lims_db=lims_db)
return cls._is_multi_plane_session(
imaging_plane_group_meta=imaging_plane_group_meta)
def _get_motion_correction():
rigid_motion_transform_file = RigidMotionTransformFile.from_lims(
ophys_experiment_id=ophys_experiment_id, db=lims_db)
return MotionCorrection.from_data_file(
rigid_motion_transform_file=rigid_motion_transform_file)
def _get_eye_tracking_table(sync_file: SyncFile):
eye_tracking_file = EyeTrackingFile.from_lims(
db=lims_db, ophys_experiment_id=ophys_experiment_id)
eye_tracking_table = EyeTrackingTable.from_data_file(
data_file=eye_tracking_file,
sync_file=sync_file,
z_threshold=eye_tracking_z_threshold,
dilation_frames=eye_tracking_dilation_frames)
return eye_tracking_table
lims_db = db_connection_creator(
fallback_credentials=LIMS_DB_CREDENTIAL_MAP)
sync_file = SyncFile.from_lims(db=lims_db,
ophys_experiment_id=ophys_experiment_id)
stimulus_timestamps = StimulusTimestamps.from_sync_file(
sync_file=sync_file)
behavior_session_id = BehaviorSessionId.from_lims(
db=lims_db, ophys_experiment_id=ophys_experiment_id)
is_multiplane_session = _is_multi_plane_session()
meta = BehaviorOphysMetadata.from_lims(
ophys_experiment_id=ophys_experiment_id,
lims_db=lims_db,
is_multiplane=is_multiplane_session)
monitor_delay = calculate_monitor_delay(
sync_file=sync_file, equipment=meta.behavior_metadata.equipment)
date_of_acquisition = DateOfAcquisitionOphys.from_lims(
ophys_experiment_id=ophys_experiment_id, lims_db=lims_db)
behavior_session = BehaviorSession.from_lims(
lims_db=lims_db,
behavior_session_id=behavior_session_id.value,
stimulus_timestamps=stimulus_timestamps,
monitor_delay=monitor_delay,
date_of_acquisition=date_of_acquisition)
if is_multiplane_session:
ophys_timestamps = OphysTimestampsMultiplane.from_sync_file(
sync_file=sync_file,
group_count=meta.ophys_metadata.imaging_plane_group_count,
plane_group=meta.ophys_metadata.imaging_plane_group)
else:
ophys_timestamps = OphysTimestamps.from_sync_file(
sync_file=sync_file)
projections = Projections.from_lims(
ophys_experiment_id=ophys_experiment_id, lims_db=lims_db)
cell_specimens = CellSpecimens.from_lims(
ophys_experiment_id=ophys_experiment_id,
lims_db=lims_db,
ophys_timestamps=ophys_timestamps,
segmentation_mask_image_spacing=projections.max_projection.spacing,
events_params=EventsParams(
filter_scale=events_filter_scale,
filter_n_time_steps=events_filter_n_time_steps),
exclude_invalid_rois=exclude_invalid_rois)
motion_correction = _get_motion_correction()
if skip_eye_tracking:
eye_tracking_table = None
eye_tracking_rig_geometry = None
else:
eye_tracking_table = _get_eye_tracking_table(sync_file=sync_file)
eye_tracking_rig_geometry = EyeTrackingRigGeometry.from_lims(
ophys_experiment_id=ophys_experiment_id, lims_db=lims_db)
return BehaviorOphysExperiment(
behavior_session=behavior_session,
cell_specimens=cell_specimens,
ophys_timestamps=ophys_timestamps,
metadata=meta,
projections=projections,
motion_correction=motion_correction,
eye_tracking_table=eye_tracking_table,
eye_tracking_rig_geometry=eye_tracking_rig_geometry,
date_of_acquisition=date_of_acquisition)
def from_lims(cls, behavior_session_id: int,
lims_db: Optional[PostgresQueryMixin] = None,
stimulus_timestamps: Optional[StimulusTimestamps] = None,
monitor_delay: Optional[float] = None,
date_of_acquisition: Optional[DateOfAcquisition] = None) \
-> "BehaviorSession":
"""
Parameters
----------
behavior_session_id
Behavior session id
lims_db
Database connection. If not provided will create a new one.
stimulus_timestamps
Stimulus timestamps. If not provided, will calculate stimulus
timestamps from stimulus file.
monitor_delay
Monitor delay. If not provided, will use an estimate.
To provide this value, see for example
allensdk.brain_observatory.behavior.data_objects.stimuli.util.
calculate_monitor_delay
date_of_acquisition
Date of acquisition. If not provided, will read from
behavior_sessions table.
Returns
-------
`BehaviorSession` instance
"""
if lims_db is None:
lims_db = db_connection_creator(
fallback_credentials=LIMS_DB_CREDENTIAL_MAP)
behavior_session_id = BehaviorSessionId(behavior_session_id)
stimulus_file = StimulusFile.from_lims(
db=lims_db, behavior_session_id=behavior_session_id.value)
if stimulus_timestamps is None:
stimulus_timestamps = StimulusTimestamps.from_stimulus_file(
stimulus_file=stimulus_file)
running_acquisition = RunningAcquisition.from_lims(
lims_db, behavior_session_id.value)
raw_running_speed = RunningSpeed.from_lims(
lims_db,
behavior_session_id.value,
filtered=False,
stimulus_timestamps=stimulus_timestamps)
running_speed = RunningSpeed.from_lims(
lims_db,
behavior_session_id.value,
stimulus_timestamps=stimulus_timestamps)
behavior_metadata = BehaviorMetadata.from_lims(
behavior_session_id=behavior_session_id, lims_db=lims_db)
if monitor_delay is None:
monitor_delay = cls._get_monitor_delay()
licks, rewards, stimuli, task_parameters, trials = \
cls._read_data_from_stimulus_file(
stimulus_file=stimulus_file,
stimulus_timestamps=stimulus_timestamps,
trial_monitor_delay=monitor_delay
)
if date_of_acquisition is None:
date_of_acquisition = DateOfAcquisition.from_lims(
behavior_session_id=behavior_session_id.value, lims_db=lims_db)
date_of_acquisition = date_of_acquisition.validate(
stimulus_file=stimulus_file,
behavior_session_id=behavior_session_id.value)
return BehaviorSession(behavior_session_id=behavior_session_id,
stimulus_timestamps=stimulus_timestamps,
running_acquisition=running_acquisition,
raw_running_speed=raw_running_speed,
running_speed=running_speed,
metadata=behavior_metadata,
licks=licks,
rewards=rewards,
stimuli=stimuli,
task_parameters=task_parameters,
trials=trials,
date_of_acquisition=date_of_acquisition)