def get_ophys_pupil_data(self,
ophys_experiment_id: int,
file_name: str = None,
suppress_pupil_data: bool = True) -> pd.DataFrame:
"""Download the h5 eye gaze mapping file for an ophys_experiment if
it hasn't already been downloaded and return it as a pandas.DataFrame.
Parameters
----------
file_name: string
File name to save/read the data set. If file_name is None,
the file_name will be pulled out of the manifest. If caching
is disabled, no file will be saved. Default is None.
ophys_experiment_id: int
id of the ophys_experiment to retrieve pupil data for.
suppress_pupil_data: bool
Whether or not to suppress pupil data from dataset.
Default is True.
Returns
-------
pd.DataFrame
If 'suppress_eye_gaze_data' is set to 'False':
Contains raw/filtered columns for gaze mapping:
*_eye_area
*_pupil_area
*_screen_coordinates_x_cm
*_screen_coordinates_y_cm
*_screen_coordinates_spherical_x_deg
*_screen_coorindates_spherical_y_deg
Otherwise:
An empty pandas DataFrame
"""
if suppress_pupil_data:
print(
"This pupil data is obtained using a new eye "
"tracking algorithm and is in the process of being validated. "
"If you would like to view the data anyways, "
"please set the 'suppress_pupil_data' parameter to 'False'.")
return pd.DataFrame()
# NOTE: This is a really ugly hack to get around the fact that warehouse does
# not have Ophys session ids associated with experiment ids. This should be
# removed when warehouse session ids have associations with experiment ids.
# ----- Start of ugly hack -----
try:
ophys_session_id = ophys_experiment_session_id_map[
ophys_experiment_id]
except KeyError:
raise RuntimeError(
f"Experiment id '{ophys_experiment_id}' has no associated session!"
)
# ----- End of ugly hack -----
file_name = self.get_cache_path(file_name, self.EYE_GAZE_DATA_KEY,
ophys_session_id)
if not file_name:
raise RuntimeError("Could not obtain a file_name for pupil data "
f"with experiment id: {ophys_experiment_id} "
f"(session id: {ophys_session_id})")
# NOTE: `save_ophys_experiment_eye_gaze_data` will also need to be
# updated to remove ophy_session_id param when ugly hack is removed.
self.api.save_ophys_experiment_eye_gaze_data(ophys_experiment_id,
ophys_session_id,
file_name,
strategy='lazy')
gaze_mapping_data = read_eye_gaze_mappings(Path(file_name))
return create_eye_gaze_mapping_dataframe(gaze_mapping_data)
def write_ecephys_nwb(output_path,
session_id,
session_start_time,
stimulus_table_path,
invalid_epochs,
probes,
running_speed_path,
session_sync_path,
eye_tracking_rig_geometry,
eye_dlc_ellipses_path,
eye_gaze_mapping_path,
pool_size,
optotagging_table_path=None,
session_metadata=None,
**kwargs):
nwbfile = pynwb.NWBFile(
session_description='Data and metadata for an Ecephys session',
identifier=f"{session_id}",
session_id=f"{session_id}",
session_start_time=session_start_time,
institution="Allen Institute for Brain Science")
if session_metadata is not None:
nwbfile = add_metadata_to_nwbfile(nwbfile, session_metadata)
stimulus_columns_to_drop = [
"colorSpace", "depth", "interpolate", "pos", "rgbPedestal", "tex",
"texRes", "flipHoriz", "flipVert", "rgb", "signalDots"
]
stimulus_table = read_stimulus_table(
stimulus_table_path, columns_to_drop=stimulus_columns_to_drop)
nwbfile = add_stimulus_timestamps(
nwbfile, stimulus_table['start_time'].values
) # TODO: patch until full timestamps are output by stim table module
nwbfile = add_stimulus_presentations(nwbfile, stimulus_table)
nwbfile = add_invalid_times(nwbfile, invalid_epochs)
if optotagging_table_path is not None:
optotagging_table = pd.read_csv(optotagging_table_path)
nwbfile = add_optotagging_table_to_nwbfile(nwbfile, optotagging_table)
nwbfile = add_probewise_data_to_nwbfile(nwbfile, probes)
running_speed, raw_running_data = read_running_speed(running_speed_path)
add_running_speed_to_nwbfile(nwbfile, running_speed)
add_raw_running_data_to_nwbfile(nwbfile, raw_running_data)
add_eye_tracking_rig_geometry_data_to_nwbfile(nwbfile,
eye_tracking_rig_geometry)
# Collect eye tracking/gaze mapping data from files
eye_tracking_frame_times = su.get_synchronized_frame_times(
session_sync_file=session_sync_path,
sync_line_label_keys=Dataset.EYE_TRACKING_KEYS)
eye_dlc_tracking_data = read_eye_dlc_tracking_ellipses(
Path(eye_dlc_ellipses_path))
if eye_gaze_mapping_path:
eye_gaze_data = read_eye_gaze_mappings(Path(eye_gaze_mapping_path))
else:
eye_gaze_data = None
add_eye_tracking_data_to_nwbfile(nwbfile, eye_tracking_frame_times,
eye_dlc_tracking_data, eye_gaze_data)
Manifest.safe_make_parent_dirs(output_path)
with pynwb.NWBHDF5IO(output_path, mode='w') as io:
logging.info(f"writing session nwb file to {output_path}")
io.write(nwbfile, cache_spec=True)
probes_with_lfp = [p for p in probes if p["lfp"] is not None]
probe_outputs = write_probewise_lfp_files(probes_with_lfp,
session_id,
session_metadata,
session_start_time,
pool_size=pool_size)
return {'nwb_path': output_path, "probe_outputs": probe_outputs}
def create_nwb_file(session, nwb_file_path):
logging.info('Staring session {}'.format(session.session_id))
nwbfile = pynwb.NWBFile(
session_description=session.description,
session_id=str(session.session_id),
identifier=str(session.experiment_metadata['experiment_id']),
session_start_time=session.start_time,
# experiment_description=str(session.experiment_metadata['experiment_id'])
)
### Build the stimulus table ###
stim_table_df = session.stimulus_table
nwbfile = add_stimulus_timestamps(nwbfile,
stim_table_df['start_time'].values)
nwbfile = add_stimulus_presentations(nwbfile, stim_table_df)
### Image Segmentation ###
optical_channel = pynwb.ophys.OpticalChannel(
name='optical_channel',
description='description',
emission_lambda=session.emission_wavelength)
# device = pynwb.device.Device(name='Allen Institute two-photon pipeline: {}'.format(session.session_metadata['rig']))
device = pynwb.device.Device(
name='{}'.format(session.session_metadata['rig']))
nwbfile.add_device(device)
imaging_plane = nwbfile.create_imaging_plane(
name=session.imaging_plane_name,
optical_channel=optical_channel,
description=session.imaging_plane_name,
device=device,
excitation_lambda=session.excitation_lambda,
imaging_rate=session.imaging_rate,
indicator=session.calcium_indicator,
location='area: {},depth: {}'.format(
session.experiment_metadata["area"],
str(session.experiment_metadata["depth"])),
unit="Fluorescence (au)",
reference_frame="Intrinsic imaging home",
)
ophys_module = nwbfile.create_processing_module(
session.ophys_module_name,
'contains optical physiology processed data')
max_projection = pynwb.ophys.TwoPhotonSeries(
name='max_project',
data=[session.max_projection],
imaging_plane=imaging_plane,
dimension=[session.image_width, session.image_height],
rate=30.0)
# nwbfile.add_acquisition(max_projection)
ophys_module.add_data_interface(max_projection)
img_seg = pynwb.ophys.ImageSegmentation(
name=session.image_segmentation_name)
ophys_module.add_data_interface(img_seg)
ps = img_seg.create_plane_segmentation(
name=session.plane_segmentation_name,
description=
"Segmentation for imaging plane (de Vries et al., 2019, Nat Neurosci)",
imaging_plane=imaging_plane,
)
for cell_id, roi_mask in session.roi_masks.items():
ps.add_roi(id=cell_id, image_mask=roi_mask)
rt_region = ps.create_roi_table_region(
description="segmented cells with cell_specimen_ids")
### Fluorescence traces ###
fluorescence = pynwb.ophys.Fluorescence()
ophys_module.add_data_interface(fluorescence)
timestamps = session.twop_timestamps
raw_traces = session.raw_traces
assert (raw_traces.shape[1] <= timestamps.shape[0])
fluorescence.create_roi_response_series(
name='raw_traces',
data=raw_traces,
rois=rt_region,
unit='lumens',
timestamps=timestamps[:raw_traces.shape[1]])
neuropil_traces = session.neuropil_traces
assert (neuropil_traces.shape[1] <= timestamps.shape[0])
fluorescence.create_roi_response_series(
name='neuropil_traces',
data=neuropil_traces,
rois=rt_region,
unit='lumens',
timestamps=timestamps[:neuropil_traces.shape[1]])
demixed_traces = session.demixed_traces
assert (demixed_traces.shape[1] <= timestamps.shape[0])
fluorescence.create_roi_response_series(
name='demixed_traces',
data=demixed_traces,
rois=rt_region,
unit='lumens',
timestamps=timestamps[:demixed_traces.shape[1]])
dff_traces = session.dff_traces
assert (dff_traces.shape[1] <= timestamps.shape[0])
fluorescence.create_roi_response_series(
name='DfOverF',
data=session.dff_traces,
rois=rt_region,
unit='lumens',
timestamps=timestamps[:dff_traces.shape[1]])
### Running Speed ###
running_velocity = pynwb.base.TimeSeries(
name="running_speed",
data=session.running_velocity[:len(session.stimulus_timestamps)],
timestamps=session.stimulus_timestamps,
unit="cm/s",
description="Speed of the mouse on a rotating wheel",
)
behavior_module = pynwb.behavior.BehavioralTimeSeries()
behavior_module.add_timeseries(running_velocity)
ophys_module.add_data_interface(behavior_module)
### Motion Correction ###
# NOTE: Older versions of pynwb had a schema bug in ophys.CorrectedImageStack class that prevents it from being
# acccessed. Unfortunately currently allensdk has frozen pynwb at version 1.0.2. To fix we need to add the
# x/y corrections as a time series rather than using CorrectImageStack.
# add_motion_correction_cis(session, ophys_module, nwbfile)
add_motion_correction_pm(session, ophys_module, nwbfile)
### Subject and lab metadata ###
sex_lu = {'F': 'female', 'M': 'male'}
subject_metadata = pynwb.file.Subject(
age=session.session_metadata['age'][1:],
genotype=session.session_metadata['full_genotype'],
sex=sex_lu.get(session.session_metadata['sex'], 'unknown'),
species='Mus musculus',
subject_id=str(session.session_metadata['specimen_id']),
description=session.session_metadata['name'])
nwbfile.subject = subject_metadata
### Events ###
# add_events_discrete(session=session, ophys_module=ophys_module, nwbfile=nwbfile)
add_events_contiguous(session=session,
ophys_module=ophys_module,
nwbfile=nwbfile,
roi_table=rt_region)
# pd.set_option('display.max_columns', None)
# nwbfile.units = pynwb.misc.Units.from_dataframe(session.units, name='units')
# # events_indices = session.event_indices
# nwbfile.units.add_column(
# name='event_times',
# data=session.event_times,
# index=session.event_indices,
# description='times (s) of detected L0 events'
# )
#
# nwbfile.units.add_column(
# name='event_amplitudes',
# data=session.event_amps,
# index=session.event_indices,
# description='amplitudes (s) of detected L0 events'
# )
### Eye Tracking ###
eye_dlc_path = session.eye_dlc_screen_mapping
if eye_dlc_path is not None:
eye_gazing_data = read_eye_gaze_mappings(eye_dlc_path)
add_eye_gaze_mapping_data_to_nwbfile(nwbfile, eye_gazing_data)
with pynwb.NWBHDF5IO(str(nwb_file_path), mode="w") as io:
io.write(nwbfile)
def write_ecephys_nwb(output_path,
session_id,
session_start_time,
stimulus_table_path,
invalid_epochs,
probes,
running_speed_path,
session_sync_path,
eye_tracking_rig_geometry,
eye_dlc_ellipses_path,
eye_gaze_mapping_path,
pool_size,
optotagging_table_path=None,
session_metadata=None,
**kwargs):
nwbfile = pynwb.NWBFile(session_description='EcephysSession',
identifier='{}'.format(session_id),
session_start_time=session_start_time)
if session_metadata is not None:
nwbfile = add_metadata_to_nwbfile(nwbfile, session_metadata)
stimulus_table = read_stimulus_table(stimulus_table_path)
nwbfile = add_stimulus_timestamps(
nwbfile, stimulus_table['start_time'].values
) # TODO: patch until full timestamps are output by stim table module
nwbfile = add_stimulus_presentations(nwbfile, stimulus_table)
nwbfile = add_invalid_times(nwbfile, invalid_epochs)
if optotagging_table_path is not None:
optotagging_table = pd.read_csv(optotagging_table_path)
nwbfile = add_optotagging_table_to_nwbfile(nwbfile, optotagging_table)
nwbfile = add_probewise_data_to_nwbfile(nwbfile, probes)
running_speed, raw_running_data = read_running_speed(running_speed_path)
add_running_speed_to_nwbfile(nwbfile, running_speed)
add_raw_running_data_to_nwbfile(nwbfile, raw_running_data)
# --- Add eye tracking ellipse fits to nwb file ---
eye_tracking_frame_times = su.get_synchronized_frame_times(
session_sync_file=session_sync_path,
sync_line_label_keys=Dataset.EYE_TRACKING_KEYS)
eye_dlc_tracking_data = read_eye_dlc_tracking_ellipses(
Path(eye_dlc_ellipses_path))
if eye_tracking_data_is_valid(eye_dlc_tracking_data=eye_dlc_tracking_data,
synced_timestamps=eye_tracking_frame_times):
add_eye_tracking_ellipse_fit_data_to_nwbfile(
nwbfile,
eye_dlc_tracking_data=eye_dlc_tracking_data,
synced_timestamps=eye_tracking_frame_times)
# --- Append eye tracking rig geometry info to nwb file (with eye tracking) ---
append_eye_tracking_rig_geometry_data_to_nwbfile(
nwbfile, eye_tracking_rig_geometry=eye_tracking_rig_geometry)
# --- Add gaze mapped positions to nwb file ---
if eye_gaze_mapping_path:
eye_gaze_data = read_eye_gaze_mappings(Path(eye_gaze_mapping_path))
add_eye_gaze_mapping_data_to_nwbfile(nwbfile,
eye_gaze_data=eye_gaze_data)
Manifest.safe_make_parent_dirs(output_path)
io = pynwb.NWBHDF5IO(output_path, mode='w')
logging.info(f"writing session nwb file to {output_path}")
io.write(nwbfile)
io.close()
probes_with_lfp = [p for p in probes if p["lfp"] is not None]
probe_outputs = write_probewise_lfp_files(probes_with_lfp,
session_start_time,
pool_size=pool_size)
return {'nwb_path': output_path, "probe_outputs": probe_outputs}