def load_max_projection_nwb(sess_files):
"""
load_max_projection_nwb(sess_files)
Returns maximum projection image of downsampled z-stack as an array, from
NWB files.
Required args:
- sess_files (Path): full path names of the session files
Returns:
- max_proj (2D array): maximum projection image across downsampled
z-stack (hei x wei), with pixel intensity
in 0 (incl) to 256 (excl) range
("uint8" datatype).
"""
ophys_file = sess_file_util.select_nwb_sess_path(sess_files, ophys=True)
with pynwb.NWBHDF5IO(str(ophys_file), "r") as f:
nwbfile_in = f.read()
ophys_module = nwbfile_in.get_processing_module("ophys")
main_field = "PlaneImages"
data_field = "max_projection"
try:
max_proj = ophys_module.get_data_interface(
main_field).get_image(data_field)[()].astype("uint8")
except KeyError as err:
raise KeyError(
"Could not find a maximum projection plane image "
f"for {ophys_file} due to: {err}"
)
return max_proj
def load_stimulus_images_nwb(sess_files, template_name="gabors", frame_ns=[0]):
"""
load_stimulus_images_nwb(sess_files)
Returns the stimulus images for the requested template, and frames.
Required args:
- sess_files (Path): full path names of the session files
Optional args:
- template_name (str): name of the stimulus template
default: "gabors"
- frame_ns (list) : frame numbers (must be unique and in increasing
order)
default: frame_ns
Returns:
- stim_images (list): list of stimulus images (grayscale)
(height x width x channel (1))
"""
stim_file = sess_file_util.select_nwb_sess_path(sess_files, stim=True)
frame_ns = gen_util.list_if_not(frame_ns)
unique_bool = (len(np.unique(frame_ns)) == len(frame_ns))
increasing_bool = (np.sort(frame_ns) == np.asarray(frame_ns)).all()
if not (unique_bool and increasing_bool):
raise ValueError(
"frame_ns must be provided in strictly increasing order, and "
"comprise only unique values."
)
with pynwb.NWBHDF5IO(str(stim_file), "r") as f:
nwbfile_in = f.read()
try:
stim_template = nwbfile_in.get_stimulus_template(template_name)
n_frames = stim_template.data.shape[0]
if max(frame_ns) > n_frames - 1:
raise ValueError(
"Some of the frames requested go beyond the number of "
"frames recorded for this template."
)
# loading the images can be slow
template_images = stim_template.data[frame_ns]
except KeyError as err:
raise KeyError(
f"Could not find frames for '{template_name}' stimulus "
f"template in {stim_file} due to: {err}"
)
template_images = list(template_images)
return template_images
def get_roi_masks_nwb(sess_files, mask_threshold=MASK_THRESHOLD,
min_n_pix=MIN_N_PIX, make_bool=True):
"""
get_roi_masks_nwb(sess_files)
Returns tracked ROIs, optionally converted to boolean.
Required args:
- sess_files (Path): full path names of the session files
Optional args:
- mask_threshold (float): minimum value in non-boolean mask to
retain a pixel in an ROI mask
default: MASK_THRESHOLD
- min_n_pix (int) : minimum number of pixels in an ROI below
which, ROI is set to be empty (calculated
before conversion to boolean, using weighted
pixels)
default: MIN_N_PIX
- make_bool (bool) : if True, ROIs are converted to boolean
before being returned
default: True
Returns:
- roi_masks (3D array): ROI masks, structured as
ROI x height x width
- roi_ids (list) : ID for each ROI
"""
ophys_file = sess_file_util.select_nwb_sess_path(sess_files, ophys=True)
with pynwb.NWBHDF5IO(str(ophys_file), "r") as f:
nwbfile_in = f.read()
ophys_module = nwbfile_in.get_processing_module("ophys")
main_field = "ImageSegmentation"
data_field = "PlaneSegmentation"
try:
plane_seg = ophys_module.get_data_interface(
main_field).get_plane_segmentation(data_field
)
except KeyError as err:
raise KeyError(
"Could not find plane segmentation data in image segmentation "
f"for {ophys_file} due to: {err}"
)
roi_masks = np.asarray(plane_seg["image_mask"].data)
roi_ids = list(plane_seg["id"].data)
roi_masks = process_roi_masks(
roi_masks, mask_threshold=mask_threshold, min_n_pix=min_n_pix,
make_bool=make_bool
)
return roi_masks, roi_ids
def load_roi_data_nwb(sess_files):
"""
load_roi_data_nwb(sess_files)
Returns ROI data from NWB files.
Required args:
- sess_files (Path): full path names of the session files
Returns:
- roi_ids (list) : ROI IDs
- nrois (int) : total number of ROIs
- tot_twop_fr (int): total number of two-photon frames recorded
"""
ophys_file = sess_file_util.select_nwb_sess_path(sess_files, ophys=True)
with pynwb.NWBHDF5IO(str(ophys_file), "r") as f:
nwbfile_in = f.read()
ophys_module = nwbfile_in.get_processing_module("ophys")
main_field = "ImageSegmentation"
data_field = "PlaneSegmentation"
try:
plane_seg = ophys_module.get_data_interface(
main_field).get_plane_segmentation(data_field
)
except KeyError as err:
raise KeyError(
"Could not find plane segmentation data in image segmentation "
f"for {ophys_file} due to: {err}"
)
# ROI IDs
roi_ids = list(plane_seg["id"].data)
# Number of ROIs and frames
main_field = "DfOverF"
data_field = "RoiResponseSeries"
try:
roi_resp_series = ophys_module.get_data_interface(
main_field).get_roi_response_series(data_field
)
except KeyError as err:
raise KeyError(
"Could not find ROI response series data in image segmentation "
f"for {ophys_file} due to: {err}"
)
tot_twop_fr, nrois = roi_resp_series.data.shape
return roi_ids, nrois, tot_twop_fr
def load_run_data_nwb(sess_files, diff_thr=50, drop_tol=0.0003, sessid=None):
"""
load_run_data_nwb(sess_files)
Returns pre-processed running velocity from NWB files.
Required args:
- sess_files (Path): full path names of the session files
Optional args:
- diff_thr (int): threshold of difference in running velocity to
identify outliers
default: 50
- drop_tol (num): the tolerance for proportion running frames
dropped. A warning is produced only if this
condition is not met.
default: 0.0003
- sessid (int) : session ID to include in the log or error
default: None
Returns:
- run_velocity (1D array): array of running velocities in cm/s for each
recorded stimulus frames
"""
behav_file = sess_file_util.select_nwb_sess_path(sess_files, behav=True)
with pynwb.NWBHDF5IO(str(behav_file), "r") as f:
nwbfile_in = f.read()
behav_module = nwbfile_in.get_processing_module("behavior")
main_field = "BehavioralTimeSeries"
data_field = "running_velocity"
try:
behav_time_series = behav_module.get_data_interface(
main_field).get_timeseries(data_field)
except KeyError as err:
raise KeyError(
"Could not find running velocity data in behavioral time "
f"series for {behav_module} due to: {err}"
)
run_velocity = np.asarray(behav_time_series.data)
run_velocity = nan_large_run_differences(
run_velocity, diff_thr, warn_nans=True, drop_tol=drop_tol,
sessid=sessid
)
return run_velocity
def load_roi_traces_nwb(sess_files, roi_ns=None, frame_ns=None):
"""
load_roi_traces_nwb(sess_files)
Returns ROI traces from NWB files (stored as frames x ROIs).
Required args:
- sess_files (list): full path names of the session files
Optional args:
- roi_ns (int or array-like) : ROIs to load (None for all)
default: None
- frame_ns (int or array-like): frames to load (None for all)
default: None
Returns:
- roi_traces (1 or 2D array): ROI traces (ROI x frame)
"""
ophys_file = sess_file_util.select_nwb_sess_path(sess_files, ophys=True)
with pynwb.NWBHDF5IO(str(ophys_file), "r") as f:
nwbfile_in = f.read()
ophys_module = nwbfile_in.get_processing_module("ophys")
main_field = "DfOverF"
data_field = "RoiResponseSeries"
try:
roi_resp_series = ophys_module.get_data_interface(
main_field).get_roi_response_series(data_field
)
except KeyError as err:
raise KeyError(
"Could not find ROI response series data in image segmentation "
f"for {ophys_file} due to: {err}"
)
roi_data_handle = roi_resp_series.data
roi_traces = load_traces_optimally(
roi_data_handle, roi_ns=roi_ns, frame_ns=frame_ns,
rois_first=False,
)
return roi_traces
def get_tracked_rois_nwb(sess_files):
"""
get_tracked_rois_nwb(sess_files)
Returns ROI tracking indices.
Required args:
- sess_files (list): full path names of the session files
Returns:
- tracked_rois (1D array): ordered indices of tracked ROIs
"""
ophys_file = sess_file_util.select_nwb_sess_path(sess_files, ophys=True)
with pynwb.NWBHDF5IO(str(ophys_file), "r") as f:
nwbfile_in = f.read()
ophys_module = nwbfile_in.get_processing_module("ophys")
main_field = "ImageSegmentation"
data_field = "PlaneSegmentation"
try:
plane_seg = ophys_module.get_data_interface(
main_field).get_plane_segmentation(data_field
)
except KeyError as err:
raise KeyError(
"Could not find plane segmentation data in image segmentation "
f"for {ophys_file} due to: {err}"
)
tracking_key = "tracking_id"
if tracking_key not in plane_seg.colnames:
raise RuntimeError(f"No tracking data in {ophys_file}.")
# tracking index for each ROI (np.nan for non tracked ROIs)
roi_tracking = np.asarray(plane_seg[tracking_key].data)
tracked_idxs = np.where(np.isfinite(roi_tracking))[0]
tracking_order = np.argsort(roi_tracking[tracked_idxs])
# ordered indices of tracked ROIs
tracked_rois = tracked_idxs[tracking_order]
return tracked_rois
def load_stimulus_table_nwb(sess_files, full_table=True):
"""
load_stimulus_table_nwb(sess_files)
Retrieves stimulus dataframe.
Arguments:
sess_files (Path): full path names of the session files
Returns:
df (pandas): stimulus table.
"""
sess_file = sess_file_util.select_nwb_sess_path(sess_files)
exclude = set() if full_table else set(FULL_TABLE_COLUMNS)
# a bit long
with pynwb.NWBHDF5IO(str(sess_file), "r") as f:
nwbfile_in = f.read()
df = nwbfile_in.trials.to_dataframe(exclude=exclude)
# rename time columns
df = df.rename(columns={
"start_time": "start_time_sec",
"stop_time": "stop_time_sec"
})
df["duration_sec"] = df["stop_time_sec"] - df["start_time_sec"]
# add 2p and stimulus frames back in, if necessary
if "start_frame_stim" not in df.columns:
twop_timestamps, stim_timestamps = \
sess_sync_util.get_frame_timestamps_nwb(sess_files)
df = add_frames_from_timestamps(df, twop_timestamps, stim_timestamps)
# sort columns
column_order = [col for col in FINAL_COLUMN_ORDER if col in df.columns]
df = df[column_order]
return df
def load_pup_data_nwb(sess_files):
"""
load_pup_data_nwb(sess_files)
Returns pre-processed pupil data from NWB files.
Required args:
- sess_files (Path): full path names of the session files
Returns:
- pup_data_df (pd DataFrame): pupil data dataframe with columns:
- frames (int) : frame number
- pup_diam (float) : median pupil diameter in pixels
"""
behav_file = sess_file_util.select_nwb_sess_path(sess_files, behav=True)
with pynwb.NWBHDF5IO(str(behav_file), "r") as f:
nwbfile_in = f.read()
behav_module = nwbfile_in.get_processing_module("behavior")
main_field = "PupilTracking"
data_field = "pupil_diameter"
try:
behav_time_series = behav_module.get_data_interface(
main_field).get_timeseries(data_field)
except KeyError as err:
raise KeyError(
"Could not find pupil diameter data in behavioral time "
f"series for {behav_module} due to: {err}"
)
pup_data = np.asarray(behav_time_series.data)
pup_data_df = pd.DataFrame()
pup_data_df["pup_diam"] = pup_data / sess_sync_util.MM_PER_PIXEL
pup_data_df.insert(0, "frames", value=range(len(pup_data_df)))
return pup_data_df
def get_frame_timestamps_nwb(sess_files):
"""
get_frame_timestamps_nwb(sess_files)
Returns time stamps for stimulus and two-photon frames.
Required args:
- sess_files (Path): full path names of the session files
Returns:
- stim_timestamps (1D array): time stamp for each stimulus frame
- twop_timestamps (1D array): time stamp for each two-photon frame
"""
behav_file = sess_file_util.select_nwb_sess_path(sess_files, behav=True)
use_ophys = False
with pynwb.NWBHDF5IO(str(behav_file), "r") as f:
nwbfile_in = f.read()
behav_module = nwbfile_in.get_processing_module("behavior")
main_field = "BehavioralTimeSeries"
data_field = "running_velocity"
try:
run_time_series = behav_module.get_data_interface(
main_field).get_timeseries(data_field)
except KeyError as err:
raise KeyError(
"Could not find running velocity data in behavioral time "
f"series for {behav_module} due to: {err}")
stim_timestamps = np.asarray(run_time_series.timestamps)
main_field = "PupilTracking"
data_field = "pupil_diameter"
try:
twop_timestamps = np.asarray(
behav_module.get_data_interface(main_field).get_timeseries(
data_field).timestamps)
except KeyError as err:
use_ophys = True
# if timestamps weren't found with pupil data, look for optical physiology
# data
if use_ophys:
ophys_file = sess_file_util.select_nwb_sess_path(sess_files,
ophys=True)
with pynwb.NWBHDF5IO(str(ophys_file), "r") as f:
nwbfile_in = f.read()
ophys_module = nwbfile_in.get_processing_module("ophys")
main_field = "DfOverF"
data_field = "RoiResponseSeries"
try:
roi_resp_series = ophys_module.get_data_interface(
main_field).get_roi_response_series(data_field)
except KeyError:
file_str = f"{behav_file} or {ophys_file}"
if behav_file == ophys_file:
file_str = behav_file
raise OSError(
"Two-photon timestamps cannot be collected, as no "
f"pupil or ROI series data was found in {file_str}.")
twop_timestamps = roi_resp_series.timestamps
return stim_timestamps, twop_timestamps
