def __init__(self, tiff_path: pathlib.Path):
self._file_path = tiff_path
self._metadata = ScanImageMetadata(tiff_path=tiff_path)
self._validate_z_stack()
self._get_z_manifest()
self._frame_shape = dict()
def test_zs_for_roi(tmp_path_factory, mock_2x3_metadata_zs,
mock_2x3_metadata_zsAllActuators,
mock_1x3_metadata_zsAllActuators,
mock_3x1_metadata_zsAllActuators):
"""
Test behavior of ScanImageMetadata.zs_for_roi
"""
tmpdir = tmp_path_factory.mktemp('test_all_zs')
tmp_path = pathlib.Path(tempfile.mkstemp(dir=tmpdir, suffix='.tiff')[1])
to_replace = 'ophys_etl.modules.mesoscope_splitting.'
to_replace += 'tiff_metadata._read_metadata'
for metadata_fixture in (mock_2x3_metadata_zs,
mock_2x3_metadata_zsAllActuators,
mock_1x3_metadata_zsAllActuators,
mock_3x1_metadata_zsAllActuators):
expected_rois = metadata_fixture[2]
with patch(to_replace, new=Mock(return_value=metadata_fixture[0])):
metadata = ScanImageMetadata(tiff_path=tmp_path)
assert metadata.n_rois == len(expected_rois)
for i_roi in range(metadata.n_rois):
assert metadata.zs_for_roi(i_roi) == expected_rois[i_roi]['zs']
with pytest.raises(ValueError, match="You asked for ROI"):
metadata.zs_for_roi(metadata.n_rois)
def test_all_zs(tmp_path_factory, mock_2x3_metadata_zs,
mock_2x3_metadata_zsAllActuators,
mock_1x3_metadata_zsAllActuators,
mock_3x1_metadata_zsAllActuators):
"""
Test that ScanImageMetadata.all_zs() returns the expected result
"""
tmpdir = tmp_path_factory.mktemp('test_all_zs')
tmp_path = pathlib.Path(tempfile.mkstemp(dir=tmpdir, suffix='.tiff')[1])
expected = [[1, 4, 2], [9, 5, 11], [2, 6, 1]]
to_replace = 'ophys_etl.modules.mesoscope_splitting.'
to_replace += 'tiff_metadata._read_metadata'
for metadata_fixture in (mock_2x3_metadata_zs,
mock_2x3_metadata_zsAllActuators,
mock_1x3_metadata_zsAllActuators,
mock_3x1_metadata_zsAllActuators):
expected = metadata_fixture[1]
with patch(to_replace, new=Mock(return_value=metadata_fixture[0])):
metadata = ScanImageMetadata(tiff_path=tmp_path)
assert expected == metadata.all_zs()
def test_all_zs_error(tmp_path_factory, mock_2x3_metadata_nozs):
"""
Test that expected exception is thrown when metadata lacks both
SI.hStackManager.zs and SI.hStackManager.zsAllActuators
"""
tmpdir = tmp_path_factory.mktemp('test_all_zs')
tmp_path = pathlib.Path(tempfile.mkstemp(dir=tmpdir, suffix='.tiff')[1])
to_replace = 'ophys_etl.modules.mesoscope_splitting.'
to_replace += 'tiff_metadata._read_metadata'
with patch(to_replace, new=Mock(return_value=mock_2x3_metadata_nozs)):
metadata = ScanImageMetadata(tiff_path=tmp_path)
with pytest.raises(ValueError, match="Cannot load all_zs"):
metadata.all_zs()
def test_no_file_error():
"""
Test that, ScanImageMetadata raises an error if you
give it a path that doesn't point to a file
"""
dummy_path = pathlib.Path('not_a_file.tiff')
with pytest.raises(ValueError, match="is not a file"):
ScanImageMetadata(tiff_path=dummy_path)
class ScanImageTiffSplitter(IntFromZMapperMixin):
"""
A class to naively split up a tiff file by just looping over
the scanfields in its ROIs
**this will not work for z-stacks**
Parameters
----------
tiff_path: pathlib.Path
Path to the TIFF file whose metadata we are parsing
"""
def __init__(self, tiff_path: pathlib.Path):
self._file_path = tiff_path
self._metadata = ScanImageMetadata(tiff_path=tiff_path)
self._validate_z_stack()
self._get_z_manifest()
self._frame_shape = dict()
def _validate_z_stack(self):
"""
Make sure that the zsAllActuators are arranged the
way we expect, i.e.
[[roi0_z0, roi0_z1, roi0_z2..., roi0_zM],
[roi0_zM+1, roi0_zM+2, ...],
[roi1_z0, roi1_z1, roi1_z2..., roi1_zM],
[roi1_zM+1, roi1_zM+2, ...],
...
[roiN_z0, roiN_z1, roiN_z2...]]
or, in the case of one ROI
[z0, z1, z2....]
"""
z_value_array = self._metadata.all_zs()
# check that z_value_array is a list of lists
if not isinstance(z_value_array, list):
msg = "Unclear how to split this TIFF\n"
msg += f"{self._file_path.resolve().absolute()}\n"
msg += f"metadata.all_zs {self._metadata.all_zs()}"
raise RuntimeError(msg)
if isinstance(z_value_array[0], list):
z_value_array = np.concatenate(z_value_array)
defined_rois = self._metadata.defined_rois
z_int_per_roi = []
msg = ""
# check that the same Z value does not appear more than
# once in the same ROI
for i_roi, roi in enumerate(defined_rois):
if isinstance(roi['zs'], list):
roi_zs = roi['zs']
else:
roi_zs = [
roi['zs'],
]
roi_z_ints = [self._int_from_z(z_value=zz) for zz in roi_zs]
z_int_set = set(roi_z_ints)
if len(z_int_set) != len(roi_zs):
msg += f"roi {i_roi} has duplicate zs: {roi['zs']}\n"
z_int_per_roi.append(z_int_set)
# check that z values in z_array occurr in ROI order
offset = 0
n_roi = len(z_int_per_roi)
n_z_per_roi = len(z_value_array) // n_roi
# check that every ROI has the same number of zs
if len(z_value_array) % n_roi != 0:
msg += "There do not appear to be an "
msg += "equal number of zs per ROI\n"
msg += f"n_z: {len(z_value_array)} "
msg += f"n_roi: {len(z_int_per_roi)}\n"
for roi_z_ints in z_int_per_roi:
these_z_ints = set([
self._int_from_z(z_value=zz)
for zz in z_value_array[offset:offset + n_z_per_roi]
])
if these_z_ints != roi_z_ints:
these_z_ints = set([
self._int_from_z(z_value=zz)
for zz in z_value_array[offset:offset + n_z_per_roi - 1]
])
# might be placeholder value == 0
odd_value = z_value_array[offset + n_z_per_roi - 1]
if np.abs(odd_value) >= 1.0e-6 or these_z_ints != roi_z_ints:
msg += "z_values from sub array "
msg += "not in correct order for ROIs; "
break
offset += n_z_per_roi
if len(msg) > 0:
full_msg = "Unclear how to split this TIFF\n"
full_msg += f"{self._file_path.resolve().absolute()}\n"
full_msg += f"{msg}"
full_msg += f"all_zs {self._metadata.all_zs()}\nfrom rois:\n"
for roi in self._metadata.defined_rois:
full_msg += f"zs: {roi['zs']}\n"
raise RuntimeError(full_msg)
def _get_z_manifest(self):
"""
Populate various member objects that help us keep
track of what z values go with what ROIs in this TIFF
"""
local_z_value_list = np.array(self._metadata.all_zs()).flatten()
defined_rois = self._metadata.defined_rois
# create a list of sets indicating which z values were actually
# scanned in the ROI (this will help us parse the placeholder
# zeros that sometimes get dropped into
# SI.hStackManager.zsAllActuators
valid_z_int_per_roi = []
valid_z_per_roi = []
for roi in defined_rois:
this_z_value = roi['zs']
if isinstance(this_z_value, int):
this_z_value = [
this_z_value,
]
z_as_int = [self._int_from_z(z_value=zz) for zz in this_z_value]
valid_z_int_per_roi.append(set(z_as_int))
valid_z_per_roi.append(this_z_value)
self._valid_z_int_per_roi = valid_z_int_per_roi
self._valid_z_per_roi = valid_z_per_roi
self._n_valid_zs = 0
self._roi_z_int_manifest = []
ct = 0
i_roi = 0
local_z_index_list = [
self._int_from_z(zz) for zz in local_z_value_list
]
for zz in local_z_index_list:
if i_roi >= len(valid_z_int_per_roi):
break
if zz in valid_z_int_per_roi[i_roi]:
roi_z = (i_roi, zz)
self._roi_z_int_manifest.append(roi_z)
self._n_valid_zs += 1
ct += 1
if ct == len(valid_z_int_per_roi[i_roi]):
i_roi += 1
ct = 0
@property
def input_path(self) -> pathlib.Path:
"""
The file this splitter is trying to split
"""
return self._file_path
def is_z_valid_for_roi(self, i_roi: int, z_value: float) -> bool:
"""
Is specified z-value valid for the specified ROI
"""
z_as_int = self._int_from_z(z_value=z_value)
return z_as_int in self._valid_z_int_per_roi[i_roi]
@property
def roi_z_int_manifest(self) -> List[Tuple[int, int]]:
"""
A list of tuples. Each tuple is a valid
(roi_index, z_as_int) pair.
"""
return self._roi_z_int_manifest
@property
def n_valid_zs(self) -> int:
"""
The total number of valid z values associated with this TIFF.
"""
return self._n_valid_zs
@property
def n_rois(self) -> int:
"""
The number of ROIs in this TIFF
"""
return self._metadata.n_rois
@property
def n_pages(self):
"""
The number of pages in this TIFF
"""
if not hasattr(self, '_n_pages'):
with tifffile.TiffFile(self._file_path, mode='rb') as tiff_file:
self._n_pages = len(tiff_file.pages)
return self._n_pages
def roi_center(self, i_roi: int) -> Tuple[float, float]:
"""
The (X, Y) center coordinates of roi_index=i_roi
"""
return self._metadata.roi_center(i_roi=i_roi)
def _get_offset(self, i_roi: int, z_value: float) -> int:
"""
Get the first page associated with the specified
i_roi, z_value pair.
"""
found_it = False
n_step_over = 0
this_roi_z = (i_roi, self._int_from_z(z_value=z_value))
for roi_z_pair in self.roi_z_int_manifest:
if roi_z_pair == this_roi_z:
found_it = True
break
n_step_over += 1
if not found_it:
msg = f"Could not find stride for {i_roi}, {z_value}\n"
msg += f"TIFF file {self._file_path.resolve().absolute()}"
raise ValueError(msg)
return n_step_over
def _get_pages(self, i_roi: int, z_value: float) -> List[np.ndarray]:
"""
Get a list of np.ndarrays representing the pages of image data
for ROI i_roi at the specified z_value
"""
if i_roi >= self.n_rois:
msg = f"You asked for ROI {i_roi}; "
msg += f"there are only {self.n_rois} ROIs "
msg += f"in {self._file_path.resolve().absolute()}"
raise ValueError(msg)
if not self.is_z_valid_for_roi(i_roi=i_roi, z_value=z_value):
msg = f"{z_value} is not a valid z value for ROI {i_roi};"
msg += f"valid z values are {self._valid_z_per_roi[i_roi]}\n"
msg += f"TIFF file {self._file_path.resolve().absolute()}"
raise ValueError(msg)
offset = self._get_offset(i_roi=i_roi, z_value=z_value)
tiff_data = []
with tifffile.TiffFile(self._file_path, mode='rb') as tiff_file:
for i_page in range(offset, self.n_pages, self.n_valid_zs):
arr = tiff_file.pages[i_page].asarray()
tiff_data.append(arr)
key_pair = (i_roi, z_value)
if key_pair in self._frame_shape:
if arr.shape != self._frame_shape[key_pair]:
msg = f"ROI {i_roi} z_value {z_value}\n"
msg += "yields inconsistent frame shape"
raise RuntimeError(msg)
else:
self._frame_shape[key_pair] = arr.shape
return tiff_data
def frame_shape(self, i_roi: int,
z_value: Optional[float]) -> Tuple[int, int]:
"""
Get the shape of the image for a specified ROI at a specified
z value
Parameters
----------
i_roi: int
index of the ROI
z_value: Optional[float]
value of z. If None, z_value will be detected automaticall
(assuming there is no ambiguity)
Returns
-------
frame_shape: Tuple[int, int]
(nrows, ncolumns)
"""
if z_value is None:
z_value = self._get_z_value(i_roi=i_roi)
key_pair = (i_roi, self._int_from_z(z_value))
if key_pair not in self._frame_shape:
offset = self._get_offset(i_roi=i_roi, z_value=z_value)
with tifffile.TiffFile(self._file_path, mode='rb') as tiff_file:
page = tiff_file.pages[offset].asarray()
self._frame_shape[key_pair] = page.shape
return self._frame_shape[key_pair]
def _get_z_value(self, i_roi: int) -> float:
"""
Return the z_value associated with i_roi, assuming
there is only one. Raises a RuntimeError if there
is more than one.
"""
# When splitting surface TIFFs, there's no sensible
# way to know the z-value ahead of time (whatever the
# operator enters is just a placeholder). The block
# of code below will scan for z-values than align with
# the specified ROI ID and select the correct z value
# (assuming there is only one)
possible_z_values = []
for pair in self.roi_z_int_manifest:
if pair[0] == i_roi:
possible_z_values.append(pair[1])
if len(possible_z_values) > 1:
msg = f"{len(possible_z_values)} possible z values "
msg += f"for ROI {i_roi}; must specify one of\n"
msg += f"{possible_z_values}"
raise RuntimeError(msg)
z_value = possible_z_values[0]
return self._z_from_int(ii=z_value)
def write_output_file(self, i_roi: int, z_value: Optional[float],
output_path: pathlib.Path) -> None:
"""
Write the image created by averaging all of the TIFF
pages associated with an (i_roi, z_value) pair to a TIFF
file.
Parameters
----------
i_roi: int
z_value: Optional[int]
If None, will be detected automatically (assuming there
is only one)
output_path: pathlib.Path
Path to file to be written
Returns
-------
None
Output is written to output_path
"""
if output_path.suffix not in ('.tif', '.tiff'):
msg = "expected .tiff output path; "
msg += f"you specified {output_path.resolve().absolute()}"
if z_value is None:
z_value = self._get_z_value(i_roi=i_roi)
data = np.array(self._get_pages(i_roi=i_roi, z_value=z_value))
avg_img = np.mean(data, axis=0)
avg_img = normalize_array(array=avg_img,
lower_cutoff=None,
upper_cutoff=None)
tifffile.imsave(output_path, avg_img)
return None
def run(self):
t0 = time.time()
output = {"column_stacks": []}
files_to_record = []
ts_path = pathlib.Path(self.args['timeseries_tif'])
timeseries_splitter = TimeSeriesSplitter(tiff_path=ts_path)
files_to_record.append(ts_path)
depth_path = pathlib.Path(self.args["depths_tif"])
depth_splitter = ScanImageTiffSplitter(tiff_path=depth_path)
files_to_record.append(depth_path)
surface_path = pathlib.Path(self.args["surface_tif"])
surface_splitter = ScanImageTiffSplitter(tiff_path=surface_path)
files_to_record.append(surface_path)
zstack_path_list = []
for plane_grp in self.args['plane_groups']:
zstack_path = pathlib.Path(plane_grp['local_z_stack_tif'])
zstack_path_list.append(zstack_path)
files_to_record.append(zstack_path)
zstack_splitter = ZStackSplitter(tiff_path_list=zstack_path_list)
ready_to_archive = set()
# Looking at recent examples of outputs from this queue,
# I do not think we have honored the 'column_z_stack_tif'
# entry in the schema for some time now. I find no examples
# in which this entry of the input.jon is ever populated.
# I am leaving it here for now to avoid the complication of
# having to modify the ruby strategy associated with this
# queue, which is out of scope for the work we have
# currently committed to.
for plane_group in self.args["plane_groups"]:
if "column_z_stack_tif" in plane_group:
msg = "'column_z_stack_tif' detected in 'plane_groups'; "
msg += "the TIFF splitting code no longer handles that file."
self.logger.warn(msg)
# There are cases where the centers for ROIs are not
# exact across modalities, so we cannot demand that the
# ROI centers be the same to within an absolute tolerance.
# Here we use the timeseries TIFF to assemble a list of all
# available ROI centers. When splitting the other TIFFs, we
# will validate them by making sure that the closest
# valid_roi_center is always what we expect.
valid_roi_centers = get_valid_roi_centers(
timeseries_splitter=timeseries_splitter)
experiment_metadata = []
for plane_group in self.args["plane_groups"]:
for experiment in plane_group["ophys_experiments"]:
this_exp_metadata = dict()
exp_id = experiment["experiment_id"]
this_exp_metadata["experiment_id"] = exp_id
for file_key in ('timeseries', 'depth_2p', 'surface_2p',
'local_z_stack'):
this_metadata = dict()
for data_key in ('offset_x', 'offset_y', 'rotation',
'resolution'):
this_metadata[data_key] = experiment[data_key]
this_exp_metadata[file_key] = this_metadata
experiment_dir = pathlib.Path(experiment["storage_directory"])
experiment_id = experiment["experiment_id"]
roi_index = experiment["roi_index"]
scanfield_z = experiment["scanfield_z"]
baseline_center = None
for (splitter, z_value, output_name, metadata_tag) in zip(
(depth_splitter, surface_splitter, zstack_splitter,
timeseries_splitter),
(scanfield_z, None, scanfield_z, scanfield_z),
(f"{experiment_id}_depth.tif",
f"{experiment_id}_surface.tif",
f"{experiment_id}_z_stack_local.h5",
f"{experiment_id}.h5"),
("depth_2p", "surface_2p", "local_z_stack", "timeseries")):
output_path = experiment_dir / output_name
roi_center = splitter.roi_center(i_roi=roi_index)
nearest_valid = get_nearest_roi_center(
this_roi_center=roi_center,
valid_roi_centers=valid_roi_centers)
if baseline_center is None:
baseline_center = nearest_valid
if nearest_valid != baseline_center:
msg = f"experiment {experiment_id}\n"
msg += "roi center inconsistent for "
msg += "input: "
msg += f"{splitter.input_path.resolve().absolute()}\n"
msg += "output: "
msg += f"{output_path.resolve().absolute()}\n"
msg += f"{baseline_center}; {roi_center}\n"
raise RuntimeError(msg)
splitter.write_output_file(i_roi=roi_index,
z_value=z_value,
output_path=output_path)
str_path = str(output_path.resolve().absolute())
this_exp_metadata[metadata_tag]['filename'] = str_path
frame_shape = splitter.frame_shape(i_roi=roi_index,
z_value=z_value)
this_exp_metadata[metadata_tag]['height'] = frame_shape[0]
this_exp_metadata[metadata_tag]['width'] = frame_shape[1]
self.logger.info("wrote "
f"{output_path.resolve().absolute()}")
experiment_metadata.append(this_exp_metadata)
output["experiment_output"] = experiment_metadata
ready_to_archive.add(self.args["surface_tif"])
ready_to_archive.add(self.args["depths_tif"])
ready_to_archive.add(self.args["timeseries_tif"])
for zstack_path in zstack_path_list:
ready_to_archive.add(str(zstack_path.resolve().absolute()))
output["ready_to_archive"] = list(ready_to_archive)
# record file metadata
file_metadata = []
for file_path in files_to_record:
tiff_metadata = ScanImageMetadata(file_path)
this_metadata = dict()
this_metadata['input_tif'] = str(file_path.resolve().absolute())
this_metadata['scanimage_metadata'] = tiff_metadata._metadata[0]
this_metadata['roi_metadata'] = tiff_metadata._metadata[1]
file_metadata.append(this_metadata)
output["file_metadata"] = file_metadata
self.output(get_sanitized_json_data(output), indent=1)
duration = time.time() - t0
self.logger.info(f"that took {duration:.2e} seconds")
def __init__(self, tiff_path_list: List[pathlib.Path]):
self._path_to_metadata = dict()
self._frame_shape = dict()
for tiff_path in tiff_path_list:
str_path = str(tiff_path.resolve().absolute())
self._path_to_metadata[str_path] = ScanImageMetadata(
tiff_path=tiff_path)
# construct lookup tables to help us map ROI index and z-value
# to a tiff path and an index in the z-array
# map (i_roi, z_value) pairs to TIFF file paths
self._roi_z_int_to_path = dict()
# map (tiff_file_path, z_value) to the index, i.e.
# to which scanned z-value *in this TIFF* does the
# z-value correspond.
self._path_z_int_to_index = dict()
# this is an internal lookup table which we will use
# to validate that every ROI is represented by a
# z-stack file
roi_to_path = dict()
for tiff_path in self._path_to_metadata.keys():
metadata = self._path_to_metadata[tiff_path]
this_roi = None
for i_roi, roi in enumerate(metadata.defined_rois):
if roi['discretePlaneMode'] == 0:
if this_roi is not None:
raise RuntimeError("More than one ROI has "
"discretePlaneMode==0 for "
"{tiff_path}")
this_roi = i_roi
if this_roi is None:
raise RuntimeError("Could not find discretePlaneMode==0 for "
f"{tiff_path}")
if this_roi not in roi_to_path:
roi_to_path[this_roi] = []
roi_to_path[this_roi].append(tiff_path)
z_array = np.array(metadata.all_zs())
if z_array.shape[1] != 2:
raise RuntimeError(f"z_array for {tiff_path} has odd shape\n"
f"{z_array}")
z_mean = z_array.mean(axis=0)
for ii, z_value in enumerate(z_mean):
roi_z = (this_roi, self._int_from_z(z_value=z_value))
self._roi_z_int_to_path[roi_z] = tiff_path
path_z = (tiff_path, self._int_from_z(z_value=z_value))
self._path_z_int_to_index[path_z] = ii
# check that every ROI has a z-stack file
for tiff_path in self._path_to_metadata:
metadata = self._path_to_metadata[tiff_path]
n_rois = len(metadata.defined_rois)
if len(roi_to_path) != n_rois:
msg = (f"There are {n_rois} ROIs; however, only "
f"{len(roi_to_path)} of them are represented in the "
"local z-stack TIFFS. Here is a mapping from i_roi to "
"TIFF paths\n"
f"{json.dumps(roi_to_path, indent=2, sort_keys=True)}"
"\n\nThis was determined by scanning the z-stack TIFFs "
"and noting which ROIs were marked with "
"discretePlaneMode==0")
raise RuntimeError(msg)
self._path_to_pages = dict()
for tiff_path in self._path_to_metadata.keys():
with tifffile.TiffFile(tiff_path, mode='rb') as tiff_file:
self._path_to_pages[tiff_path] = len(tiff_file.pages)