def get_aligned_tileset():
alignedTileset = TileSet(
[Axes.X, Axes.Y, Axes.CH, Axes.ZPLANE, Axes.ROUND], {
Axes.CH: NUM_CH,
Axes.ROUND: NUM_ROUND,
Axes.ZPLANE: NUM_Z
}, {
Axes.Y: HEIGHT,
Axes.X: WIDTH
})
for r in range(NUM_ROUND):
for ch in range(NUM_CH):
for z in range(NUM_Z):
tile = Tile(
{
Coordinates.X: 1,
Coordinates.Y: 4,
Coordinates.Z: 3,
}, {
Axes.ROUND: r,
Axes.CH: ch,
Axes.ZPLANE: z,
})
tile.numpy_array = np.zeros((100, 100))
alignedTileset.add_tile(tile)
return alignedTileset
def get_un_aligned_tileset():
unAlignedTileset = TileSet(
[Axes.X, Axes.Y, Axes.CH, Axes.ZPLANE, Axes.ROUND], {
Axes.CH: NUM_CH,
Axes.ROUND: NUM_ROUND,
Axes.ZPLANE: NUM_Z
}, {
Axes.Y: HEIGHT,
Axes.X: WIDTH
})
for r in range(NUM_ROUND):
for ch in range(NUM_CH):
for z in range(NUM_Z):
tile = Tile(
{
# The round_to methods generate coordinates
# based on the r value, therefore the coords vary
# throughout the tileset
Coordinates.X:
round_to_x(r),
Coordinates.Y:
round_to_y(r),
Coordinates.Z:
round_to_z(r),
},
{
Axes.ROUND: r,
Axes.CH: ch,
Axes.ZPLANE: z,
})
tile.numpy_array = np.zeros((HEIGHT, WIDTH))
unAlignedTileset.add_tile(tile)
return unAlignedTileset
def build_image(fov_count, hyb_count, ch_count, z_count,
image_fetcher: ImageFetcher, default_shape: Tuple[int, int]):
"""
Build and returns an image set with the following characteristics:
Parameters
----------
fov_count : int
Number of fields of view in this image set.
hyb_count : int
Number for hybridizations in this image set.
ch_count : int
Number for channels in this image set.
z_count : int
Number of z-layers in this image set.
default_shape : Tuple[int, int]
Default shape of the individual tiles in this image set.
Returns
-------
The slicedimage collection representing the image.
"""
collection = Collection()
for fov_ix in range(fov_count):
fov_images = TileSet(
[Coordinates.X, Coordinates.Y, Indices.Z, Indices.HYB, Indices.CH],
{
Indices.HYB: hyb_count,
Indices.CH: ch_count,
Indices.Z: z_count
},
default_shape,
ImageFormat.TIFF,
)
for z_ix in range(z_count):
for hyb_ix in range(hyb_count):
for ch_ix in range(ch_count):
image = image_fetcher.get_image(fov_ix, hyb_ix, ch_ix,
z_ix)
tile = Tile(
{
Coordinates.X: (0.0, 0.0001),
Coordinates.Y: (0.0, 0.0001),
Coordinates.Z: (0.0, 0.0001),
},
{
Indices.Z: z_ix,
Indices.HYB: hyb_ix,
Indices.CH: ch_ix,
},
image.shape,
)
tile.set_source_fh_contextmanager(image.image_data_handle,
image.format)
fov_images.add_tile(tile)
collection.add_partition("fov_{:03}".format(fov_ix), fov_images)
return collection
def synthetic_stack(
cls,
num_hyb: int = 4,
num_ch: int = 4,
num_z: int = 12,
tile_height: int = 50,
tile_width: int = 40,
tile_data_provider: Callable[[int, int, int, int, int],
np.ndarray] = None,
tile_extras_provider: Callable[[int, int, int], Any] = None,
) -> "ImageStack":
"""generate a synthetic ImageStack
Returns
-------
ImageStack :
imagestack containing a tensor whose default shape is (2, 3, 4, 30, 20)
and whose default values are all 1.
"""
if tile_data_provider is None:
tile_data_provider = cls._default_tile_data_provider
if tile_extras_provider is None:
tile_extras_provider = cls._default_tile_extras_provider
img = TileSet(
{Coordinates.X, Coordinates.Y, Indices.HYB, Indices.CH, Indices.Z},
{
Indices.HYB: num_hyb,
Indices.CH: num_ch,
Indices.Z: num_z,
},
default_tile_shape=(tile_height, tile_width),
)
for hyb in range(num_hyb):
for ch in range(num_ch):
for z in range(num_z):
tile = Tile(
{
Coordinates.X: (0.0, 0.001),
Coordinates.Y: (0.0, 0.001),
Coordinates.Z: (0.0, 0.001),
},
{
Indices.HYB: hyb,
Indices.CH: ch,
Indices.Z: z,
},
extras=tile_extras_provider(hyb, ch, z),
)
tile.numpy_array = tile_data_provider(
hyb, ch, z, tile_height, tile_width)
img.add_tile(tile)
stack = cls(img)
return stack
def synthetic_stack(
num_hyb: int=DEFAULT_NUM_HYB,
num_ch: int=DEFAULT_NUM_CH,
num_z: int=DEFAULT_NUM_Z,
tile_height: int=DEFAULT_HEIGHT,
tile_width: int=DEFAULT_WIDTH,
tile_data_provider: Callable[[int, int, int, int, int], np.ndarray]=default_tile_data_provider,
tile_extras_provider: Callable[[int, int, int], Any]=default_tile_extras_provider
) -> ImageStack:
"""generate a synthetic ImageStack
Returns
-------
ImageStack :
imagestack containing a tensor of (2, 3, 4, 30, 20) whose values are all 1.
"""
img = TileSet(
{Coordinates.X, Coordinates.Y, Indices.HYB, Indices.CH, Indices.Z},
{
Indices.HYB: num_hyb,
Indices.CH: num_ch,
Indices.Z: num_z,
},
default_tile_shape=(tile_height, tile_width),
)
for hyb in range(num_hyb):
for ch in range(num_ch):
for z in range(num_z):
tile = Tile(
{
Coordinates.X: (0.0, 0.001),
Coordinates.Y: (0.0, 0.001),
Coordinates.Z: (0.0, 0.001),
},
{
Indices.HYB: hyb,
Indices.CH: ch,
Indices.Z: z,
},
extras=tile_extras_provider(hyb, ch, z),
)
tile.numpy_array = tile_data_provider(hyb, ch, z, tile_height, tile_width)
img.add_tile(tile)
stack = ImageStack(img)
return stack
def parse_coordinate_groups(tileset: TileSet) -> List["CropParameters"]:
"""Takes a tileset and compares the physical coordinates on each tile to
create aligned coordinate groups (groups of tiles that have the same physical coordinates)
Returns
-------
A list of CropParameters. Each entry describes the r/ch/z values of tiles that are aligned
(have matching coordinates)
"""
coord_groups: OrderedDict[tuple, CropParameters] = OrderedDict()
for tile in tileset.tiles():
x_y_coords = (tile.coordinates[Coordinates.X][0],
tile.coordinates[Coordinates.X][1],
tile.coordinates[Coordinates.Y][0],
tile.coordinates[Coordinates.Y][1])
# A tile with this (x, y) has already been seen, add tile's Indices to CropParameters
if x_y_coords in coord_groups:
crop_params = coord_groups[x_y_coords]
crop_params._add_permitted_axes(Axes.CH, tile.indices[Axes.CH])
crop_params._add_permitted_axes(Axes.ROUND,
tile.indices[Axes.ROUND])
if Axes.ZPLANE in tile.indices:
crop_params._add_permitted_axes(Axes.ZPLANE,
tile.indices[Axes.ZPLANE])
else:
coord_groups[x_y_coords] = CropParameters(
permitted_chs=[tile.indices[Axes.CH]],
permitted_rounds=[tile.indices[Axes.ROUND]],
permitted_zplanes=[tile.indices[Axes.ZPLANE]]
if Axes.ZPLANE in tile.indices else None)
return list(coord_groups.values())
def __init__(self, tileset: TileSet) -> None:
self.tiles: MutableMapping[TileKey, Tile] = dict()
for tile in tileset.tiles():
key = TileKey(
round=tile.indices[Axes.ROUND],
ch=tile.indices[Axes.CH],
zplane=tile.indices.get(Axes.ZPLANE, 0))
self.tiles[key] = tile
self._extras = tileset.extras
self._expectations = _TileSetConsistencyDetector()
def __init__(self, tileset: TileSet) -> None:
tile_extras: MutableMapping[TileKey, dict] = dict()
for tile in tileset.tiles():
round_ = tile.indices[Indices.ROUND]
ch = tile.indices[Indices.CH]
z = tile.indices.get(Indices.Z, 0)
tile_extras[TileKey(round=round_, ch=ch, z=z)] = tile.extras
self.tile_extras: Mapping[TileKey, dict] = tile_extras
self._extras = tileset.extras
def set_aux(self, key, img):
if key in self.auxiliary_images:
old_img = self.auxiliary_images[key]
if old_img.shape != img.shape:
msg = "Shape mismatch. Current data shape: {}, new data shape: {}".format(
old_img.shape, img.shape)
raise AttributeError(msg)
self.auxiliary_images[key].numpy_array = img
else:
# TODO: (ttung) major hack alert. we don't have a convenient mechanism to build an ImageStack from a single
# numpy array, which we probably should.
tileset = TileSet(
{
Indices.HYB,
Indices.CH,
Indices.Z,
Coordinates.X,
Coordinates.Y,
}, {
Indices.HYB: 1,
Indices.CH: 1,
Indices.Z: 1,
})
tile = Tile(
{
Coordinates.X: (0.000, 0.001),
Coordinates.Y: (0.000, 0.001),
},
{
Indices.HYB: 0,
Indices.CH: 0,
Indices.Z: 0,
},
img.shape,
)
tile.numpy_array = img
tileset.add_tile(tile)
self.auxiliary_images[key] = ImageStack(tileset)
self.org['auxiliary_images'][key] = f"{key}.json"
def __init__(self, tileset: TileSet) -> None:
self._tile_shape = tileset.default_tile_shape
self.tiles: MutableMapping[TileKey, Tile] = dict()
for tile in tileset.tiles():
key = TileKey(round=tile.indices[Axes.ROUND],
ch=tile.indices[Axes.CH],
zplane=tile.indices.get(Axes.ZPLANE, 0))
self.tiles[key] = tile
# if we don't have the tile shape, then we peek at the tile and get its shape.
if self._tile_shape is None:
self._tile_shape = tile.tile_shape
self._extras = tileset.extras
def export(self,
filepath: str,
tile_opener=None,
tile_format: ImageFormat = ImageFormat.NUMPY) -> None:
"""write the image tensor to disk in spaceTx format
Parameters
----------
filepath : str
Path + prefix for the images and primary_images.json written by this function
tile_opener : TODO ttung: doc me.
tile_format : ImageFormat
Format in which each 2D plane should be written.
"""
tileset = TileSet(
dimensions={
Indices.ROUND,
Indices.CH,
Indices.Z,
Indices.Y,
Indices.X,
},
shape={
Indices.ROUND: self.num_rounds,
Indices.CH: self.num_chs,
Indices.Z: self.num_zlayers,
},
default_tile_shape=self._tile_shape,
extras=self._tile_metadata.extras,
)
for round_ in range(self.num_rounds):
for ch in range(self.num_chs):
for zlayer in range(self.num_zlayers):
tilekey = TileKey(round=round_, ch=ch, z=zlayer)
extras: dict = self._tile_metadata[tilekey]
tile_indices = {
Indices.ROUND: round_,
Indices.CH: ch,
Indices.Z: zlayer,
}
coordinates: MutableMapping[Coordinates,
Tuple[Number,
Number]] = dict()
x_coordinates = self.tile_coordinates(
tile_indices, Coordinates.X)
y_coordinates = self.tile_coordinates(
tile_indices, Coordinates.Y)
z_coordinates = self.tile_coordinates(
tile_indices, Coordinates.Z)
coordinates[Coordinates.X] = x_coordinates
coordinates[Coordinates.Y] = y_coordinates
if z_coordinates[0] != np.nan and z_coordinates[
1] != np.nan:
coordinates[Coordinates.Z] = z_coordinates
tile = Tile(
coordinates=coordinates,
indices=tile_indices,
extras=extras,
)
tile.numpy_array, _ = self.get_slice(indices={
Indices.ROUND: round_,
Indices.CH: ch,
Indices.Z: zlayer
})
tileset.add_tile(tile)
if tile_opener is None:
def tile_opener(tileset_path, tile, ext):
tile_basename = os.path.splitext(tileset_path)[0]
if Indices.Z in tile.indices:
zval = tile.indices[Indices.Z]
zstr = "-Z{}".format(zval)
else:
zstr = ""
return open(
"{}-H{}-C{}{}.{}".format(
tile_basename,
tile.indices[Indices.ROUND],
tile.indices[Indices.CH],
zstr,
ext,
), "wb")
if not filepath.endswith('.json'):
filepath += '.json'
Writer.write_to_path(tileset,
filepath,
pretty=True,
tile_opener=tile_opener,
tile_format=tile_format)
def _get_dimension_size(tileset: TileSet, dimension: Axes):
axis_data = AXES_DATA[dimension]
if dimension in tileset.dimensions or axis_data.required:
return tileset.get_dimension_shape(dimension)
return 1
def export(self,
filepath: str,
tile_opener=None,
tile_format: ImageFormat = ImageFormat.NUMPY) -> None:
"""write the image tensor to disk in spaceTx format
Parameters
----------
filepath : str
Path + prefix for the images and primary_images.json written by this function
tile_opener : TODO ttung: doc me.
tile_format : ImageFormat
Format in which each 2D plane should be written.
"""
# Add log data to extras
self._tile_data.extras[STARFISH_EXTRAS_KEY] = logging.LogEncoder(
).encode({LOG: self.log})
tileset = TileSet(
dimensions={
Axes.ROUND,
Axes.CH,
Axes.ZPLANE,
Axes.Y,
Axes.X,
},
shape={
Axes.ROUND: self.num_rounds,
Axes.CH: self.num_chs,
Axes.ZPLANE: self.num_zplanes,
},
default_tile_shape={
Axes.Y: self.tile_shape[0],
Axes.X: self.tile_shape[1]
},
extras=self._tile_data.extras,
)
for axis_val_map in self._iter_axes({Axes.ROUND, Axes.CH,
Axes.ZPLANE}):
tilekey = TileKey(round=axis_val_map[Axes.ROUND],
ch=axis_val_map[Axes.CH],
zplane=axis_val_map[Axes.ZPLANE])
round_, ch, zplane = tilekey.round, tilekey.ch, tilekey.z
extras: dict = self._tile_data[tilekey]
selector = {
Axes.ROUND: round_,
Axes.CH: ch,
Axes.ZPLANE: zplane,
}
coordinates: MutableMapping[Coordinates, Union[Tuple[Number,
Number],
Number]] = dict()
x_coordinates = (float(self.xarray[Coordinates.X.value][0]),
float(self.xarray[Coordinates.X.value][-1]))
y_coordinates = (float(self.xarray[Coordinates.Y.value][0]),
float(self.xarray[Coordinates.Y.value][-1]))
coordinates[Coordinates.X] = x_coordinates
coordinates[Coordinates.Y] = y_coordinates
if Coordinates.Z in self.xarray.coords:
# set the z coord to the calculated value from the associated z plane
z_coordinates = float(self.xarray[Coordinates.Z.value][zplane])
coordinates[Coordinates.Z] = z_coordinates
tile = Tile(
coordinates=coordinates,
indices=selector,
extras=extras,
)
tile.numpy_array, _ = self.get_slice(selector={
Axes.ROUND: round_,
Axes.CH: ch,
Axes.ZPLANE: zplane
})
tileset.add_tile(tile)
if tile_opener is None:
def tile_opener(tileset_path: Path, tile, ext):
base = tileset_path.parent / tileset_path.stem
if Axes.ZPLANE in tile.indices:
zval = tile.indices[Axes.ZPLANE]
zstr = "-Z{}".format(zval)
else:
zstr = ""
return open(
"{}-H{}-C{}{}.{}".format(
str(base),
tile.indices[Axes.ROUND],
tile.indices[Axes.CH],
zstr,
ext,
), "wb")
if not filepath.endswith('.json'):
filepath += '.json'
Writer.write_to_path(tileset,
filepath,
pretty=True,
tile_opener=tile_opener,
tile_format=tile_format)
def parse_aligned_groups(
tileset: TileSet,
rounds: Optional[Collection[int]] = None,
chs: Optional[Collection[int]] = None,
zplanes: Optional[Collection[int]] = None,
x: Optional[Union[int, slice]] = None,
y: Optional[Union[int, slice]] = None) -> List["CropParameters"]:
"""Takes a tileset and any optional selected axes lists compares the physical coordinates on each
tile to create aligned coordinate groups (groups of tiles that have the same physical
coordinates)
Parameters
----------
tileset: TileSet
The TileSet to parse
rounds: Optional[Collection[int]]
The rounds in the tileset to include in the final aligned groups. If this is not set,
then all rounds are included.
chs: Optional[Collection[int]]
The chs in the tileset to include in the final aligned groups. If this is not set,
then all chs are included.
zplanes: Optional[Collection[int]]
The zplanes in the tileset to include in the final aligned groups. If this is not set,
then all zplanes are included.
x: Optional[Union[int, slice]]
The x-range in the x-y tile to include in the final aligned groups. If this is not set,
then the entire x-y tile is included.
y: Optional[Union[int, slice]]
The y-range in the x-y tile to include in the final aligned groups. If this is not set,
then the entire x-y tile is included.
Returns
-------
List["CropParameters"]
A list of CropParameters. Each entry describes the r/ch/z values of tiles that are
aligned (have matching coordinates) and are within the selected_axes if provided.
"""
coord_groups: OrderedDict[tuple, CropParameters] = OrderedDict()
for tile in tileset.tiles():
if CropParameters.tile_in_selected_axes(tile, rounds, chs,
zplanes):
x_y_coords = (tile.coordinates[Coordinates.X][0],
tile.coordinates[Coordinates.X][1],
tile.coordinates[Coordinates.Y][0],
tile.coordinates[Coordinates.Y][1])
# A tile with this (x, y) has already been seen, add tile's indices to
# CropParameters
if x_y_coords in coord_groups:
crop_params = coord_groups[x_y_coords]
crop_params._add_permitted_axes(Axes.CH,
tile.indices[Axes.CH])
crop_params._add_permitted_axes(Axes.ROUND,
tile.indices[Axes.ROUND])
if Axes.ZPLANE in tile.indices:
crop_params._add_permitted_axes(
Axes.ZPLANE, tile.indices[Axes.ZPLANE])
else:
coord_groups[x_y_coords] = CropParameters(
permitted_chs=[tile.indices[Axes.CH]],
permitted_rounds=[tile.indices[Axes.ROUND]],
permitted_zplanes=[tile.indices[Axes.ZPLANE]]
if Axes.ZPLANE in tile.indices else None,
x_slice=x,
y_slice=y)
return list(coord_groups.values())
def build_image(
fov_count: int,
round_count: int,
ch_count: int,
z_count: int,
image_fetcher: TileFetcher,
default_shape: Optional[Tuple[int, int]] = None,
dimension_order: Sequence[Indices] = DEFAULT_DIMENSION_ORDER,
) -> Collection:
"""
Build and returns an image set with the following characteristics:
Parameters
----------
fov_count : int
Number of fields of view in this image set.
round_count : int
Number for rounds in this image set.
ch_count : int
Number for channels in this image set.
z_count : int
Number of z-layers in this image set.
image_fetcher : TileFetcher
Instance of TileFetcher that provides the data for the tile.
default_shape : Optional[Tuple[int, int]]
Default shape of the individual tiles in this image set.
dimension_order : Sequence[Indices]
Ordering for which dimensions vary, in order of the slowest changing dimension to the
fastest. For instance, if the order is (ROUND, Z, CH) and each dimension has size 2, then
the sequence is:
(ROUND=0, CH=0, Z=0)
(ROUND=0, CH=1, Z=0)
(ROUND=0, CH=0, Z=1)
(ROUND=0, CH=1, Z=1)
(ROUND=1, CH=0, Z=0)
(ROUND=1, CH=1, Z=0)
(ROUND=1, CH=0, Z=1)
(ROUND=1, CH=1, Z=1)
(default = (Indices.Z, Indices.ROUND, Indices.CH))
Returns
-------
The slicedimage collection representing the image.
"""
dimension_sizes = join_dimension_sizes(
dimension_order,
size_for_round=round_count,
size_for_ch=ch_count,
size_for_z=z_count,
)
collection = Collection()
for fov_ix in range(fov_count):
fov_images = TileSet(
[
Coordinates.X,
Coordinates.Y,
Coordinates.Z,
Indices.Z,
Indices.ROUND,
Indices.CH,
Indices.X,
Indices.Y,
],
{
Indices.ROUND: round_count,
Indices.CH: ch_count,
Indices.Z: z_count
},
default_shape,
ImageFormat.TIFF,
)
for dimension_indices in ordered_iterator(dimension_sizes):
image = image_fetcher.get_tile(fov_ix,
dimension_indices[Indices.ROUND],
dimension_indices[Indices.CH],
dimension_indices[Indices.Z])
tile = Tile(
image.coordinates,
{
Indices.Z: (dimension_indices[Indices.Z]),
Indices.ROUND: (dimension_indices[Indices.ROUND]),
Indices.CH: (dimension_indices[Indices.CH]),
},
image.shape,
extras=image.extras,
)
tile.set_numpy_array_future(image.tile_data)
fov_images.add_tile(tile)
collection.add_partition("fov_{:03}".format(fov_ix), fov_images)
return collection
def build_image(
fovs: Sequence[int],
rounds: Sequence[int],
chs: Sequence[int],
zplanes: Sequence[int],
image_fetcher: TileFetcher,
default_shape: Optional[Mapping[Axes, int]]=None,
axes_order: Sequence[Axes]=DEFAULT_DIMENSION_ORDER,
) -> Collection:
"""
Build and returns an image set with the following characteristics:
Parameters
----------
fovs : Sequence[int]
Sequence of field of view ids in this image set.
rounds : Sequence[int]
Sequence of the round numbers in this image set.
chs : Sequence[int]
Sequence of the ch numbers in this image set.
zplanes : Sequence[int]
Sequence of the zplane numbers in this image set.
image_fetcher : TileFetcher
Instance of TileFetcher that provides the data for the tile.
default_shape : Optional[Tuple[int, int]]
Default shape of the individual tiles in this image set.
axes_order : Sequence[Axes]
Ordering for which axes vary, in order of the slowest changing axis to the fastest. For
instance, if the order is (ROUND, Z, CH) and each dimension has size 2, then the sequence
is:
(ROUND=0, CH=0, Z=0)
(ROUND=0, CH=1, Z=0)
(ROUND=0, CH=0, Z=1)
(ROUND=0, CH=1, Z=1)
(ROUND=1, CH=0, Z=0)
(ROUND=1, CH=1, Z=0)
(ROUND=1, CH=0, Z=1)
(ROUND=1, CH=1, Z=1)
(default = (Axes.Z, Axes.ROUND, Axes.CH))
Returns
-------
The slicedimage collection representing the image.
"""
axes_sizes = join_axes_labels(
axes_order, rounds=rounds, chs=chs, zplanes=zplanes)
collection = Collection()
for fov_id in fovs:
fov_images = TileSet(
[
Coordinates.X,
Coordinates.Y,
Coordinates.Z,
Axes.ZPLANE,
Axes.ROUND,
Axes.CH,
Axes.X,
Axes.Y,
],
{Axes.ROUND: len(rounds), Axes.CH: len(chs), Axes.ZPLANE: len(zplanes)},
default_shape,
ImageFormat.TIFF,
)
for selector in ordered_iterator(axes_sizes):
image = image_fetcher.get_tile(
fov_id,
selector[Axes.ROUND],
selector[Axes.CH],
selector[Axes.ZPLANE])
tile = Tile(
image.coordinates,
{
Axes.ZPLANE: (selector[Axes.ZPLANE]),
Axes.ROUND: (selector[Axes.ROUND]),
Axes.CH: (selector[Axes.CH]),
},
image.shape,
extras=image.extras,
)
tile.set_numpy_array_future(image.tile_data)
fov_images.add_tile(tile)
collection.add_partition("fov_{:03}".format(fov_id), fov_images)
return collection
def build_irregular_image(
tile_identifiers: Iterable[TileIdentifier],
image_fetcher: TileFetcher,
default_shape: Optional[Mapping[Axes, int]] = None,
) -> Collection:
"""
Build and returns an image set that can potentially be irregular (i.e., the cardinality of the
dimensions are not always consistent). It can also build a regular image.
Parameters
----------
tile_identifiers : Iterable[TileIdentifier]
Iterable of all the TileIdentifier that are valid in the image.
image_fetcher : TileFetcher
Instance of TileFetcher that provides the data for the tile.
default_shape : Optional[Tuple[int, int]]
Default shape of the individual tiles in this image set.
Returns
-------
The slicedimage collection representing the image.
"""
def reducer_to_sets(
accumulated: Sequence[MutableSet[int]], update: TileIdentifier,
) -> Sequence[MutableSet[int]]:
"""Reduces to a list of sets of tile identifiers, in the order of FOV, round, ch, and
zplane."""
result: MutableSequence[MutableSet[int]] = list()
for accumulated_elem, update_elem in zip(accumulated, astuple(update)):
accumulated_elem.add(update_elem)
result.append(accumulated_elem)
return result
initial_value: Sequence[MutableSet[int]] = tuple(set() for _ in range(4))
fovs, rounds, chs, zplanes = functools.reduce(
reducer_to_sets, tile_identifiers, initial_value)
collection = Collection()
for expected_fov in fovs:
fov_images = TileSet(
[
Coordinates.X,
Coordinates.Y,
Coordinates.Z,
Axes.ZPLANE,
Axes.ROUND,
Axes.CH,
Axes.X,
Axes.Y,
],
{Axes.ROUND: len(rounds), Axes.CH: len(chs), Axes.ZPLANE: len(zplanes)},
default_shape,
ImageFormat.TIFF,
)
for tile_identifier in tile_identifiers:
current_fov, current_round, current_ch, current_zplane = astuple(tile_identifier)
# filter out the fovs that are not the one we are currently processing
if expected_fov != current_fov:
continue
image = image_fetcher.get_tile(
current_fov,
current_round,
current_ch,
current_zplane
)
tile = Tile(
image.coordinates,
{
Axes.ZPLANE: current_zplane,
Axes.ROUND: current_round,
Axes.CH: current_ch,
},
image.shape,
extras=image.extras,
)
tile.set_numpy_array_future(image.tile_data)
# Astute readers might wonder why we set this variable. This is to support in-place
# experiment construction. We monkey-patch slicedimage's Tile class such that checksum
# computation is done by finding the FetchedTile object, which allows us to calculate
# the checksum of the original file.
tile.provider = image
fov_images.add_tile(tile)
collection.add_partition("fov_{:03}".format(expected_fov), fov_images)
return collection
def synthesize() -> Tuple[Stack, list]:
"""Synthesize synthetic spatial image-based transcriptomics data
Returns
-------
Stack :
starfish Stack containing synthetic spots
list :
codebook matching the synthetic data
"""
# set random seed so that data is consistent across tests
random.seed(2)
np.random.seed(2)
NUM_HYB = 4
NUM_CH = 2
NUM_Z = 1
HEIGHT = 100
WIDTH = 100
assert WIDTH == HEIGHT # for compatibility with the parameterization of the code
def choose(n, k):
if n == k:
return [[1] * k]
subsets = [[0] + a for a in choose(n - 1, k)]
if k > 0:
subsets += [[1] + a for a in choose(n - 1, k - 1)]
return subsets
def graham_sloane_codes(n):
# n is length of codeword
# number of on bits is 4
def code_sum(codeword):
return sum([i * c for i, c in enumerate(codeword)]) % n
return [c for c in choose(n, 4) if code_sum(c) == 0]
p = {
# number of on bits (not used with current codebook)
'N_high': 4,
# length of barcode
'N_barcode': NUM_CH * NUM_HYB,
# mean number of flourophores per transcripts - depends on amplification strategy (e.g HCR, bDNA)
'N_flour': 200,
# mean number of photons per flourophore - depends on exposure time, bleaching rate of dye
'N_photons_per_flour': 50,
# mean number of background photons per pixel - depends on tissue clearing and autoflourescence
'N_photon_background': 1000,
# quantum efficiency of the camera detector units number of electrons per photon
'detection_efficiency': .25,
# camera read noise per pixel in units electrons
'N_background_electrons': 1,
# number of RNA puncta; keep this low to reduce overlap probability
'N_spots': 20,
# height and width of image in pixel units
'N_size': WIDTH,
# standard devitation of gaussian in pixel units
'psf': 2,
# dynamic range of camera sensor 37,000 assuming a 16-bit AD converter
'graylevel': 37000.0 / 2 ** 16,
# 16-bit AD converter
'bits': 16
}
codebook = graham_sloane_codes(p['N_barcode'])
def generate_spot(p):
position = rand(2)
gene = random.choice(range(len(codebook)))
barcode = array(codebook[gene])
photons = [poisson(p['N_photons_per_flour']) * poisson(p['N_flour']) * b for b in barcode]
return DataFrame({'position': [position], 'barcode': [barcode], 'photons': [photons], 'gene': gene})
# right now there is no jitter on x-y positions of the spots, we might want to make it a vector
spots = concat([generate_spot(p) for _ in range(p['N_spots'])]) # type: ignore
image = zeros((p['N_barcode'], p['N_size'], p['N_size'],))
for s in spots.itertuples():
image[:, int(p['N_size'] * s.position[0]), int(p['N_size'] * s.position[1])] = s.photons
image_with_background = image + poisson(p['N_photon_background'], size=image.shape)
filtered = array([gaussian(im, p['psf']) for im in image_with_background])
filtered = filtered * p['detection_efficiency'] + normal(scale=p['N_background_electrons'], size=filtered.shape)
signal = np.array([(x / p['graylevel']).astype(int).clip(0, 2 ** p['bits']) for x in filtered])
def select_uint_dtype(array):
"""choose appropriate dtype based on values of an array"""
max_val = np.max(array)
for dtype in [np.uint8, np.uint16, np.uint32, np.uint64]:
if max_val <= dtype(-1):
return array.astype(dtype)
raise ValueError('value exceeds dynamic range of largest numpy type')
corrected_signal = select_uint_dtype(signal)
rescaled_signal: np.ndarray = rescale_intensity(corrected_signal)
# set up the tile set
image_data = TileSet(
{Coordinates.X, Coordinates.Y, Indices.HYB, Indices.CH, Indices.Z},
{
Indices.HYB: NUM_HYB,
Indices.CH: NUM_CH,
Indices.Z: NUM_Z,
},
default_tile_shape=(HEIGHT, WIDTH),
)
# fill the TileSet
experiment_indices = list(product(range(NUM_HYB), range(NUM_CH), range(NUM_Z)))
for i, (hyb, ch, z) in enumerate(experiment_indices):
tile = Tile(
{
Coordinates.X: (0.0, 0.001),
Coordinates.Y: (0.0, 0.001),
Coordinates.Z: (0.0, 0.001),
},
{
Indices.HYB: hyb,
Indices.CH: ch,
Indices.Z: z,
}
)
tile.numpy_array = rescaled_signal[i]
image_data.add_tile(tile)
data_stack = ImageStack(image_data)
# make a max projection and pretend that's the dots image, which we'll create another ImageStack for this
dots_data = TileSet(
{Coordinates.X, Coordinates.Y, Indices.HYB, Indices.CH, Indices.Z},
{
Indices.HYB: 1,
Indices.CH: 1,
Indices.Z: 1,
},
default_tile_shape=(HEIGHT, WIDTH),
)
tile = Tile(
{
Coordinates.X: (0.0, 0.001),
Coordinates.Y: (0.0, 0.001),
Coordinates.Z: (0.0, 0.001),
},
{
Indices.HYB: 0,
Indices.CH: 0,
Indices.Z: 0,
}
)
tile.numpy_array = np.max(rescaled_signal, 0)
dots_data.add_tile(tile)
dots_stack = ImageStack(dots_data)
# TODO can we mock up a nuclei image somehow?
# put the data together into a top-level Stack
results = Stack.from_data(data_stack, aux_dict={'dots': dots_stack})
# make the codebook(s)
codebook = []
for _, code_record in spots.iterrows():
codeword = []
for code_value, (hyb, ch, z) in zip(code_record['barcode'], experiment_indices):
if code_value != 0:
codeword.append({
Indices.HYB: hyb,
Indices.CH: ch,
Indices.Z: z,
"v": code_value
})
codebook.append(
{
'codeword': codeword,
'gene_name': code_record['gene']
}
)
return results, codebook
def __init__(self, tileset: TileSet) -> None:
self._num_rounds = ImageStack._get_dimension_size(
tileset, Indices.ROUND)
self._num_chs = ImageStack._get_dimension_size(tileset, Indices.CH)
self._num_zlayers = ImageStack._get_dimension_size(tileset, Indices.Z)
self._tile_metadata = TileSetData(tileset)
self._tile_shape = tileset.default_tile_shape
# Examine the tiles to figure out the right kind (int, float, etc.) and size. We require
# that all the tiles have the same kind of data type, but we do not require that they all
# have the same size of data type. The # allocated array is the highest size we encounter.
kind = None
max_size = 0
for tile in tqdm(tileset.tiles(),
disable=(not StarfishConfig().verbose)):
dtype = tile.numpy_array.dtype
if kind is None:
kind = dtype.kind
else:
if kind != dtype.kind:
raise TypeError(
"All tiles should have the same kind of dtype")
if dtype.itemsize > max_size:
max_size = dtype.itemsize
if self._tile_shape is None:
self._tile_shape = tile.tile_shape
elif tile.tile_shape is not None and self._tile_shape != tile.tile_shape:
raise ValueError(
"Starfish does not support tiles that are not identical in shape"
)
shape: MutableSequence[int] = []
dims: MutableSequence[str] = []
coordinates_shape: MutableSequence[int] = []
coordinates_dimensions: MutableSequence[str] = []
coordinates_tick_marks: MutableMapping[str,
Sequence[Union[int,
str]]] = dict()
for ix in range(N_AXES):
size_for_axis: Optional[int] = None
dim_for_axis: Optional[Indices] = None
for axis_name, axis_data in AXES_DATA.items():
if ix == axis_data.order:
size_for_axis = ImageStack._get_dimension_size(
tileset, axis_name)
dim_for_axis = axis_name
break
if size_for_axis is None or dim_for_axis is None:
raise ValueError(
f"Could not find entry for the {ix}th axis in AXES_DATA")
shape.append(size_for_axis)
dims.append(dim_for_axis.value)
coordinates_shape.append(size_for_axis)
coordinates_dimensions.append(dim_for_axis.value)
coordinates_tick_marks[dim_for_axis.value] = list(
range(size_for_axis))
shape.extend(self._tile_shape)
dims.extend([Indices.Y.value, Indices.X.value])
coordinates_shape.append(6)
coordinates_dimensions.append(PHYSICAL_COORDINATE_DIMENSION)
coordinates_tick_marks[PHYSICAL_COORDINATE_DIMENSION] = [
PhysicalCoordinateTypes.X_MIN.value,
PhysicalCoordinateTypes.X_MAX.value,
PhysicalCoordinateTypes.Y_MIN.value,
PhysicalCoordinateTypes.Y_MAX.value,
PhysicalCoordinateTypes.Z_MIN.value,
PhysicalCoordinateTypes.Z_MAX.value,
]
# now that we know the tile data type (kind and size), we can allocate the data array.
self._data = MPDataArray.from_shape_and_dtype(
shape=shape,
dtype=np.float32,
initial_value=0,
dims=dims,
)
self._coordinates = xr.DataArray(
np.empty(
shape=coordinates_shape,
dtype=np.float32,
),
dims=coordinates_dimensions,
coords=coordinates_tick_marks,
)
# iterate through the tiles and set the data.
for tile in tileset.tiles():
h = tile.indices[Indices.ROUND]
c = tile.indices[Indices.CH]
zlayer = tile.indices.get(Indices.Z, 0)
data = tile.numpy_array
if max_size != data.dtype.itemsize:
warnings.warn(
f"Tile "
f"(R: {tile.indices[Indices.ROUND]} C: {tile.indices[Indices.CH]} "
f"Z: {tile.indices[Indices.Z]}) has "
f"dtype {data.dtype}. One or more tiles is of a larger dtype "
f"{self._data.dtype}.", DataFormatWarning)
data = img_as_float32(data)
self.set_slice(indices={
Indices.ROUND: h,
Indices.CH: c,
Indices.Z: zlayer
},
data=data)
coordinate_selector = {
Indices.ROUND.value: h,
Indices.CH.value: c,
Indices.Z.value: zlayer,
}
coordinates_values = [
tile.coordinates[Coordinates.X][0],
tile.coordinates[Coordinates.X][1],
tile.coordinates[Coordinates.Y][0],
tile.coordinates[Coordinates.Y][1],
]
if Coordinates.Z in tile.coordinates:
coordinates_values.extend([
tile.coordinates[Coordinates.Z][0],
tile.coordinates[Coordinates.Z][1],
])
else:
coordinates_values.extend([np.nan, np.nan])
self._coordinates.loc[coordinate_selector] = np.array(
coordinates_values)
def build_image(
fov_count: int,
round_count: int,
ch_count: int,
z_count: int,
image_fetcher: TileFetcher,
default_shape: Optional[Tuple[int, int]] = None,
) -> Collection:
"""
Build and returns an image set with the following characteristics:
Parameters
----------
fov_count : int
Number of fields of view in this image set.
round_count : int
Number for rounds in this image set.
ch_count : int
Number for channels in this image set.
z_count : int
Number of z-layers in this image set.
image_fetcher : TileFetcher
Instance of TileFetcher that provides the data for the tile.
default_shape : Optional[Tuple[int, int]]
Default shape of the individual tiles in this image set.
Returns
-------
The slicedimage collection representing the image.
"""
collection = Collection()
for fov_ix in range(fov_count):
fov_images = TileSet(
[
Coordinates.X,
Coordinates.Y,
Coordinates.Z,
Indices.Z,
Indices.ROUND,
Indices.CH,
Indices.X,
Indices.Y,
],
{
Indices.ROUND: round_count,
Indices.CH: ch_count,
Indices.Z: z_count
},
default_shape,
ImageFormat.TIFF,
)
for z_ix in range(z_count):
for round_ix in range(round_count):
for ch_ix in range(ch_count):
image = image_fetcher.get_tile(fov_ix, round_ix, ch_ix,
z_ix)
tile = Tile(
image.coordinates,
{
Indices.Z: z_ix,
Indices.ROUND: round_ix,
Indices.CH: ch_ix,
},
image.shape,
extras=image.extras,
)
tile.set_numpy_array_future(image.tile_data)
fov_images.add_tile(tile)
collection.add_partition("fov_{:03}".format(fov_ix), fov_images)
return collection
def export(self,
filepath: str,
tile_opener: Optional[Callable[[PurePath, Tile, str], BinaryIO]] = None,
tile_format: ImageFormat=ImageFormat.NUMPY) -> None:
"""write the image tensor to disk in spaceTx format
Parameters
----------
filepath : str
Path + prefix for the images and primary_images.json written by this function
tile_opener : Optional[Callable[[PurePath, Tile, str], BinaryIO]]
A callable responsible for opening the file that a tile's data is to be written to. The
callable should accept three arguments -- the path of the tileset, the tile data, and
the expected file extension. If this is not specified, a reasonable default is provided.
tile_format : ImageFormat
Format in which each 2D plane should be written.
"""
# Add log data to extras
tileset_extras = self._tile_data.extras if self._tile_data else {}
tileset_extras[STARFISH_EXTRAS_KEY] = self.log.encode()
tileset = TileSet(
dimensions={
Axes.ROUND,
Axes.CH,
Axes.ZPLANE,
Axes.Y,
Axes.X,
},
shape={
Axes.ROUND: self.num_rounds,
Axes.CH: self.num_chs,
Axes.ZPLANE: self.num_zplanes,
},
default_tile_shape={Axes.Y: self.tile_shape[0], Axes.X: self.tile_shape[1]},
extras=tileset_extras,
)
for selector in self._iter_axes({Axes.ROUND, Axes.CH, Axes.ZPLANE}):
tilekey = TileKey(
round=selector[Axes.ROUND],
ch=selector[Axes.CH],
zplane=selector[Axes.ZPLANE])
extras: dict = self._tile_data[tilekey] if self._tile_data else {}
coordinates: MutableMapping[Coordinates, Union[Tuple[Number, Number], Number]] = dict()
x_coordinates = (float(self.xarray[Coordinates.X.value][0]),
float(self.xarray[Coordinates.X.value][-1]))
y_coordinates = (float(self.xarray[Coordinates.Y.value][0]),
float(self.xarray[Coordinates.Y.value][-1]))
coordinates[Coordinates.X] = x_coordinates
coordinates[Coordinates.Y] = y_coordinates
if Coordinates.Z in self.xarray.coords:
# set the z coord to the calculated value from the associated z plane
z_coordinates = float(self.xarray[Coordinates.Z.value][selector[Axes.ZPLANE]])
coordinates[Coordinates.Z] = z_coordinates
tile = Tile(
coordinates=coordinates,
indices=selector,
extras=extras,
)
tile.numpy_array, _ = self.get_slice(selector)
tileset.add_tile(tile)
if tile_opener is None:
def tile_opener(tileset_path: PurePath, tile: Tile, ext: str):
base = tileset_path.parent / tileset_path.stem
if Axes.ZPLANE in tile.indices:
zval = tile.indices[Axes.ZPLANE]
zstr = "-Z{}".format(zval)
else:
zstr = ""
return open(
"{}-H{}-C{}{}.{}".format(
str(base),
tile.indices[Axes.ROUND],
tile.indices[Axes.CH],
zstr,
ext,
),
"wb")
if not filepath.endswith('.json'):
filepath += '.json'
Writer.write_to_path(
tileset,
filepath,
pretty=True,
tile_opener=tile_opener,
tile_format=tile_format)