def run_command(cls, args):
_, name, baseurl = resolve_path_or_url(args.in_url)
slicedimage = Reader.parse_doc(name, baseurl)
Writer.write_to_path(slicedimage,
args.out_path,
pretty=args.pretty,
tile_opener=fake_file_opener,
tile_writer=identity_writer)
def write(self, filepath: str, tile_opener=None) -> None:
"""write the image tensor to disk in spaceTx format
Parameters
----------
filepath : str
Path + prefix for the images and primary_images.json written by this function
tile_opener : TODO ttung: doc me.
"""
for tile in self._image_partition.tiles():
h = tile.indices[Indices.ROUND]
c = tile.indices[Indices.CH]
zlayer = tile.indices.get(Indices.Z, 0)
tile.numpy_array, axes = self.get_slice(indices={
Indices.ROUND: h,
Indices.CH: c,
Indices.Z: zlayer
})
assert len(axes) == 0
seen_x_coords, seen_y_coords, seen_z_coords = set(), set(), set()
for tile in self._image_partition.tiles():
seen_x_coords.add(tile.coordinates[Coordinates.X])
seen_y_coords.add(tile.coordinates[Coordinates.Y])
z_coords = tile.coordinates.get(Coordinates.Z, None)
if z_coords is not None:
seen_z_coords.add(z_coords)
sorted_x_coords = sorted(seen_x_coords)
sorted_y_coords = sorted(seen_y_coords)
sorted_z_coords = sorted(seen_z_coords)
x_coords_to_idx = {
coords: idx
for idx, coords in enumerate(sorted_x_coords)
}
y_coords_to_idx = {
coords: idx
for idx, coords in enumerate(sorted_y_coords)
}
z_coords_to_idx = {
coords: idx
for idx, coords in enumerate(sorted_z_coords)
}
if tile_opener is None:
def tile_opener(tileset_path, tile, ext):
tile_basename = os.path.splitext(tileset_path)[0]
xcoord = tile.coordinates[Coordinates.X]
ycoord = tile.coordinates[Coordinates.Y]
zcoord = tile.coordinates.get(Coordinates.Z, None)
xcoord = tuple(xcoord) if isinstance(xcoord, list) else xcoord
ycoord = tuple(ycoord) if isinstance(ycoord, list) else ycoord
xval = x_coords_to_idx[xcoord]
yval = y_coords_to_idx[ycoord]
if zcoord is not None:
zval = z_coords_to_idx[zcoord]
zstr = "-Z{}".format(zval)
else:
zstr = ""
return open(
"{}-X{}-Y{}{}-H{}-C{}.{}".format(
tile_basename,
xval,
yval,
zstr,
tile.indices[Indices.ROUND],
tile.indices[Indices.CH],
ext,
), "wb")
if not filepath.endswith('.json'):
filepath += '.json'
Writer.write_to_path(self._image_partition,
filepath,
pretty=True,
tile_opener=tile_opener)
def write_experiment_json(
path,
fov_count,
hyb_dimensions,
aux_name_to_dimensions,
hyb_image_fetcher=None,
aux_image_fetcher=None,
postprocess_func=None,
default_shape=(1536, 1024),
):
"""
Build and returns a top-level experiment description with the following characteristics:
Parameters
----------
path : str
Directory to write the files to.
fov_count : int
Number of fields of view in this experiment.
hyb_dimensions : Mapping[str, int]
Dictionary mapping dimension name to dimension size for the hybridization image.
aux_name_to_dimensions : Mapping[str, Mapping[str, int]]
Dictionary mapping the auxiliary image type to dictionaries, which map from dimension name to dimension size.
hyb_image_fetcher : Optional[ImageFetcher]
ImageFetcher for hybridization images. Set this if you want specific image data to be set for the hybridization
images. If not provided, the image data is set to random noise via :class:`RandomNoiseImageFetcher`.
aux_image_fetcher : Optional[Mapping[str, ImageFetcher]]
ImageFetchers for auxiliary images. Set this if you want specific image data to be set for one or more aux
image types. If not provided for any given aux image, the image data is set to random noise via
:class:`RandomNoiseImageFetcher`.
postprocess_func : Optional[Callable[[dict], dict]]
If provided, this is called with the experiment document for any postprocessing. An example of this would be to
add something to one of the top-level extras field. The callable should return what is to be written as the
experiment document.
default_shape : Tuple[int, int] (default = (1536, 1024))
Default shape for the tiles in this experiment.
"""
if hyb_image_fetcher is None:
hyb_image_fetcher = RandomNoiseImageFetcher()
if aux_image_fetcher is None:
aux_image_fetcher = {}
if postprocess_func is None:
postprocess_func = lambda doc: doc
experiment_doc = {
'version': "0.0.0",
'auxiliary_images': {},
'extras': {},
}
hybridization_image = build_image(
fov_count,
hyb_dimensions[Indices.HYB], hyb_dimensions[Indices.CH], hyb_dimensions[Indices.Z],
hyb_image_fetcher,
default_shape=default_shape,
)
Writer.write_to_path(
hybridization_image,
os.path.join(path, "hybridization.json"),
pretty=True,
partition_path_generator=fov_path_generator,
tile_opener=tile_opener,
tile_writer=tile_writer,
)
experiment_doc['hybridization_images'] = "hybridization.json"
for aux_name, aux_dimensions in aux_name_to_dimensions.items():
if aux_dimensions is None:
continue
auxiliary_image = build_image(
fov_count,
aux_dimensions[Indices.HYB], aux_dimensions[Indices.CH], aux_dimensions[Indices.Z],
aux_image_fetcher.get(aux_name, RandomNoiseImageFetcher()),
default_shape=default_shape,
)
Writer.write_to_path(
auxiliary_image,
os.path.join(path, "{}.json".format(aux_name)),
pretty=True,
partition_path_generator=fov_path_generator,
tile_opener=tile_opener,
tile_writer=tile_writer,
)
experiment_doc['auxiliary_images'][aux_name] = "{}.json".format(aux_name)
experiment_doc = postprocess_func(experiment_doc)
with open(os.path.join(path, "experiment.json"), "w") as fh:
json.dump(experiment_doc, fh, indent=4)
def export(self,
filepath: str,
tile_opener=None,
tile_format: ImageFormat = ImageFormat.NUMPY) -> None:
"""write the image tensor to disk in spaceTx format
Parameters
----------
filepath : str
Path + prefix for the images and primary_images.json written by this function
tile_opener : TODO ttung: doc me.
tile_format : ImageFormat
Format in which each 2D plane should be written.
"""
# Add log data to extras
self._tile_data.extras[STARFISH_EXTRAS_KEY] = logging.LogEncoder(
).encode({LOG: self.log})
tileset = TileSet(
dimensions={
Axes.ROUND,
Axes.CH,
Axes.ZPLANE,
Axes.Y,
Axes.X,
},
shape={
Axes.ROUND: self.num_rounds,
Axes.CH: self.num_chs,
Axes.ZPLANE: self.num_zplanes,
},
default_tile_shape={
Axes.Y: self.tile_shape[0],
Axes.X: self.tile_shape[1]
},
extras=self._tile_data.extras,
)
for axis_val_map in self._iter_axes({Axes.ROUND, Axes.CH,
Axes.ZPLANE}):
tilekey = TileKey(round=axis_val_map[Axes.ROUND],
ch=axis_val_map[Axes.CH],
zplane=axis_val_map[Axes.ZPLANE])
round_, ch, zplane = tilekey.round, tilekey.ch, tilekey.z
extras: dict = self._tile_data[tilekey]
selector = {
Axes.ROUND: round_,
Axes.CH: ch,
Axes.ZPLANE: zplane,
}
coordinates: MutableMapping[Coordinates, Union[Tuple[Number,
Number],
Number]] = dict()
x_coordinates = (float(self.xarray[Coordinates.X.value][0]),
float(self.xarray[Coordinates.X.value][-1]))
y_coordinates = (float(self.xarray[Coordinates.Y.value][0]),
float(self.xarray[Coordinates.Y.value][-1]))
coordinates[Coordinates.X] = x_coordinates
coordinates[Coordinates.Y] = y_coordinates
if Coordinates.Z in self.xarray.coords:
# set the z coord to the calculated value from the associated z plane
z_coordinates = float(self.xarray[Coordinates.Z.value][zplane])
coordinates[Coordinates.Z] = z_coordinates
tile = Tile(
coordinates=coordinates,
indices=selector,
extras=extras,
)
tile.numpy_array, _ = self.get_slice(selector={
Axes.ROUND: round_,
Axes.CH: ch,
Axes.ZPLANE: zplane
})
tileset.add_tile(tile)
if tile_opener is None:
def tile_opener(tileset_path: Path, tile, ext):
base = tileset_path.parent / tileset_path.stem
if Axes.ZPLANE in tile.indices:
zval = tile.indices[Axes.ZPLANE]
zstr = "-Z{}".format(zval)
else:
zstr = ""
return open(
"{}-H{}-C{}{}.{}".format(
str(base),
tile.indices[Axes.ROUND],
tile.indices[Axes.CH],
zstr,
ext,
), "wb")
if not filepath.endswith('.json'):
filepath += '.json'
Writer.write_to_path(tileset,
filepath,
pretty=True,
tile_opener=tile_opener,
tile_format=tile_format)
def write_experiment_json(
path: str,
fov_count: int,
tile_format: ImageFormat,
*,
primary_image_dimensions: Mapping[Union[str, Axes], int],
aux_name_to_dimensions: Mapping[str, Mapping[Union[str, Axes], int]],
primary_tile_fetcher: Optional[TileFetcher]=None,
aux_tile_fetcher: Optional[Mapping[str, TileFetcher]]=None,
postprocess_func: Optional[Callable[[dict], dict]]=None,
default_shape: Optional[Mapping[Axes, int]]=None,
dimension_order: Sequence[Axes]=(Axes.ZPLANE, Axes.ROUND, Axes.CH),
) -> None:
"""
Build and returns a top-level experiment description with the following characteristics:
Parameters
----------
path : str
Directory to write the files to.
fov_count : int
Number of fields of view in this experiment.
primary_image_dimensions : Mapping[Union[str, Axes], int]
Dictionary mapping dimension name to dimension size for the primary image.
aux_name_to_dimensions : Mapping[str, Mapping[Union[str, Axes], int]]
Dictionary mapping the auxiliary image type to dictionaries, which map from dimension name
to dimension size.
primary_tile_fetcher : Optional[TileFetcher]
TileFetcher for primary images. Set this if you want specific image data to be set for the
primary images. If not provided, the image data is set to random noise via
:class:`RandomNoiseTileFetcher`.
aux_tile_fetcher : Optional[Mapping[str, TileFetcher]]
TileFetchers for auxiliary images. Set this if you want specific image data to be set for
one or more aux image types. If not provided for any given aux image, the image data is
set to random noise via :class:`RandomNoiseTileFetcher`.
postprocess_func : Optional[Callable[[dict], dict]]
If provided, this is called with the experiment document for any postprocessing.
An example of this would be to add something to one of the top-level extras field.
The callable should return what is to be written as the experiment document.
default_shape : Optional[Tuple[int, int]] (default = None)
Default shape for the tiles in this experiment.
dimension_order : Sequence[Axes]
Ordering for which dimensions vary, in order of the slowest changing dimension to the
fastest. For instance, if the order is (ROUND, Z, CH) and each dimension has size 2, then
the sequence is:
(ROUND=0, CH=0, Z=0)
(ROUND=0, CH=1, Z=0)
(ROUND=0, CH=0, Z=1)
(ROUND=0, CH=1, Z=1)
(ROUND=1, CH=0, Z=0)
(ROUND=1, CH=1, Z=0)
(ROUND=1, CH=0, Z=1)
(ROUND=1, CH=1, Z=1)
(default = (Axes.Z, Axes.ROUND, Axes.CH))
"""
if primary_tile_fetcher is None:
primary_tile_fetcher = tile_fetcher_factory(RandomNoiseTile)
if aux_tile_fetcher is None:
aux_tile_fetcher = {}
if postprocess_func is None:
postprocess_func = lambda doc: doc
experiment_doc: Dict[str, Any] = {
'version': str(CURRENT_VERSION),
'images': {},
'extras': {},
}
primary_image = build_image(
range(fov_count),
range(primary_image_dimensions[Axes.ROUND]),
range(primary_image_dimensions[Axes.CH]),
range(primary_image_dimensions[Axes.ZPLANE]),
primary_tile_fetcher,
axes_order=dimension_order,
default_shape=default_shape,
)
Writer.write_to_path(
primary_image,
os.path.join(path, "primary_images.json"),
pretty=True,
partition_path_generator=_fov_path_generator,
tile_opener=_tile_opener,
tile_format=tile_format,
)
experiment_doc['images']['primary'] = "primary_images.json"
for aux_name, aux_dimensions in aux_name_to_dimensions.items():
if aux_dimensions is None:
continue
auxiliary_image = build_image(
range(fov_count),
range(aux_dimensions[Axes.ROUND]),
range(aux_dimensions[Axes.CH]),
range(aux_dimensions[Axes.ZPLANE]),
aux_tile_fetcher.get(aux_name, tile_fetcher_factory(RandomNoiseTile)),
axes_order=dimension_order,
default_shape=default_shape,
)
Writer.write_to_path(
auxiliary_image,
os.path.join(path, "{}.json".format(aux_name)),
pretty=True,
partition_path_generator=_fov_path_generator,
tile_opener=_tile_opener,
tile_format=tile_format,
)
experiment_doc['images'][aux_name] = "{}.json".format(aux_name)
experiment_doc["codebook"] = "codebook.json"
codebook_array = [
{
"codeword": [
{"r": 0, "c": 0, "v": 1},
],
"target": "PLEASE_REPLACE_ME"
},
]
codebook = Codebook.from_code_array(codebook_array)
codebook_json_filename = "codebook.json"
codebook.to_json(os.path.join(path, codebook_json_filename))
experiment_doc = postprocess_func(experiment_doc)
with open(os.path.join(path, "experiment.json"), "w") as fh:
json.dump(experiment_doc, fh, indent=4)
def write_irregular_experiment_json(
path: str,
tile_format: ImageFormat,
*,
image_tile_identifiers: Mapping[str, Iterable[TileIdentifier]],
tile_fetchers: Mapping[str, TileFetcher],
postprocess_func: Optional[Callable[[dict], dict]]=None,
default_shape: Optional[Mapping[Axes, int]]=None,
fov_path_generator: Callable[[Path, str], Path] = None,
tile_opener: Optional[Callable[[Path, Tile, str], BinaryIO]] = None,
) -> None:
"""
Build and returns a top-level experiment description with the following characteristics:
Parameters
----------
path : str
Directory to write the files to.
tile_format : ImageFormat
File format to write the tiles as.
image_tile_identifiers : Mapping[str, Iterable[TileIdentifier]]
Dictionary mapping the image type to an iterable of TileIdentifiers.
tile_fetchers : Mapping[str, TileFetcher]
Dictionary mapping the image type to a TileFetcher.
postprocess_func : Optional[Callable[[dict], dict]]
If provided, this is called with the experiment document for any postprocessing.
An example of this would be to add something to one of the top-level extras field.
The callable should return what is to be written as the experiment document.
default_shape : Optional[Tuple[int, int]] (default = None)
Default shape for the tiles in this experiment.
fov_path_generator : Optional[Callable[[Path, str], Path]]
Generates the path for a FOV's json file. If one is not provided, the default generates
the FOV's json file at the same level as the top-level json file for an image. If this is
not provided, a reasonable default will be provided.
tile_opener : Optional[Callable[[Path, Tile, str], BinaryIO]]
Callable that gets invoked with the following arguments: 1. the directory of the experiment
that is being constructed, 2. the tile that is being written, and 3. the file extension
that the tile should be written with. The callable is expected to return an open file
handle. If this is not provided, a reasonable default will be provided.
"""
if postprocess_func is None:
postprocess_func = lambda doc: doc
if fov_path_generator is None:
fov_path_generator = _fov_path_generator
if tile_opener is None:
tile_opener = _tile_opener
experiment_doc: Dict[str, Any] = {
'version': str(CURRENT_VERSION),
'images': {},
'extras': {},
}
for image_type, tile_identifiers in image_tile_identifiers.items():
tile_fetcher = tile_fetchers[image_type]
image = build_irregular_image(tile_identifiers, tile_fetcher, default_shape)
Writer.write_to_path(
image,
os.path.join(path, f"{image_type}.json"),
pretty=True,
partition_path_generator=fov_path_generator,
tile_opener=tile_opener,
tile_format=tile_format,
)
experiment_doc['images'][image_type] = f"{image_type}.json"
experiment_doc["codebook"] = "codebook.json"
codebook_array = [
{
"codeword": [
{"r": 0, "c": 0, "v": 1},
],
"target": "PLEASE_REPLACE_ME"
},
]
codebook = Codebook.from_code_array(codebook_array)
codebook_json_filename = "codebook.json"
codebook.to_json(os.path.join(path, codebook_json_filename))
experiment_doc = postprocess_func(experiment_doc)
with open(os.path.join(path, "experiment.json"), "w") as fh:
json.dump(experiment_doc, fh, indent=4)
def export(self,
filepath: str,
tile_opener=None,
tile_format: ImageFormat = ImageFormat.NUMPY) -> None:
"""write the image tensor to disk in spaceTx format
Parameters
----------
filepath : str
Path + prefix for the images and primary_images.json written by this function
tile_opener : TODO ttung: doc me.
tile_format : ImageFormat
Format in which each 2D plane should be written.
"""
tileset = TileSet(
dimensions={
Indices.ROUND,
Indices.CH,
Indices.Z,
Indices.Y,
Indices.X,
},
shape={
Indices.ROUND: self.num_rounds,
Indices.CH: self.num_chs,
Indices.Z: self.num_zlayers,
},
default_tile_shape=self._tile_shape,
extras=self._tile_metadata.extras,
)
for round_ in range(self.num_rounds):
for ch in range(self.num_chs):
for zlayer in range(self.num_zlayers):
tilekey = TileKey(round=round_, ch=ch, z=zlayer)
extras: dict = self._tile_metadata[tilekey]
tile_indices = {
Indices.ROUND: round_,
Indices.CH: ch,
Indices.Z: zlayer,
}
coordinates: MutableMapping[Coordinates,
Tuple[Number,
Number]] = dict()
x_coordinates = self.tile_coordinates(
tile_indices, Coordinates.X)
y_coordinates = self.tile_coordinates(
tile_indices, Coordinates.Y)
z_coordinates = self.tile_coordinates(
tile_indices, Coordinates.Z)
coordinates[Coordinates.X] = x_coordinates
coordinates[Coordinates.Y] = y_coordinates
if z_coordinates[0] != np.nan and z_coordinates[
1] != np.nan:
coordinates[Coordinates.Z] = z_coordinates
tile = Tile(
coordinates=coordinates,
indices=tile_indices,
extras=extras,
)
tile.numpy_array, _ = self.get_slice(indices={
Indices.ROUND: round_,
Indices.CH: ch,
Indices.Z: zlayer
})
tileset.add_tile(tile)
if tile_opener is None:
def tile_opener(tileset_path, tile, ext):
tile_basename = os.path.splitext(tileset_path)[0]
if Indices.Z in tile.indices:
zval = tile.indices[Indices.Z]
zstr = "-Z{}".format(zval)
else:
zstr = ""
return open(
"{}-H{}-C{}{}.{}".format(
tile_basename,
tile.indices[Indices.ROUND],
tile.indices[Indices.CH],
zstr,
ext,
), "wb")
if not filepath.endswith('.json'):
filepath += '.json'
Writer.write_to_path(tileset,
filepath,
pretty=True,
tile_opener=tile_opener,
tile_format=tile_format)
def export(self,
filepath: str,
tile_opener: Optional[Callable[[PurePath, Tile, str], BinaryIO]] = None,
tile_format: ImageFormat=ImageFormat.NUMPY) -> None:
"""write the image tensor to disk in spaceTx format
Parameters
----------
filepath : str
Path + prefix for the images and primary_images.json written by this function
tile_opener : Optional[Callable[[PurePath, Tile, str], BinaryIO]]
A callable responsible for opening the file that a tile's data is to be written to. The
callable should accept three arguments -- the path of the tileset, the tile data, and
the expected file extension. If this is not specified, a reasonable default is provided.
tile_format : ImageFormat
Format in which each 2D plane should be written.
"""
# Add log data to extras
tileset_extras = self._tile_data.extras if self._tile_data else {}
tileset_extras[STARFISH_EXTRAS_KEY] = self.log.encode()
tileset = TileSet(
dimensions={
Axes.ROUND,
Axes.CH,
Axes.ZPLANE,
Axes.Y,
Axes.X,
},
shape={
Axes.ROUND: self.num_rounds,
Axes.CH: self.num_chs,
Axes.ZPLANE: self.num_zplanes,
},
default_tile_shape={Axes.Y: self.tile_shape[0], Axes.X: self.tile_shape[1]},
extras=tileset_extras,
)
for selector in self._iter_axes({Axes.ROUND, Axes.CH, Axes.ZPLANE}):
tilekey = TileKey(
round=selector[Axes.ROUND],
ch=selector[Axes.CH],
zplane=selector[Axes.ZPLANE])
extras: dict = self._tile_data[tilekey] if self._tile_data else {}
coordinates: MutableMapping[Coordinates, Union[Tuple[Number, Number], Number]] = dict()
x_coordinates = (float(self.xarray[Coordinates.X.value][0]),
float(self.xarray[Coordinates.X.value][-1]))
y_coordinates = (float(self.xarray[Coordinates.Y.value][0]),
float(self.xarray[Coordinates.Y.value][-1]))
coordinates[Coordinates.X] = x_coordinates
coordinates[Coordinates.Y] = y_coordinates
if Coordinates.Z in self.xarray.coords:
# set the z coord to the calculated value from the associated z plane
z_coordinates = float(self.xarray[Coordinates.Z.value][selector[Axes.ZPLANE]])
coordinates[Coordinates.Z] = z_coordinates
tile = Tile(
coordinates=coordinates,
indices=selector,
extras=extras,
)
tile.numpy_array, _ = self.get_slice(selector)
tileset.add_tile(tile)
if tile_opener is None:
def tile_opener(tileset_path: PurePath, tile: Tile, ext: str):
base = tileset_path.parent / tileset_path.stem
if Axes.ZPLANE in tile.indices:
zval = tile.indices[Axes.ZPLANE]
zstr = "-Z{}".format(zval)
else:
zstr = ""
return open(
"{}-H{}-C{}{}.{}".format(
str(base),
tile.indices[Axes.ROUND],
tile.indices[Axes.CH],
zstr,
ext,
),
"wb")
if not filepath.endswith('.json'):
filepath += '.json'
Writer.write_to_path(
tileset,
filepath,
pretty=True,
tile_opener=tile_opener,
tile_format=tile_format)