def write_components(self, destination_dir: Path):
""" Write out the estimated bleed-through components.
These bleed-through components can be subtracted from the original
images to achieve bleed-through correction.
Args:
destination_dir: Path to the directory where the output images will
be written.
"""
with ProcessPoolExecutor(max_workers=utils.NUM_THREADS) as executor:
processes = list()
for source_index, input_path in enumerate(self.__files):
writer_name = utils.replace_extension(input_path.name)
processes.append(
executor.submit(
self._write_components_thread,
destination_dir,
writer_name,
source_index,
))
for process in processes:
process.result()
return
def test_resize_gif(self):
basename = os.path.basename(PREVIEW_GIF)
size = (200, None)
gallery.resize(PREVIEW_GIF, os.getcwd(), "", basename, size, 1)
self.assertTrue(
os.path.exists(utils.replace_extension(basename, ".gif")))
output_size = gallery.get_size(basename, 1)
self.assertEqual(output_size["width"], size[0])
def test_resize_jpeg(self):
basename = os.path.basename(IMG_3870_JPEG)
size = (800, None)
gallery.resize(IMG_3870_JPEG, os.getcwd(), "", basename, size, 1)
self.assertTrue(
os.path.exists(utils.replace_extension(basename, ".jpeg")))
output_size = gallery.get_size(basename, 1)
self.assertEqual(output_size["width"], size[0])
def test_resize_tiff(self):
basename = os.path.basename(IMG_6218_TIFF)
size = (1600, None)
gallery.resize(IMG_6218_TIFF, os.getcwd(), "", basename, size, 1)
expected_basename = utils.replace_extension(basename, ".tiff")
self.assertTrue(os.path.exists(expected_basename))
output_size = gallery.get_size(expected_basename, 1)
self.assertEqual(output_size["width"], size[0])
def main(
input_dir: Path,
file_pattern: str,
output_dir: Path,
):
fp = filepattern.FilePattern(input_dir, file_pattern)
files = [Path(file[0]['file']).resolve() for file in fp]
files = list(filter(
lambda file_path: file_path.name.endswith('.ome.tif') or file_path.name.endswith('.ome.zarr'),
files
))
executor = (ThreadPoolExecutor if utils.USE_GPU else ProcessPoolExecutor)(utils.NUM_THREADS)
processes: List[Future[bool]] = list()
for in_file in files:
with BioReader(in_file) as reader:
x_shape, y_shape, z_shape = reader.X, reader.Y, reader.Z
metadata = reader.metadata
ndims = 2 if z_shape == 1 else 3
out_file = output_dir.joinpath(utils.replace_extension(in_file, extension='_flow.ome.zarr'))
init_zarr_file(out_file, ndims, metadata)
tile_count = 0
for z in range(0, z_shape, utils.TILE_SIZE):
z = None if ndims == 2 else z
for y in range(0, y_shape, utils.TILE_SIZE):
for x in range(0, x_shape, utils.TILE_SIZE):
coordinates = x, y, z
device = (tile_count % utils.NUM_THREADS) if utils.USE_GPU else None
tile_count += 1
# flow_thread(in_file, out_file, coordinates, device)
processes.append(executor.submit(
flow_thread,
in_file,
out_file,
coordinates,
device,
))
done, not_done = wait(processes, 0)
while len(not_done) > 0:
logger.info(f'Percent complete: {100 * len(done) / len(processes):6.3f}%')
for r in done:
r.result()
done, not_done = wait(processes, 5)
executor.shutdown()
return
def process_cm_data(data, ext_libs=[], user_args={}):
"""Processes data field from jsonnet config map.
Iterates through jsonnet files in configMap (.libsonnet files first)
and generates json data.
Args:
data (dict): Data from config map labeled as jsonnet code.
ext_libs (:obj:`list of str`, optional): List of paths to
external jsonnet libs.
user_args (:obj:`dict`, optional): Keyword arguments to jsonnet build function.
Returns:
list of (str, dict): Generated json data.
Raises:
JsonnetConfigMapError: Raised if jsonnet evaluation fails.
"""
libsonnet_folder = "./libsonnets"
jsons = []
# sort by extension: .libsonnet fields first, .jsonnet second
for dataKey in sorted(data.keys(),
key=lambda x: x.split(".")[1],
reverse=True):
_, extension = os.path.splitext(dataKey)
if extension == ".libsonnet":
utils.save_text_to_file(libsonnet_folder, dataKey, data[dataKey])
continue
try:
jsonnet_code = data[dataKey]
json_ = _jsonnet.evaluate_snippet(dataKey,
jsonnet_code,
jpathdir=ext_libs,
**user_args)
except RuntimeError as e:
log.error(f"{dataKey} is not a valid jsonnet, raised error: {e}")
if os.path.exists(libsonnet_folder):
utils.remove_folder(libsonnet_folder)
raise JsonnetConfigMapError
else:
json_filename = utils.replace_extension(dataKey, "json")
jsons.append((json_filename, json.loads(json_)))
if os.path.exists(libsonnet_folder):
utils.remove_folder(libsonnet_folder)
return jsons
def vector_to_label(
*,
in_path: Path,
flow_magnitude_threshold: float,
output_dir: Path,
):
# TODO: This next line breaks in the docker container because the base pytorch container comes with python3.7
# Apparently, it's a classic serialization bug on thread-locks that was fixed for python3.9.
# Until pytorch provides a container with python3.9 or polus provides a container with python3.9 and pytorch,
# we are stuck with this.
# executor = (ThreadPoolExecutor if utils.USE_GPU else ProcessPoolExecutor)(utils.NUM_THREADS)
executor = ThreadPoolExecutor(utils.NUM_THREADS)
with BioReader(in_path) as reader:
reader_shape = (reader.Z, reader.Y, reader.X)
metadata = reader.metadata
zarr_path = output_dir.joinpath(
utils.replace_extension(in_path, extension='_tmp.ome.zarr'))
init_zarr_file(zarr_path, metadata)
threads: Dict[Tuple[int, int, int], Future[ThreadFuture]] = dict()
thread_kwargs: Dict[str, Any] = {
'in_path': in_path,
'zarr_path': zarr_path,
'coordinates': (0, 0, 0),
'reader_shape': reader_shape,
'flow_magnitude_threshold': flow_magnitude_threshold,
'future_z': None,
'future_y': None,
'future_x': None,
}
tile_count = 0
for z_index, z in enumerate(range(0, reader_shape[0], utils.TILE_SIZE)):
for y_index, y in enumerate(range(0, reader_shape[1],
utils.TILE_SIZE)):
for x_index, x in enumerate(
range(0, reader_shape[2], utils.TILE_SIZE)):
tile_count += 1
thread_kwargs['coordinates'] = x, y, z
thread_kwargs['future_z'] = None if z_index == 0 else threads[(
z_index - 1, y_index, x_index)]
thread_kwargs['future_y'] = None if y_index == 0 else threads[(
z_index, y_index - 1, x_index)]
thread_kwargs['future_x'] = None if x_index == 0 else threads[(
z_index, y_index, x_index - 1)]
threads[(z_index, y_index,
x_index)] = executor.submit(vector_thread,
**thread_kwargs)
done, not_done = wait(threads.values(), 0)
while len(not_done) > 0:
logger.info(
f'File {in_path.name}, Progress: {100 * len(done) / len(threads):6.3f}%'
)
[future.result() for future in done]
done, not_done = wait(threads.values(), 15)
executor.shutdown()
out_path = output_dir.joinpath(utils.replace_extension(zarr_path))
zarr_to_tif(zarr_path, out_path)
return