def spim_to_blockfs(stack: SpimStack,
directory: Directory,
n_workers: int,
read_fn: READ_FUNCTION_T = tifffile.imread):
global DIRECTORY
DIRECTORY = directory
dependents = make_s2b_dependents(stack, directory, read_fn)
pipeline = Pipeline(dependents)
with multiprocessing.Pool(n_workers) as pool:
pipeline.run(pool)
directory.close()
DIRECTORY = None
def main(args: typing.Sequence[str] = sys.argv[1:]):
global ARRAY_READER, DIRECTORY
opts = parse_args(args)
x_step_size = opts.x_step_size
y_voxel_size = opts.y_voxel_size
r = x_step_size / y_voxel_size * np.sqrt(2)
ARRAY_READER = ArrayReader(pathlib.Path(opts.input).as_uri(),
format="blockfs")
dest_path = pathlib.Path(opts.output)
dest_path.mkdir(parents=True, exist_ok=True)
#
# Find the size of the destination volume by looking
# at the x/z corners
#
x00d = xz2xd(0, 0, r)
x01d = xz2xd(0, ARRAY_READER.shape[0], r)
x10d = xz2xd(ARRAY_READER.shape[2], 0, r)
x11d = xz2xd(ARRAY_READER.shape[2], ARRAY_READER.shape[0], r)
z00d = xz2zd(0, 0, r)
z01d = xz2zd(0, ARRAY_READER.shape[0], r)
z10d = xz2zd(ARRAY_READER.shape[2], 0, r)
z11d = xz2zd(ARRAY_READER.shape[2], ARRAY_READER.shape[0], r)
x0d = int(np.min([x00d, x01d, x10d, x11d]))
x1d = int(np.ceil(np.max([x00d, x01d, x10d, x11d])))
z0d = int(np.min([z00d, z01d, z10d, z11d]))
z1d = int(np.ceil(np.max([z00d, z01d, z10d, z11d])))
output_shape = (z1d - z0d, ARRAY_READER.shape[1], x1d - x0d)
#
# Get the blockfs destination started
#
blockfs_stack = BlockfsStack(output_shape, opts.output)
voxel_size = (1000. * x_step_size, 1000. * y_voxel_size,
1000. * x_step_size)
blockfs_stack.write_info_file(opts.levels, voxel_size)
bfs_level1_dir = \
pathlib.Path(opts.output) / "1_1_1" / BlockfsStack.DIRECTORY_FILENAME
bfs_level1_dir.parent.mkdir(parents=True, exist_ok=True)
DIRECTORY = Directory(output_shape[2],
output_shape[1],
output_shape[0],
ARRAY_READER.dtype,
str(bfs_level1_dir),
n_filenames=opts.n_writers)
DIRECTORY.create()
DIRECTORY.start_writer_processes()
xds = np.arange(0, output_shape[2], DIRECTORY.x_block_size)
yds = np.arange(0, output_shape[1], DIRECTORY.y_block_size)
zds = np.arange(0, output_shape[0], DIRECTORY.z_block_size)
with multiprocessing.Pool(opts.n_cores) as pool:
futures = []
for x0di, y0di, z0di in itertools.product(xds, yds, zds):
futures.append(
pool.apply_async(do_one, (x0di, y0di, z0di, x0d, z0d, r)))
for future in tqdm.tqdm(futures):
future.get()
DIRECTORY.close()
for level in range(2, opts.levels + 1):
blockfs_stack.write_level_n(level, n_cores=opts.n_writers)
def do_GET(self):
if args.format == FORMAT_RAW or self.path.startswith("/socket"):
print("Handing request to simple http request handler")
super(RequestHandler, self).do_GET()
elif self.path.find("/info") >= 0:
size = pathlib.Path("info").stat().st_size
self.send_response(HTTPStatus.OK)
self.send_header("Content-type", 'application/octet-stream')
self.send_header("Content-Length", str(size))
self.end_headers()
with open("info", "rb") as fd:
self.copyfile(fd, self.wfile)
elif args.format == FORMAT_TIFF:
import tifffile
path = self.path[1:] + ".tiff"
print(path)
if not os.path.exists(path):
super(RequestHandler, self).do_GET()
return
chunk = tifffile.imread(path)
self.send_chunk(chunk)
elif args.format == FORMAT_ZARR:
import zarr
level, path = self.path[1:].split('/')
if not os.path.exists(level):
super(RequestHandler, self).do_GET()
return
x0, y0, z0 = self.parse_path(path)
store = zarr.NestedDirectoryStore(level)
z_arr = zarr.open(store, mode='r')
chunk = z_arr[z0:z1, y0:y1, x0:x1]
self.send_chunk(chunk)
elif args.format == FORMAT_BLOCKFS:
level, path = self.path[1:].split('/')
if not os.path.exists(level):
super(RequestHandler, self).do_GET()
return
x0, y0, z0 = self.parse_path(path)
directory = Directory.open(
os.path.join(level, "precomputed.blockfs"))
chunk = directory.read_block(x0, y0, z0)
self.send_chunk(chunk)
elif args.format == FORMAT_NGFF:
import zarr
level, path = self.path[1:].split('/')
x0, y0, z0 = self.parse_path(path)
store = zarr.NestedDirectoryStore(".")
group = zarr.group(store)
lx, ly, lz = [int(_) for _ in level.split("_")]
llevel = int(np.round(np.log2(lx), 0))
a = group[llevel]
_, _, zs, ys, xs = a.chunks
z1 = min(a.shape[2], z0 + zs)
y1 = min(a.shape[3], y0 + zs)
x1 = min(a.shape[4], x0 + xs)
chunk = a[0, 0, z0:z1, y0:y1, x0:x1]
self.send_chunk(chunk)
else:
raise ValueError('Invalid format specified')
def __init__(self, path:str, x_step_size:float, yum:float, z0:int,
is_oblique:bool=True):
"""
:param path: Path to a blockfs directory file. The path contains
position metadata for the volume - the path should be something like
%d_%d/1_1_1/precomputed.blockfs
:param x_step_size: stepper size in the X direction. We assume that the
X pixel size is sqrt(2) * x_step_size, so the Z pixel size is the
same as the stepper size.
:param yum: Y pixel size of pixels in the original planes
:param z0: z0 of the stack.
:param is_oblique: True if oblique, False if doing non-oblique stitch
"""
self.key = uuid.uuid4()
self.path = path
self.directory = Directory.open(path)
self.xum = x_step_size
self.zum = x_step_size
self.yum = yum
z_metadata_dir = os.path.dirname(os.path.dirname(path))
metadata_dir = os.path.split(os.path.dirname(z_metadata_dir))[-1]
self.x0, self.y0 = [int(_) / 10 for _ in metadata_dir.split("_")]
self.z0 = z0
self.is_oblique = is_oblique
def main(args: typing.Sequence[str] = sys.argv[1:]):
global ARRAY_READER, DIRECTORY
opts = parse_args(args)
if opts.kernel_size is None:
klen = kernel_size(opts.power)
else:
klen = opts.kernel_size
voxel_size = [float(_) * 1000 for _ in opts.voxel_size.split(",")]
ARRAY_READER = ArrayReader(pathlib.Path(opts.input).as_uri(),
format=opts.input_format)
bfs_stack = BlockfsStack(ARRAY_READER.shape, opts.output)
bfs_stack.write_info_file(opts.levels, voxel_size=voxel_size)
bfs_level1_dir = \
pathlib.Path(opts.output) / "1_1_1" / BlockfsStack.DIRECTORY_FILENAME
bfs_level1_dir.parent.mkdir(parents=True, exist_ok=True)
DIRECTORY = Directory(ARRAY_READER.shape[2],
ARRAY_READER.shape[1],
ARRAY_READER.shape[0],
ARRAY_READER.dtype,
str(bfs_level1_dir),
n_filenames=opts.n_writers)
DIRECTORY.create()
DIRECTORY.start_writer_processes()
nblks = opts.n_blocks_per_process
xsize = DIRECTORY.x_block_size * nblks
ysize = DIRECTORY.y_block_size * nblks
zsize = DIRECTORY.z_block_size * nblks
xr0 = np.arange(0, DIRECTORY.x_extent, xsize)
yr0 = np.arange(0, DIRECTORY.y_extent, ysize)
zr0 = np.arange(0, DIRECTORY.z_extent, zsize)
xr1 = np.minimum(xr0 + xsize, DIRECTORY.x_extent)
yr1 = np.minimum(yr0 + ysize, DIRECTORY.y_extent)
zr1 = np.minimum(zr0 + zsize, DIRECTORY.z_extent)
futures = []
if opts.use_gpu:
cores = 1
else:
cores = opts.n_cores
with multiprocessing.Pool(cores) as pool:
for (x0, x1), (y0,
y1), (z0,
z1) in itertools.product(zip(xr0, xr1),
zip(yr0, yr1),
zip(zr0, zr1)):
futures.append(
pool.apply_async(do_one,
(x0, y0, z0, x1, y1, z1, opts.power, klen,
opts.iterations, opts.use_gpu)))
for future in tqdm.tqdm(futures, desc="Writing blocks"):
future.get()
DIRECTORY.close()
for level in range(2, opts.levels + 1):
bfs_stack.write_level_n(level, n_cores=opts.n_writers)
def test_blockfs(self):
with make_case(np.uint16, (256, 256, 256), return_path=True) as \
(glob_expr, dest, volume):
main([
"--source", glob_expr, "--dest", dest, "--levels", "2",
"--format", "blockfs"
])
directory = Directory.open(
os.path.join(dest, "1_1_1", BlockfsStack.DIRECTORY_FILENAME))
block = directory.read_block(0, 0, 0)
np.testing.assert_array_equal(block, volume[:64, :64, :64])
main([
"--source", glob_expr, "--dest", dest, "--levels", "2",
"--format", "blockfs", "--chunk-size", "128,128,128"
])
directory = Directory.open(
os.path.join(dest, "1_1_1", BlockfsStack.DIRECTORY_FILENAME))
block = directory.read_block(0, 0, 0)
np.testing.assert_array_equal(block, volume[:128, :128, :128])
def make_case(volume_shape:typing.Tuple[int, int, int],
block_shape:typing.Tuple[int, int, int])\
->typing.Tuple[typing.Sequence[str], Directory, np.ndarray]:
volume = np.random.RandomState(np.prod(volume_shape))\
.randint(0, 65535, volume_shape).astype(np.uint16)
tempdir = tempfile.mkdtemp()
directory = Directory(volume_shape[2] + volume_shape[0],
volume_shape[1],
volume_shape[2],
np.uint16,
os.path.join(tempdir, "test.blockfs"),
n_filenames=1,
x_block_size=block_shape[2],
y_block_size=block_shape[1],
z_block_size=block_shape[0])
directory.create()
directory.start_writer_processes()
paths = []
for i, plane in enumerate(volume):
path = os.path.join(tempdir, "img_%04d.tiff" % i)
paths.append(path)
tifffile.imsave(path, plane)
yield paths, directory, volume
shutil.rmtree(tempdir, ignore_errors=True)
def main(args=sys.argv[1:]):
opts = parse_args(args)
logging.basicConfig(level=getattr(logging, opts.log_level))
paths = sorted(glob.glob(opts.input))
if len(paths) == 0:
print("No files were found for expression: %s" % opts.input)
return
frame = StackFrame(paths[0], 0, 0, 0)
stack = SpimStack(paths, frame.x0, frame.x1, frame.y0, frame.y1, 0)
zs, ys, xs, dtype = get_blockfs_dims(stack)
bfs_stack = BlockfsStack((zs, ys, xs), opts.output)
bfs_stack.write_info_file(opts.levels)
bfs_level1_dir = os.path.join(
opts.output, "1_1_1", BlockfsStack.DIRECTORY_FILENAME)
if not os.path.exists(os.path.dirname(bfs_level1_dir)):
os.mkdir(os.path.dirname(bfs_level1_dir))
directory = Directory(xs,
ys,
zs,
dtype,
bfs_level1_dir,
n_filenames=opts.n_writers)
directory.create()
directory.start_writer_processes()
spim_to_blockfs(stack, directory, opts.n_workers)
for level in range(2, opts.levels+1):
bfs_stack.write_level_n(level, n_cores=opts.n_writers)
def __init__(self, path, x1, y1, z1):
x0 = y0 = z0 = 0
trailing_oblique_start = x1 - z1 + z0
z, y, x = np.mgrid[z0:z1, y0:y1, x0:x1 - z1]
x = x + z
volume = np.zeros((z1 - z0, y1 - y0, x1 - x0), np.uint16)
r = np.random.RandomState(1234)
volume[z.flatten(), y.flatten(), x.flatten()] = \
r.randint(0, 65535, np.prod(z.shape))
blockfs_path = os.path.join(path, "1_1_1", "precomputed.blockfs")
os.makedirs(os.path.dirname(blockfs_path))
directory = Directory(x1 - x0,
y1 - y0,
z1 - x0,
np.uint16,
blockfs_path,
n_filenames=1,
x_block_size=4,
y_block_size=4,
z_block_size=4)
directory.create()
directory.start_writer_processes()
for xs in range(0, x1 - x0, 4):
for ys in range(0, y1 - y0, 4):
for zs in range(0, z1 - z0, 4):
directory.write_block(
volume[zs:zs + 4, ys:ys + 4, xs:xs + 4], xs, ys, zs)
directory.close()
self.volume = volume
self.path = path
self.x0 = x0
self.x1 = x1
self.y0 = y0
self.y1 = y1
self.z0 = z0
self.z1 = z1
def main(args=sys.argv[1:]):
opts = parse_args(args)
logging.basicConfig(level=getattr(logging, opts.log_level))
volume_paths = []
zs = []
for root, folders, files in os.walk(opts.input, followlinks=True):
if os.path.split(root)[-1] == "1_1_1":
for file in files:
if file == BlockfsStack.DIRECTORY_FILENAME:
volume_paths.append(os.path.join(root, file))
try:
zs.append(int(os.path.split(os.path.dirname(root))[1]))
except ValueError:
logging.warning(
"Non-numeric Z found in stack path: %s" % root)
all_z = sorted(set(zs))
z_offsets = [opts.z_offset * all_z.index(z) * opts.x_step_size for z in zs]
volumes = [
StitchSrcVolume(volume_path, opts.x_step_size, opts.y_voxel_size,
z_offset)
for volume_path, z_offset in zip(volume_paths, z_offsets)
]
z_too = adjust_alignments(opts, volumes)
StitchSrcVolume.rebase_all(volumes, z_too=z_too)
if opts.compute_y_illum_corr:
y_illum_corr = StitchSrcVolume.compute_illum_corr(
volumes,
n_patches=opts.n_y_illum_patches,
min_mean=opts.min_y_illum_mean,
min_corr_coef=opts.min_y_illum_corr_coef,
n_workers=opts.n_workers)
elif opts.y_illum_corr is not None:
y_illum_corr = opts.y_illum_corr
else:
y_illum_corr = None
if y_illum_corr is not None:
y_illum_corr = \
(1 - y_illum_corr) * (2047 - np.arange(2048)) / 2047 + \
y_illum_corr
if opts.output_size is None:
zs, ys, xs = get_output_size(volumes)
x0 = y0 = z0 = 0
else:
xs, ys, zs = [int(_) for _ in opts.output_size.split(",")]
if opts.output_offset is None:
x0 = y0 = z0 = 0
else:
x0, y0, z0 = [int(_) for _ in opts.output_offset.split(",")]
if not os.path.exists(opts.output):
os.mkdir(opts.output)
l1_dir = os.path.join(opts.output, "1_1_1")
if not os.path.exists(l1_dir):
os.mkdir(l1_dir)
output = BlockfsStack((zs, ys, xs), opts.output)
voxel_size = (opts.x_step_size * 1000, opts.y_voxel_size * 1000,
opts.x_step_size * 1000)
output.write_info_file(opts.levels, voxel_size)
directory_path = os.path.join(l1_dir, BlockfsStack.DIRECTORY_FILENAME)
directory = Directory(xs,
ys,
zs,
volumes[0].directory.dtype,
directory_path,
n_filenames=opts.n_writers)
directory.create()
directory.start_writer_processes()
run(volumes, directory, x0, y0, z0, opts.n_workers, opts.silent,
y_illum_corr)
directory.close()
for level in range(2, opts.levels + 1):
output.write_level_n(level, opts.silent, opts.n_writers)