def test_write_chunk_128(self):
with make_case(np.uint16, (200, 200, 200), return_path=True) \
as (glob_expr, dest, stack):
bfs = BlockfsStack(glob_expr, dest, chunk_size=(128, 128, 128))
bfs.write_level_1(silent=True, n_cores=1)
directory_filename = \
os.path.join(dest, "1_1_1", BlockfsStack.DIRECTORY_FILENAME)
directory = Directory.open(directory_filename)
np.testing.assert_array_equal(stack[:128, :128, :128],
directory.read_block(0, 0, 0))
np.testing.assert_array_equal(stack[128:, 128:, 128:],
directory.read_block(128, 128, 128))
def test_write_oddly_shaped(self):
with make_case(np.uint16, (100, 100, 100), return_path=True)\
as (glob_expr, dest, stack):
bfs = BlockfsStack(glob_expr, dest)
bfs.write_level_1(silent=True, n_cores=1)
directory_filename =\
os.path.join(dest, "1_1_1", BlockfsStack.DIRECTORY_FILENAME)
directory = Directory.open(directory_filename)
np.testing.assert_array_equal(stack[:64, :64, :64],
directory.read_block(0, 0, 0))
np.testing.assert_array_equal(stack[64:, 64:, 64:],
directory.read_block(64, 64, 64))
def main(args=sys.argv[1:]):
opts = parse_args(args)
logging.basicConfig(level=getattr(logging, opts.log_level))
paths = sorted(glob.glob(opts.input))
if len(paths) == 0:
print("No files were found for expression: %s" % opts.input)
return
frame = StackFrame(paths[0], 0, 0, 0)
stack = SpimStack(paths, frame.x0, frame.x1, frame.y0, frame.y1, 0)
zs, ys, xs, dtype = get_blockfs_dims(stack)
bfs_stack = BlockfsStack((zs, ys, xs), opts.output)
bfs_stack.write_info_file(opts.levels)
bfs_level1_dir = os.path.join(
opts.output, "1_1_1", BlockfsStack.DIRECTORY_FILENAME)
if not os.path.exists(os.path.dirname(bfs_level1_dir)):
os.mkdir(os.path.dirname(bfs_level1_dir))
directory = Directory(xs,
ys,
zs,
dtype,
bfs_level1_dir,
n_filenames=opts.n_writers)
directory.create()
directory.start_writer_processes()
spim_to_blockfs(stack, directory, opts.n_workers)
for level in range(2, opts.levels+1):
bfs_stack.write_level_n(level, n_cores=opts.n_writers)
def main(args: typing.Sequence[str] = sys.argv[1:]):
global ARRAY_READER, DIRECTORY
opts = parse_args(args)
x_step_size = opts.x_step_size
y_voxel_size = opts.y_voxel_size
r = x_step_size / y_voxel_size * np.sqrt(2)
ARRAY_READER = ArrayReader(pathlib.Path(opts.input).as_uri(),
format="blockfs")
dest_path = pathlib.Path(opts.output)
dest_path.mkdir(parents=True, exist_ok=True)
#
# Find the size of the destination volume by looking
# at the x/z corners
#
x00d = xz2xd(0, 0, r)
x01d = xz2xd(0, ARRAY_READER.shape[0], r)
x10d = xz2xd(ARRAY_READER.shape[2], 0, r)
x11d = xz2xd(ARRAY_READER.shape[2], ARRAY_READER.shape[0], r)
z00d = xz2zd(0, 0, r)
z01d = xz2zd(0, ARRAY_READER.shape[0], r)
z10d = xz2zd(ARRAY_READER.shape[2], 0, r)
z11d = xz2zd(ARRAY_READER.shape[2], ARRAY_READER.shape[0], r)
x0d = int(np.min([x00d, x01d, x10d, x11d]))
x1d = int(np.ceil(np.max([x00d, x01d, x10d, x11d])))
z0d = int(np.min([z00d, z01d, z10d, z11d]))
z1d = int(np.ceil(np.max([z00d, z01d, z10d, z11d])))
output_shape = (z1d - z0d, ARRAY_READER.shape[1], x1d - x0d)
#
# Get the blockfs destination started
#
blockfs_stack = BlockfsStack(output_shape, opts.output)
voxel_size = (1000. * x_step_size, 1000. * y_voxel_size,
1000. * x_step_size)
blockfs_stack.write_info_file(opts.levels, voxel_size)
bfs_level1_dir = \
pathlib.Path(opts.output) / "1_1_1" / BlockfsStack.DIRECTORY_FILENAME
bfs_level1_dir.parent.mkdir(parents=True, exist_ok=True)
DIRECTORY = Directory(output_shape[2],
output_shape[1],
output_shape[0],
ARRAY_READER.dtype,
str(bfs_level1_dir),
n_filenames=opts.n_writers)
DIRECTORY.create()
DIRECTORY.start_writer_processes()
xds = np.arange(0, output_shape[2], DIRECTORY.x_block_size)
yds = np.arange(0, output_shape[1], DIRECTORY.y_block_size)
zds = np.arange(0, output_shape[0], DIRECTORY.z_block_size)
with multiprocessing.Pool(opts.n_cores) as pool:
futures = []
for x0di, y0di, z0di in itertools.product(xds, yds, zds):
futures.append(
pool.apply_async(do_one, (x0di, y0di, z0di, x0d, z0d, r)))
for future in tqdm.tqdm(futures):
future.get()
DIRECTORY.close()
for level in range(2, opts.levels + 1):
blockfs_stack.write_level_n(level, n_cores=opts.n_writers)
def main(args: typing.Sequence[str] = sys.argv[1:]):
global ARRAY_READER, DIRECTORY
opts = parse_args(args)
if opts.kernel_size is None:
klen = kernel_size(opts.power)
else:
klen = opts.kernel_size
voxel_size = [float(_) * 1000 for _ in opts.voxel_size.split(",")]
ARRAY_READER = ArrayReader(pathlib.Path(opts.input).as_uri(),
format=opts.input_format)
bfs_stack = BlockfsStack(ARRAY_READER.shape, opts.output)
bfs_stack.write_info_file(opts.levels, voxel_size=voxel_size)
bfs_level1_dir = \
pathlib.Path(opts.output) / "1_1_1" / BlockfsStack.DIRECTORY_FILENAME
bfs_level1_dir.parent.mkdir(parents=True, exist_ok=True)
DIRECTORY = Directory(ARRAY_READER.shape[2],
ARRAY_READER.shape[1],
ARRAY_READER.shape[0],
ARRAY_READER.dtype,
str(bfs_level1_dir),
n_filenames=opts.n_writers)
DIRECTORY.create()
DIRECTORY.start_writer_processes()
nblks = opts.n_blocks_per_process
xsize = DIRECTORY.x_block_size * nblks
ysize = DIRECTORY.y_block_size * nblks
zsize = DIRECTORY.z_block_size * nblks
xr0 = np.arange(0, DIRECTORY.x_extent, xsize)
yr0 = np.arange(0, DIRECTORY.y_extent, ysize)
zr0 = np.arange(0, DIRECTORY.z_extent, zsize)
xr1 = np.minimum(xr0 + xsize, DIRECTORY.x_extent)
yr1 = np.minimum(yr0 + ysize, DIRECTORY.y_extent)
zr1 = np.minimum(zr0 + zsize, DIRECTORY.z_extent)
futures = []
if opts.use_gpu:
cores = 1
else:
cores = opts.n_cores
with multiprocessing.Pool(cores) as pool:
for (x0, x1), (y0,
y1), (z0,
z1) in itertools.product(zip(xr0, xr1),
zip(yr0, yr1),
zip(zr0, zr1)):
futures.append(
pool.apply_async(do_one,
(x0, y0, z0, x1, y1, z1, opts.power, klen,
opts.iterations, opts.use_gpu)))
for future in tqdm.tqdm(futures, desc="Writing blocks"):
future.get()
DIRECTORY.close()
for level in range(2, opts.levels + 1):
bfs_stack.write_level_n(level, n_cores=opts.n_writers)
def main(args=sys.argv[1:]):
global MY_OPTS, MY_DCIMG, FLAT
MY_OPTS = parse_args(args)
destripe_method = MY_OPTS.destripe_method
if not (destripe_method is None or destripe_method == "lightsheet"
or destripe_method == "wavelet"):
print("--destripe-method must be \"lightsheet\", \"wavelet\" or blank",
file=sys.stderr)
sys.exit(-1)
if MY_OPTS.flat is not None:
FLAT = normalize_flat(tifffile.imread(MY_OPTS.flat))
if MY_OPTS.jp2k:
paths = sorted(glob.glob(MY_OPTS.input))
fn = do_one_jp2000
img = glymur.Jp2k(paths[0])
x_extent = img.shape[1]
y_extent = img.shape[0]
else:
MY_DCIMG = DCIMG(MY_OPTS.input)
start = MY_OPTS.start
stop = MY_OPTS.stop or MY_DCIMG.n_frames
x_extent = int(MY_DCIMG.x_dim)
y_extent = int(MY_DCIMG.y_dim)
paths = [str(i) for i in range(start, stop)]
fn = do_one_dcimg
stack = SpimStack(paths, 0, 0, x_extent, y_extent, 0)
#
# The stack dimensions are a little elongated because of the
# parallelogram
#
z_extent, y_extent, x_extent, dtype = get_blockfs_dims(
stack, x_extent, y_extent)
bfs_stack = BlockfsStack((z_extent, y_extent, x_extent), MY_OPTS.output)
bfs_stack.write_info_file(MY_OPTS.levels)
bfs_level1_dir = os.path.join(MY_OPTS.output, "1_1_1",
BlockfsStack.DIRECTORY_FILENAME)
if not os.path.exists(os.path.dirname(bfs_level1_dir)):
os.mkdir(os.path.dirname(bfs_level1_dir))
directory = Directory(x_extent,
y_extent,
z_extent,
np.uint16,
bfs_level1_dir,
n_filenames=MY_OPTS.n_writers)
directory.create()
directory.start_writer_processes()
spim_to_blockfs(stack, directory, MY_OPTS.n_workers, read_fn=fn)
for level in range(2, MY_OPTS.levels + 1):
bfs_stack.write_level_n(level, n_cores=MY_OPTS.n_writers)
def convert_to_tif_and_blockfs(
precomputed_path,
output_pattern:str,
volume:VExtent=None,
dtype=None,
compression=4,
cores=multiprocessing.cpu_count(),
io_cores=multiprocessing.cpu_count(),
voxel_size=(1800, 1800, 2000),
n_levels:int=5):
if volume is None:
volume = V.volume
if dtype is None:
dtype = V.dtype
blockfs_stack = BlockfsStack(volume.shape, precomputed_path)
blockfs_stack.write_info_file(n_levels, voxel_size)
directory = blockfs_stack.make_l1_directory(io_cores)
directory.create()
directory.start_writer_processes()
sm = SharedMemory((directory.z_block_size,
volume.y1 - volume.y0,
volume.x1 - volume.x0), dtype)
with multiprocessing.Pool(cores) as pool:
for z0 in tqdm.tqdm(
range(volume.z0, volume.z1, directory.z_block_size)):
z1 = min(volume.z1, z0 + directory.z_block_size)
futures = []
for z in range(z0, z1):
futures.append(pool.apply_async(
do_plane,
(volume, z0, z, sm, output_pattern % z, compression)))
for future in futures:
future.get()
x0 = np.arange(0, sm.shape[2], directory.x_block_size)
x1 = np.minimum(sm.shape[2], x0 + directory.x_block_size)
y0 = np.arange(0, sm.shape[1], directory.y_block_size)
y1 = np.minimum(sm.shape[1], y0 + directory.y_block_size)
with sm.txn() as memory:
for (x0a, x1a), (y0a, y1a) in itertools.product(
zip(x0, x1),zip(y0, y1)):
directory.write_block(memory[:z1-z0, y0a:y1a, x0a:x1a],
x0a, y0a, z0)
directory.close()
for level in range(2, n_levels+1):
blockfs_stack.write_level_n(level, n_cores=io_cores)
def main(args=sys.argv[1:]):
opts = parse_args(args)
logging.basicConfig(level=getattr(logging, opts.log_level))
volume_paths = []
zs = []
for root, folders, files in os.walk(opts.input, followlinks=True):
if os.path.split(root)[-1] == "1_1_1":
for file in files:
if file == BlockfsStack.DIRECTORY_FILENAME:
volume_paths.append(os.path.join(root, file))
try:
zs.append(int(os.path.split(os.path.dirname(root))[1]))
except ValueError:
logging.warning(
"Non-numeric Z found in stack path: %s" % root)
all_z = sorted(set(zs))
z_offsets = [opts.z_offset * all_z.index(z) * opts.x_step_size for z in zs]
volumes = [
StitchSrcVolume(volume_path, opts.x_step_size, opts.y_voxel_size,
z_offset)
for volume_path, z_offset in zip(volume_paths, z_offsets)
]
z_too = adjust_alignments(opts, volumes)
StitchSrcVolume.rebase_all(volumes, z_too=z_too)
if opts.compute_y_illum_corr:
y_illum_corr = StitchSrcVolume.compute_illum_corr(
volumes,
n_patches=opts.n_y_illum_patches,
min_mean=opts.min_y_illum_mean,
min_corr_coef=opts.min_y_illum_corr_coef,
n_workers=opts.n_workers)
elif opts.y_illum_corr is not None:
y_illum_corr = opts.y_illum_corr
else:
y_illum_corr = None
if y_illum_corr is not None:
y_illum_corr = \
(1 - y_illum_corr) * (2047 - np.arange(2048)) / 2047 + \
y_illum_corr
if opts.output_size is None:
zs, ys, xs = get_output_size(volumes)
x0 = y0 = z0 = 0
else:
xs, ys, zs = [int(_) for _ in opts.output_size.split(",")]
if opts.output_offset is None:
x0 = y0 = z0 = 0
else:
x0, y0, z0 = [int(_) for _ in opts.output_offset.split(",")]
if not os.path.exists(opts.output):
os.mkdir(opts.output)
l1_dir = os.path.join(opts.output, "1_1_1")
if not os.path.exists(l1_dir):
os.mkdir(l1_dir)
output = BlockfsStack((zs, ys, xs), opts.output)
voxel_size = (opts.x_step_size * 1000, opts.y_voxel_size * 1000,
opts.x_step_size * 1000)
output.write_info_file(opts.levels, voxel_size)
directory_path = os.path.join(l1_dir, BlockfsStack.DIRECTORY_FILENAME)
directory = Directory(xs,
ys,
zs,
volumes[0].directory.dtype,
directory_path,
n_filenames=opts.n_writers)
directory.create()
directory.start_writer_processes()
run(volumes, directory, x0, y0, z0, opts.n_workers, opts.silent,
y_illum_corr)
directory.close()
for level in range(2, opts.levels + 1):
output.write_level_n(level, opts.silent, opts.n_writers)