def test_mv(self):
for i in range(2):
a = np.random.randint(0, 65535, (256, 256, 256), np.uint16)
with make_files(1) as (dir_file, block_files):
directory = Directory(256,
256,
256,
np.uint16,
dir_file,
compression=Compression.zstd,
block_filenames=block_files)
directory.create()
for x, y, z in itertools.product(range(0, 256, 64),
range(0, 256, 64),
range(0, 256, 64)):
directory.write_block(a[z:z + 64, y:y + 64, x:x + 64], x,
y, z)
directory.close()
dest = tempfile.mkdtemp()
try:
if i == 0:
main([dir_file, dest])
else:
subprocess.check_call(["blockfs-mv", dir_file, dest])
dest_dir_file = \
os.path.join(dest, os.path.split(dir_file)[1])
dest_directory = Directory.open(dest_dir_file)
for x, y, z in itertools.product(range(0, 256, 64),
range(0, 256, 64),
range(0, 256, 64)):
block = dest_directory.read_block(x, y, z)
np.testing.assert_array_equal(
a[z:z + 64, y:y + 64, x:x + 64], block)
finally:
shutil.rmtree(dest)
def test_04_write_read(self):
a = np.random.randint(0, 65535, (64, 64, 64), np.uint16)
with make_files(1) as (dir_file, block_files):
directory = Directory(1024, 1024, 1024, np.uint16, dir_file,
compression=Compression.zstd,
block_filenames=block_files)
directory.create()
directory.write_block(a, 64, 128, 192)
directory.close()
a_out = Directory.open(dir_file).read_block(64, 128, 192)
np.testing.assert_array_equal(a, a_out)
def test_02_01_encode_decode(self):
with make_files(1) as (dir_file, block_files):
directory = Directory(1024, 1024, 1024, np.uint16, dir_file,
block_filenames=block_files)
directory.create()
try:
test_cases = ((0, 524304), (524304, 524304))
for offset, size in test_cases:
a = np.zeros(directory.directory_entry_size, np.uint8)
directory.encode_directory_entry(a, offset, size)
offset_out, size_out = directory.decode_directory_entry(a)
self.assertEqual(offset, offset_out)
self.assertEqual(size, size_out)
finally:
directory.close()
def test_case(self):
src_dir = pathlib.Path(tempfile.mkdtemp())
dest_dir_parent = pathlib.Path(tempfile.mkdtemp())
dest_dir = dest_dir_parent / "dest"
dest_dir.mkdir()
all_files = []
rs = np.random.RandomState(1234)
a = rs.randint(0, 65535, (256, 256, 256), dtype=np.uint16)
try:
dir_file = src_dir / "my.blockfs"
block_files = [src_dir / ("my.blockfs.%d" % i) for i in range(4)]
directory = Directory(256, 256, 256, np.uint16, str(dir_file),
compression=Compression.zstd,
block_filenames=
[str(_) for _ in block_files])
directory.create()
for x, y, z in itertools.product(range(0, 256, 64),
range(0, 256, 64),
range(0, 256, 64)):
directory.write_block(
a[z:z + 64, y:y + 64, x:x + 64], x, y, z)
directory.close()
for path in [dir_file] + list(block_files):
dest_path = dest_dir / path.name
path.replace(dest_path)
all_files.append(dest_path)
dest_directory_file = dest_dir / pathlib.Path(dir_file).name
main([str(dest_directory_file)])
directory = Directory.open(str(dest_directory_file))
for x, y, z in itertools.product(range(0, 256, 64),
range(0, 256, 64),
range(0, 256, 64)):
np.testing.assert_array_equal(a[z:z+64, y:y+64, x:x+64],
directory.read_block(x, y, z))
finally:
try:
for path in all_files:
path.unlink()
dest_dir.rmdir()
dest_dir_parent.rmdir()
except:
traceback.print_exc()
print("Failed to remove files")
def test_write_chunk_128(self):
with make_case(np.uint16, (200, 200, 200), return_path=True) \
as (glob_expr, dest, stack):
bfs = BlockfsStack(glob_expr, dest, chunk_size=(128, 128, 128))
bfs.write_level_1(silent=True, n_cores=1)
directory_filename = \
os.path.join(dest, "1_1_1", BlockfsStack.DIRECTORY_FILENAME)
directory = Directory.open(directory_filename)
np.testing.assert_array_equal(stack[:128, :128, :128],
directory.read_block(0, 0, 0))
np.testing.assert_array_equal(stack[128:, 128:, 128:],
directory.read_block(128, 128, 128))
def test_write_oddly_shaped(self):
with make_case(np.uint16, (100, 100, 100), return_path=True)\
as (glob_expr, dest, stack):
bfs = BlockfsStack(glob_expr, dest)
bfs.write_level_1(silent=True, n_cores=1)
directory_filename =\
os.path.join(dest, "1_1_1", BlockfsStack.DIRECTORY_FILENAME)
directory = Directory.open(directory_filename)
np.testing.assert_array_equal(stack[:64, :64, :64],
directory.read_block(0, 0, 0))
np.testing.assert_array_equal(stack[64:, 64:, 64:],
directory.read_block(64, 64, 64))
def test_02_create_and_open(self):
with make_files(1) as (dir_file, block_files):
directory = Directory(1024, 1024, 1024, np.uint16, dir_file,
block_filenames=block_files)
directory.create()
directory.close()
directory = Directory.open(dir_file)
self.assertEqual(directory.x_extent, 1024)
self.assertEqual(directory.y_extent, 1024)
self.assertEqual(directory.z_extent, 1024)
def test_03_write(self):
a = np.random.randint(0, 65535, (64, 64, 64), np.uint16)
with make_files(1) as (dir_file, block_files):
directory = Directory(1024, 1024, 1024, np.uint16, dir_file,
compression=Compression.zstd,
block_filenames=block_files)
directory.create()
directory.write_block(a, 64, 128, 192)
directory.close()
with open(block_files[0], "rb") as fd:
block = fd.read()
blosc = Blosc("zstd")
a_out = np.frombuffer(blosc.decode(block), np.uint16)\
.reshape(64, 64, 64)
np.testing.assert_array_equal(a, a_out)
def test_01_create(self):
with make_files(1) as (dir_file, block_files):
directory = Directory(1024, 1024, 1024, np.uint16, dir_file,
block_filenames=block_files)
directory.create()
directory.close()
with open(dir_file, "rb") as fd:
header = fd.read(len(Directory.HEADER))
self.assertEqual(header, Directory.HEADER)
def main(args=sys.argv[1:]):
global MY_OPTS, MY_DCIMG, FLAT
MY_OPTS = parse_args(args)
destripe_method = MY_OPTS.destripe_method
if not (destripe_method is None or destripe_method == "lightsheet"
or destripe_method == "wavelet"):
print("--destripe-method must be \"lightsheet\", \"wavelet\" or blank",
file=sys.stderr)
sys.exit(-1)
if MY_OPTS.flat is not None:
FLAT = normalize_flat(tifffile.imread(MY_OPTS.flat))
if MY_OPTS.jp2k:
paths = sorted(glob.glob(MY_OPTS.input))
fn = do_one_jp2000
img = glymur.Jp2k(paths[0])
x_extent = img.shape[1]
y_extent = img.shape[0]
else:
MY_DCIMG = DCIMG(MY_OPTS.input)
start = MY_OPTS.start
stop = MY_OPTS.stop or MY_DCIMG.n_frames
x_extent = int(MY_DCIMG.x_dim)
y_extent = int(MY_DCIMG.y_dim)
paths = [str(i) for i in range(start, stop)]
fn = do_one_dcimg
stack = SpimStack(paths, 0, 0, x_extent, y_extent, 0)
#
# The stack dimensions are a little elongated because of the
# parallelogram
#
z_extent, y_extent, x_extent, dtype = get_blockfs_dims(
stack, x_extent, y_extent)
bfs_stack = BlockfsStack((z_extent, y_extent, x_extent), MY_OPTS.output)
bfs_stack.write_info_file(MY_OPTS.levels)
bfs_level1_dir = os.path.join(MY_OPTS.output, "1_1_1",
BlockfsStack.DIRECTORY_FILENAME)
if not os.path.exists(os.path.dirname(bfs_level1_dir)):
os.mkdir(os.path.dirname(bfs_level1_dir))
directory = Directory(x_extent,
y_extent,
z_extent,
np.uint16,
bfs_level1_dir,
n_filenames=MY_OPTS.n_writers)
directory.create()
directory.start_writer_processes()
spim_to_blockfs(stack, directory, MY_OPTS.n_workers, read_fn=fn)
for level in range(2, MY_OPTS.levels + 1):
bfs_stack.write_level_n(level, n_cores=MY_OPTS.n_writers)
def read_image_values(src: str, xs: np.ndarray, ys: np.ndarray,
zs: np.ndarray) -> np.ndarray:
global block
"""
Read image values from a blockfs file.
:param src: the precomputed.blockfs file
:param xs: the X coordinates of the image values to read
:param ys: the Y coordinates of the image values to read
:param zs: the Z coordinates of the image values to read
:return: a 1-d array of the values at each x, y and z
"""
directory = Directory.open(src)
xb = np.floor(xs / directory.x_block_size).astype(np.int16)
yb = np.floor(ys / directory.y_block_size).astype(np.int16)
zb = np.floor(zs / directory.z_block_size).astype(np.int16)
df = pandas.DataFrame(
dict(x=xs.astype(np.int32),
y=ys.astype(np.int32),
z=zs.astype(np.int32),
xb=xb,
yb=yb,
zb=zb))
dg = df.groupby(["xb", "yb", "zb"])
result = np.zeros(len(xs), directory.dtype)
for (xi, yi, zi), idxs in dg.groups.items():
x_off = xi * directory.x_block_size
y_off = yi * directory.y_block_size
z_off = zi * directory.z_block_size
if xi < 0 or yi < 0 or zi < 0 or\
x_off >= directory.x_extent or \
y_off >= directory.y_extent or \
z_off >= directory.z_extent:
continue
block = directory.read_block(x_off, y_off, z_off)
sub_df = dg.get_group((xi, yi, zi))
values = ndimage.map_coordinates(block, [
sub_df.z.values - z_off, sub_df.y.values - y_off,
sub_df.x.values - x_off
])
result[idxs] = values
return result
def main(args=sys.argv[1:]):
opts = parse_args(args)
src_path = pathlib.Path(opts.index_file).absolute().resolve()
src_directory = Directory.open(str(src_path))
directory_offset = src_directory.directory_offset
dest_path = src_path.parent / (src_path.name + '.new')
src_directory.block_filenames = [
str(src_path.parent / pathlib.Path(_).name)
for _ in src_directory.block_filenames
]
src_directory.directory_filename = str(dest_path)
src_directory.create(create_shards=False)
with src_path.open("rb") as src_fd:
src_fd.seek(directory_offset)
with dest_path.open("ab+") as dest_fd:
for offset in range(src_directory.directory_offset,
src_path.stat().st_size, opts.block_size):
data = src_fd.read(opts.block_size)
dest_fd.write(data)
dest_path.replace(src_path)
def read_chunk(url, x0, x1, y0, y1, z0, z1, level=1, format="tiff"):
"""Read an arbitrary chunk of data
:param url: Base URL of the precomputed data source
:param x0: starting X coordinate, in the level's coordinate space
:param x1: ending X coordinate (non-inclusive)
:param y0: starting Y coordinate
:param y1: ending Y cooridinate
:param z0: starting Z coordinate
:param z1: ending Z coordinate
:param level: mipmap level
:param format: the read format if it's a file URL. Defaults to tiff, but
you can use "blockfs"
:return: a Numpy array containing the data
"""
is_file = urlparse(url).scheme.lower() == "file"
info = get_info(url)
scale = info.get_scale(level)
result = np.zeros((z1 - z0, y1 - y0, x1 - x0), info.data_type)
shape = np.array(scale.shape)
offset = np.array(scale.offset)
stride = np.array(scale.chunk_sizes)
end = offset + shape
x0d = _chunk_start(x0, offset[0], stride[0])
x1d = _chunk_end(x1, offset[0], stride[0], end[0])
y0d = _chunk_start(y0, offset[1], stride[1])
y1d = _chunk_end(y1, offset[1], stride[1], end[1])
z0d = _chunk_start(z0, offset[2], stride[2])
z1d = _chunk_end(z1, offset[2], stride[2], end[2])
for x0c, y0c, z0c in itertools.product(range(x0d, x1d, stride[0]),
range(y0d, y1d, stride[1]),
range(z0d, z1d, stride[2])):
x1c = min(x1d, x0c + stride[0])
y1c = min(y1d, y0c + stride[1])
z1c = min(z1d, z0c + stride[2])
chunk_url = url + "/" + scale.key + "/%d-%d_%d-%d_%d-%d" % (
x0c, x1c, y0c, y1c, z0c, z1c)
if is_file:
if format == "tiff":
chunk_url += ".tiff"
with urlopen(chunk_url) as fd:
chunk = tifffile.imread(fd)
elif format == "blockfs":
from blockfs import Directory
from .blockfs_stack import BlockfsStack
directory_url = url + "/" + scale.key + "/" +\
BlockfsStack.DIRECTORY_FILENAME
directory_parse = urlparse(directory_url)
directory_path = os.path.join(directory_parse.netloc,
unquote(directory_parse.path))
directory = Directory.open(directory_path)
chunk = directory.read_block(x0c, y0c, z0c)
elif format == 'ngff':
group = get_ngff_group_from_url(url)
key = str(int(np.log2(level)))
dataset = group[key]
dataset.read_only = True
chunk = dataset[0, 0, z0c:z1c, y0c:y1c, x0c:x1c]
elif format == 'zarr':
zarr_url = url + "/" + scale.key
zarr_parse = urlparse(zarr_url)
zarr_path = os.path.join(zarr_parse.netloc,
unquote(zarr_parse.path))
storage = zarr.NestedDirectoryStore(zarr_path)
dataset = zarr.Array(storage)
chunk = dataset[z0c:z1c, y0c:y1c, x0c:x1c]
else:
raise NotImplementedError("Can't read %s yet" % format)
else:
response = urlopen(chunk_url)
data = response.read()
chunk = np.frombuffer(data, info.data_type).reshape(
(z1c - z0c, y1c - y0c, x1c - x0c))
if z0c < z0:
chunk = chunk[z0 - z0c:]
z0c = z0
if z1c > z1:
chunk = chunk[:z1 - z0c]
z1c = z1
if y0c < y0:
chunk = chunk[:, y0 - y0c:]
y0c = y0
if y1c > y1:
chunk = chunk[:, :y1 - y0c]
y1c = y1
if x0c < x0:
chunk = chunk[:, :, x0 - x0c:]
x0c = x0
if x1c > x1:
chunk = chunk[:, :, :x1 - x0c]
x1c = x1
result[z0c - z0:z0c - z0 + chunk.shape[0],
y0c - y0:y0c - y0 + chunk.shape[1],
x0c - x0:x0c - x0 + chunk.shape[2]] = chunk
return result
def serve_precomputed(environ, start_response, config_file):
config = get_config(config_file)
path_info = environ["PATH_INFO"]
if path_info == "/":
return serve_directory(start_response, config_file)
for source in config:
if path_info[1:].startswith(source["name"] + "/"):
try:
filename = path_info[2 + len(source["name"]):]
dest = os.path.join(source["directory"])
if filename == "mesh/info":
return file_not_found(filename, start_response)
elif filename == "info":
logging.info("Serving %s" % source["name"])
destpath = os.path.join(dest, "info")
if not os.path.exists(destpath):
return file_not_found(destpath, start_response)
with open(destpath, "rb") as fd:
data = fd.read()
start_response("200 OK",
[("Content-type", "application/json"),
("Content-Length", str(len(data))),
('Access-Control-Allow-Origin', '*')])
return [data]
elif source["format"] == "tiff":
import tifffile
path = os.path.join(dest, filename + ".tiff")
if not os.path.exists(path):
return file_not_found(path, start_response)
img = tifffile.imread(path)
data = img.tostring("C")
start_response(
"200 OK",
[("Content-type", "application/octet-stream"),
("Content-Length", str(len(data))),
('Access-Control-Allow-Origin', '*')])
return [data]
elif source["format"] == "zarr":
import zarr
filename, x0, x1, y0, y1, z0, z1 = \
parse_filename(dest, filename)
if not os.path.exists(filename):
return file_not_found(filename, start_response)
store = zarr.NestedDirectoryStore(filename)
z_arr = zarr.open(store, mode='r')
chunk = z_arr[z0:z1, y0:y1, x0:x1]
data = chunk.tostring("C")
start_response(
"200 OK",
[("Content-type", "application/octet-stream"),
("Content-Length", str(len(data))),
('Access-Control-Allow-Origin', '*')])
return [data]
elif source["format"] == "blockfs":
from blockfs import Directory
filename, x0, x1, y0, y1, z0, z1 = \
parse_filename(dest, filename)
filename = os.path.join(filename, "precomputed.blockfs")
directory = Directory.open(filename)
chunk = directory.read_block(x0, y0, z0)
data = chunk.tostring("C")
start_response(
"200 OK",
[("Content-type", "application/octet-stream"),
("Content-Length", str(len(data))),
('Access-Control-Allow-Origin', '*')])
return [data]
elif source["format"] == "ngff":
import zarr
filename, x0, x1, y0, y1, z0, z1 = \
parse_filename(dest, filename)
root, level = os.path.split(filename)
lx, ly, lz = [int(_) for _ in level.split("_")]
llevel = int(np.round(np.log2(lx), 0))
store = zarr.NestedDirectoryStore(root)
group = zarr.group(store)
a = group[llevel]
_, _, zs, ys, xs = a.chunks
z1 = min(a.shape[2], z0 + zs)
y1 = min(a.shape[3], y0 + ys)
x1 = min(a.shape[4], x0 + xs)
chunk = a[0, 0, z0:z1, y0:y1, x0:x1]
data = chunk.tostring("C")
start_response(
"200 OK",
[("Content-type", "application/octet-stream"),
("Content-Length", str(len(data))),
('Access-Control-Allow-Origin', '*')])
return [data]
except ParseFilenameError:
return file_not_found(path_info, start_response)
else:
return file_not_found(path_info, start_response)
def test_05_write_not_there(self):
a = np.random.randint(0, 65535, (64, 64, 64), np.uint16)
with make_files(1) as (dir_file, block_files):
directory = Directory(1024, 1024, 1024, np.uint16, dir_file,
compression=Compression.zstd,
block_filenames=block_files)
directory.create()
directory.write_block(a, 64, 128, 192)
directory.close()
a_out = directory.read_block(192, 128, 64)
np.testing.assert_array_equal(a_out, 0)
#
# Test for read in directory beyond EOF
#
with make_files(1) as (dir_file, block_files):
directory = Directory(1024, 1024, 1024, np.uint16, dir_file,
compression=Compression.zstd,
block_filenames=block_files)
directory.create()
directory.write_block(a, 192, 128, 64)
directory.close()
a_out = directory.read_block(64, 128, 192)
np.testing.assert_array_equal(a_out, 0)