def test_mv(self):
for i in range(2):
a = np.random.randint(0, 65535, (256, 256, 256), np.uint16)
with make_files(1) as (dir_file, block_files):
directory = Directory(256,
256,
256,
np.uint16,
dir_file,
compression=Compression.zstd,
block_filenames=block_files)
directory.create()
for x, y, z in itertools.product(range(0, 256, 64),
range(0, 256, 64),
range(0, 256, 64)):
directory.write_block(a[z:z + 64, y:y + 64, x:x + 64], x,
y, z)
directory.close()
dest = tempfile.mkdtemp()
try:
if i == 0:
main([dir_file, dest])
else:
subprocess.check_call(["blockfs-mv", dir_file, dest])
dest_dir_file = \
os.path.join(dest, os.path.split(dir_file)[1])
dest_directory = Directory.open(dest_dir_file)
for x, y, z in itertools.product(range(0, 256, 64),
range(0, 256, 64),
range(0, 256, 64)):
block = dest_directory.read_block(x, y, z)
np.testing.assert_array_equal(
a[z:z + 64, y:y + 64, x:x + 64], block)
finally:
shutil.rmtree(dest)
def test_01_create(self):
with make_files(1) as (dir_file, block_files):
directory = Directory(1024, 1024, 1024, np.uint16, dir_file,
block_filenames=block_files)
directory.create()
directory.close()
with open(dir_file, "rb") as fd:
header = fd.read(len(Directory.HEADER))
self.assertEqual(header, Directory.HEADER)
def test_04_write_read(self):
a = np.random.randint(0, 65535, (64, 64, 64), np.uint16)
with make_files(1) as (dir_file, block_files):
directory = Directory(1024, 1024, 1024, np.uint16, dir_file,
compression=Compression.zstd,
block_filenames=block_files)
directory.create()
directory.write_block(a, 64, 128, 192)
directory.close()
a_out = Directory.open(dir_file).read_block(64, 128, 192)
np.testing.assert_array_equal(a, a_out)
def test_02_create_and_open(self):
with make_files(1) as (dir_file, block_files):
directory = Directory(1024, 1024, 1024, np.uint16, dir_file,
block_filenames=block_files)
directory.create()
directory.close()
directory = Directory.open(dir_file)
self.assertEqual(directory.x_extent, 1024)
self.assertEqual(directory.y_extent, 1024)
self.assertEqual(directory.z_extent, 1024)
def main(args=sys.argv[1:]):
global MY_OPTS, MY_DCIMG, FLAT
MY_OPTS = parse_args(args)
destripe_method = MY_OPTS.destripe_method
if not (destripe_method is None or destripe_method == "lightsheet"
or destripe_method == "wavelet"):
print("--destripe-method must be \"lightsheet\", \"wavelet\" or blank",
file=sys.stderr)
sys.exit(-1)
if MY_OPTS.flat is not None:
FLAT = normalize_flat(tifffile.imread(MY_OPTS.flat))
if MY_OPTS.jp2k:
paths = sorted(glob.glob(MY_OPTS.input))
fn = do_one_jp2000
img = glymur.Jp2k(paths[0])
x_extent = img.shape[1]
y_extent = img.shape[0]
else:
MY_DCIMG = DCIMG(MY_OPTS.input)
start = MY_OPTS.start
stop = MY_OPTS.stop or MY_DCIMG.n_frames
x_extent = int(MY_DCIMG.x_dim)
y_extent = int(MY_DCIMG.y_dim)
paths = [str(i) for i in range(start, stop)]
fn = do_one_dcimg
stack = SpimStack(paths, 0, 0, x_extent, y_extent, 0)
#
# The stack dimensions are a little elongated because of the
# parallelogram
#
z_extent, y_extent, x_extent, dtype = get_blockfs_dims(
stack, x_extent, y_extent)
bfs_stack = BlockfsStack((z_extent, y_extent, x_extent), MY_OPTS.output)
bfs_stack.write_info_file(MY_OPTS.levels)
bfs_level1_dir = os.path.join(MY_OPTS.output, "1_1_1",
BlockfsStack.DIRECTORY_FILENAME)
if not os.path.exists(os.path.dirname(bfs_level1_dir)):
os.mkdir(os.path.dirname(bfs_level1_dir))
directory = Directory(x_extent,
y_extent,
z_extent,
np.uint16,
bfs_level1_dir,
n_filenames=MY_OPTS.n_writers)
directory.create()
directory.start_writer_processes()
spim_to_blockfs(stack, directory, MY_OPTS.n_workers, read_fn=fn)
for level in range(2, MY_OPTS.levels + 1):
bfs_stack.write_level_n(level, n_cores=MY_OPTS.n_writers)
def test_02_01_encode_decode(self):
with make_files(1) as (dir_file, block_files):
directory = Directory(1024, 1024, 1024, np.uint16, dir_file,
block_filenames=block_files)
directory.create()
try:
test_cases = ((0, 524304), (524304, 524304))
for offset, size in test_cases:
a = np.zeros(directory.directory_entry_size, np.uint8)
directory.encode_directory_entry(a, offset, size)
offset_out, size_out = directory.decode_directory_entry(a)
self.assertEqual(offset, offset_out)
self.assertEqual(size, size_out)
finally:
directory.close()
def test_03_write(self):
a = np.random.randint(0, 65535, (64, 64, 64), np.uint16)
with make_files(1) as (dir_file, block_files):
directory = Directory(1024, 1024, 1024, np.uint16, dir_file,
compression=Compression.zstd,
block_filenames=block_files)
directory.create()
directory.write_block(a, 64, 128, 192)
directory.close()
with open(block_files[0], "rb") as fd:
block = fd.read()
blosc = Blosc("zstd")
a_out = np.frombuffer(blosc.decode(block), np.uint16)\
.reshape(64, 64, 64)
np.testing.assert_array_equal(a, a_out)
def test_case(self):
src_dir = pathlib.Path(tempfile.mkdtemp())
dest_dir_parent = pathlib.Path(tempfile.mkdtemp())
dest_dir = dest_dir_parent / "dest"
dest_dir.mkdir()
all_files = []
rs = np.random.RandomState(1234)
a = rs.randint(0, 65535, (256, 256, 256), dtype=np.uint16)
try:
dir_file = src_dir / "my.blockfs"
block_files = [src_dir / ("my.blockfs.%d" % i) for i in range(4)]
directory = Directory(256, 256, 256, np.uint16, str(dir_file),
compression=Compression.zstd,
block_filenames=
[str(_) for _ in block_files])
directory.create()
for x, y, z in itertools.product(range(0, 256, 64),
range(0, 256, 64),
range(0, 256, 64)):
directory.write_block(
a[z:z + 64, y:y + 64, x:x + 64], x, y, z)
directory.close()
for path in [dir_file] + list(block_files):
dest_path = dest_dir / path.name
path.replace(dest_path)
all_files.append(dest_path)
dest_directory_file = dest_dir / pathlib.Path(dir_file).name
main([str(dest_directory_file)])
directory = Directory.open(str(dest_directory_file))
for x, y, z in itertools.product(range(0, 256, 64),
range(0, 256, 64),
range(0, 256, 64)):
np.testing.assert_array_equal(a[z:z+64, y:y+64, x:x+64],
directory.read_block(x, y, z))
finally:
try:
for path in all_files:
path.unlink()
dest_dir.rmdir()
dest_dir_parent.rmdir()
except:
traceback.print_exc()
print("Failed to remove files")
def test_05_write_not_there(self):
a = np.random.randint(0, 65535, (64, 64, 64), np.uint16)
with make_files(1) as (dir_file, block_files):
directory = Directory(1024, 1024, 1024, np.uint16, dir_file,
compression=Compression.zstd,
block_filenames=block_files)
directory.create()
directory.write_block(a, 64, 128, 192)
directory.close()
a_out = directory.read_block(192, 128, 64)
np.testing.assert_array_equal(a_out, 0)
#
# Test for read in directory beyond EOF
#
with make_files(1) as (dir_file, block_files):
directory = Directory(1024, 1024, 1024, np.uint16, dir_file,
compression=Compression.zstd,
block_filenames=block_files)
directory.create()
directory.write_block(a, 192, 128, 64)
directory.close()
a_out = directory.read_block(64, 128, 192)
np.testing.assert_array_equal(a_out, 0)