def test_mv(self): for i in range(2): a = np.random.randint(0, 65535, (256, 256, 256), np.uint16) with make_files(1) as (dir_file, block_files): directory = Directory(256, 256, 256, np.uint16, dir_file, compression=Compression.zstd, block_filenames=block_files) directory.create() for x, y, z in itertools.product(range(0, 256, 64), range(0, 256, 64), range(0, 256, 64)): directory.write_block(a[z:z + 64, y:y + 64, x:x + 64], x, y, z) directory.close() dest = tempfile.mkdtemp() try: if i == 0: main([dir_file, dest]) else: subprocess.check_call(["blockfs-mv", dir_file, dest]) dest_dir_file = \ os.path.join(dest, os.path.split(dir_file)[1]) dest_directory = Directory.open(dest_dir_file) for x, y, z in itertools.product(range(0, 256, 64), range(0, 256, 64), range(0, 256, 64)): block = dest_directory.read_block(x, y, z) np.testing.assert_array_equal( a[z:z + 64, y:y + 64, x:x + 64], block) finally: shutil.rmtree(dest)
def test_01_create(self): with make_files(1) as (dir_file, block_files): directory = Directory(1024, 1024, 1024, np.uint16, dir_file, block_filenames=block_files) directory.create() directory.close() with open(dir_file, "rb") as fd: header = fd.read(len(Directory.HEADER)) self.assertEqual(header, Directory.HEADER)
def test_04_write_read(self): a = np.random.randint(0, 65535, (64, 64, 64), np.uint16) with make_files(1) as (dir_file, block_files): directory = Directory(1024, 1024, 1024, np.uint16, dir_file, compression=Compression.zstd, block_filenames=block_files) directory.create() directory.write_block(a, 64, 128, 192) directory.close() a_out = Directory.open(dir_file).read_block(64, 128, 192) np.testing.assert_array_equal(a, a_out)
def test_02_create_and_open(self): with make_files(1) as (dir_file, block_files): directory = Directory(1024, 1024, 1024, np.uint16, dir_file, block_filenames=block_files) directory.create() directory.close() directory = Directory.open(dir_file) self.assertEqual(directory.x_extent, 1024) self.assertEqual(directory.y_extent, 1024) self.assertEqual(directory.z_extent, 1024)
def main(args=sys.argv[1:]): global MY_OPTS, MY_DCIMG, FLAT MY_OPTS = parse_args(args) destripe_method = MY_OPTS.destripe_method if not (destripe_method is None or destripe_method == "lightsheet" or destripe_method == "wavelet"): print("--destripe-method must be \"lightsheet\", \"wavelet\" or blank", file=sys.stderr) sys.exit(-1) if MY_OPTS.flat is not None: FLAT = normalize_flat(tifffile.imread(MY_OPTS.flat)) if MY_OPTS.jp2k: paths = sorted(glob.glob(MY_OPTS.input)) fn = do_one_jp2000 img = glymur.Jp2k(paths[0]) x_extent = img.shape[1] y_extent = img.shape[0] else: MY_DCIMG = DCIMG(MY_OPTS.input) start = MY_OPTS.start stop = MY_OPTS.stop or MY_DCIMG.n_frames x_extent = int(MY_DCIMG.x_dim) y_extent = int(MY_DCIMG.y_dim) paths = [str(i) for i in range(start, stop)] fn = do_one_dcimg stack = SpimStack(paths, 0, 0, x_extent, y_extent, 0) # # The stack dimensions are a little elongated because of the # parallelogram # z_extent, y_extent, x_extent, dtype = get_blockfs_dims( stack, x_extent, y_extent) bfs_stack = BlockfsStack((z_extent, y_extent, x_extent), MY_OPTS.output) bfs_stack.write_info_file(MY_OPTS.levels) bfs_level1_dir = os.path.join(MY_OPTS.output, "1_1_1", BlockfsStack.DIRECTORY_FILENAME) if not os.path.exists(os.path.dirname(bfs_level1_dir)): os.mkdir(os.path.dirname(bfs_level1_dir)) directory = Directory(x_extent, y_extent, z_extent, np.uint16, bfs_level1_dir, n_filenames=MY_OPTS.n_writers) directory.create() directory.start_writer_processes() spim_to_blockfs(stack, directory, MY_OPTS.n_workers, read_fn=fn) for level in range(2, MY_OPTS.levels + 1): bfs_stack.write_level_n(level, n_cores=MY_OPTS.n_writers)
def test_02_01_encode_decode(self): with make_files(1) as (dir_file, block_files): directory = Directory(1024, 1024, 1024, np.uint16, dir_file, block_filenames=block_files) directory.create() try: test_cases = ((0, 524304), (524304, 524304)) for offset, size in test_cases: a = np.zeros(directory.directory_entry_size, np.uint8) directory.encode_directory_entry(a, offset, size) offset_out, size_out = directory.decode_directory_entry(a) self.assertEqual(offset, offset_out) self.assertEqual(size, size_out) finally: directory.close()
def test_03_write(self): a = np.random.randint(0, 65535, (64, 64, 64), np.uint16) with make_files(1) as (dir_file, block_files): directory = Directory(1024, 1024, 1024, np.uint16, dir_file, compression=Compression.zstd, block_filenames=block_files) directory.create() directory.write_block(a, 64, 128, 192) directory.close() with open(block_files[0], "rb") as fd: block = fd.read() blosc = Blosc("zstd") a_out = np.frombuffer(blosc.decode(block), np.uint16)\ .reshape(64, 64, 64) np.testing.assert_array_equal(a, a_out)
def test_case(self): src_dir = pathlib.Path(tempfile.mkdtemp()) dest_dir_parent = pathlib.Path(tempfile.mkdtemp()) dest_dir = dest_dir_parent / "dest" dest_dir.mkdir() all_files = [] rs = np.random.RandomState(1234) a = rs.randint(0, 65535, (256, 256, 256), dtype=np.uint16) try: dir_file = src_dir / "my.blockfs" block_files = [src_dir / ("my.blockfs.%d" % i) for i in range(4)] directory = Directory(256, 256, 256, np.uint16, str(dir_file), compression=Compression.zstd, block_filenames= [str(_) for _ in block_files]) directory.create() for x, y, z in itertools.product(range(0, 256, 64), range(0, 256, 64), range(0, 256, 64)): directory.write_block( a[z:z + 64, y:y + 64, x:x + 64], x, y, z) directory.close() for path in [dir_file] + list(block_files): dest_path = dest_dir / path.name path.replace(dest_path) all_files.append(dest_path) dest_directory_file = dest_dir / pathlib.Path(dir_file).name main([str(dest_directory_file)]) directory = Directory.open(str(dest_directory_file)) for x, y, z in itertools.product(range(0, 256, 64), range(0, 256, 64), range(0, 256, 64)): np.testing.assert_array_equal(a[z:z+64, y:y+64, x:x+64], directory.read_block(x, y, z)) finally: try: for path in all_files: path.unlink() dest_dir.rmdir() dest_dir_parent.rmdir() except: traceback.print_exc() print("Failed to remove files")
def test_05_write_not_there(self): a = np.random.randint(0, 65535, (64, 64, 64), np.uint16) with make_files(1) as (dir_file, block_files): directory = Directory(1024, 1024, 1024, np.uint16, dir_file, compression=Compression.zstd, block_filenames=block_files) directory.create() directory.write_block(a, 64, 128, 192) directory.close() a_out = directory.read_block(192, 128, 64) np.testing.assert_array_equal(a_out, 0) # # Test for read in directory beyond EOF # with make_files(1) as (dir_file, block_files): directory = Directory(1024, 1024, 1024, np.uint16, dir_file, compression=Compression.zstd, block_filenames=block_files) directory.create() directory.write_block(a, 192, 128, 64) directory.close() a_out = directory.read_block(64, 128, 192) np.testing.assert_array_equal(a_out, 0)