def test_copy_dataset_custom(self):
from z5py.util import copy_dataset
in_path = os.path.join(self.tmp_dir, 'in.n5')
out_path = os.path.join(self.tmp_dir, 'out.n5')
# create input file
in_file = z5py.File(in_path, use_zarr_format=False)
ds_in = in_file.create_dataset('data',
dtype='float32',
shape=self.shape,
chunks=self.chunks,
compression='gzip')
# write test data
data = np.arange(ds_in.size).reshape(ds_in.shape).astype(ds_in.dtype)
ds_in[:] = data
# copy_dataset
new_chunks = (20, 20, 20)
for compression in ('raw', 'gzip'):
for dtype in ('float64', 'int32', 'uint32'):
ds_name = 'ds_%s_%s' % (compression, dtype)
copy_dataset(in_path,
out_path,
'data',
ds_name,
chunks=new_chunks,
n_threads=8,
compression=compression,
dtype=dtype)
# make sure that new data agrees
out_file = z5py.File(out_path, use_zarr_format=False)
ds_out = out_file[ds_name]
data_out = ds_out[:]
self.assertEqual(data_out.shape, data.shape)
self.assertEqual(ds_out.chunks, new_chunks)
self.assertTrue(np.allclose(data, data_out))
def test_copy_dataset_with_roi(self):
from z5py.util import copy_dataset
in_path = os.path.join(self.tmp_dir, 'in.n5')
out_path = os.path.join(self.tmp_dir, 'out.n5')
in_file = z5py.File(in_path, use_zarr_format=False)
out_file = z5py.File(out_path, use_zarr_format=False)
# create input dataset
ds_in = in_file.create_dataset('data',
dtype='float32',
shape=self.shape,
chunks=self.chunks,
compression='gzip')
# write test data
data = np.arange(ds_in.size).reshape(ds_in.shape).astype(ds_in.dtype)
ds_in[:] = data
# define roi
roi = np.s_[5:45, 9:83, 10:60]
out_of_roi_mask = np.ones(self.shape, dtype='bool')
out_of_roi_mask[roi] = False
roi_shape = tuple(rr.stop - rr.start for rr in roi)
# copy dataset with roi
copy_dataset(in_path,
out_path,
'data',
'data_roi',
n_threads=8,
roi=roi,
fit_to_roi=False)
ds_out = out_file['data_roi']
data_out = ds_out[:]
self.assertEqual(data_out.shape, data.shape)
self.assertTrue(np.allclose(data_out[roi], data[roi]))
self.assertTrue(np.allclose(data_out[out_of_roi_mask], 0))
# copy dataset with roi and fit_to_roi
copy_dataset(in_path,
out_path,
'data',
'data_roi_fit',
n_threads=8,
roi=roi,
fit_to_roi=True)
ds_out = out_file['data_roi_fit']
data_out = ds_out[:]
self.assertEqual(data_out.shape, roi_shape)
self.assertTrue(np.allclose(data_out, data[roi]))
def test_copy_dataset_default(self):
from z5py.util import copy_dataset
in_path = os.path.join(self.tmp_dir, 'in.n5')
out_path = os.path.join(self.tmp_dir, 'out.n5')
# create input file
in_file = z5py.File(in_path, use_zarr_format=False)
ds_in = in_file.create_dataset('data',
dtype='float32',
shape=self.shape,
chunks=self.chunks,
compression='gzip')
# write test data
data = np.arange(ds_in.size).reshape(ds_in.shape).astype(ds_in.dtype)
ds_in[:] = data
# copy_dataset for different out blocks
out_file = z5py.File(out_path, use_zarr_format=False)
new_chunks = (20, 20, 20)
# NOTE we can only choose out blocks that align with the chunks
# because otherwise we run into issues due to not thread safe blocking
for out_blocks in (None, (40, 40, 40), (60, 60, 60)):
ds_str = 'none' if out_blocks is None else '_'.join(
map(str, out_blocks))
ds_name = 'data_%s' % ds_str
copy_dataset(in_path,
out_path,
'data',
ds_name,
chunks=new_chunks,
block_shape=out_blocks,
n_threads=8)
# make sure that new data agrees
ds_out = out_file[ds_name]
data_out = ds_out[:]
self.assertEqual(data_out.shape, data.shape)
self.assertEqual(ds_out.chunks, new_chunks)
self.assertTrue(np.allclose(data, data_out))
def check(out_path):
path = '/g/kreshuk/data/schwab/parapodia_rachel/data.n5'
in_key = 'volumes/paintera/lmc_nuclei_v1_fragments/label-to-block-mapping/s0'
out_key = 'volumes/segmentation/labs'
copy_dataset(path, out_path, in_key, out_key, 8)