def test_copy_dataset_custom(self): from z5py.util import copy_dataset in_path = os.path.join(self.tmp_dir, 'in.n5') out_path = os.path.join(self.tmp_dir, 'out.n5') # create input file in_file = z5py.File(in_path, use_zarr_format=False) ds_in = in_file.create_dataset('data', dtype='float32', shape=self.shape, chunks=self.chunks, compression='gzip') # write test data data = np.arange(ds_in.size).reshape(ds_in.shape).astype(ds_in.dtype) ds_in[:] = data # copy_dataset new_chunks = (20, 20, 20) for compression in ('raw', 'gzip'): for dtype in ('float64', 'int32', 'uint32'): ds_name = 'ds_%s_%s' % (compression, dtype) copy_dataset(in_path, out_path, 'data', ds_name, chunks=new_chunks, n_threads=8, compression=compression, dtype=dtype) # make sure that new data agrees out_file = z5py.File(out_path, use_zarr_format=False) ds_out = out_file[ds_name] data_out = ds_out[:] self.assertEqual(data_out.shape, data.shape) self.assertEqual(ds_out.chunks, new_chunks) self.assertTrue(np.allclose(data, data_out))
def test_copy_dataset_with_roi(self): from z5py.util import copy_dataset in_path = os.path.join(self.tmp_dir, 'in.n5') out_path = os.path.join(self.tmp_dir, 'out.n5') in_file = z5py.File(in_path, use_zarr_format=False) out_file = z5py.File(out_path, use_zarr_format=False) # create input dataset ds_in = in_file.create_dataset('data', dtype='float32', shape=self.shape, chunks=self.chunks, compression='gzip') # write test data data = np.arange(ds_in.size).reshape(ds_in.shape).astype(ds_in.dtype) ds_in[:] = data # define roi roi = np.s_[5:45, 9:83, 10:60] out_of_roi_mask = np.ones(self.shape, dtype='bool') out_of_roi_mask[roi] = False roi_shape = tuple(rr.stop - rr.start for rr in roi) # copy dataset with roi copy_dataset(in_path, out_path, 'data', 'data_roi', n_threads=8, roi=roi, fit_to_roi=False) ds_out = out_file['data_roi'] data_out = ds_out[:] self.assertEqual(data_out.shape, data.shape) self.assertTrue(np.allclose(data_out[roi], data[roi])) self.assertTrue(np.allclose(data_out[out_of_roi_mask], 0)) # copy dataset with roi and fit_to_roi copy_dataset(in_path, out_path, 'data', 'data_roi_fit', n_threads=8, roi=roi, fit_to_roi=True) ds_out = out_file['data_roi_fit'] data_out = ds_out[:] self.assertEqual(data_out.shape, roi_shape) self.assertTrue(np.allclose(data_out, data[roi]))
def test_copy_dataset_default(self): from z5py.util import copy_dataset in_path = os.path.join(self.tmp_dir, 'in.n5') out_path = os.path.join(self.tmp_dir, 'out.n5') # create input file in_file = z5py.File(in_path, use_zarr_format=False) ds_in = in_file.create_dataset('data', dtype='float32', shape=self.shape, chunks=self.chunks, compression='gzip') # write test data data = np.arange(ds_in.size).reshape(ds_in.shape).astype(ds_in.dtype) ds_in[:] = data # copy_dataset for different out blocks out_file = z5py.File(out_path, use_zarr_format=False) new_chunks = (20, 20, 20) # NOTE we can only choose out blocks that align with the chunks # because otherwise we run into issues due to not thread safe blocking for out_blocks in (None, (40, 40, 40), (60, 60, 60)): ds_str = 'none' if out_blocks is None else '_'.join( map(str, out_blocks)) ds_name = 'data_%s' % ds_str copy_dataset(in_path, out_path, 'data', ds_name, chunks=new_chunks, block_shape=out_blocks, n_threads=8) # make sure that new data agrees ds_out = out_file[ds_name] data_out = ds_out[:] self.assertEqual(data_out.shape, data.shape) self.assertEqual(ds_out.chunks, new_chunks) self.assertTrue(np.allclose(data, data_out))
def check(out_path): path = '/g/kreshuk/data/schwab/parapodia_rachel/data.n5' in_key = 'volumes/paintera/lmc_nuclei_v1_fragments/label-to-block-mapping/s0' out_key = 'volumes/segmentation/labs' copy_dataset(path, out_path, in_key, out_key, 8)