def test_dataset_creation(self): for ndim in range(1, 4): max_nchunks = random_int_tuple(1, 4, ndim) for chunk_index in product_range(max_nchunks): chunk_shape = random_int_tuple(1, 10, ndim) for fill_level in chain((None,), product_range((1,) * ndim, chunk_shape)): if fill_level is None: total_shape = tuple(n * (i + 1) for n, i in zip(chunk_shape, chunk_index)) else: total_shape = tuple(n * i + fl for n, i, fl in zip(chunk_shape, chunk_index, fill_level)) chunk_data = np.random.uniform(-10, 10, chunk_shape).astype(random.choice((float, int))) stream = BytesIO() buffer = ChunkBuffer(stream, "data", data=chunk_data, maxshape=(None,) * ndim) buffer.select(chunk_index) buffer.create_dataset(stream if random.random() < 0.5 else None, filemode="w", write=True, fill_level=fill_level) with h5.File(stream, "r") as h5f: dataset = h5f["data"] self.assertEqual(dataset.shape, total_shape) self.assertEqual(dataset.chunks, chunk_shape) self.assertEqual(dataset.dtype, chunk_data.dtype) self.assertEqual(dataset.maxshape, buffer.maxshape) fill_slices = tuple(map(slice, fill_level)) if fill_level is not None else ... np.testing.assert_allclose(ChunkBuffer.load(h5f, "data", chunk_index).data[fill_slices], chunk_data[fill_slices])
def test_real_files(self): with TemporaryDirectory() as tempdir: filename = Path(tempdir) / "test_file.h5" chunk_shape = (1, 2, 3) array = np.random.uniform(-10, 10, chunk_shape) buffer = ChunkBuffer(filename, "data", data=array) buffer.create_dataset(filemode="w") self.assertTrue(filename.exists()) with h5.File(filename, "r") as h5f: np.testing.assert_allclose(h5f["data"][()], array) # extend dataset with stored filename array = np.random.uniform(-10, 10, chunk_shape) buffer.select((1, 0, 0)) buffer.data[...] = array buffer.write(must_exist=False) with h5.File(filename, "r") as h5f: np.testing.assert_allclose(h5f["data"][1:, :, :], array) # extend dataset with passed in filename array = np.random.uniform(-10, 10, chunk_shape) buffer.select((1, 1, 0)) buffer.data[...] = array buffer.write(must_exist=False, file=filename) with h5.File(filename, "r") as h5f: np.testing.assert_allclose(h5f["data"][1:, 2:, :], array) # extend dataset with passed in dataset array = np.random.uniform(-10, 10, chunk_shape) buffer.select((1, 0, 1)) buffer.data[...] = array with h5.File(filename, "r+") as h5f: dataset = h5f["data"] buffer.write(must_exist=False, dataset=dataset) np.testing.assert_allclose(dataset[1:, :2, 3:], array) # wrong filename with self.assertRaises(ValueError): buffer.write(must_exist=False, file="wrong_file.h5") # wrong dataset with h5.File(filename, "a") as h5f: wrong_dataset = h5f.create_dataset("wrong_data", (1,)) with self.assertRaises(ValueError): buffer.write(must_exist=False, dataset=wrong_dataset)