def test_dataset_creation(self):
for ndim in range(1, 4):
max_nchunks = random_int_tuple(1, 4, ndim)
for chunk_index in product_range(max_nchunks):
chunk_shape = random_int_tuple(1, 10, ndim)
for fill_level in chain((None,), product_range((1,) * ndim, chunk_shape)):
if fill_level is None:
total_shape = tuple(n * (i + 1)
for n, i in zip(chunk_shape, chunk_index))
else:
total_shape = tuple(n * i + fl
for n, i, fl in zip(chunk_shape, chunk_index, fill_level))
chunk_data = np.random.uniform(-10, 10, chunk_shape).astype(random.choice((float, int)))
stream = BytesIO()
buffer = ChunkBuffer(stream, "data", data=chunk_data, maxshape=(None,) * ndim)
buffer.select(chunk_index)
buffer.create_dataset(stream if random.random() < 0.5 else None, filemode="w",
write=True, fill_level=fill_level)
with h5.File(stream, "r") as h5f:
dataset = h5f["data"]
self.assertEqual(dataset.shape, total_shape)
self.assertEqual(dataset.chunks, chunk_shape)
self.assertEqual(dataset.dtype, chunk_data.dtype)
self.assertEqual(dataset.maxshape, buffer.maxshape)
fill_slices = tuple(map(slice, fill_level)) if fill_level is not None else ...
np.testing.assert_allclose(ChunkBuffer.load(h5f, "data", chunk_index).data[fill_slices],
chunk_data[fill_slices])
def test_real_files(self):
with TemporaryDirectory() as tempdir:
filename = Path(tempdir) / "test_file.h5"
chunk_shape = (1, 2, 3)
array = np.random.uniform(-10, 10, chunk_shape)
buffer = ChunkBuffer(filename, "data", data=array)
buffer.create_dataset(filemode="w")
self.assertTrue(filename.exists())
with h5.File(filename, "r") as h5f:
np.testing.assert_allclose(h5f["data"][()], array)
# extend dataset with stored filename
array = np.random.uniform(-10, 10, chunk_shape)
buffer.select((1, 0, 0))
buffer.data[...] = array
buffer.write(must_exist=False)
with h5.File(filename, "r") as h5f:
np.testing.assert_allclose(h5f["data"][1:, :, :], array)
# extend dataset with passed in filename
array = np.random.uniform(-10, 10, chunk_shape)
buffer.select((1, 1, 0))
buffer.data[...] = array
buffer.write(must_exist=False, file=filename)
with h5.File(filename, "r") as h5f:
np.testing.assert_allclose(h5f["data"][1:, 2:, :], array)
# extend dataset with passed in dataset
array = np.random.uniform(-10, 10, chunk_shape)
buffer.select((1, 0, 1))
buffer.data[...] = array
with h5.File(filename, "r+") as h5f:
dataset = h5f["data"]
buffer.write(must_exist=False, dataset=dataset)
np.testing.assert_allclose(dataset[1:, :2, 3:], array)
# wrong filename
with self.assertRaises(ValueError):
buffer.write(must_exist=False, file="wrong_file.h5")
# wrong dataset
with h5.File(filename, "a") as h5f:
wrong_dataset = h5f.create_dataset("wrong_data", (1,))
with self.assertRaises(ValueError):
buffer.write(must_exist=False, dataset=wrong_dataset)