def test_write_overwrite(self): for ndim in range(1, 4): chunk_shape = random_int_tuple(1, 10, ndim) nchunks = random_int_tuple(1, 4, ndim) total_shape = tuple(n * c for n, c in zip(chunk_shape, nchunks)) stream = BytesIO() chunk = np.random.uniform(-10, 10, chunk_shape).astype(random.choice((int, float))) file_content = np.random.uniform(-10, 10, total_shape).astype(chunk.dtype) with h5.File(stream, "w") as h5f: h5f.create_dataset("data", data=file_content, chunks=chunk_shape, maxshape=(None,) * ndim) buffer = ChunkBuffer(stream, "data", data=chunk) # valid indices for chunk_index in product_range(nchunks): with h5.File(stream, "a") as h5f: h5f["data"][...] = file_content buffer.select(chunk_index) buffer.write(must_exist=True) desired_file_content = file_content.copy() desired_file_content[_chunk_slices(chunk_index, chunk_shape)] = chunk with h5.File(stream, "r") as h5f: np.testing.assert_allclose(h5f["data"][()], desired_file_content) # index out of bounds for dim in range(ndim): chunk_index = tuple(map(lambda n: random.randint(0, n - 1), nchunks)) chunk_index = chunk_index[:dim] + (nchunks[dim] + random.randint(1, 10),) + chunk_index[dim + 1:] buffer.select(chunk_index) with self.assertRaises(RuntimeError): buffer.write(must_exist=True)
def test_real_files(self): with TemporaryDirectory() as tempdir: filename = Path(tempdir) / "test_file.h5" chunk_shape = (1, 2, 3) array = np.random.uniform(-10, 10, chunk_shape) buffer = ChunkBuffer(filename, "data", data=array) buffer.create_dataset(filemode="w") self.assertTrue(filename.exists()) with h5.File(filename, "r") as h5f: np.testing.assert_allclose(h5f["data"][()], array) # extend dataset with stored filename array = np.random.uniform(-10, 10, chunk_shape) buffer.select((1, 0, 0)) buffer.data[...] = array buffer.write(must_exist=False) with h5.File(filename, "r") as h5f: np.testing.assert_allclose(h5f["data"][1:, :, :], array) # extend dataset with passed in filename array = np.random.uniform(-10, 10, chunk_shape) buffer.select((1, 1, 0)) buffer.data[...] = array buffer.write(must_exist=False, file=filename) with h5.File(filename, "r") as h5f: np.testing.assert_allclose(h5f["data"][1:, 2:, :], array) # extend dataset with passed in dataset array = np.random.uniform(-10, 10, chunk_shape) buffer.select((1, 0, 1)) buffer.data[...] = array with h5.File(filename, "r+") as h5f: dataset = h5f["data"] buffer.write(must_exist=False, dataset=dataset) np.testing.assert_allclose(dataset[1:, :2, 3:], array) # wrong filename with self.assertRaises(ValueError): buffer.write(must_exist=False, file="wrong_file.h5") # wrong dataset with h5.File(filename, "a") as h5f: wrong_dataset = h5f.create_dataset("wrong_data", (1,)) with self.assertRaises(ValueError): buffer.write(must_exist=False, dataset=wrong_dataset)
def test_write_extend(self): for ndim in range(1, 4): chunk_shape = random_int_tuple(1, 10, ndim) nchunks = random_int_tuple(1, 5, ndim) chunks = [] stream = BytesIO() with h5.File(stream, "w") as h5f: h5f.create_dataset("data", shape=chunk_shape, dtype=float, chunks=chunk_shape, maxshape=(None,) * ndim) buffer = ChunkBuffer(stream, "data", shape=chunk_shape, dtype=float) for chunk_index in product_range(nchunks): chunks.append((_chunk_slices(chunk_index, chunk_shape), np.random.uniform(-10, 10, chunk_shape))) buffer.select(chunk_index) buffer.data[...] = chunks[-1][1] buffer.write(must_exist=False) with h5.File(stream, "r") as f: dataset = f["data"] for chunk_slice, expected in chunks: np.testing.assert_allclose(dataset[chunk_slice], expected)