def test_load(self): for ndim in range(1, 4): chunk_shape = random_int_tuple(1, 10, ndim) nchunks = random_int_tuple(1, 4, ndim) total_shape = tuple(n * c for n, c in zip(chunk_shape, nchunks)) array = np.random.uniform(-10, 10, total_shape) stream = BytesIO() with h5.File(stream, "w") as h5f: h5f.create_dataset("data", data=array, chunks=chunk_shape) # valid, load all chunks, positive indices for chunk_index in product_range(nchunks): buffer = ChunkBuffer.load(stream, "data", chunk_index) np.testing.assert_allclose(buffer.data, array[_chunk_slices(chunk_index, chunk_shape)], err_msg=capture_variables(ndim=ndim, chunk_shape=chunk_shape, nchunks=nchunks, chunk_index=chunk_index)) # negative index neg_index = (-1,) * ndim pos_index = tuple(n - 1 for n in nchunks) buffer = ChunkBuffer.load(stream, "data", neg_index) np.testing.assert_allclose(buffer.data, array[_chunk_slices(pos_index, chunk_shape)], err_msg=capture_variables(ndim=ndim, chunk_shape=chunk_shape, nchunks=nchunks, chunk_index=neg_index)) # invalid, load non-existent chunk # outside of maxshape, discoverable through maxshape with self.assertRaises(IndexError): ChunkBuffer.load(stream, "data", nchunks) # outside of maxshape, not discoverable through maxshape with self.assertRaises(IndexError): ChunkBuffer.load(stream, "data", (nchunks[0] + 1,) + nchunks[1:]) # within maxshape but not stored with h5.File(stream, "w") as h5f: h5f.create_dataset("partially_filled", shape=total_shape, chunks=chunk_shape, maxshape=tuple(n * 2 for n in total_shape)) with self.assertRaises(IndexError): ChunkBuffer.load(stream, "partially_filled", (nchunks[0] + 1,) + nchunks[1:]) # invalid, contiguous dataset stream = BytesIO() with h5.File(stream, "w") as h5f: h5f.create_dataset("data", data=np.random.uniform(-10, 10, (5, 3))) with self.assertRaises(RuntimeError): ChunkBuffer.load(stream, "data", (0, 0))
def test_dataset_creation(self): for ndim in range(1, 4): max_nchunks = random_int_tuple(1, 4, ndim) for chunk_index in product_range(max_nchunks): chunk_shape = random_int_tuple(1, 10, ndim) for fill_level in chain((None,), product_range((1,) * ndim, chunk_shape)): if fill_level is None: total_shape = tuple(n * (i + 1) for n, i in zip(chunk_shape, chunk_index)) else: total_shape = tuple(n * i + fl for n, i, fl in zip(chunk_shape, chunk_index, fill_level)) chunk_data = np.random.uniform(-10, 10, chunk_shape).astype(random.choice((float, int))) stream = BytesIO() buffer = ChunkBuffer(stream, "data", data=chunk_data, maxshape=(None,) * ndim) buffer.select(chunk_index) buffer.create_dataset(stream if random.random() < 0.5 else None, filemode="w", write=True, fill_level=fill_level) with h5.File(stream, "r") as h5f: dataset = h5f["data"] self.assertEqual(dataset.shape, total_shape) self.assertEqual(dataset.chunks, chunk_shape) self.assertEqual(dataset.dtype, chunk_data.dtype) self.assertEqual(dataset.maxshape, buffer.maxshape) fill_slices = tuple(map(slice, fill_level)) if fill_level is not None else ... np.testing.assert_allclose(ChunkBuffer.load(h5f, "data", chunk_index).data[fill_slices], chunk_data[fill_slices])