def bench_read_caterva(fname, planes_idx, copy): t0 = time() a = cat.open(fname, copy=copy) t1 = time() print("Time for opening the on-disk frame (caterva, copy=%s): %.3fs" % (copy, (t1 - t0))) t0 = time() for i in planes_idx: rbytes = a[:, i, :] block = np.frombuffer(rbytes, dtype=dtype).reshape( (shape[0], shape[2])) del a t1 = time() print("Time for reading with getitem (caterva, copy=%s): %.3fs" % (copy, (t1 - t0)))
def bench_read_caterva(fname, copy): if macosx: os.system("/usr/sbin/purge") t0 = time() a = cat.open(fname, copy=copy) t1 = time() print("Time for opening the on-disk frame (caterva, copy=%s): %.3fs" % (copy, (t1 - t0))) if macosx: os.system("/usr/sbin/purge") t0 = time() acc = 0 for (block, info) in a.iter_read(): block = np.frombuffer(block, dtype=dtype).reshape(info.shape) acc += np.sum(block) del a t1 = time() print("Time for reducing with (caterva, copy=%s): %.3fs" % (copy, (t1 - t0))) return acc
def test_persistency(shape, chunks, blocks, urlpath, sequencial, dtype): if os.path.exists(urlpath): cat.remove(urlpath) size = int(np.prod(shape)) nparray = np.arange(size, dtype=dtype).reshape(shape) _ = cat.asarray(nparray, chunks=chunks, blocks=blocks, urlpath=urlpath, sequencial=sequencial) b = cat.open(urlpath) bc = b[:] nparray2 = np.asarray(bc).view(dtype) np.testing.assert_almost_equal(nparray, nparray2) cat.remove(urlpath)
h5f = tables.open_file(fname_h5, 'w', driver='H5FD_CORE', driver_core_backing_store=0) h5ca = h5f.create_carray(h5f.root, 'carray', filters=filters, chunkshape=chunkshape, obj=content) h5f.flush() h5ratio = h5ca.size_in_memory / h5ca.size_on_disk if persistent: h5f.close() t1 = time() print("Time for filling array (hdf5): %.3fs ; CRatio: %.1fx" % ((t1 - t0), h5ratio)) # Setup the coordinates for random planes planes_idx = np.random.randint(0, shape[1], 100) # Time getitem with caterva t0 = time() if persistent: a = cat.open(fname_cat) # reopen for i in planes_idx: rbytes = a[:, i, :] del a t1 = time() print("Time for reading with getitem (caterva): %.3fs" % (t1 - t0)) # Time getitem with zarr t0 = time() if persistent: z = zarr.open(fname_zarr, mode='r') for i in planes_idx: block = z[:, i, :] del z t1 = time() print("Time for reading with getitem (zarr): %.3fs" % (t1 - t0))
if os.path.exists(urlpath): cat.remove(urlpath) dtype = np.dtype(np.complex128) itemsize = dtype.itemsize # Create a numpy array nparray = np.arange(int(np.prod(shape)), dtype=dtype).reshape(shape) # Create a caterva array from a numpy array (on disk) a = cat.from_buffer(bytes(nparray), nparray.shape, itemsize, chunks=chunks, blocks=blocks, urlpath=urlpath, sequencial=False) # Read a caterva array from disk b = cat.open(urlpath) # Convert a caterva array to a numpy array nparray2 = np.asarray(cat.from_buffer(b.to_buffer(), b.shape, b.itemsize)).view(dtype) np.testing.assert_almost_equal(nparray, nparray2) # Remove file on disk if os.path.exists(urlpath): cat.remove(urlpath)