def bench_read_caterva(fname, planes_idx, copy):
t0 = time()
a = cat.open(fname, copy=copy)
t1 = time()
print("Time for opening the on-disk frame (caterva, copy=%s): %.3fs" %
(copy, (t1 - t0)))
t0 = time()
for i in planes_idx:
rbytes = a[:, i, :]
block = np.frombuffer(rbytes, dtype=dtype).reshape(
(shape[0], shape[2]))
del a
t1 = time()
print("Time for reading with getitem (caterva, copy=%s): %.3fs" %
(copy, (t1 - t0)))
def bench_read_caterva(fname, copy):
if macosx: os.system("/usr/sbin/purge")
t0 = time()
a = cat.open(fname, copy=copy)
t1 = time()
print("Time for opening the on-disk frame (caterva, copy=%s): %.3fs" %
(copy, (t1 - t0)))
if macosx: os.system("/usr/sbin/purge")
t0 = time()
acc = 0
for (block, info) in a.iter_read():
block = np.frombuffer(block, dtype=dtype).reshape(info.shape)
acc += np.sum(block)
del a
t1 = time()
print("Time for reducing with (caterva, copy=%s): %.3fs" % (copy,
(t1 - t0)))
return acc
def test_persistency(shape, chunks, blocks, urlpath, sequencial, dtype):
if os.path.exists(urlpath):
cat.remove(urlpath)
size = int(np.prod(shape))
nparray = np.arange(size, dtype=dtype).reshape(shape)
_ = cat.asarray(nparray,
chunks=chunks,
blocks=blocks,
urlpath=urlpath,
sequencial=sequencial)
b = cat.open(urlpath)
bc = b[:]
nparray2 = np.asarray(bc).view(dtype)
np.testing.assert_almost_equal(nparray, nparray2)
cat.remove(urlpath)
h5f = tables.open_file(fname_h5, 'w', driver='H5FD_CORE', driver_core_backing_store=0)
h5ca = h5f.create_carray(h5f.root, 'carray', filters=filters, chunkshape=chunkshape, obj=content)
h5f.flush()
h5ratio = h5ca.size_in_memory / h5ca.size_on_disk
if persistent:
h5f.close()
t1 = time()
print("Time for filling array (hdf5): %.3fs ; CRatio: %.1fx" % ((t1 - t0), h5ratio))
# Setup the coordinates for random planes
planes_idx = np.random.randint(0, shape[1], 100)
# Time getitem with caterva
t0 = time()
if persistent:
a = cat.open(fname_cat) # reopen
for i in planes_idx:
rbytes = a[:, i, :]
del a
t1 = time()
print("Time for reading with getitem (caterva): %.3fs" % (t1 - t0))
# Time getitem with zarr
t0 = time()
if persistent:
z = zarr.open(fname_zarr, mode='r')
for i in planes_idx:
block = z[:, i, :]
del z
t1 = time()
print("Time for reading with getitem (zarr): %.3fs" % (t1 - t0))
if os.path.exists(urlpath):
cat.remove(urlpath)
dtype = np.dtype(np.complex128)
itemsize = dtype.itemsize
# Create a numpy array
nparray = np.arange(int(np.prod(shape)), dtype=dtype).reshape(shape)
# Create a caterva array from a numpy array (on disk)
a = cat.from_buffer(bytes(nparray),
nparray.shape,
itemsize,
chunks=chunks,
blocks=blocks,
urlpath=urlpath,
sequencial=False)
# Read a caterva array from disk
b = cat.open(urlpath)
# Convert a caterva array to a numpy array
nparray2 = np.asarray(cat.from_buffer(b.to_buffer(), b.shape,
b.itemsize)).view(dtype)
np.testing.assert_almost_equal(nparray, nparray2)
# Remove file on disk
if os.path.exists(urlpath):
cat.remove(urlpath)