def test_persistency(shape, chunkshape, blockshape, filename, dtype, copy):
if os.path.exists(filename):
os.remove(filename)
size = int(np.prod(shape))
nparray = np.arange(size, dtype=dtype).reshape(shape)
cat.asarray(nparray,
chunkshape=chunkshape,
blockshape=blockshape,
filename=filename)
b = cat.from_file(filename, copy)
nparray2 = np.asarray(b.copy())
np.testing.assert_almost_equal(nparray, nparray2)
os.remove(filename)
def bench_read_caterva(fname, planes_idx, copy):
t0 = time()
a = cat.from_file(fname, copy=copy)
t1 = time()
print("Time for opening the on-disk frame (caterva, copy=%s): %.3fs" %
(copy, (t1 - t0)))
t0 = time()
for i in planes_idx:
rbytes = a[:, i, :]
block = np.frombuffer(rbytes, dtype=dtype).reshape(
(shape[0], shape[2]))
del a
t1 = time()
print("Time for reading with getitem (caterva, copy=%s): %.3fs" %
(copy, (t1 - t0)))
def bench_read_caterva(fname, copy):
if macosx: os.system("/usr/sbin/purge")
t0 = time()
a = cat.from_file(fname, copy=copy)
t1 = time()
print("Time for opening the on-disk frame (caterva, copy=%s): %.3fs" % (copy, (t1 - t0)))
if macosx: os.system("/usr/sbin/purge")
t0 = time()
acc = 0
for (block, info) in a.iter_read():
block = np.frombuffer(block, dtype=dtype).reshape(info.shape)
acc += np.sum(block)
del a
t1 = time()
print("Time for reducing with (caterva, copy=%s): %.3fs" % (copy, (t1 - t0)))
return acc
if persistent:
h5f = tables.open_file(fname_h5, 'w')
else:
h5f = tables.open_file(fname_h5, 'w', driver='H5FD_CORE', driver_core_backing_store=0)
h5ca = h5f.create_carray(h5f.root, 'carray', filters=filters, chunkshape=chunkshape, obj=content)
h5f.flush()
h5ratio = h5ca.size_in_memory / h5ca.size_on_disk
if persistent:
h5f.close()
t1 = time()
print("Time for filling array (hdf5): %.3fs ; CRatio: %.1fx" % ((t1 - t0), h5ratio))
# Check that the contents are the same
t0 = time()
if persistent:
a = cat.from_file(fname_cat, copy=False) # reopen
z = zarr.open(fname_zarr, mode='r')
h5f = tables.open_file(fname_h5, 'r', filters=filters)
h5ca = h5f.root.carray
for block, info in a.iter_read(chunkshape):
block_cat = block
block_zarr = z[info.slice]
np.testing.assert_array_almost_equal(block_cat, block_zarr)
block_h5 = h5ca[info.slice]
np.testing.assert_array_almost_equal(block_cat, block_h5)
if persistent:
del a
del z
h5f.close()
t1 = time()
print("Time for checking contents: %.3fs" % (t1 - t0))
shape = (128, 128)
chunkshape = (32, 32)
blockshape = (8, 8)
filename = "ex_persistency.cat"
if os.path.exists(filename):
# Remove file on disk
os.remove(filename)
dtype = np.dtype(np.complex128)
itemsize = dtype.itemsize
# Create a numpy array
nparray = np.arange(int(np.prod(shape)), dtype=dtype).reshape(shape)
# Create a caterva array from a numpy array (on disk)
a = cat.from_buffer(bytes(nparray), nparray.shape, chunkshape=chunkshape, blockshape=blockshape,
filename=filename, itemsize=itemsize)
# Read a caterva array from disk
b = cat.from_file(filename)
# Convert a caterva array to a numpy array
nparray2 = np.asarray(cat.from_buffer(b.to_buffer(), b.shape, b.itemsize, str(dtype)))
np.testing.assert_almost_equal(nparray, nparray2)
# Remove file on disk
os.remove(filename)