def test_metalayers(shape, chunks, blocks, urlpath, sequencial, dtype):
if os.path.exists(urlpath):
cat.remove(urlpath)
numpy_meta = packb({b"dtype": str(np.dtype(dtype))})
test_meta = packb({b"lorem": 1234})
# Create an empty caterva array (on disk)
itemsize = np.dtype(dtype).itemsize
a = cat.empty(shape,
itemsize,
chunks=chunks,
blocks=blocks,
urlpath=urlpath,
sequencial=sequencial,
meta={
"numpy": numpy_meta,
"test": test_meta
})
assert ("numpy" in a.meta)
assert ("error" not in a.meta)
assert (a.meta["numpy"] == numpy_meta)
assert ("test" in a.meta)
assert (a.meta["test"] == test_meta)
test_meta = packb({b"lorem": 4231})
a.meta["test"] = test_meta
assert (a.meta["test"] == test_meta)
# Remove file on disk
cat.remove(urlpath)
def test_empty(shape, chunks, blocks, itemsize, codec, clevel, use_dict,
nthreads, filters):
a = cat.empty(shape,
chunks=chunks,
blocks=blocks,
itemsize=itemsize,
codec=codec,
clevel=clevel,
use_dict=use_dict,
nthreads=nthreads,
filters=filters)
if chunks is not None:
assert a.chunks == chunks
assert a.blocks == blocks
assert a.shape == shape
assert a.itemsize == itemsize
assert a.codec == (codec if chunks is not None else None)
assert a.clevel == (clevel if chunks is not None else 1)
if chunks is not None:
assert a.filters[-len(filters):] == filters
else:
assert a.filters is None
# content = np.arange(int(np.prod(shape)), dtype=dtype).reshape(shape)
t1 = time()
print("Time for filling array (numpy): %.3fs" % (t1 - t0))
t0 = time()
np.save(fname_npy, content)
t1 = time()
print("Time for storing array on-disk (numpy): %.3fs" % (t1 - t0))
# Create and fill a caterva array using a block iterator
t0 = time()
a = cat.empty(shape,
chunkshape=chunkshape,
blockshape=blockshape,
itemsize=content.itemsize,
filename=fname_cat,
cname=cname,
clevel=clevel,
filters=[filter],
nthreads=nthreads)
for block, info in a.iter_write():
nparray = content[info.slice]
block[:] = bytes(nparray)
acratio = a.cratio
del a
t1 = time()
print("Time for storing array on-disk (caterva, iter): %.3fs ; CRatio: %.1fx" %
((t1 - t0), acratio))
print()
#######################################################################
# Copyright (C) 2019-present, Blosc Development team <*****@*****.**>
# All rights reserved.
#
# This source code is licensed under a BSD-style license (found in the
# LICENSE file in the root directory of this source tree)
#######################################################################
import caterva as cat
import numpy as np
np.random.seed(123)
shape, chunks, blocks, itemsize, codec, clevel, use_dict, nthreads, filters = (
(400, 399, 401), (20, 10, 130), (6, 6, 26), 3, cat.Codec.BLOSCLZ, 5, False,
2, [cat.Filter.DELTA, cat.Filter.TRUNC_PREC])
a = cat.empty(shape,
chunks=chunks,
blocks=blocks,
itemsize=itemsize,
codec=codec,
clevel=clevel,
use_dict=use_dict,
nthreads=nthreads,
filters=filters)
print("HOLA")
t1 = time()
print("Time for filling array (numpy): %.3fs" % (t1 - t0))
t0 = time()
arr2 = arr.copy()
t1 = time()
print("Time for copying array in-memory (numpy): %.3fs" % (t1 - t0))
# Create and fill a caterva array using a block iterator and an in-memory frame
t0 = time()
carr = cat.empty(shape,
np.dtype(dtype).itemsize,
dtype=dtype,
chunkshape=chunkshape,
blockshape=blockshape,
enforceframe=True,
cname=cname,
clevel=clevel,
filters=[filter],
cnthreads=nthreads,
dnthreads=nthreads)
for block, info in carr.iter_write():
nparray = arr[info.slice]
block[:] = bytes(nparray)
acratio = carr.cratio
t1 = time()
print(
"Time for creating an array in-memory (numpy -> caterva, copy): %.3fs ; CRatio: %.1fx"
% ((t1 - t0), acratio))
print()
dtype = np.float64
persistent = bool(sys.argv[1]) if len(sys.argv) > 1 else False
if persistent:
filename = "bench_getitem.cat"
if os.path.exists(filename):
# Remove file on disk
os.remove(filename)
else:
filename = None
itemsize = np.dtype(dtype).itemsize
# Create an empty caterva array
a = cat.empty(shape, itemsize, dtype=str(np.dtype(dtype)), chunkshape=chunkshape, blockshape=blockshape,
filename=filename, compcode=0)
# Fill an empty caterva array using a block iterator
t0 = time()
count = 0
for block, info in a.iter_write():
nparray = np.arange(count, count + info.nitems, dtype=dtype).reshape(info.shape)
block[:] = bytes(nparray)
count += info.nitems
t1 = time()
print("Time for filling: %.3fs" % (t1 - t0))
# Check that the retrieved items are correct
t0 = time()
for block, info in a.iter_read(chunkshape):
pass
if os.path.exists(fname_cat):
os.remove(fname_cat)
fname_zarr = "compare_getslice.zarr"
if os.path.exists(fname_zarr):
shutil.rmtree(fname_zarr)
fname_h5 = "compare_getslice.h5"
if os.path.exists(fname_h5):
os.remove(fname_h5)
# Create content for populating arrays
content = np.random.normal(0, 1, int(np.prod(shape))).reshape(shape)
# Create and fill a caterva array using a block iterator
t0 = time()
a = cat.empty(shape, content.itemsize, chunkshape=chunkshape, blockshape=blockshape,
dtype=str(content.dtype), urlpath=fname_cat,
cname=cname, clevel=clevel, filters=[filter], nthreads=nthreads)
a[:] = content
acratio = a.cratio
if persistent:
del a
t1 = time()
print("Time for filling array (caterva, iter): %.3fs ; CRatio: %.1fx" % ((t1 - t0), acratio))
# Create and fill a zarr array
t0 = time()
compressor = numcodecs.Blosc(cname=cname, clevel=clevel, shuffle=zfilter, blocksize=blocksize)
numcodecs.blosc.set_nthreads(nthreads)
if persistent:
z = zarr.open(fname_zarr, mode='w', shape=shape, chunks=chunkshape, dtype=dtype, compressor=compressor)
else: