def test_empty_numpy(shape, chunkshape, blockshape, dtype, cname, clevel, use_dict, nthreads,
filters):
dtype = np.dtype(dtype)
a = cat.empty(shape, dtype.itemsize,
dtype=str(dtype),
chunkshape=chunkshape,
blockshape=blockshape,
cname=cname,
clevel=clevel,
use_dict=use_dict,
nthreads=nthreads,
filters=filters)
if chunkshape is not None:
assert a.chunkshape == chunkshape
assert a.blockshape == blockshape
assert a.shape == shape
assert a._dtype == dtype
assert a.itemsize == dtype.itemsize
assert a.cname == (cname if chunkshape is not None else None)
assert a.clevel == (clevel if chunkshape is not None else 1)
if chunkshape is not None:
assert a.filters[-len(filters):] == filters
else:
assert a.filters is None
def test_iters_numpy(shape, chunkshape1, blockshape1, chunkshape2, blockshape2, dtype):
dtype = np.dtype(dtype)
size = int(np.prod(shape))
nparray = np.ones(size, dtype=dtype).reshape(shape)
a = cat.asarray(nparray, chunkshape=chunkshape1, blockshape=blockshape1) # creates a NPArray
b = cat.empty(shape, dtype.itemsize, str(dtype), chunkshape=chunkshape2, blockshape=blockshape2)
itershape = chunkshape2 if chunkshape2 is not None else b.shape
for (block_r, info_r), (block_w, info_w) in lzip(a.iter_read(itershape), b.iter_write()):
block_w[:] = bytearray(np.asarray(block_r))
nparray2 = np.asarray(b.copy())
np.testing.assert_equal(nparray, nparray2)
def test_metalayers(shape, chunkshape, blockshape, filename, dtype):
if os.path.exists(filename):
os.remove(filename)
# Create an empty caterva array (on disk)
itemsize = np.dtype(dtype).itemsize
a = cat.empty(shape,
chunkshape=chunkshape,
blockshape=blockshape,
filename=filename,
itemsize=itemsize,
metalayers={
"numpy": {
b"dtype": str(np.dtype(dtype))
},
"test": {
b"lorem": 1234
}
})
assert (a.has_metalayer("numpy") is True)
assert (a.get_metalayer("error") is None)
assert (a.get_metalayer("numpy") == {
b"dtype": bytes(str(np.dtype(dtype)), "utf-8")
})
assert (a.has_metalayer("test") is True)
assert (a.get_metalayer("test") == {b"lorem": 1234})
assert (a.update_metalayer("test", {b"lorem": 4321}) >= 0)
assert (a.get_metalayer("test") == {b"lorem": 4321})
# Fill an empty caterva array using a block iterator
nparray = np.arange(int(np.prod(shape)), dtype=dtype).reshape(shape)
for block, info in a.iter_write():
block[:] = bytes(nparray[info.slice])
assert (a.update_usermeta({
b"author": b"cat4py example",
b"description": b"lorem ipsum"
}) >= 0)
assert (a.get_usermeta() == {
b"author": b"cat4py example",
b"description": b"lorem ipsum"
})
assert (a.update_usermeta({b"author": b"cat4py example"}) >= 0)
assert (a.get_usermeta() == {b"author": b"cat4py example"})
# Remove file on disk
os.remove(filename)
def test_iters(shape, chunkshape1, blockshape1, chunkshape2, blockshape2,
dtype):
size = int(np.prod(shape))
nparray = np.ones(size, dtype=dtype).reshape(shape)
a = cat.from_buffer(bytes(nparray),
nparray.shape,
itemsize=nparray.itemsize,
chunkshape=chunkshape1,
blockshape=blockshape1)
itemsize = np.dtype(dtype).itemsize
b = cat.empty(shape,
chunkshape=chunkshape2,
blockshape=blockshape2,
itemsize=itemsize)
itershape = chunkshape2 if chunkshape2 is not None else b.shape
for (block_r, info_r), (block_w, info_w) in lzip(a.iter_read(itershape),
b.iter_write()):
block_w[:] = block_r
nparray2 = b.to_numpy(dtype)
np.testing.assert_equal(nparray, nparray2)
blockshape=blockshape,
enforceframe=True)
# Create an empty caterva array (on disk)
metalayers = {
"numpy": {
b"dtype": str(np.dtype(dtype))
},
"test": {
b"lorem": 1234
}
}
b = cat.empty(shape,
itemsize,
str(dtype),
chunkshape=chunkshape,
blockshape=blockshape,
filename=filename,
metalayers=metalayers)
assert (b.has_metalayer("numpy") is True)
assert (b.get_metalayer("numpy") == {
b"dtype": bytes(str(np.dtype(dtype)), "utf-8")
})
assert (b.has_metalayer("test") is True)
assert (b.get_metalayer("test") == {b"lorem": 1234})
assert (b.update_metalayer("test", {b"lorem": 4321}) >= 0)
assert (b.get_metalayer("test") == {b"lorem": 4321})
# Fill an empty caterva array using a block iterator
for block, info in b.iter_write():
shape = (512, 512)
chunkshape = (121, 99)
blockshape = (12, 31)
dtype = np.float64
itemsize = np.dtype(dtype).itemsize
# Create a numpy array
nparray = np.linspace(0, 1, int(np.prod(shape)), dtype=dtype).reshape(shape)
# Create a caterva array from a numpy array
a = cat.from_numpy(nparray,
chunkshape=chunkshape,
blockshape=blockshape,
enforceframe=True)
# Create an empty caterva array (on disk)
b = cat.empty(shape, dtype=dtype, itemsize=itemsize)
# Fill an empty caterva array using a block iterator
for block, info in b.iter_write():
block[:] = nparray[info.slice]
# Assert both caterva arrays
for (block1, info1), (block2, info2) in lzip(a.iter_read(blockshape),
b.iter_read(blockshape)):
np.testing.assert_almost_equal(block1, block2)
print(b[5:10, 5:10])
# t0 = time()
# a = cat.from_buffer(bytes(content), shape, pshape=pshape, itemsize=content.itemsize, filename=fname_cat,
# cname=cname, clevel=clevel, filters=[filter],
# cnthreads=nthreads, dnthreads=nthreads)
# if persistent:
# del a
# t1 = time()
# print("Time for filling array (caterva, from_buffer): %.3fs" % (t1 - t0))
# if fname_cat is not None and os.path.exists(fname_cat):
# os.remove(fname_cat)
# Create and fill a caterva array using a block iterator
t0 = time()
a = cat.empty(shape, chunkshape=chunkshape, blockshape=blockshape,
dtype=content.dtype, filename=fname_cat,
cname=cname, clevel=clevel, filters=[filter], nthreads=nthreads)
for block, info in a.iter_write():
block[:] = content[info.slice]
acratio = a.cratio
if persistent:
del a
t1 = time()
print("Time for filling array (caterva, iter): %.3fs ; CRatio: %.1fx" % ((t1 - t0), acratio))
# Create and fill a zarr array
t0 = time()
compressor = numcodecs.Blosc(cname=cname, clevel=clevel, shuffle=filter, blocksize=blocksize)
numcodecs.blosc.set_nthreads(nthreads)
if persistent:
z = zarr.open(fname_zarr, mode='w', shape=shape, chunks=chunkshape, dtype=dtype, compressor=compressor)
if persistent:
filename = "bench_getitem.cat"
if os.path.exists(filename):
# Remove file on disk
os.remove(filename)
else:
filename = None
itemsize = np.dtype(dtype).itemsize
# Create an empty caterva array
a = cat.empty(shape,
itemsize,
dtype=str(np.dtype(dtype)),
chunkshape=chunkshape,
blockshape=blockshape,
filename=filename,
compcode=0)
# Fill an empty caterva array using a block iterator
t0 = time()
count = 0
for block, info in a.iter_write():
nparray = np.arange(count, count + info.nitems,
dtype=dtype).reshape(info.shape)
block[:] = bytes(nparray)
count += info.nitems
t1 = time()
print("Time for filling: %.3fs" % (t1 - t0))
# Remove file on disk
os.remove(filename)
dtype = np.dtype(np.complex128)
itemsize = dtype.itemsize
# Create a numpy array
nparray = np.arange(int(np.prod(shape)), dtype=dtype).reshape(shape)
# Create a caterva array from a numpy array
a = cat.asarray(nparray)
# Create an empty caterva array (on disk)
b = cat.empty(shape,
itemsize,
dtype=str(dtype),
chunkshape=chunkshape,
blockshape=blockshape,
filename=filename)
# Fill an empty caterva array using a block iterator
for block, info in b.iter_write():
block[:] = bytes(nparray[info.slice])
# Load file
c = cat.from_file(filename)
# Assert both caterva arrays
itershape = (5, 5)
for (block1, info1), (block2, info2) in lzip(a.iter_read(itershape),
c.iter_read(itershape)):
np.testing.assert_equal(np.asarray(block1), np.asarray(block2))
t1 = time()
print("Time for filling array (numpy): %.3fs" % (t1 - t0))
t0 = time()
arr2 = arr.copy()
t1 = time()
print("Time for copying array in-memory (numpy): %.3fs" % (t1 - t0))
# Create and fill a caterva array using a block iterator and an in-memory frame
t0 = time()
carr = cat.empty(shape,
np.dtype(dtype).itemsize,
dtype=dtype,
chunkshape=chunkshape,
blockshape=blockshape,
enforceframe=True,
cname=cname,
clevel=clevel,
filters=[filter],
cnthreads=nthreads,
dnthreads=nthreads)
for block, info in carr.iter_write():
nparray = arr[info.slice]
block[:] = bytes(nparray)
acratio = carr.cratio
t1 = time()
print(
"Time for creating an array in-memory (numpy -> caterva, copy): %.3fs ; CRatio: %.1fx"
% ((t1 - t0), acratio))
print()
# Remove file on disk
os.remove(filename)
dtype = np.complex128
itemsize = np.dtype(dtype).itemsize
# Create a numpy array
nparray = np.arange(int(np.prod(shape)), dtype=dtype).reshape(shape)
# Create a caterva array from a numpy array
a = cat.from_numpy(nparray)
# Create an empty caterva array (on disk)
b = cat.empty(shape,
chunkshape=chunkshape,
blockshape=blockshape,
filename=filename,
itemsize=itemsize)
# Fill an empty caterva array using a block iterator
for block, info in b.iter_write():
block[:] = bytes(nparray[info.slice])
# Load file
c = cat.from_file(filename)
# Assert both caterva arrays
itershape = (5, 5)
for (block1, info1), (block2, info2) in lzip(a.iter_read(itershape),
c.iter_read(itershape)):
assert bytes(block1) == block2
dtype = np.float64
persistent = bool(sys.argv[1]) if len(sys.argv) > 1 else False
if persistent:
filename = "bench_getitem.cat"
if os.path.exists(filename):
# Remove file on disk
os.remove(filename)
else:
filename = None
itemsize = np.dtype(dtype).itemsize
# Create an empty caterva array
a = cat.empty(shape, dtype=dtype, chunkshape=chunkshape, blockshape=blockshape,
filename=filename, compcode=0)
# Fill an empty caterva array using a block iterator
t0 = time()
count = 0
for block, info in a.iter_write():
nparray = np.arange(count, count + info.size, dtype=dtype).reshape(info.shape)
block[:] = nparray
count += info.size
t1 = time()
print("Time for filling: %.3fs" % (t1 - t0))
# Check that the retrieved items are correct
t0 = time()
for block, info in a.iter_read(chunkshape):
pass
shape = (512, 512)
chunkshape = (121, 99)
blockshape = (12, 31)
dtype = np.dtype(np.float64)
itemsize = np.dtype(dtype).itemsize
# Create a numpy array
nparray = np.linspace(0, 1, int(np.prod(shape)), dtype=dtype).reshape(shape)
# Create a caterva array from a numpy array
a = cat.asarray(nparray,
chunkshape=chunkshape,
blockshape=blockshape,
enforceframe=True)
# Create an empty caterva array (on disk)
b = cat.empty(shape, itemsize, dtype=str(dtype))
# Fill an empty caterva array using a block iterator
for block, info in b.iter_write():
block[:] = bytes(nparray[info.slice])
# Assert both caterva arrays
for (block1, info1), (block2, info2) in lzip(a.iter_read(blockshape),
b.iter_read(blockshape)):
np.testing.assert_equal(np.asarray(block1), np.asarray(block2))
print(np.asarray(b[5:10, 5:10]))
