def test_persistency(shape, chunkshape, blockshape, filename, dtype, copy):
if os.path.exists(filename):
os.remove(filename)
size = int(np.prod(shape))
nparray = np.arange(size, dtype=dtype).reshape(shape)
cat.asarray(nparray,
chunkshape=chunkshape,
blockshape=blockshape,
filename=filename)
b = cat.from_file(filename, copy)
nparray2 = np.asarray(b.copy())
np.testing.assert_almost_equal(nparray, nparray2)
os.remove(filename)
def test_getitem_numpy(shape, chunkshape, blockshape, slices, dtype):
size = int(np.prod(shape))
nparray = np.arange(size, dtype=dtype).reshape(shape)
a = cat.asarray(nparray, chunkshape=chunkshape, blockshape=blockshape)
nparray_slice = nparray[slices]
a_slice = a[slices]
np.testing.assert_almost_equal(a_slice, nparray_slice)
def test_iters_numpy(shape, chunkshape1, blockshape1, chunkshape2, blockshape2, dtype):
dtype = np.dtype(dtype)
size = int(np.prod(shape))
nparray = np.ones(size, dtype=dtype).reshape(shape)
a = cat.asarray(nparray, chunkshape=chunkshape1, blockshape=blockshape1) # creates a NPArray
b = cat.empty(shape, dtype.itemsize, str(dtype), chunkshape=chunkshape2, blockshape=blockshape2)
itershape = chunkshape2 if chunkshape2 is not None else b.shape
for (block_r, info_r), (block_w, info_w) in lzip(a.iter_read(itershape), b.iter_write()):
block_w[:] = bytearray(np.asarray(block_r))
nparray2 = np.asarray(b.copy())
np.testing.assert_equal(nparray, nparray2)
def test_frame_numpy(shape, chunkshape, blockshape, dtype, enforceframe,
filename, copy_sframe):
if filename is not None and os.path.exists(filename):
os.remove(filename)
size = int(np.prod(shape))
nparray = np.arange(size, dtype=dtype).reshape(shape)
a = cat.asarray(nparray,
chunkshape=chunkshape,
blockshape=blockshape,
enforceframe=enforceframe,
filename=filename)
sframe1 = a.to_sframe()
buffer1 = a.to_buffer()
# Size of a compressed frame should be less than the plain buffer for the cases here
assert len(sframe1) < len(buffer1)
b = cat.from_sframe(sframe1, copy=copy_sframe)
sframe2 = b.to_sframe()
# For some reason, the size of sframe1 and sframe2 are not equal when copies are made,
# but the important thing is that the length of the frame should be stable in multiple
# round-trips after the first one.
# assert len(sframe2) == len(sframe1)
sframe3 = sframe2
c = b
for i in range(10):
c = cat.from_sframe(sframe2, copy=copy_sframe)
sframe3 = c.to_sframe()
if not copy_sframe:
# When the frame is not copied, we *need* a copy for the next iteration
sframe3 = bytes(sframe3)
assert len(sframe3) == len(sframe2)
buffer2 = b.to_buffer()
assert buffer2 == buffer1
buffer3 = c.to_buffer()
assert buffer3 == buffer1
if filename is not None and os.path.exists(filename):
os.remove(filename)
blockshape = (12, 7)
filename = "ex_metalayers.cat"
if os.path.exists(filename):
# Remove file on disk
os.remove(filename)
dtype = np.dtype(np.int32)
itemsize = np.dtype(dtype).itemsize
# Create a numpy array
nparray = np.arange(int(np.prod(shape)), dtype=dtype).reshape(shape)
# Create a caterva array from a numpy array
a = cat.asarray(nparray,
chunkshape=chunkshape,
blockshape=blockshape,
enforceframe=True)
# Create an empty caterva array (on disk)
metalayers = {
"numpy": {
b"dtype": str(np.dtype(dtype))
},
"test": {
b"lorem": 1234
}
}
b = cat.empty(shape,
itemsize,
str(dtype),
chunkshape=chunkshape,
import cat4py as cat import numpy as np shape = (200, 132) chunkshape = (55, 23) blockshape = (5, 11) slices = (5, ..., 25) dtype = np.float64 itemsize = np.dtype(dtype).itemsize # Create a numpy array nparray = np.arange(int(np.prod(shape)), dtype=dtype).reshape(shape) # Create a caterva array from a numpy array a = cat.asarray(nparray, chunkshape=chunkshape, blockshape=blockshape) # Get a slice buffer = np.asarray(a[slices]) buffer2 = nparray[slices] np.testing.assert_almost_equal(buffer, buffer2)
chunkshape = (10, 10)
blockshape = (10, 10)
filename = "ex_iters.cat"
if os.path.exists(filename):
# Remove file on disk
os.remove(filename)
dtype = np.dtype(np.complex128)
itemsize = dtype.itemsize
# Create a numpy array
nparray = np.arange(int(np.prod(shape)), dtype=dtype).reshape(shape)
# Create a caterva array from a numpy array
a = cat.asarray(nparray)
# Create an empty caterva array (on disk)
b = cat.empty(shape,
itemsize,
dtype=str(dtype),
chunkshape=chunkshape,
blockshape=blockshape,
filename=filename)
# Fill an empty caterva array using a block iterator
for block, info in b.iter_write():
block[:] = bytes(nparray[info.slice])
# Load file
c = cat.from_file(filename)
dtype = np.dtype(np.float64)
# Create a buffer
buffer = bytes(np.arange(int(np.prod(shape)), dtype=dtype).reshape(shape))
# Create a caterva array from a buffer
a = cat.from_buffer(buffer, shape, dtype.itemsize, dtype=str(dtype),
chunkshape=chunkshape, blockshape=blockshape)
# Get a copy of a caterva array (plainbuffer)
b = cat.copy(a)
d = b.copy()
aux = np.asarray(b)
aux[1, 2] = 0
aux2 = cat.asarray(aux)
print(np.asarray(aux2))
c = np.asarray(b)
c[3:5, 2:7] = 0
print(c)
del b
print(c)
# Convert the copy to a buffer
buffer1 = a.to_buffer()
buffer2 = d.to_buffer()
def test_numpy(shape, chunkshape, blockshape, dtype):
size = int(np.prod(shape))
nparray = np.arange(size, dtype=dtype).reshape(shape)
a = cat.asarray(nparray, chunkshape=chunkshape, blockshape=blockshape)
nparray2 = np.asarray(a.copy())
np.testing.assert_almost_equal(nparray, nparray2)
