def test_empty_numpy(shape, chunkshape, blockshape, dtype, cname, clevel, use_dict, nthreads, filters): dtype = np.dtype(dtype) a = cat.empty(shape, dtype.itemsize, dtype=str(dtype), chunkshape=chunkshape, blockshape=blockshape, cname=cname, clevel=clevel, use_dict=use_dict, nthreads=nthreads, filters=filters) if chunkshape is not None: assert a.chunkshape == chunkshape assert a.blockshape == blockshape assert a.shape == shape assert a._dtype == dtype assert a.itemsize == dtype.itemsize assert a.cname == (cname if chunkshape is not None else None) assert a.clevel == (clevel if chunkshape is not None else 1) if chunkshape is not None: assert a.filters[-len(filters):] == filters else: assert a.filters is None
def test_iters_numpy(shape, chunkshape1, blockshape1, chunkshape2, blockshape2, dtype): dtype = np.dtype(dtype) size = int(np.prod(shape)) nparray = np.ones(size, dtype=dtype).reshape(shape) a = cat.asarray(nparray, chunkshape=chunkshape1, blockshape=blockshape1) # creates a NPArray b = cat.empty(shape, dtype.itemsize, str(dtype), chunkshape=chunkshape2, blockshape=blockshape2) itershape = chunkshape2 if chunkshape2 is not None else b.shape for (block_r, info_r), (block_w, info_w) in lzip(a.iter_read(itershape), b.iter_write()): block_w[:] = bytearray(np.asarray(block_r)) nparray2 = np.asarray(b.copy()) np.testing.assert_equal(nparray, nparray2)
def test_metalayers(shape, chunkshape, blockshape, filename, dtype): if os.path.exists(filename): os.remove(filename) # Create an empty caterva array (on disk) itemsize = np.dtype(dtype).itemsize a = cat.empty(shape, chunkshape=chunkshape, blockshape=blockshape, filename=filename, itemsize=itemsize, metalayers={ "numpy": { b"dtype": str(np.dtype(dtype)) }, "test": { b"lorem": 1234 } }) assert (a.has_metalayer("numpy") is True) assert (a.get_metalayer("error") is None) assert (a.get_metalayer("numpy") == { b"dtype": bytes(str(np.dtype(dtype)), "utf-8") }) assert (a.has_metalayer("test") is True) assert (a.get_metalayer("test") == {b"lorem": 1234}) assert (a.update_metalayer("test", {b"lorem": 4321}) >= 0) assert (a.get_metalayer("test") == {b"lorem": 4321}) # Fill an empty caterva array using a block iterator nparray = np.arange(int(np.prod(shape)), dtype=dtype).reshape(shape) for block, info in a.iter_write(): block[:] = bytes(nparray[info.slice]) assert (a.update_usermeta({ b"author": b"cat4py example", b"description": b"lorem ipsum" }) >= 0) assert (a.get_usermeta() == { b"author": b"cat4py example", b"description": b"lorem ipsum" }) assert (a.update_usermeta({b"author": b"cat4py example"}) >= 0) assert (a.get_usermeta() == {b"author": b"cat4py example"}) # Remove file on disk os.remove(filename)
def test_iters(shape, chunkshape1, blockshape1, chunkshape2, blockshape2, dtype): size = int(np.prod(shape)) nparray = np.ones(size, dtype=dtype).reshape(shape) a = cat.from_buffer(bytes(nparray), nparray.shape, itemsize=nparray.itemsize, chunkshape=chunkshape1, blockshape=blockshape1) itemsize = np.dtype(dtype).itemsize b = cat.empty(shape, chunkshape=chunkshape2, blockshape=blockshape2, itemsize=itemsize) itershape = chunkshape2 if chunkshape2 is not None else b.shape for (block_r, info_r), (block_w, info_w) in lzip(a.iter_read(itershape), b.iter_write()): block_w[:] = block_r nparray2 = b.to_numpy(dtype) np.testing.assert_equal(nparray, nparray2)
blockshape=blockshape, enforceframe=True) # Create an empty caterva array (on disk) metalayers = { "numpy": { b"dtype": str(np.dtype(dtype)) }, "test": { b"lorem": 1234 } } b = cat.empty(shape, itemsize, str(dtype), chunkshape=chunkshape, blockshape=blockshape, filename=filename, metalayers=metalayers) assert (b.has_metalayer("numpy") is True) assert (b.get_metalayer("numpy") == { b"dtype": bytes(str(np.dtype(dtype)), "utf-8") }) assert (b.has_metalayer("test") is True) assert (b.get_metalayer("test") == {b"lorem": 1234}) assert (b.update_metalayer("test", {b"lorem": 4321}) >= 0) assert (b.get_metalayer("test") == {b"lorem": 4321}) # Fill an empty caterva array using a block iterator for block, info in b.iter_write():
shape = (512, 512) chunkshape = (121, 99) blockshape = (12, 31) dtype = np.float64 itemsize = np.dtype(dtype).itemsize # Create a numpy array nparray = np.linspace(0, 1, int(np.prod(shape)), dtype=dtype).reshape(shape) # Create a caterva array from a numpy array a = cat.from_numpy(nparray, chunkshape=chunkshape, blockshape=blockshape, enforceframe=True) # Create an empty caterva array (on disk) b = cat.empty(shape, dtype=dtype, itemsize=itemsize) # Fill an empty caterva array using a block iterator for block, info in b.iter_write(): block[:] = nparray[info.slice] # Assert both caterva arrays for (block1, info1), (block2, info2) in lzip(a.iter_read(blockshape), b.iter_read(blockshape)): np.testing.assert_almost_equal(block1, block2) print(b[5:10, 5:10])
# t0 = time() # a = cat.from_buffer(bytes(content), shape, pshape=pshape, itemsize=content.itemsize, filename=fname_cat, # cname=cname, clevel=clevel, filters=[filter], # cnthreads=nthreads, dnthreads=nthreads) # if persistent: # del a # t1 = time() # print("Time for filling array (caterva, from_buffer): %.3fs" % (t1 - t0)) # if fname_cat is not None and os.path.exists(fname_cat): # os.remove(fname_cat) # Create and fill a caterva array using a block iterator t0 = time() a = cat.empty(shape, chunkshape=chunkshape, blockshape=blockshape, dtype=content.dtype, filename=fname_cat, cname=cname, clevel=clevel, filters=[filter], nthreads=nthreads) for block, info in a.iter_write(): block[:] = content[info.slice] acratio = a.cratio if persistent: del a t1 = time() print("Time for filling array (caterva, iter): %.3fs ; CRatio: %.1fx" % ((t1 - t0), acratio)) # Create and fill a zarr array t0 = time() compressor = numcodecs.Blosc(cname=cname, clevel=clevel, shuffle=filter, blocksize=blocksize) numcodecs.blosc.set_nthreads(nthreads) if persistent: z = zarr.open(fname_zarr, mode='w', shape=shape, chunks=chunkshape, dtype=dtype, compressor=compressor)
if persistent: filename = "bench_getitem.cat" if os.path.exists(filename): # Remove file on disk os.remove(filename) else: filename = None itemsize = np.dtype(dtype).itemsize # Create an empty caterva array a = cat.empty(shape, itemsize, dtype=str(np.dtype(dtype)), chunkshape=chunkshape, blockshape=blockshape, filename=filename, compcode=0) # Fill an empty caterva array using a block iterator t0 = time() count = 0 for block, info in a.iter_write(): nparray = np.arange(count, count + info.nitems, dtype=dtype).reshape(info.shape) block[:] = bytes(nparray) count += info.nitems t1 = time() print("Time for filling: %.3fs" % (t1 - t0))
# Remove file on disk os.remove(filename) dtype = np.dtype(np.complex128) itemsize = dtype.itemsize # Create a numpy array nparray = np.arange(int(np.prod(shape)), dtype=dtype).reshape(shape) # Create a caterva array from a numpy array a = cat.asarray(nparray) # Create an empty caterva array (on disk) b = cat.empty(shape, itemsize, dtype=str(dtype), chunkshape=chunkshape, blockshape=blockshape, filename=filename) # Fill an empty caterva array using a block iterator for block, info in b.iter_write(): block[:] = bytes(nparray[info.slice]) # Load file c = cat.from_file(filename) # Assert both caterva arrays itershape = (5, 5) for (block1, info1), (block2, info2) in lzip(a.iter_read(itershape), c.iter_read(itershape)): np.testing.assert_equal(np.asarray(block1), np.asarray(block2))
t1 = time() print("Time for filling array (numpy): %.3fs" % (t1 - t0)) t0 = time() arr2 = arr.copy() t1 = time() print("Time for copying array in-memory (numpy): %.3fs" % (t1 - t0)) # Create and fill a caterva array using a block iterator and an in-memory frame t0 = time() carr = cat.empty(shape, np.dtype(dtype).itemsize, dtype=dtype, chunkshape=chunkshape, blockshape=blockshape, enforceframe=True, cname=cname, clevel=clevel, filters=[filter], cnthreads=nthreads, dnthreads=nthreads) for block, info in carr.iter_write(): nparray = arr[info.slice] block[:] = bytes(nparray) acratio = carr.cratio t1 = time() print( "Time for creating an array in-memory (numpy -> caterva, copy): %.3fs ; CRatio: %.1fx" % ((t1 - t0), acratio)) print()
# Remove file on disk os.remove(filename) dtype = np.complex128 itemsize = np.dtype(dtype).itemsize # Create a numpy array nparray = np.arange(int(np.prod(shape)), dtype=dtype).reshape(shape) # Create a caterva array from a numpy array a = cat.from_numpy(nparray) # Create an empty caterva array (on disk) b = cat.empty(shape, chunkshape=chunkshape, blockshape=blockshape, filename=filename, itemsize=itemsize) # Fill an empty caterva array using a block iterator for block, info in b.iter_write(): block[:] = bytes(nparray[info.slice]) # Load file c = cat.from_file(filename) # Assert both caterva arrays itershape = (5, 5) for (block1, info1), (block2, info2) in lzip(a.iter_read(itershape), c.iter_read(itershape)): assert bytes(block1) == block2
dtype = np.float64 persistent = bool(sys.argv[1]) if len(sys.argv) > 1 else False if persistent: filename = "bench_getitem.cat" if os.path.exists(filename): # Remove file on disk os.remove(filename) else: filename = None itemsize = np.dtype(dtype).itemsize # Create an empty caterva array a = cat.empty(shape, dtype=dtype, chunkshape=chunkshape, blockshape=blockshape, filename=filename, compcode=0) # Fill an empty caterva array using a block iterator t0 = time() count = 0 for block, info in a.iter_write(): nparray = np.arange(count, count + info.size, dtype=dtype).reshape(info.shape) block[:] = nparray count += info.size t1 = time() print("Time for filling: %.3fs" % (t1 - t0)) # Check that the retrieved items are correct t0 = time() for block, info in a.iter_read(chunkshape): pass
shape = (512, 512) chunkshape = (121, 99) blockshape = (12, 31) dtype = np.dtype(np.float64) itemsize = np.dtype(dtype).itemsize # Create a numpy array nparray = np.linspace(0, 1, int(np.prod(shape)), dtype=dtype).reshape(shape) # Create a caterva array from a numpy array a = cat.asarray(nparray, chunkshape=chunkshape, blockshape=blockshape, enforceframe=True) # Create an empty caterva array (on disk) b = cat.empty(shape, itemsize, dtype=str(dtype)) # Fill an empty caterva array using a block iterator for block, info in b.iter_write(): block[:] = bytes(nparray[info.slice]) # Assert both caterva arrays for (block1, info1), (block2, info2) in lzip(a.iter_read(blockshape), b.iter_read(blockshape)): np.testing.assert_equal(np.asarray(block1), np.asarray(block2)) print(np.asarray(b[5:10, 5:10]))