def test_update(contiguous, urlpath, nchunks, nupdates, copy, create_chunk):
storage = {
"contiguous": contiguous,
"urlpath": urlpath,
"cparams": {"nthreads": 2},
"dparams": {"nthreads": 2},
}
blosc2.remove_urlpath(urlpath)
nbytes = 23401
schunk = blosc2.SChunk(chunksize=nbytes * 2, **storage)
for i in range(nchunks):
bytes_obj = b"i " * nbytes
nchunks_ = schunk.append_data(bytes_obj)
assert nchunks_ == (i + 1)
for i in range(nupdates):
pos = random.randint(0, nchunks - 1)
bytes_obj = b"i " * nbytes
if create_chunk:
chunk = blosc2.compress2(bytes_obj)
schunk.update_chunk(pos, chunk)
else:
schunk.update_data(pos, bytes_obj, copy)
res = schunk.decompress_chunk(pos)
assert res == bytes_obj
for i in range(nchunks):
schunk.decompress_chunk(i)
blosc2.remove_urlpath(urlpath)
def test_schunk(contiguous, urlpath, nbytes, cparams, dparams, nchunks):
storage = {
"contiguous": contiguous,
"urlpath": urlpath,
"cparams": cparams,
"dparams": dparams
}
blosc2.remove_urlpath(urlpath)
schunk = blosc2.SChunk(chunksize=2 * nbytes, **storage)
for i in range(nchunks):
bytes_obj = b"i " * nbytes
nchunks_ = schunk.append_data(bytes_obj)
assert nchunks_ == (i + 1)
for i in range(nchunks):
bytes_obj = b"i " * nbytes
res = schunk.decompress_chunk(i)
assert res == bytes_obj
dest = bytearray(bytes_obj)
schunk.decompress_chunk(i, dst=dest)
assert dest == bytes_obj
for i in range(nchunks):
schunk.get_chunk(i)
blosc2.remove_urlpath(urlpath)
def test_schunk_delete(contiguous, urlpath, nchunks, ndeletes):
storage = {
"contiguous": contiguous,
"urlpath": urlpath,
"cparams": {
"nthreads": 2
},
"dparams": {
"nthreads": 2
},
}
blosc2.remove_urlpath(urlpath)
nbytes = 23401
schunk = blosc2.SChunk(chunksize=nbytes * 2, **storage)
for i in range(nchunks):
bytes_obj = b"i " * nbytes
nchunks_ = schunk.append_data(bytes_obj)
assert nchunks_ == (i + 1)
for i in range(ndeletes):
pos = random.randint(0, nchunks - 1)
if pos != (nchunks - 1):
buff = schunk.decompress_chunk(pos + 1)
nchunks_ = schunk.delete_chunk(pos)
assert nchunks_ == (nchunks - 1)
if pos != (nchunks - 1):
buff_ = schunk.decompress_chunk(pos)
assert buff == buff_
nchunks -= 1
for i in range(nchunks):
schunk.decompress_chunk(i)
blosc2.remove_urlpath(urlpath)
def test_schunk_update_numpy(contiguous, urlpath, nchunks, nupdates, copy, create_chunk):
storage = {
"contiguous": contiguous,
"urlpath": urlpath,
"cparams": {"nthreads": 2},
"dparams": {"nthreads": 2},
}
blosc2.remove_urlpath(urlpath)
schunk = blosc2.SChunk(chunksize=200 * 1000 * 4, **storage)
for i in range(nchunks):
buffer = i * numpy.arange(200 * 1000, dtype="int32")
nchunks_ = schunk.append_data(buffer)
assert nchunks_ == (i + 1)
for i in range(nupdates):
pos = random.randint(0, nchunks - 1)
buffer = pos * numpy.arange(200 * 1000, dtype="int32")
if create_chunk:
chunk = blosc2.compress2(buffer)
schunk.update_chunk(pos, chunk)
else:
schunk.update_data(pos, buffer, copy)
chunk_ = schunk.decompress_chunk(pos)
bytes_obj = buffer.tobytes()
assert chunk_ == bytes_obj
dest = numpy.empty(buffer.shape, buffer.dtype)
schunk.decompress_chunk(pos, dest)
assert numpy.array_equal(buffer, dest)
for i in range(nchunks):
schunk.decompress_chunk(i)
blosc2.remove_urlpath(urlpath)
def test_schunk(contiguous, urlpath, cparams, dparams, chunksize):
storage = {
"contiguous": contiguous,
"urlpath": urlpath,
"cparams": cparams,
"dparams": dparams
}
blosc2.remove_urlpath(urlpath)
nrep = 1000
nchunks = 5 * nrep // chunksize + 1 if nrep * 5 % chunksize != 0 else 5 * nrep // chunksize
buffer = b"1234 " * nrep
schunk = blosc2.SChunk(chunksize=chunksize, data=buffer, **storage)
for i in range(nchunks):
start = i * chunksize
if i == (nchunks - 1):
end = len(buffer)
else:
end = (i + 1) * chunksize
bytes_obj = buffer[start:end]
res = schunk.decompress_chunk(i)
assert res == bytes_obj
dest = bytearray(bytes_obj)
schunk.decompress_chunk(i, dst=dest)
assert dest == bytes_obj
for i in range(nchunks):
schunk.get_chunk(i)
blosc2.remove_urlpath(urlpath)
def test_schunk_delete_numpy(contiguous, urlpath, nchunks, ndeletes):
storage = {
"contiguous": contiguous,
"urlpath": urlpath,
"cparams": {
"nthreads": 2
},
"dparams": {
"nthreads": 2
},
}
blosc2.remove_urlpath(urlpath)
schunk = blosc2.SChunk(chunksize=200 * 1000 * 4, **storage)
for i in range(nchunks):
buffer = i * numpy.arange(200 * 1000, dtype="int32")
nchunks_ = schunk.append_data(buffer)
assert nchunks_ == (i + 1)
for i in range(ndeletes):
pos = random.randint(0, nchunks - 1)
if pos != (nchunks - 1):
buff = schunk.decompress_chunk(pos + 1)
nchunks_ = schunk.delete_chunk(pos)
assert nchunks_ == (nchunks - 1)
if pos != (nchunks - 1):
buff_ = schunk.decompress_chunk(pos)
assert buff == buff_
nchunks -= 1
for i in range(nchunks):
schunk.decompress_chunk(i)
blosc2.remove_urlpath(urlpath)
def test_schunk(contiguous, urlpath, nbytes, cparams, dparams, nchunks):
storage = {
"contiguous": contiguous,
"urlpath": urlpath,
"cparams": cparams,
"dparams": dparams
}
blosc2.remove_urlpath(urlpath)
schunk = blosc2.SChunk(chunksize=2 * nbytes, **storage)
for i in range(nchunks):
bytes_obj = b"i " * nbytes
nchunks_ = schunk.append_data(bytes_obj)
assert nchunks_ == (i + 1)
add(schunk)
delete(schunk)
blosc2.remove_urlpath(urlpath)
def test_schunk_numpy(contiguous, urlpath, cparams, dparams, nchunks):
storage = {
"contiguous": contiguous,
"urlpath": urlpath,
"cparams": cparams,
"dparams": dparams
}
blosc2.remove_urlpath(urlpath)
schunk = blosc2.SChunk(chunksize=200 * 1000 * 4, **storage)
for i in range(nchunks):
buffer = i * numpy.arange(200 * 1000, dtype="int32")
nchunks_ = schunk.append_data(buffer)
assert nchunks_ == (i + 1)
add(schunk)
delete(schunk)
blosc2.remove_urlpath(urlpath)
def test_schunk_numpy(contiguous, urlpath, cparams, dparams, chunksize):
storage = {
"contiguous": contiguous,
"urlpath": urlpath,
"cparams": cparams,
"dparams": dparams
}
blosc2.remove_urlpath(urlpath)
num_elem = 20 * 1000
nchunks = num_elem * 4 // chunksize + 1 if num_elem * 4 % chunksize != 0 else num_elem * 4 // chunksize
data = numpy.arange(num_elem, dtype="int32")
bytes_obj = data.tobytes()
schunk = blosc2.SChunk(chunksize=chunksize, data=data, **storage)
for i in range(nchunks):
start = i * chunksize
np_start = start // 4
if i == (nchunks - 1):
end = len(bytes_obj)
else:
end = (i + 1) * chunksize
np_end = end // 4
res = schunk.decompress_chunk(i)
assert res == bytes_obj[start:end]
dest = numpy.empty(np_end - np_start, dtype=data.dtype)
schunk.decompress_chunk(i, dest)
assert numpy.array_equal(data[np_start:np_end], dest)
schunk.decompress_chunk(i, memoryview(dest))
assert numpy.array_equal(data[np_start:np_end], dest)
dest = bytearray(data)
schunk.decompress_chunk(i, dest[start:end])
assert dest[start:end] == bytes_obj[start:end]
for i in range(nchunks):
schunk.get_chunk(i)
blosc2.remove_urlpath(urlpath)
def test_schunk_numpy(contiguous, urlpath, cparams, dparams, nchunks):
storage = {
"contiguous": contiguous,
"urlpath": urlpath,
"cparams": cparams,
"dparams": dparams
}
blosc2.remove_urlpath(urlpath)
schunk = blosc2.SChunk(chunksize=200 * 1000 * 4, **storage)
for i in range(nchunks):
buffer = i * numpy.arange(200 * 1000, dtype="int32")
nchunks_ = schunk.append_data(buffer)
assert nchunks_ == (i + 1)
for i in range(nchunks):
buffer = i * numpy.arange(200 * 1000, dtype="int32")
bytes_obj = buffer.tobytes()
res = schunk.decompress_chunk(i)
assert res == bytes_obj
dest = numpy.empty(buffer.shape, buffer.dtype)
schunk.decompress_chunk(i, dest)
assert numpy.array_equal(buffer, dest)
schunk.decompress_chunk(i, memoryview(dest))
assert numpy.array_equal(buffer, dest)
dest = bytearray(buffer)
schunk.decompress_chunk(i, dest)
assert dest == bytes_obj
for i in range(nchunks):
schunk.get_chunk(i)
blosc2.remove_urlpath(urlpath)
######################################################################## # # Created: July 28, 2021 # Author: The Blosc development team - [email protected] # ######################################################################## import numpy import blosc2 nchunks = 10 schunk = blosc2.SChunk(chunksize=200 * 1000 * 4) for i in range(nchunks): buffer = i * numpy.arange(200 * 1000, dtype="int32") nchunks_ = schunk.append_data(buffer) assert nchunks_ == (i + 1) # Initially the vlmeta is empty print(schunk.vlmeta.vlmeta) # Add a vlmeta schunk.vlmeta["meta1"] = "first vlmetalayer" print(schunk.vlmeta.vlmeta) # Update the vlmeta schunk.vlmeta["meta1"] = "new vlmetalayer" print(schunk.vlmeta.vlmeta) # Add another vlmeta schunk.vlmeta["vlmeta2"] = "second vlmeta" # Check that it has been added assert ("vlmeta2" in schunk.vlmeta) == True
# Set the compression and decompression parameters
cparams = {"compcode": blosc2.Codec.LZ4HC, "typesize": 4}
dparams = {}
contiguous = True
urlpath = "filename"
storage = {
"contiguous": contiguous,
"urlpath": urlpath,
"cparams": cparams,
"dparams": dparams
}
blosc2.remove_urlpath(urlpath)
# Create the empty SChunk
schunk = blosc2.SChunk(chunksize=200 * 1000 * 4, **storage)
# Append some chunks
for i in range(nchunks):
buffer = i * numpy.arange(200 * 1000, dtype="int32")
nchunks_ = schunk.append_data(buffer)
assert nchunks_ == (i + 1)
# Decompress the second chunk in different ways
buffer = 1 * numpy.arange(200 * 1000, dtype="int32")
bytes_obj = buffer.tobytes()
res = schunk.decompress_chunk(1)
assert res == bytes_obj
dest = numpy.empty(buffer.shape, buffer.dtype)
schunk.decompress_chunk(1, dest)
assert numpy.array_equal(buffer, dest)
