def test_BloscpackHeader_encode():
# the raw encoded header as produces w/o any kwargs
format_version = struct.pack('<B', FORMAT_VERSION)
raw = MAGIC + format_version + \
b'\x00\x00\x00\xff\xff\xff\xff\xff\xff\xff\xff' + \
b'\xff\xff\xff\xff\xff\xff\xff\xff\x00\x00\x00\x00\x00\x00\x00\x00'
# modify the raw encoded header with the value starting at offset
def mod_raw(offset, replacement):
return raw[0:offset] + replacement + \
raw[offset+len(replacement):]
# test with no arguments
yield nt.assert_equal, raw, BloscpackHeader().encode()
for offset, replacement, kwargs in [
(4, struct.pack('<B', 23), {'format_version': 23}),
# test with options
(5, b'\x01', {'offsets': True}),
(5, b'\x02', {'metadata': True}),
(5, b'\x03', {'offsets': True, 'metadata': True}),
# test with checksum
(6, b'\x01', {'checksum': 'adler32'}),
(6, b'\x08', {'checksum': 'sha512'}),
# test with typesize
(7, b'\x01', {'typesize': 1}),
(7, b'\x02', {'typesize': 2}),
(7, b'\x04', {'typesize': 4}),
(7, b'\x10', {'typesize': 16}),
(7, b'\xff', {'typesize': 255}),
# test with chunksize
(8, b'\xff\xff\xff\xff', {'chunk_size': -1}),
(8, b'\x01\x00\x00\x00', {'chunk_size': 1}),
(8, b'\x00\x00\x10\x00', {'chunk_size': reverse_pretty('1M')}),
(8, b'\xef\xff\xff\x7f', {'chunk_size': blosc.BLOSC_MAX_BUFFERSIZE}),
# test with last_chunk
(12, b'\xff\xff\xff\xff', {'last_chunk': -1}),
(12, b'\x01\x00\x00\x00', {'last_chunk': 1}),
(12, b'\x00\x00\x10\x00', {'last_chunk': reverse_pretty('1M')}),
(12, b'\xef\xff\xff\x7f', {'last_chunk': blosc.BLOSC_MAX_BUFFERSIZE}),
# test nchunks
(16, b'\xff\xff\xff\xff\xff\xff\xff\xff', {'nchunks': -1}),
(16, b'\x00\x00\x00\x00\x00\x00\x00\x00', {'nchunks': 0}),
(16, b'\x01\x00\x00\x00\x00\x00\x00\x00', {'nchunks': 1}),
(16, b'\x7f\x00\x00\x00\x00\x00\x00\x00', {'nchunks': 127}),
(16, b'\xff\xff\xff\xff\xff\xff\xff\x7f', {'nchunks': MAX_CHUNKS}),
# test max_app_chunks
(16, b'\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00',
{'nchunks': 1, 'max_app_chunks': 0}),
(16, b'\x01\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00',
{'nchunks': 1, 'max_app_chunks': 1}),
(16, b'\x01\x00\x00\x00\x00\x00\x00\x00\x7f\x00\x00\x00\x00\x00\x00\x00',
{'nchunks': 1, 'max_app_chunks': 127}),
# Maximum value is MAX_CHUNKS - 1 since nchunks is already 1
(16, b'\x01\x00\x00\x00\x00\x00\x00\x00\xfe\xff\xff\xff\xff\xff\xff\x7f',
{'nchunks': 1, 'max_app_chunks': MAX_CHUNKS-1}),
]:
yield nt.assert_equal, mod_raw(offset, replacement), \
BloscpackHeader(**kwargs).encode()
def test_calculate_nchunks():
# check for zero or negative chunk_size
nt.assert_raises(ValueError, calculate_nchunks,
23, chunk_size=0)
nt.assert_raises(ValueError, calculate_nchunks,
23, chunk_size=-1)
nt.assert_equal((9, 1, 1), calculate_nchunks(9, chunk_size=1))
nt.assert_equal((5, 2, 1), calculate_nchunks(9, chunk_size=2))
nt.assert_equal((3, 3, 3), calculate_nchunks(9, chunk_size=3))
nt.assert_equal((3, 4, 1), calculate_nchunks(9, chunk_size=4))
nt.assert_equal((2, 5, 4), calculate_nchunks(9, chunk_size=5))
nt.assert_equal((2, 6, 3), calculate_nchunks(9, chunk_size=6))
nt.assert_equal((2, 7, 2), calculate_nchunks(9, chunk_size=7))
nt.assert_equal((2, 8, 1), calculate_nchunks(9, chunk_size=8))
nt.assert_equal((1, 9, 9), calculate_nchunks(9, chunk_size=9))
# check downgrade
nt.assert_equal((1, 23, 23), calculate_nchunks(23, chunk_size=24))
# single byte file
nt.assert_equal((1, 1, 1),
calculate_nchunks(1, chunk_size=1))
# check that a zero length input is handled correctly
nt.assert_equal((1, 0, 0),
calculate_nchunks(0, chunk_size=1))
# check that the chunk_size is ignored in this case
nt.assert_equal((1, 0, 0),
calculate_nchunks(0, chunk_size=512))
# in_file_size must be strictly positive
nt.assert_raises(ValueError, calculate_nchunks, -1)
# check overflow of nchunks due to chunk_size being too small
# and thus stuff not fitting into the header
nt.assert_raises(ChunkingException, calculate_nchunks,
MAX_CHUNKS+1, chunk_size=1)
# check that strings are converted correctly
nt.assert_equal((6, 1048576, 209715),
calculate_nchunks(reverse_pretty('5.2M')))
nt.assert_equal((3, 2097152, 1258291),
calculate_nchunks(reverse_pretty('5.2M'),
chunk_size='2M'))
def test_pack_unpack_mem():
pack_unpack_mem(1, chunk_size=reverse_pretty('1M'))
pack_unpack_mem(1, chunk_size=reverse_pretty('2M'))
pack_unpack_mem(1, chunk_size=reverse_pretty('4M'))
pack_unpack_mem(1, chunk_size=reverse_pretty('8M'))
metadata = {"dtype": "float64", "shape": [1024], "others": []}
pack_unpack_mem(1, chunk_size=reverse_pretty('1M'), metadata=metadata)
pack_unpack_mem(1, chunk_size=reverse_pretty('2M'), metadata=metadata)
pack_unpack_mem(1, chunk_size=reverse_pretty('4M'), metadata=metadata)
pack_unpack_mem(1, chunk_size=reverse_pretty('8M'), metadata=metadata)
def test_pack_unpack_mem():
pack_unpack_mem(1, chunk_size=reverse_pretty('1M'))
pack_unpack_mem(1, chunk_size=reverse_pretty('2M'))
pack_unpack_mem(1, chunk_size=reverse_pretty('4M'))
pack_unpack_mem(1, chunk_size=reverse_pretty('8M'))
metadata = {"dtype": "float64", "shape": [1024], "others": []}
pack_unpack_mem(1, chunk_size=reverse_pretty('1M'), metadata=metadata)
pack_unpack_mem(1, chunk_size=reverse_pretty('2M'), metadata=metadata)
pack_unpack_mem(1, chunk_size=reverse_pretty('4M'), metadata=metadata)
pack_unpack_mem(1, chunk_size=reverse_pretty('8M'), metadata=metadata)
def test_pretty_filesieze():
nt.assert_equal('0B', pretty_size(0))
nt.assert_equal('9.0T', pretty_size(9898989898879))
nt.assert_equal('4.78G', pretty_size(5129898234))
nt.assert_equal('12.3M', pretty_size(12898234))
nt.assert_equal('966.7K', pretty_size(989898))
nt.assert_equal('128.0B', pretty_size(128))
nt.assert_equal(0, reverse_pretty('0B'))
nt.assert_equal(8, reverse_pretty('8B'))
nt.assert_equal(8192, reverse_pretty('8K'))
nt.assert_equal(134217728, reverse_pretty('128M'))
nt.assert_equal(2147483648, reverse_pretty('2G'))
nt.assert_equal(2199023255552, reverse_pretty('2T'))
# can't handle Petabytes, yet
nt.assert_raises(ValueError, reverse_pretty, '2P')
def pack_unpack_hard():
""" Test on somewhat larger arrays, but be nice to memory. """
# Array is apprx. 1.5 GB large
# should make apprx 1536 chunks
pack_unpack(100, chunk_size=reverse_pretty('1M'), progress=simple_progress)
def test_pack_unpack_fp():
pack_unpack_fp(1, chunk_size=reverse_pretty('1M'))
pack_unpack_fp(1, chunk_size=reverse_pretty('2M'))
pack_unpack_fp(1, chunk_size=reverse_pretty('4M'))
pack_unpack_fp(1, chunk_size=reverse_pretty('8M'))
def test_BloscpackHeader_decode():
format_version = struct.pack('<B', FORMAT_VERSION)
raw = MAGIC + format_version + \
b'\x00\x00\x00\xff\xff\xff\xff\xff\xff\xff\xff' + \
b'\xff\xff\xff\xff\xff\xff\xff\xff\x00\x00\x00\x00\x00\x00\x00\x00'
def mod_raw(offset, replacement):
return raw[0:offset] + replacement + \
raw[offset+len(replacement):]
# check with no args
yield nt.assert_equal, BloscpackHeader(), BloscpackHeader.decode(raw)
for kwargs, offset, replacement in [
# check with format_version
({
'format_version': 23
}, 4, b'\x17'),
# check with options
({
'offsets': True
}, 5, b'\x01'),
({
'metadata': True
}, 5, b'\x02'),
({
'metadata': True,
'offsets': True
}, 5, b'\x03'),
# check with checksum
({
'checksum': 'adler32'
}, 6, b'\x01'),
({
'checksum': 'sha384'
}, 6, b'\x07'),
# check with typesize
({
'typesize': 1
}, 7, b'\x01'),
({
'typesize': 2
}, 7, b'\x02'),
({
'typesize': 4
}, 7, b'\x04'),
({
'typesize': 8
}, 7, b'\x08'),
({
'typesize': blosc.BLOSC_MAX_TYPESIZE
}, 7, b'\xff'),
# check with chunk_size
({
'chunk_size': 1
}, 8, b'\x01\x00\x00\x00'),
({
'chunk_size': reverse_pretty('1M')
}, 8, b'\x00\x00\x10\x00'),
({
'chunk_size': blosc.BLOSC_MAX_BUFFERSIZE
}, 8, b'\xef\xff\xff\x7f'),
# check with last_chunk
({
'last_chunk': 1
}, 12, b'\x01\x00\x00\x00'),
({
'last_chunk': reverse_pretty('1M')
}, 12, b'\x00\x00\x10\x00'),
({
'last_chunk': blosc.BLOSC_MAX_BUFFERSIZE
}, 12, b'\xef\xff\xff\x7f'),
# check with nchunks
({
'nchunks': 1
}, 16, b'\x01\x00\x00\x00\x00\x00\x00\x00'),
({
'nchunks': reverse_pretty('1M')
}, 16, b'\x00\x00\x10\x00\x00\x00\x00\x00'),
({
'nchunks': MAX_CHUNKS
}, 16, b'\xff\xff\xff\xff\xff\xff\xff\x7f'),
# check with max_app_chunks
({
'nchunks': 1,
'max_app_chunks': 0
}, 16,
b'\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'),
({
'nchunks': 1,
'max_app_chunks': 1
}, 16,
b'\x01\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00'),
({
'nchunks': 1,
'max_app_chunks': reverse_pretty('1M')
}, 16,
b'\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x00\x00\x00\x00\x00'),
# Maximum value is MAX_CHUNKS - 1 since nchunks is already 1
({
'nchunks': 1,
'max_app_chunks': MAX_CHUNKS - 1
}, 16,
b'\x01\x00\x00\x00\x00\x00\x00\x00\xfe\xff\xff\xff\xff\xff\xff\x7f'),
]:
yield (nt.assert_equal, BloscpackHeader(**kwargs),
BloscpackHeader.decode(mod_raw(offset, replacement)))
def test_BloscpackHeader_encode():
# the raw encoded header as produces w/o any kwargs
format_version = struct.pack('<B', FORMAT_VERSION)
raw = MAGIC + format_version + \
b'\x00\x00\x00\xff\xff\xff\xff\xff\xff\xff\xff' + \
b'\xff\xff\xff\xff\xff\xff\xff\xff\x00\x00\x00\x00\x00\x00\x00\x00'
# modify the raw encoded header with the value starting at offset
def mod_raw(offset, replacement):
return raw[0:offset] + replacement + \
raw[offset+len(replacement):]
# test with no arguments
yield nt.assert_equal, raw, BloscpackHeader().encode()
for offset, replacement, kwargs in [
(4, struct.pack('<B', 23), {
'format_version': 23
}),
# test with options
(5, b'\x01', {
'offsets': True
}),
(5, b'\x02', {
'metadata': True
}),
(5, b'\x03', {
'offsets': True,
'metadata': True
}),
# test with checksum
(6, b'\x01', {
'checksum': 'adler32'
}),
(6, b'\x08', {
'checksum': 'sha512'
}),
# test with typesize
(7, b'\x01', {
'typesize': 1
}),
(7, b'\x02', {
'typesize': 2
}),
(7, b'\x04', {
'typesize': 4
}),
(7, b'\x10', {
'typesize': 16
}),
(7, b'\xff', {
'typesize': 255
}),
# test with chunksize
(8, b'\xff\xff\xff\xff', {
'chunk_size': -1
}),
(8, b'\x01\x00\x00\x00', {
'chunk_size': 1
}),
(8, b'\x00\x00\x10\x00', {
'chunk_size': reverse_pretty('1M')
}),
(8, b'\xef\xff\xff\x7f', {
'chunk_size': blosc.BLOSC_MAX_BUFFERSIZE
}),
# test with last_chunk
(12, b'\xff\xff\xff\xff', {
'last_chunk': -1
}),
(12, b'\x01\x00\x00\x00', {
'last_chunk': 1
}),
(12, b'\x00\x00\x10\x00', {
'last_chunk': reverse_pretty('1M')
}),
(12, b'\xef\xff\xff\x7f', {
'last_chunk': blosc.BLOSC_MAX_BUFFERSIZE
}),
# test nchunks
(16, b'\xff\xff\xff\xff\xff\xff\xff\xff', {
'nchunks': -1
}),
(16, b'\x00\x00\x00\x00\x00\x00\x00\x00', {
'nchunks': 0
}),
(16, b'\x01\x00\x00\x00\x00\x00\x00\x00', {
'nchunks': 1
}),
(16, b'\x7f\x00\x00\x00\x00\x00\x00\x00', {
'nchunks': 127
}),
(16, b'\xff\xff\xff\xff\xff\xff\xff\x7f', {
'nchunks': MAX_CHUNKS
}),
# test max_app_chunks
(16,
b'\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00', {
'nchunks': 1,
'max_app_chunks': 0
}),
(16,
b'\x01\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00', {
'nchunks': 1,
'max_app_chunks': 1
}),
(16,
b'\x01\x00\x00\x00\x00\x00\x00\x00\x7f\x00\x00\x00\x00\x00\x00\x00', {
'nchunks': 1,
'max_app_chunks': 127
}),
# Maximum value is MAX_CHUNKS - 1 since nchunks is already 1
(16,
b'\x01\x00\x00\x00\x00\x00\x00\x00\xfe\xff\xff\xff\xff\xff\xff\x7f', {
'nchunks': 1,
'max_app_chunks': MAX_CHUNKS - 1
}),
]:
yield nt.assert_equal, mod_raw(offset, replacement), \
BloscpackHeader(**kwargs).encode()
def pack_unpack_hard():
""" Test on somewhat larger arrays, but be nice to memory. """
# Array is apprx. 1.5 GB large
# should make apprx 1536 chunks
pack_unpack(100, chunk_size=reverse_pretty('1M'), progress=simple_progress)
def test_pack_unpack_fp():
pack_unpack_fp(1, chunk_size=reverse_pretty('1M'))
pack_unpack_fp(1, chunk_size=reverse_pretty('2M'))
pack_unpack_fp(1, chunk_size=reverse_pretty('4M'))
pack_unpack_fp(1, chunk_size=reverse_pretty('8M'))
def test_BloscpackHeader_decode():
format_version = struct.pack('<B', FORMAT_VERSION)
raw = MAGIC + format_version + \
b'\x00\x00\x00\xff\xff\xff\xff\xff\xff\xff\xff' + \
b'\xff\xff\xff\xff\xff\xff\xff\xff\x00\x00\x00\x00\x00\x00\x00\x00'
def mod_raw(offset, replacement):
return raw[0:offset] + replacement + \
raw[offset+len(replacement):]
# check with no args
yield nt.assert_equal, BloscpackHeader(), BloscpackHeader.decode(raw)
for kwargs, offset, replacement in [
# check with format_version
({'format_version': 23}, 4, b'\x17'),
# check with options
({'offsets': True}, 5, b'\x01'),
({'metadata': True}, 5, b'\x02'),
({'metadata': True, 'offsets': True}, 5, b'\x03'),
# check with checksum
({'checksum': 'adler32'}, 6, b'\x01'),
({'checksum': 'sha384'}, 6, b'\x07'),
# check with typesize
({'typesize': 1}, 7, b'\x01'),
({'typesize': 2}, 7, b'\x02'),
({'typesize': 4}, 7, b'\x04'),
({'typesize': 8}, 7, b'\x08'),
({'typesize': blosc.BLOSC_MAX_TYPESIZE}, 7, b'\xff'),
# check with chunk_size
({'chunk_size': 1},
8, b'\x01\x00\x00\x00'),
({'chunk_size': reverse_pretty('1M')},
8, b'\x00\x00\x10\x00'),
({'chunk_size': blosc.BLOSC_MAX_BUFFERSIZE},
8, b'\xef\xff\xff\x7f'),
# check with last_chunk
({'last_chunk': 1},
12, b'\x01\x00\x00\x00'),
({'last_chunk': reverse_pretty('1M')},
12, b'\x00\x00\x10\x00'),
({'last_chunk': blosc.BLOSC_MAX_BUFFERSIZE},
12, b'\xef\xff\xff\x7f'),
# check with nchunks
({'nchunks': 1},
16, b'\x01\x00\x00\x00\x00\x00\x00\x00'),
({'nchunks': reverse_pretty('1M')},
16, b'\x00\x00\x10\x00\x00\x00\x00\x00'),
({'nchunks': MAX_CHUNKS},
16, b'\xff\xff\xff\xff\xff\xff\xff\x7f'),
# check with max_app_chunks
({'nchunks': 1, 'max_app_chunks': 0},
16, b'\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'),
({'nchunks': 1, 'max_app_chunks': 1},
16, b'\x01\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00'),
({'nchunks': 1, 'max_app_chunks': reverse_pretty('1M')},
16, b'\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x00\x00\x00\x00\x00'),
# Maximum value is MAX_CHUNKS - 1 since nchunks is already 1
({'nchunks': 1, 'max_app_chunks': MAX_CHUNKS-1},
16, b'\x01\x00\x00\x00\x00\x00\x00\x00\xfe\xff\xff\xff\xff\xff\xff\x7f'),
]:
yield (nt.assert_equal,
BloscpackHeader(**kwargs),
BloscpackHeader.decode(mod_raw(offset, replacement)))
def test_pack_unpack_fp():
pack_unpack_fp(1, chunk_size=reverse_pretty("1M"))
pack_unpack_fp(1, chunk_size=reverse_pretty("2M"))
pack_unpack_fp(1, chunk_size=reverse_pretty("4M"))
pack_unpack_fp(1, chunk_size=reverse_pretty("8M"))
