def test_BloscpackHeader_encode(): # the raw encoded header as produces w/o any kwargs format_version = struct.pack('<B', FORMAT_VERSION) raw = MAGIC + format_version + \ b'\x00\x00\x00\xff\xff\xff\xff\xff\xff\xff\xff' + \ b'\xff\xff\xff\xff\xff\xff\xff\xff\x00\x00\x00\x00\x00\x00\x00\x00' # modify the raw encoded header with the value starting at offset def mod_raw(offset, replacement): return raw[0:offset] + replacement + \ raw[offset+len(replacement):] # test with no arguments yield nt.assert_equal, raw, BloscpackHeader().encode() for offset, replacement, kwargs in [ (4, struct.pack('<B', 23), {'format_version': 23}), # test with options (5, b'\x01', {'offsets': True}), (5, b'\x02', {'metadata': True}), (5, b'\x03', {'offsets': True, 'metadata': True}), # test with checksum (6, b'\x01', {'checksum': 'adler32'}), (6, b'\x08', {'checksum': 'sha512'}), # test with typesize (7, b'\x01', {'typesize': 1}), (7, b'\x02', {'typesize': 2}), (7, b'\x04', {'typesize': 4}), (7, b'\x10', {'typesize': 16}), (7, b'\xff', {'typesize': 255}), # test with chunksize (8, b'\xff\xff\xff\xff', {'chunk_size': -1}), (8, b'\x01\x00\x00\x00', {'chunk_size': 1}), (8, b'\x00\x00\x10\x00', {'chunk_size': reverse_pretty('1M')}), (8, b'\xef\xff\xff\x7f', {'chunk_size': blosc.BLOSC_MAX_BUFFERSIZE}), # test with last_chunk (12, b'\xff\xff\xff\xff', {'last_chunk': -1}), (12, b'\x01\x00\x00\x00', {'last_chunk': 1}), (12, b'\x00\x00\x10\x00', {'last_chunk': reverse_pretty('1M')}), (12, b'\xef\xff\xff\x7f', {'last_chunk': blosc.BLOSC_MAX_BUFFERSIZE}), # test nchunks (16, b'\xff\xff\xff\xff\xff\xff\xff\xff', {'nchunks': -1}), (16, b'\x00\x00\x00\x00\x00\x00\x00\x00', {'nchunks': 0}), (16, b'\x01\x00\x00\x00\x00\x00\x00\x00', {'nchunks': 1}), (16, b'\x7f\x00\x00\x00\x00\x00\x00\x00', {'nchunks': 127}), (16, b'\xff\xff\xff\xff\xff\xff\xff\x7f', {'nchunks': MAX_CHUNKS}), # test max_app_chunks (16, b'\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00', {'nchunks': 1, 'max_app_chunks': 0}), (16, b'\x01\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00', {'nchunks': 1, 'max_app_chunks': 1}), (16, b'\x01\x00\x00\x00\x00\x00\x00\x00\x7f\x00\x00\x00\x00\x00\x00\x00', {'nchunks': 1, 'max_app_chunks': 127}), # Maximum value is MAX_CHUNKS - 1 since nchunks is already 1 (16, b'\x01\x00\x00\x00\x00\x00\x00\x00\xfe\xff\xff\xff\xff\xff\xff\x7f', {'nchunks': 1, 'max_app_chunks': MAX_CHUNKS-1}), ]: yield nt.assert_equal, mod_raw(offset, replacement), \ BloscpackHeader(**kwargs).encode()
def test_calculate_nchunks(): # check for zero or negative chunk_size nt.assert_raises(ValueError, calculate_nchunks, 23, chunk_size=0) nt.assert_raises(ValueError, calculate_nchunks, 23, chunk_size=-1) nt.assert_equal((9, 1, 1), calculate_nchunks(9, chunk_size=1)) nt.assert_equal((5, 2, 1), calculate_nchunks(9, chunk_size=2)) nt.assert_equal((3, 3, 3), calculate_nchunks(9, chunk_size=3)) nt.assert_equal((3, 4, 1), calculate_nchunks(9, chunk_size=4)) nt.assert_equal((2, 5, 4), calculate_nchunks(9, chunk_size=5)) nt.assert_equal((2, 6, 3), calculate_nchunks(9, chunk_size=6)) nt.assert_equal((2, 7, 2), calculate_nchunks(9, chunk_size=7)) nt.assert_equal((2, 8, 1), calculate_nchunks(9, chunk_size=8)) nt.assert_equal((1, 9, 9), calculate_nchunks(9, chunk_size=9)) # check downgrade nt.assert_equal((1, 23, 23), calculate_nchunks(23, chunk_size=24)) # single byte file nt.assert_equal((1, 1, 1), calculate_nchunks(1, chunk_size=1)) # check that a zero length input is handled correctly nt.assert_equal((1, 0, 0), calculate_nchunks(0, chunk_size=1)) # check that the chunk_size is ignored in this case nt.assert_equal((1, 0, 0), calculate_nchunks(0, chunk_size=512)) # in_file_size must be strictly positive nt.assert_raises(ValueError, calculate_nchunks, -1) # check overflow of nchunks due to chunk_size being too small # and thus stuff not fitting into the header nt.assert_raises(ChunkingException, calculate_nchunks, MAX_CHUNKS+1, chunk_size=1) # check that strings are converted correctly nt.assert_equal((6, 1048576, 209715), calculate_nchunks(reverse_pretty('5.2M'))) nt.assert_equal((3, 2097152, 1258291), calculate_nchunks(reverse_pretty('5.2M'), chunk_size='2M'))
def test_pack_unpack_mem(): pack_unpack_mem(1, chunk_size=reverse_pretty('1M')) pack_unpack_mem(1, chunk_size=reverse_pretty('2M')) pack_unpack_mem(1, chunk_size=reverse_pretty('4M')) pack_unpack_mem(1, chunk_size=reverse_pretty('8M')) metadata = {"dtype": "float64", "shape": [1024], "others": []} pack_unpack_mem(1, chunk_size=reverse_pretty('1M'), metadata=metadata) pack_unpack_mem(1, chunk_size=reverse_pretty('2M'), metadata=metadata) pack_unpack_mem(1, chunk_size=reverse_pretty('4M'), metadata=metadata) pack_unpack_mem(1, chunk_size=reverse_pretty('8M'), metadata=metadata)
def test_pretty_filesieze(): nt.assert_equal('0B', pretty_size(0)) nt.assert_equal('9.0T', pretty_size(9898989898879)) nt.assert_equal('4.78G', pretty_size(5129898234)) nt.assert_equal('12.3M', pretty_size(12898234)) nt.assert_equal('966.7K', pretty_size(989898)) nt.assert_equal('128.0B', pretty_size(128)) nt.assert_equal(0, reverse_pretty('0B')) nt.assert_equal(8, reverse_pretty('8B')) nt.assert_equal(8192, reverse_pretty('8K')) nt.assert_equal(134217728, reverse_pretty('128M')) nt.assert_equal(2147483648, reverse_pretty('2G')) nt.assert_equal(2199023255552, reverse_pretty('2T')) # can't handle Petabytes, yet nt.assert_raises(ValueError, reverse_pretty, '2P')
def pack_unpack_hard(): """ Test on somewhat larger arrays, but be nice to memory. """ # Array is apprx. 1.5 GB large # should make apprx 1536 chunks pack_unpack(100, chunk_size=reverse_pretty('1M'), progress=simple_progress)
def test_pack_unpack_fp(): pack_unpack_fp(1, chunk_size=reverse_pretty('1M')) pack_unpack_fp(1, chunk_size=reverse_pretty('2M')) pack_unpack_fp(1, chunk_size=reverse_pretty('4M')) pack_unpack_fp(1, chunk_size=reverse_pretty('8M'))
def test_BloscpackHeader_decode(): format_version = struct.pack('<B', FORMAT_VERSION) raw = MAGIC + format_version + \ b'\x00\x00\x00\xff\xff\xff\xff\xff\xff\xff\xff' + \ b'\xff\xff\xff\xff\xff\xff\xff\xff\x00\x00\x00\x00\x00\x00\x00\x00' def mod_raw(offset, replacement): return raw[0:offset] + replacement + \ raw[offset+len(replacement):] # check with no args yield nt.assert_equal, BloscpackHeader(), BloscpackHeader.decode(raw) for kwargs, offset, replacement in [ # check with format_version ({ 'format_version': 23 }, 4, b'\x17'), # check with options ({ 'offsets': True }, 5, b'\x01'), ({ 'metadata': True }, 5, b'\x02'), ({ 'metadata': True, 'offsets': True }, 5, b'\x03'), # check with checksum ({ 'checksum': 'adler32' }, 6, b'\x01'), ({ 'checksum': 'sha384' }, 6, b'\x07'), # check with typesize ({ 'typesize': 1 }, 7, b'\x01'), ({ 'typesize': 2 }, 7, b'\x02'), ({ 'typesize': 4 }, 7, b'\x04'), ({ 'typesize': 8 }, 7, b'\x08'), ({ 'typesize': blosc.BLOSC_MAX_TYPESIZE }, 7, b'\xff'), # check with chunk_size ({ 'chunk_size': 1 }, 8, b'\x01\x00\x00\x00'), ({ 'chunk_size': reverse_pretty('1M') }, 8, b'\x00\x00\x10\x00'), ({ 'chunk_size': blosc.BLOSC_MAX_BUFFERSIZE }, 8, b'\xef\xff\xff\x7f'), # check with last_chunk ({ 'last_chunk': 1 }, 12, b'\x01\x00\x00\x00'), ({ 'last_chunk': reverse_pretty('1M') }, 12, b'\x00\x00\x10\x00'), ({ 'last_chunk': blosc.BLOSC_MAX_BUFFERSIZE }, 12, b'\xef\xff\xff\x7f'), # check with nchunks ({ 'nchunks': 1 }, 16, b'\x01\x00\x00\x00\x00\x00\x00\x00'), ({ 'nchunks': reverse_pretty('1M') }, 16, b'\x00\x00\x10\x00\x00\x00\x00\x00'), ({ 'nchunks': MAX_CHUNKS }, 16, b'\xff\xff\xff\xff\xff\xff\xff\x7f'), # check with max_app_chunks ({ 'nchunks': 1, 'max_app_chunks': 0 }, 16, b'\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'), ({ 'nchunks': 1, 'max_app_chunks': 1 }, 16, b'\x01\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00'), ({ 'nchunks': 1, 'max_app_chunks': reverse_pretty('1M') }, 16, b'\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x00\x00\x00\x00\x00'), # Maximum value is MAX_CHUNKS - 1 since nchunks is already 1 ({ 'nchunks': 1, 'max_app_chunks': MAX_CHUNKS - 1 }, 16, b'\x01\x00\x00\x00\x00\x00\x00\x00\xfe\xff\xff\xff\xff\xff\xff\x7f'), ]: yield (nt.assert_equal, BloscpackHeader(**kwargs), BloscpackHeader.decode(mod_raw(offset, replacement)))
def test_BloscpackHeader_encode(): # the raw encoded header as produces w/o any kwargs format_version = struct.pack('<B', FORMAT_VERSION) raw = MAGIC + format_version + \ b'\x00\x00\x00\xff\xff\xff\xff\xff\xff\xff\xff' + \ b'\xff\xff\xff\xff\xff\xff\xff\xff\x00\x00\x00\x00\x00\x00\x00\x00' # modify the raw encoded header with the value starting at offset def mod_raw(offset, replacement): return raw[0:offset] + replacement + \ raw[offset+len(replacement):] # test with no arguments yield nt.assert_equal, raw, BloscpackHeader().encode() for offset, replacement, kwargs in [ (4, struct.pack('<B', 23), { 'format_version': 23 }), # test with options (5, b'\x01', { 'offsets': True }), (5, b'\x02', { 'metadata': True }), (5, b'\x03', { 'offsets': True, 'metadata': True }), # test with checksum (6, b'\x01', { 'checksum': 'adler32' }), (6, b'\x08', { 'checksum': 'sha512' }), # test with typesize (7, b'\x01', { 'typesize': 1 }), (7, b'\x02', { 'typesize': 2 }), (7, b'\x04', { 'typesize': 4 }), (7, b'\x10', { 'typesize': 16 }), (7, b'\xff', { 'typesize': 255 }), # test with chunksize (8, b'\xff\xff\xff\xff', { 'chunk_size': -1 }), (8, b'\x01\x00\x00\x00', { 'chunk_size': 1 }), (8, b'\x00\x00\x10\x00', { 'chunk_size': reverse_pretty('1M') }), (8, b'\xef\xff\xff\x7f', { 'chunk_size': blosc.BLOSC_MAX_BUFFERSIZE }), # test with last_chunk (12, b'\xff\xff\xff\xff', { 'last_chunk': -1 }), (12, b'\x01\x00\x00\x00', { 'last_chunk': 1 }), (12, b'\x00\x00\x10\x00', { 'last_chunk': reverse_pretty('1M') }), (12, b'\xef\xff\xff\x7f', { 'last_chunk': blosc.BLOSC_MAX_BUFFERSIZE }), # test nchunks (16, b'\xff\xff\xff\xff\xff\xff\xff\xff', { 'nchunks': -1 }), (16, b'\x00\x00\x00\x00\x00\x00\x00\x00', { 'nchunks': 0 }), (16, b'\x01\x00\x00\x00\x00\x00\x00\x00', { 'nchunks': 1 }), (16, b'\x7f\x00\x00\x00\x00\x00\x00\x00', { 'nchunks': 127 }), (16, b'\xff\xff\xff\xff\xff\xff\xff\x7f', { 'nchunks': MAX_CHUNKS }), # test max_app_chunks (16, b'\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00', { 'nchunks': 1, 'max_app_chunks': 0 }), (16, b'\x01\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00', { 'nchunks': 1, 'max_app_chunks': 1 }), (16, b'\x01\x00\x00\x00\x00\x00\x00\x00\x7f\x00\x00\x00\x00\x00\x00\x00', { 'nchunks': 1, 'max_app_chunks': 127 }), # Maximum value is MAX_CHUNKS - 1 since nchunks is already 1 (16, b'\x01\x00\x00\x00\x00\x00\x00\x00\xfe\xff\xff\xff\xff\xff\xff\x7f', { 'nchunks': 1, 'max_app_chunks': MAX_CHUNKS - 1 }), ]: yield nt.assert_equal, mod_raw(offset, replacement), \ BloscpackHeader(**kwargs).encode()
def test_BloscpackHeader_decode(): format_version = struct.pack('<B', FORMAT_VERSION) raw = MAGIC + format_version + \ b'\x00\x00\x00\xff\xff\xff\xff\xff\xff\xff\xff' + \ b'\xff\xff\xff\xff\xff\xff\xff\xff\x00\x00\x00\x00\x00\x00\x00\x00' def mod_raw(offset, replacement): return raw[0:offset] + replacement + \ raw[offset+len(replacement):] # check with no args yield nt.assert_equal, BloscpackHeader(), BloscpackHeader.decode(raw) for kwargs, offset, replacement in [ # check with format_version ({'format_version': 23}, 4, b'\x17'), # check with options ({'offsets': True}, 5, b'\x01'), ({'metadata': True}, 5, b'\x02'), ({'metadata': True, 'offsets': True}, 5, b'\x03'), # check with checksum ({'checksum': 'adler32'}, 6, b'\x01'), ({'checksum': 'sha384'}, 6, b'\x07'), # check with typesize ({'typesize': 1}, 7, b'\x01'), ({'typesize': 2}, 7, b'\x02'), ({'typesize': 4}, 7, b'\x04'), ({'typesize': 8}, 7, b'\x08'), ({'typesize': blosc.BLOSC_MAX_TYPESIZE}, 7, b'\xff'), # check with chunk_size ({'chunk_size': 1}, 8, b'\x01\x00\x00\x00'), ({'chunk_size': reverse_pretty('1M')}, 8, b'\x00\x00\x10\x00'), ({'chunk_size': blosc.BLOSC_MAX_BUFFERSIZE}, 8, b'\xef\xff\xff\x7f'), # check with last_chunk ({'last_chunk': 1}, 12, b'\x01\x00\x00\x00'), ({'last_chunk': reverse_pretty('1M')}, 12, b'\x00\x00\x10\x00'), ({'last_chunk': blosc.BLOSC_MAX_BUFFERSIZE}, 12, b'\xef\xff\xff\x7f'), # check with nchunks ({'nchunks': 1}, 16, b'\x01\x00\x00\x00\x00\x00\x00\x00'), ({'nchunks': reverse_pretty('1M')}, 16, b'\x00\x00\x10\x00\x00\x00\x00\x00'), ({'nchunks': MAX_CHUNKS}, 16, b'\xff\xff\xff\xff\xff\xff\xff\x7f'), # check with max_app_chunks ({'nchunks': 1, 'max_app_chunks': 0}, 16, b'\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'), ({'nchunks': 1, 'max_app_chunks': 1}, 16, b'\x01\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00'), ({'nchunks': 1, 'max_app_chunks': reverse_pretty('1M')}, 16, b'\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x00\x00\x00\x00\x00'), # Maximum value is MAX_CHUNKS - 1 since nchunks is already 1 ({'nchunks': 1, 'max_app_chunks': MAX_CHUNKS-1}, 16, b'\x01\x00\x00\x00\x00\x00\x00\x00\xfe\xff\xff\xff\xff\xff\xff\x7f'), ]: yield (nt.assert_equal, BloscpackHeader(**kwargs), BloscpackHeader.decode(mod_raw(offset, replacement)))
def test_pack_unpack_fp(): pack_unpack_fp(1, chunk_size=reverse_pretty("1M")) pack_unpack_fp(1, chunk_size=reverse_pretty("2M")) pack_unpack_fp(1, chunk_size=reverse_pretty("4M")) pack_unpack_fp(1, chunk_size=reverse_pretty("8M"))