def pack_unpack_mem(repeats, chunk_size=DEFAULT_CHUNK_SIZE, progress=False, metadata=None): in_fp, out_fp, dcmp_fp = StringIO(), StringIO(), StringIO() if progress: print("Creating test array") create_array_fp(repeats, in_fp, progress=progress) in_fp_size = in_fp.tell() if progress: print("Compressing") in_fp.seek(0) nchunks, chunk_size, last_chunk_size = \ calculate_nchunks(in_fp_size, chunk_size) # let us play merry go round source = PlainFPSource(in_fp) sink = CompressedMemorySink() pack(source, sink, nchunks, chunk_size, last_chunk_size, metadata=metadata) source = CompressedMemorySource(sink) sink = PlainMemorySink() unpack(source, sink) nt.assert_equal(metadata, source.metadata) source = PlainMemorySource(sink.chunks) sink = CompressedFPSink(out_fp) pack(source, sink, nchunks, chunk_size, last_chunk_size, metadata=metadata) out_fp.seek(0) source = CompressedFPSource(out_fp) sink = PlainFPSink(dcmp_fp) unpack(source, sink) nt.assert_equal(metadata, source.metadata) in_fp.seek(0) dcmp_fp.seek(0) cmp_fp(in_fp, dcmp_fp) return source.metadata
def pack_unpack_fp(repeats, chunk_size=DEFAULT_CHUNK_SIZE, progress=False, metadata=None): in_fp, out_fp, dcmp_fp = StringIO(), StringIO(), StringIO() if progress: print("Creating test array") create_array_fp(repeats, in_fp, progress=progress) in_fp_size = in_fp.tell() if progress: print("Compressing") in_fp.seek(0) nchunks, chunk_size, last_chunk_size = \ calculate_nchunks(in_fp_size, chunk_size) source = PlainFPSource(in_fp) sink = CompressedFPSink(out_fp) pack(source, sink, nchunks, chunk_size, last_chunk_size, metadata=metadata) out_fp.seek(0) if progress: print("Decompressing") source = CompressedFPSource(out_fp) sink = PlainFPSink(dcmp_fp) unpack(source, sink) if progress: print("Verifying") cmp_fp(in_fp, dcmp_fp) return source.metadata
def test_disable_offsets(): in_fp, out_fp, dcmp_fp = StringIO(), StringIO(), StringIO() create_array_fp(1, in_fp) in_fp_size = in_fp.tell() in_fp.seek(0) bloscpack_args = BloscpackArgs(offsets=False) source = PlainFPSource(in_fp) sink = CompressedFPSink(out_fp) pack(source, sink, *calculate_nchunks(in_fp_size), bloscpack_args=bloscpack_args) out_fp.seek(0) bloscpack_header, metadata, metadata_header, offsets = _read_beginning(out_fp) nt.assert_true(len(offsets) == 0)
def test_append_mix_shuffle(): orig, new, new_size, dcmp = prep_array_for_append() # use the typesize from the file # deactivate shuffle # crank up the clevel to ensure compression happens, otherwise the flags # will be screwed later on blosc_args = BloscArgs(typesize=None, shuffle=False, clevel=9) # need to create something that will be compressible even without shuffle, # the linspace used in 'new' doesn't work anymore as of python-blosc 1.6.1 to_append = np.zeros(int(2e6)) to_append_fp = StringIO() to_append_fp.write(to_append.tostring()) to_append_fp_size = to_append_fp.tell() to_append_fp.seek(0) # now do the append reset_append_fp(orig, to_append_fp, to_append_fp_size, blosc_args=blosc_args) # decompress 'orig' so that we can examine it source = CompressedFPSource(orig) sink = PlainFPSink(dcmp) unpack(source, sink) orig.seek(0) dcmp.seek(0) new.seek(0) new_str = new.read() dcmp_str = dcmp.read() # now sanity check the length and content of the decompressed nt.assert_equal(len(dcmp_str), len(new_str) + to_append_fp_size) nt.assert_equal(dcmp_str, new_str + to_append.tostring()) # now get the first and the last chunk and check that the shuffle doesn't # match bloscpack_header, offsets = reset_read_beginning(orig)[0:4:3] orig.seek(offsets[0]) checksum_impl = CHECKSUMS_LOOKUP[bloscpack_header['checksum']] compressed_zero, blosc_header_zero, digest = \ _read_compressed_chunk_fp(orig, checksum_impl) decompressed_zero = blosc.decompress(compressed_zero) orig.seek(offsets[-1]) compressed_last, blosc_header_last, digest = \ _read_compressed_chunk_fp(orig, checksum_impl) decompressed_last = blosc.decompress(compressed_last) # first chunk has shuffle active nt.assert_equal(blosc_header_zero['flags'], 1) # last chunk doesn't nt.assert_equal(blosc_header_last['flags'], 0)
def test_offsets(): with create_tmp_files() as (tdir, in_file, out_file, dcmp_file): create_array(1, in_file) pack_file(in_file, out_file, chunk_size='2M') with open(out_file, 'r+b') as input_fp: bloscpack_header = _read_bloscpack_header(input_fp) total_entries = bloscpack_header.total_prospective_chunks offsets = _read_offsets(input_fp, bloscpack_header) # First chunks should start after header and offsets first = BLOSCPACK_HEADER_LENGTH + 8 * total_entries # We assume that the others are correct nt.assert_equal(offsets[0], first) nt.assert_equal([ 736, 368207, 633319, 902306, 1173771, 1419535, 1666981, 1913995 ], offsets) # try to read the second header input_fp.seek(offsets[1], 0) blosc_header_raw = input_fp.read(BLOSC_HEADER_LENGTH) expected = { 'versionlz': 1, 'blocksize': 262144, 'ctbytes': 265108, 'version': 2, 'flags': 1, 'nbytes': 2097152, 'typesize': 8 } blosc_header = decode_blosc_header(blosc_header_raw) nt.assert_equal(expected, blosc_header) # now check the same thing again, but w/o any max_app_chunks input_fp, output_fp = StringIO(), StringIO() create_array_fp(1, input_fp) nchunks, chunk_size, last_chunk_size = \ calculate_nchunks(input_fp.tell(), chunk_size='2M') input_fp.seek(0, 0) bloscpack_args = BloscpackArgs(max_app_chunks=0) source = PlainFPSource(input_fp) sink = CompressedFPSink(output_fp) pack(source, sink, nchunks, chunk_size, last_chunk_size, bloscpack_args=bloscpack_args) output_fp.seek(0, 0) bloscpack_header = _read_bloscpack_header(output_fp) nt.assert_equal(0, bloscpack_header.max_app_chunks) offsets = _read_offsets(output_fp, bloscpack_header) nt.assert_equal( [96, 367567, 632679, 901666, 1173131, 1418895, 1666341, 1913355], offsets)
def test_offsets(): with create_tmp_files() as (tdir, in_file, out_file, dcmp_file): create_array(1, in_file) pack_file_to_file(in_file, out_file, chunk_size='2M') with open(out_file, 'r+b') as input_fp: bloscpack_header = _read_bloscpack_header(input_fp) total_entries = bloscpack_header.total_prospective_chunks offsets = _read_offsets(input_fp, bloscpack_header) # First chunks should start after header and offsets first = BLOSCPACK_HEADER_LENGTH + 8 * total_entries # We assume that the others are correct nt.assert_equal(offsets[0], first) nt.assert_equal(736, offsets[0]) # try to read the second header input_fp.seek(offsets[1], 0) blosc_header_raw = input_fp.read(BLOSC_HEADER_LENGTH) expected = { 'versionlz': 1, 'version': 2, 'flags': 1, 'nbytes': 2097152, 'typesize': 8 } blosc_header = decode_blosc_header(blosc_header_raw) blosc_header_slice = dict( (k, blosc_header[k]) for k in expected.keys()) nt.assert_equal(expected, blosc_header_slice) # now check the same thing again, but w/o any max_app_chunks input_fp, output_fp = StringIO(), StringIO() create_array_fp(1, input_fp) nchunks, chunk_size, last_chunk_size = \ calculate_nchunks(input_fp.tell(), chunk_size='2M') input_fp.seek(0, 0) bloscpack_args = BloscpackArgs(max_app_chunks=0) source = PlainFPSource(input_fp) sink = CompressedFPSink(output_fp) pack(source, sink, nchunks, chunk_size, last_chunk_size, bloscpack_args=bloscpack_args) output_fp.seek(0, 0) bloscpack_header = _read_bloscpack_header(output_fp) nt.assert_equal(0, bloscpack_header.max_app_chunks) offsets = _read_offsets(output_fp, bloscpack_header) nt.assert_equal(96, offsets[0])
def test_offsets(): with create_tmp_files() as (tdir, in_file, out_file, dcmp_file): create_array(1, in_file) pack_file(in_file, out_file, chunk_size='2M') with open(out_file, 'r+b') as input_fp: bloscpack_header = _read_bloscpack_header(input_fp) total_entries = bloscpack_header.total_prospective_chunks offsets = _read_offsets(input_fp, bloscpack_header) # First chunks should start after header and offsets first = BLOSCPACK_HEADER_LENGTH + 8 * total_entries # We assume that the others are correct nt.assert_equal(offsets[0], first) nt.assert_equal([736, 368207, 633319, 902306, 1173771, 1419535, 1666981, 1913995], offsets) # try to read the second header input_fp.seek(offsets[1], 0) blosc_header_raw = input_fp.read(BLOSC_HEADER_LENGTH) expected = {'versionlz': 1, 'blocksize': 262144, 'ctbytes': 265108, 'version': 2, 'flags': 1, 'nbytes': 2097152, 'typesize': 8} blosc_header = decode_blosc_header(blosc_header_raw) nt.assert_equal(expected, blosc_header) # now check the same thing again, but w/o any max_app_chunks input_fp, output_fp = StringIO(), StringIO() create_array_fp(1, input_fp) nchunks, chunk_size, last_chunk_size = \ calculate_nchunks(input_fp.tell(), chunk_size='2M') input_fp.seek(0, 0) bloscpack_args = BloscpackArgs(max_app_chunks=0) source = PlainFPSource(input_fp) sink = CompressedFPSink(output_fp) pack(source, sink, nchunks, chunk_size, last_chunk_size, bloscpack_args=bloscpack_args ) output_fp.seek(0, 0) bloscpack_header = _read_bloscpack_header(output_fp) nt.assert_equal(0, bloscpack_header.max_app_chunks) offsets = _read_offsets(output_fp, bloscpack_header) nt.assert_equal([96, 367567, 632679, 901666, 1173131, 1418895, 1666341, 1913355], offsets)
def test_disable_offsets(): in_fp, out_fp, dcmp_fp = StringIO(), StringIO(), StringIO() create_array_fp(1, in_fp) in_fp_size = in_fp.tell() in_fp.seek(0) bloscpack_args = BloscpackArgs(offsets=False) source = PlainFPSource(in_fp) sink = CompressedFPSink(out_fp) pack(source, sink, *calculate_nchunks(in_fp_size), bloscpack_args=bloscpack_args) out_fp.seek(0) bloscpack_header, metadata, metadata_header, offsets = \ _read_beginning(out_fp) nt.assert_true(len(offsets) == 0)
def test_offsets(): with create_tmp_files() as (tdir, in_file, out_file, dcmp_file): create_array(1, in_file) pack_file(in_file, out_file, chunk_size='2M') with open(out_file, 'r+b') as input_fp: bloscpack_header = _read_bloscpack_header(input_fp) total_entries = bloscpack_header.total_prospective_chunks offsets = _read_offsets(input_fp, bloscpack_header) # First chunks should start after header and offsets first = BLOSCPACK_HEADER_LENGTH + 8 * total_entries # We assume that the others are correct nt.assert_equal(offsets[0], first) nt.assert_equal(736, offsets[0]) # try to read the second header input_fp.seek(offsets[1], 0) blosc_header_raw = input_fp.read(BLOSC_HEADER_LENGTH) expected = {'versionlz': 1, 'version': 2, 'flags': 1, 'nbytes': 2097152, 'typesize': 8} blosc_header = decode_blosc_header(blosc_header_raw) blosc_header_slice = dict((k, blosc_header[k]) for k in expected.keys()) nt.assert_equal(expected, blosc_header_slice) # now check the same thing again, but w/o any max_app_chunks input_fp, output_fp = StringIO(), StringIO() create_array_fp(1, input_fp) nchunks, chunk_size, last_chunk_size = \ calculate_nchunks(input_fp.tell(), chunk_size='2M') input_fp.seek(0, 0) bloscpack_args = BloscpackArgs(max_app_chunks=0) source = PlainFPSource(input_fp) sink = CompressedFPSink(output_fp) pack(source, sink, nchunks, chunk_size, last_chunk_size, bloscpack_args=bloscpack_args ) output_fp.seek(0, 0) bloscpack_header = _read_bloscpack_header(output_fp) nt.assert_equal(0, bloscpack_header.max_app_chunks) offsets = _read_offsets(output_fp, bloscpack_header) nt.assert_equal(96, offsets[0])