def pack_unpack_fp(repeats, chunk_size=DEFAULT_CHUNK_SIZE, progress=False, metadata=None): in_fp, out_fp, dcmp_fp = StringIO(), StringIO(), StringIO() if progress: print("Creating test array") create_array_fp(repeats, in_fp, progress=progress) in_fp_size = in_fp.tell() if progress: print("Compressing") in_fp.seek(0) nchunks, chunk_size, last_chunk_size = \ calculate_nchunks(in_fp_size, chunk_size) source = PlainFPSource(in_fp) sink = CompressedFPSink(out_fp) pack(source, sink, nchunks, chunk_size, last_chunk_size, metadata=metadata) out_fp.seek(0) if progress: print("Decompressing") source = CompressedFPSource(out_fp) sink = PlainFPSink(dcmp_fp) unpack(source, sink) if progress: print("Verifying") cmp_fp(in_fp, dcmp_fp) return source.metadata
def pack_unpack_mem(repeats, chunk_size=DEFAULT_CHUNK_SIZE, progress=False, metadata=None): in_fp, out_fp, dcmp_fp = StringIO(), StringIO(), StringIO() if progress: print("Creating test array") create_array_fp(repeats, in_fp, progress=progress) in_fp_size = in_fp.tell() if progress: print("Compressing") in_fp.seek(0) nchunks, chunk_size, last_chunk_size = \ calculate_nchunks(in_fp_size, chunk_size) # let us play merry go round source = PlainFPSource(in_fp) sink = CompressedMemorySink() pack(source, sink, nchunks, chunk_size, last_chunk_size, metadata=metadata) source = CompressedMemorySource(sink) sink = PlainMemorySink() unpack(source, sink) nt.assert_equal(metadata, source.metadata) source = PlainMemorySource(sink.chunks) sink = CompressedFPSink(out_fp) pack(source, sink, nchunks, chunk_size, last_chunk_size, metadata=metadata) out_fp.seek(0) source = CompressedFPSource(out_fp) sink = PlainFPSink(dcmp_fp) unpack(source, sink) nt.assert_equal(metadata, source.metadata) in_fp.seek(0) dcmp_fp.seek(0) cmp_fp(in_fp, dcmp_fp) return source.metadata