def pack_unpack_fp(repeats, chunk_size=DEFAULT_CHUNK_SIZE,
progress=False, metadata=None):
in_fp, out_fp, dcmp_fp = StringIO(), StringIO(), StringIO()
if progress:
print("Creating test array")
create_array_fp(repeats, in_fp, progress=progress)
in_fp_size = in_fp.tell()
if progress:
print("Compressing")
in_fp.seek(0)
nchunks, chunk_size, last_chunk_size = \
calculate_nchunks(in_fp_size, chunk_size)
source = PlainFPSource(in_fp)
sink = CompressedFPSink(out_fp)
pack(source, sink,
nchunks, chunk_size, last_chunk_size,
metadata=metadata)
out_fp.seek(0)
if progress:
print("Decompressing")
source = CompressedFPSource(out_fp)
sink = PlainFPSink(dcmp_fp)
unpack(source, sink)
if progress:
print("Verifying")
cmp_fp(in_fp, dcmp_fp)
return source.metadata
def pack_unpack_mem(repeats, chunk_size=DEFAULT_CHUNK_SIZE,
progress=False, metadata=None):
in_fp, out_fp, dcmp_fp = StringIO(), StringIO(), StringIO()
if progress:
print("Creating test array")
create_array_fp(repeats, in_fp, progress=progress)
in_fp_size = in_fp.tell()
if progress:
print("Compressing")
in_fp.seek(0)
nchunks, chunk_size, last_chunk_size = \
calculate_nchunks(in_fp_size, chunk_size)
# let us play merry go round
source = PlainFPSource(in_fp)
sink = CompressedMemorySink()
pack(source, sink, nchunks, chunk_size, last_chunk_size, metadata=metadata)
source = CompressedMemorySource(sink)
sink = PlainMemorySink()
unpack(source, sink)
nt.assert_equal(metadata, source.metadata)
source = PlainMemorySource(sink.chunks)
sink = CompressedFPSink(out_fp)
pack(source, sink, nchunks, chunk_size, last_chunk_size, metadata=metadata)
out_fp.seek(0)
source = CompressedFPSource(out_fp)
sink = PlainFPSink(dcmp_fp)
unpack(source, sink)
nt.assert_equal(metadata, source.metadata)
in_fp.seek(0)
dcmp_fp.seek(0)
cmp_fp(in_fp, dcmp_fp)
return source.metadata
def pack_unpack_fp(repeats,
chunk_size=DEFAULT_CHUNK_SIZE,
progress=False,
metadata=None):
in_fp, out_fp, dcmp_fp = StringIO(), StringIO(), StringIO()
if progress:
print("Creating test array")
create_array_fp(repeats, in_fp, progress=progress)
in_fp_size = in_fp.tell()
if progress:
print("Compressing")
in_fp.seek(0)
nchunks, chunk_size, last_chunk_size = \
calculate_nchunks(in_fp_size, chunk_size)
source = PlainFPSource(in_fp)
sink = CompressedFPSink(out_fp)
pack(source, sink, nchunks, chunk_size, last_chunk_size, metadata=metadata)
out_fp.seek(0)
if progress:
print("Decompressing")
source = CompressedFPSource(out_fp)
sink = PlainFPSink(dcmp_fp)
unpack(source, sink)
if progress:
print("Verifying")
cmp_fp(in_fp, dcmp_fp)
return source.metadata
def pack_unpack_mem(repeats,
chunk_size=DEFAULT_CHUNK_SIZE,
progress=False,
metadata=None):
in_fp, out_fp, dcmp_fp = StringIO(), StringIO(), StringIO()
if progress:
print("Creating test array")
create_array_fp(repeats, in_fp, progress=progress)
in_fp_size = in_fp.tell()
if progress:
print("Compressing")
in_fp.seek(0)
nchunks, chunk_size, last_chunk_size = \
calculate_nchunks(in_fp_size, chunk_size)
# let us play merry go round
source = PlainFPSource(in_fp)
sink = CompressedMemorySink()
pack(source, sink, nchunks, chunk_size, last_chunk_size, metadata=metadata)
source = CompressedMemorySource(sink)
sink = PlainMemorySink()
unpack(source, sink)
nt.assert_equal(metadata, source.metadata)
source = PlainMemorySource(sink.chunks)
sink = CompressedFPSink(out_fp)
pack(source, sink, nchunks, chunk_size, last_chunk_size, metadata=metadata)
out_fp.seek(0)
source = CompressedFPSource(out_fp)
sink = PlainFPSink(dcmp_fp)
unpack(source, sink)
nt.assert_equal(metadata, source.metadata)
in_fp.seek(0)
dcmp_fp.seek(0)
cmp_fp(in_fp, dcmp_fp)
return source.metadata
