def test_bounds(self):
zstd.CompressionParameters(zstd.WINDOWLOG_MIN, zstd.CHAINLOG_MIN,
zstd.HASHLOG_MIN, zstd.SEARCHLOG_MIN,
zstd.SEARCHLENGTH_MIN + 1,
zstd.TARGETLENGTH_MIN, zstd.STRATEGY_FAST)
zstd.CompressionParameters(zstd.WINDOWLOG_MAX, zstd.CHAINLOG_MAX,
zstd.HASHLOG_MAX, zstd.SEARCHLOG_MAX,
zstd.SEARCHLENGTH_MAX - 1,
zstd.TARGETLENGTH_MAX, zstd.STRATEGY_BTOPT)
def test_bounds(self):
zstd.CompressionParameters(window_log=zstd.WINDOWLOG_MIN,
chain_log=zstd.CHAINLOG_MIN,
hash_log=zstd.HASHLOG_MIN,
search_log=zstd.SEARCHLOG_MIN,
min_match=zstd.SEARCHLENGTH_MIN + 1,
target_length=zstd.TARGETLENGTH_MIN,
compression_strategy=zstd.STRATEGY_FAST)
zstd.CompressionParameters(window_log=zstd.WINDOWLOG_MAX,
chain_log=zstd.CHAINLOG_MAX,
hash_log=zstd.HASHLOG_MAX,
search_log=zstd.SEARCHLOG_MAX,
min_match=zstd.SEARCHLENGTH_MAX - 1,
compression_strategy=zstd.STRATEGY_BTULTRA)
def test_compression_params(self):
params = zstd.CompressionParameters(
window_log=20,
chain_log=6,
hash_log=12,
min_match=5,
search_log=4,
target_length=10,
compression_strategy=zstd.STRATEGY_FAST)
buffer = io.BytesIO()
cctx = zstd.ZstdCompressor(compression_params=params)
with cctx.write_to(buffer) as compressor:
self.assertEqual(compressor.write(b'foo'), 0)
self.assertEqual(compressor.write(b'bar'), 0)
self.assertEqual(compressor.write(b'foobar' * 16384), 0)
compressed = buffer.getvalue()
params = zstd.get_frame_parameters(compressed)
self.assertEqual(params.content_size, zstd.CONTENTSIZE_UNKNOWN)
self.assertEqual(params.window_size, 1048576)
self.assertEqual(params.dict_id, 0)
self.assertFalse(params.has_checksum)
h = hashlib.sha1(compressed).hexdigest()
self.assertEqual(h, '2a8111d72eb5004cdcecbdac37da9f26720d30ef')
def test_valid_init(self, windowlog, chainlog, hashlog, searchlog,
searchlength, targetlength, strategy):
# ZSTD_checkCParams moves the goal posts on us from what's advertised
# in the constants. So move along with them.
if searchlength == zstd.SEARCHLENGTH_MIN and strategy in (
zstd.STRATEGY_FAST, zstd.STRATEGY_GREEDY):
searchlength += 1
elif searchlength == zstd.SEARCHLENGTH_MAX and strategy != zstd.STRATEGY_FAST:
searchlength -= 1
# 32-bit machines may have trouble allocating larger sizes. So cap
# them. This reduces test coverage for this machine type. But 64-bit
# should still have us covered.
maxint = sys.maxint if hasattr(sys, 'maxint') else sys.maxsize
if maxint <= 2**32:
windowlog = min(windowlog, 25)
hashlog = min(hashlog, 25)
chainlog = min(chainlog, 26)
searchlog = min(searchlog, 24)
p = zstd.CompressionParameters(windowlog, chainlog, hashlog, searchlog,
searchlength, targetlength, strategy)
cctx = zstd.ZstdCompressor(compression_params=p)
with cctx.write_to(io.BytesIO()):
pass
def test_estimated_compression_context_size(self):
p = zstd.CompressionParameters(20, 16, 17, 1, 5, 16,
zstd.STRATEGY_DFAST)
# 32-bit has slightly different values from 64-bit.
self.assertAlmostEqual(p.estimated_compression_context_size(),
1289132,
delta=130)
def test_bad_precompute_compress(self):
d = zstd.train_dictionary(8192, generate_samples(), k=64, d=16)
with self.assertRaisesRegexp(ValueError, 'must specify one of level or '):
d.precompute_compress()
with self.assertRaisesRegexp(ValueError, 'must only specify one of level or '):
d.precompute_compress(level=3,
compression_params=zstd.CompressionParameters())
def test_valid_init(self, windowlog, chainlog, hashlog, searchlog,
searchlength, targetlength, strategy):
zstd.CompressionParameters(window_log=windowlog,
chain_log=chainlog,
hash_log=hashlog,
search_log=searchlog,
min_match=searchlength,
target_length=targetlength,
compression_strategy=strategy)
def test_members(self):
p = zstd.CompressionParameters(10, 6, 7, 4, 5, 8, 1)
self.assertEqual(p.window_log, 10)
self.assertEqual(p.chain_log, 6)
self.assertEqual(p.hash_log, 7)
self.assertEqual(p.search_log, 4)
self.assertEqual(p.search_length, 5)
self.assertEqual(p.target_length, 8)
self.assertEqual(p.strategy, 1)
def test_estimate_compression_context_size(self, windowlog, chainlog,
hashlog, searchlog,
searchlength, targetlength,
strategy):
if searchlength == zstd.SEARCHLENGTH_MIN and strategy in (
zstd.STRATEGY_FAST, zstd.STRATEGY_GREEDY):
searchlength += 1
elif searchlength == zstd.SEARCHLENGTH_MAX and strategy != zstd.STRATEGY_FAST:
searchlength -= 1
p = zstd.CompressionParameters(windowlog, chainlog, hashlog, searchlog,
searchlength, targetlength, strategy)
size = zstd.estimate_compression_context_size(p)
def test_estimated_compression_context_size(self):
p = zstd.CompressionParameters(
window_log=20,
chain_log=16,
hash_log=17,
search_log=1,
min_match=5,
target_length=16,
compression_strategy=zstd.STRATEGY_DFAST)
# 32-bit has slightly different values from 64-bit.
self.assertAlmostEqual(p.estimated_compression_context_size(),
1294072,
delta=250)
def test_bad_precompute_compress(self):
samples = generate_samples()
d = zstd.train_dictionary(get_optimal_dict_size_heuristically(samples),
samples,
k=64,
d=8)
with self.assertRaisesRegex(ValueError,
"must specify one of level or "):
d.precompute_compress()
with self.assertRaisesRegex(ValueError,
"must only specify one of level or "):
d.precompute_compress(
level=3, compression_params=zstd.CompressionParameters())
def test_valid_init(self, windowlog, chainlog, hashlog, searchlog,
searchlength, targetlength, strategy):
# ZSTD_checkCParams moves the goal posts on us from what's advertised
# in the constants. So move along with them.
if searchlength == zstd.SEARCHLENGTH_MIN and strategy in (
zstd.STRATEGY_FAST, zstd.STRATEGY_GREEDY):
searchlength += 1
elif searchlength == zstd.SEARCHLENGTH_MAX and strategy != zstd.STRATEGY_FAST:
searchlength -= 1
p = zstd.CompressionParameters(windowlog, chainlog, hashlog, searchlog,
searchlength, targetlength, strategy)
cctx = zstd.ZstdCompressor(compression_params=p)
with cctx.write_to(io.BytesIO()):
pass
def test_estimated_compression_context_size(self, windowlog, chainlog,
hashlog, searchlog,
searchlength, targetlength,
strategy):
if searchlength == zstd.SEARCHLENGTH_MIN and strategy in (
zstd.STRATEGY_FAST, zstd.STRATEGY_GREEDY):
searchlength += 1
elif searchlength == zstd.SEARCHLENGTH_MAX and strategy != zstd.STRATEGY_FAST:
searchlength -= 1
p = zstd.CompressionParameters(window_log=windowlog,
chain_log=chainlog,
hash_log=hashlog,
search_log=searchlog,
min_match=searchlength,
target_length=targetlength,
compression_strategy=strategy)
size = p.estimated_compression_context_size()
def test_compression_params(self):
params = zstd.CompressionParameters(20, 6, 12, 5, 4, 10,
zstd.STRATEGY_FAST)
buffer = io.BytesIO()
cctx = zstd.ZstdCompressor(compression_params=params)
with cctx.write_to(buffer) as compressor:
self.assertEqual(compressor.write(b'foo'), 0)
self.assertEqual(compressor.write(b'bar'), 0)
self.assertEqual(compressor.write(b'foobar' * 16384), 0)
compressed = buffer.getvalue()
params = zstd.get_frame_parameters(compressed)
self.assertEqual(params.content_size, 0)
self.assertEqual(params.window_size, 1048576)
self.assertEqual(params.dict_id, 0)
self.assertFalse(params.has_checksum)
h = hashlib.sha1(compressed).hexdigest()
self.assertEqual(h, '1ae31f270ed7de14235221a604b31ecd517ebd99')
def __init__(self, min_compression_size, *, strategy=zstd.STRATEGY_FAST):
self._min_compression_size = min_compression_size
self._params = zstd.CompressionParameters(strategy=strategy)
self._compress_ctx = zstd.ZstdCompressor(
compression_params=self._params)
def test_members(self):
p = zstd.CompressionParameters(window_log=10,
chain_log=6,
hash_log=7,
search_log=4,
min_match=5,
target_length=8,
compression_strategy=1)
self.assertEqual(p.window_log, 10)
self.assertEqual(p.chain_log, 6)
self.assertEqual(p.hash_log, 7)
self.assertEqual(p.search_log, 4)
self.assertEqual(p.min_match, 5)
self.assertEqual(p.target_length, 8)
self.assertEqual(p.compression_strategy, 1)
p = zstd.CompressionParameters(compression_level=2)
self.assertEqual(p.compression_level, 2)
p = zstd.CompressionParameters(threads=4)
self.assertEqual(p.threads, 4)
p = zstd.CompressionParameters(threads=2,
job_size=1048576,
overlap_size_log=6)
self.assertEqual(p.threads, 2)
self.assertEqual(p.job_size, 1048576)
self.assertEqual(p.overlap_size_log, 6)
p = zstd.CompressionParameters(compression_level=2)
self.assertEqual(p.compress_literals, 1)
p = zstd.CompressionParameters(compress_literals=False)
self.assertEqual(p.compress_literals, 0)
p = zstd.CompressionParameters(compression_level=-1)
self.assertEqual(p.compression_level, -1)
self.assertEqual(p.compress_literals, 0)
p = zstd.CompressionParameters(compression_level=-2,
compress_literals=True)
self.assertEqual(p.compression_level, -2)
self.assertEqual(p.compress_literals, 1)
p = zstd.CompressionParameters(force_max_window=True)
self.assertEqual(p.force_max_window, 1)
p = zstd.CompressionParameters(enable_ldm=True)
self.assertEqual(p.enable_ldm, 1)
p = zstd.CompressionParameters(ldm_hash_log=7)
self.assertEqual(p.ldm_hash_log, 7)
p = zstd.CompressionParameters(ldm_min_match=6)
self.assertEqual(p.ldm_min_match, 6)
p = zstd.CompressionParameters(ldm_bucket_size_log=7)
self.assertEqual(p.ldm_bucket_size_log, 7)
p = zstd.CompressionParameters(ldm_hash_every_log=8)
self.assertEqual(p.ldm_hash_every_log, 8)
def test_init_bad_arg_type(self):
with self.assertRaises(TypeError):
zstd.CompressionParameters()
with self.assertRaises(TypeError):
zstd.CompressionParameters(0, 1)
