def test_bounds(self): zstd.CompressionParameters(zstd.WINDOWLOG_MIN, zstd.CHAINLOG_MIN, zstd.HASHLOG_MIN, zstd.SEARCHLOG_MIN, zstd.SEARCHLENGTH_MIN + 1, zstd.TARGETLENGTH_MIN, zstd.STRATEGY_FAST) zstd.CompressionParameters(zstd.WINDOWLOG_MAX, zstd.CHAINLOG_MAX, zstd.HASHLOG_MAX, zstd.SEARCHLOG_MAX, zstd.SEARCHLENGTH_MAX - 1, zstd.TARGETLENGTH_MAX, zstd.STRATEGY_BTOPT)
def test_bounds(self): zstd.CompressionParameters(window_log=zstd.WINDOWLOG_MIN, chain_log=zstd.CHAINLOG_MIN, hash_log=zstd.HASHLOG_MIN, search_log=zstd.SEARCHLOG_MIN, min_match=zstd.SEARCHLENGTH_MIN + 1, target_length=zstd.TARGETLENGTH_MIN, compression_strategy=zstd.STRATEGY_FAST) zstd.CompressionParameters(window_log=zstd.WINDOWLOG_MAX, chain_log=zstd.CHAINLOG_MAX, hash_log=zstd.HASHLOG_MAX, search_log=zstd.SEARCHLOG_MAX, min_match=zstd.SEARCHLENGTH_MAX - 1, compression_strategy=zstd.STRATEGY_BTULTRA)
def test_compression_params(self): params = zstd.CompressionParameters( window_log=20, chain_log=6, hash_log=12, min_match=5, search_log=4, target_length=10, compression_strategy=zstd.STRATEGY_FAST) buffer = io.BytesIO() cctx = zstd.ZstdCompressor(compression_params=params) with cctx.write_to(buffer) as compressor: self.assertEqual(compressor.write(b'foo'), 0) self.assertEqual(compressor.write(b'bar'), 0) self.assertEqual(compressor.write(b'foobar' * 16384), 0) compressed = buffer.getvalue() params = zstd.get_frame_parameters(compressed) self.assertEqual(params.content_size, zstd.CONTENTSIZE_UNKNOWN) self.assertEqual(params.window_size, 1048576) self.assertEqual(params.dict_id, 0) self.assertFalse(params.has_checksum) h = hashlib.sha1(compressed).hexdigest() self.assertEqual(h, '2a8111d72eb5004cdcecbdac37da9f26720d30ef')
def test_valid_init(self, windowlog, chainlog, hashlog, searchlog, searchlength, targetlength, strategy): # ZSTD_checkCParams moves the goal posts on us from what's advertised # in the constants. So move along with them. if searchlength == zstd.SEARCHLENGTH_MIN and strategy in ( zstd.STRATEGY_FAST, zstd.STRATEGY_GREEDY): searchlength += 1 elif searchlength == zstd.SEARCHLENGTH_MAX and strategy != zstd.STRATEGY_FAST: searchlength -= 1 # 32-bit machines may have trouble allocating larger sizes. So cap # them. This reduces test coverage for this machine type. But 64-bit # should still have us covered. maxint = sys.maxint if hasattr(sys, 'maxint') else sys.maxsize if maxint <= 2**32: windowlog = min(windowlog, 25) hashlog = min(hashlog, 25) chainlog = min(chainlog, 26) searchlog = min(searchlog, 24) p = zstd.CompressionParameters(windowlog, chainlog, hashlog, searchlog, searchlength, targetlength, strategy) cctx = zstd.ZstdCompressor(compression_params=p) with cctx.write_to(io.BytesIO()): pass
def test_estimated_compression_context_size(self): p = zstd.CompressionParameters(20, 16, 17, 1, 5, 16, zstd.STRATEGY_DFAST) # 32-bit has slightly different values from 64-bit. self.assertAlmostEqual(p.estimated_compression_context_size(), 1289132, delta=130)
def test_bad_precompute_compress(self): d = zstd.train_dictionary(8192, generate_samples(), k=64, d=16) with self.assertRaisesRegexp(ValueError, 'must specify one of level or '): d.precompute_compress() with self.assertRaisesRegexp(ValueError, 'must only specify one of level or '): d.precompute_compress(level=3, compression_params=zstd.CompressionParameters())
def test_valid_init(self, windowlog, chainlog, hashlog, searchlog, searchlength, targetlength, strategy): zstd.CompressionParameters(window_log=windowlog, chain_log=chainlog, hash_log=hashlog, search_log=searchlog, min_match=searchlength, target_length=targetlength, compression_strategy=strategy)
def test_members(self): p = zstd.CompressionParameters(10, 6, 7, 4, 5, 8, 1) self.assertEqual(p.window_log, 10) self.assertEqual(p.chain_log, 6) self.assertEqual(p.hash_log, 7) self.assertEqual(p.search_log, 4) self.assertEqual(p.search_length, 5) self.assertEqual(p.target_length, 8) self.assertEqual(p.strategy, 1)
def test_estimate_compression_context_size(self, windowlog, chainlog, hashlog, searchlog, searchlength, targetlength, strategy): if searchlength == zstd.SEARCHLENGTH_MIN and strategy in ( zstd.STRATEGY_FAST, zstd.STRATEGY_GREEDY): searchlength += 1 elif searchlength == zstd.SEARCHLENGTH_MAX and strategy != zstd.STRATEGY_FAST: searchlength -= 1 p = zstd.CompressionParameters(windowlog, chainlog, hashlog, searchlog, searchlength, targetlength, strategy) size = zstd.estimate_compression_context_size(p)
def test_estimated_compression_context_size(self): p = zstd.CompressionParameters( window_log=20, chain_log=16, hash_log=17, search_log=1, min_match=5, target_length=16, compression_strategy=zstd.STRATEGY_DFAST) # 32-bit has slightly different values from 64-bit. self.assertAlmostEqual(p.estimated_compression_context_size(), 1294072, delta=250)
def test_bad_precompute_compress(self): samples = generate_samples() d = zstd.train_dictionary(get_optimal_dict_size_heuristically(samples), samples, k=64, d=8) with self.assertRaisesRegex(ValueError, "must specify one of level or "): d.precompute_compress() with self.assertRaisesRegex(ValueError, "must only specify one of level or "): d.precompute_compress( level=3, compression_params=zstd.CompressionParameters())
def test_valid_init(self, windowlog, chainlog, hashlog, searchlog, searchlength, targetlength, strategy): # ZSTD_checkCParams moves the goal posts on us from what's advertised # in the constants. So move along with them. if searchlength == zstd.SEARCHLENGTH_MIN and strategy in ( zstd.STRATEGY_FAST, zstd.STRATEGY_GREEDY): searchlength += 1 elif searchlength == zstd.SEARCHLENGTH_MAX and strategy != zstd.STRATEGY_FAST: searchlength -= 1 p = zstd.CompressionParameters(windowlog, chainlog, hashlog, searchlog, searchlength, targetlength, strategy) cctx = zstd.ZstdCompressor(compression_params=p) with cctx.write_to(io.BytesIO()): pass
def test_estimated_compression_context_size(self, windowlog, chainlog, hashlog, searchlog, searchlength, targetlength, strategy): if searchlength == zstd.SEARCHLENGTH_MIN and strategy in ( zstd.STRATEGY_FAST, zstd.STRATEGY_GREEDY): searchlength += 1 elif searchlength == zstd.SEARCHLENGTH_MAX and strategy != zstd.STRATEGY_FAST: searchlength -= 1 p = zstd.CompressionParameters(window_log=windowlog, chain_log=chainlog, hash_log=hashlog, search_log=searchlog, min_match=searchlength, target_length=targetlength, compression_strategy=strategy) size = p.estimated_compression_context_size()
def test_compression_params(self): params = zstd.CompressionParameters(20, 6, 12, 5, 4, 10, zstd.STRATEGY_FAST) buffer = io.BytesIO() cctx = zstd.ZstdCompressor(compression_params=params) with cctx.write_to(buffer) as compressor: self.assertEqual(compressor.write(b'foo'), 0) self.assertEqual(compressor.write(b'bar'), 0) self.assertEqual(compressor.write(b'foobar' * 16384), 0) compressed = buffer.getvalue() params = zstd.get_frame_parameters(compressed) self.assertEqual(params.content_size, 0) self.assertEqual(params.window_size, 1048576) self.assertEqual(params.dict_id, 0) self.assertFalse(params.has_checksum) h = hashlib.sha1(compressed).hexdigest() self.assertEqual(h, '1ae31f270ed7de14235221a604b31ecd517ebd99')
def __init__(self, min_compression_size, *, strategy=zstd.STRATEGY_FAST): self._min_compression_size = min_compression_size self._params = zstd.CompressionParameters(strategy=strategy) self._compress_ctx = zstd.ZstdCompressor( compression_params=self._params)
def test_members(self): p = zstd.CompressionParameters(window_log=10, chain_log=6, hash_log=7, search_log=4, min_match=5, target_length=8, compression_strategy=1) self.assertEqual(p.window_log, 10) self.assertEqual(p.chain_log, 6) self.assertEqual(p.hash_log, 7) self.assertEqual(p.search_log, 4) self.assertEqual(p.min_match, 5) self.assertEqual(p.target_length, 8) self.assertEqual(p.compression_strategy, 1) p = zstd.CompressionParameters(compression_level=2) self.assertEqual(p.compression_level, 2) p = zstd.CompressionParameters(threads=4) self.assertEqual(p.threads, 4) p = zstd.CompressionParameters(threads=2, job_size=1048576, overlap_size_log=6) self.assertEqual(p.threads, 2) self.assertEqual(p.job_size, 1048576) self.assertEqual(p.overlap_size_log, 6) p = zstd.CompressionParameters(compression_level=2) self.assertEqual(p.compress_literals, 1) p = zstd.CompressionParameters(compress_literals=False) self.assertEqual(p.compress_literals, 0) p = zstd.CompressionParameters(compression_level=-1) self.assertEqual(p.compression_level, -1) self.assertEqual(p.compress_literals, 0) p = zstd.CompressionParameters(compression_level=-2, compress_literals=True) self.assertEqual(p.compression_level, -2) self.assertEqual(p.compress_literals, 1) p = zstd.CompressionParameters(force_max_window=True) self.assertEqual(p.force_max_window, 1) p = zstd.CompressionParameters(enable_ldm=True) self.assertEqual(p.enable_ldm, 1) p = zstd.CompressionParameters(ldm_hash_log=7) self.assertEqual(p.ldm_hash_log, 7) p = zstd.CompressionParameters(ldm_min_match=6) self.assertEqual(p.ldm_min_match, 6) p = zstd.CompressionParameters(ldm_bucket_size_log=7) self.assertEqual(p.ldm_bucket_size_log, 7) p = zstd.CompressionParameters(ldm_hash_every_log=8) self.assertEqual(p.ldm_hash_every_log, 8)
def test_init_bad_arg_type(self): with self.assertRaises(TypeError): zstd.CompressionParameters() with self.assertRaises(TypeError): zstd.CompressionParameters(0, 1)