def compress(_str): """ Compress a string By default LZ4 mode is standard in interactive mode, and high compresion in applications/scripts """ return lz4_compress(_str)
def test_performance_sequential(n, length): _str = random_string(length) _strarr = [_str for _ in range(n)] now = dt.now() [c.decompress(y) for y in [c.compressHC(x) for x in _strarr]] clz4_time = (dt.now() - now).total_seconds() now = dt.now() c.decompress_array(c.compressHC_array(_strarr)) clz4_time_p = (dt.now() - now).total_seconds() now = dt.now() [lz4_decompress(y) for y in [lz4_compress(x) for x in _strarr]] lz4_time = (dt.now() - now).total_seconds() print() print("LZ4 Test %sx len:%s" % (n, length)) print(" LZ4 HC %s s" % clz4_time) print(" LZ4 HC Parallel %s s" % clz4_time_p) print(" LZ4 %s s" % lz4_time)
def mozlz4_compress(data): return b'mozLz40\0' + lz4_compress(data)
""" Copyright (C) 2020 Man Group For a list of authors, see README.md; for the license, see file LICENSE in project root directory. """ import logging from multiprocessing.pool import ThreadPool try: from lz4.block import compress as lz4_compress, decompress as lz4_decompress lz4_compressHC = lambda _str: lz4_compress(_str, mode='high_compression') except ImportError as e: from lz4 import compress as lz4_compress, compressHC as lz4_compressHC, decompress as lz4_decompress # ENABLE_PARALLEL mutated in global_scope. Do not remove. from ._config import ENABLE_PARALLEL, LZ4_HIGH_COMPRESSION, LZ4_WORKERS, LZ4_N_PARALLEL, LZ4_MINSZ_PARALLEL, \ BENCHMARK_MODE # noqa # pylint: disable=unused-import logger = logging.getLogger(__name__) _compress_thread_pool = None def enable_parallel_lz4(mode): """ Set the global multithread compression mode Parameters ---------- mode: `bool` True: Use parallel compression. False: Use sequential compression """
# Deprecated since pandas 0.23.4 from pandas.core.internals.construction import arrays_to_mgr as _arrays_to_mgr try: from pandas.api.types import infer_dtype except ImportError: from pandas.lib import infer_dtype from ..date import DateRange, to_pandas_closed_closed, mktz, datetime_to_ms, ms_to_datetime, CLOSED_CLOSED, to_dt, utc_dt_to_local_dt from ..decorators import mongo_retry from ..exceptions import OverlappingDataException, NoDataFoundException, UnorderedDataException, UnhandledDtypeException, ArcticException from .._util import indent try: from lz4.block import compress as lz4_compress, decompress as lz4_decompress lz4_compressHC = lambda _str: lz4_compress(_str, mode='high_compression') except ImportError as e: from lz4 import compress as lz4_compress, compressHC as lz4_compressHC, decompress as lz4_decompress logger = logging.getLogger(__name__) # Example-Schema: # -------------- # {ID: ObjectId('52b1d39eed5066ab5e87a56d'), # SYMBOL: u'symbol' # INDEX: Binary('...', 0), # IMAGE_DOC: { IMAGE: { # 'ASK': 10. # ... # }