def compress(_str):
"""
Compress a string
By default LZ4 mode is standard in interactive mode,
and high compresion in applications/scripts
"""
return lz4_compress(_str)
def compress(_str):
"""
Compress a string
By default LZ4 mode is standard in interactive mode,
and high compresion in applications/scripts
"""
return lz4_compress(_str)
def test_performance_sequential(n, length):
_str = random_string(length)
_strarr = [_str for _ in range(n)]
now = dt.now()
[c.decompress(y) for y in [c.compressHC(x) for x in _strarr]]
clz4_time = (dt.now() - now).total_seconds()
now = dt.now()
c.decompress_array(c.compressHC_array(_strarr))
clz4_time_p = (dt.now() - now).total_seconds()
now = dt.now()
[lz4_decompress(y) for y in [lz4_compress(x) for x in _strarr]]
lz4_time = (dt.now() - now).total_seconds()
print()
print("LZ4 Test %sx len:%s" % (n, length))
print(" LZ4 HC %s s" % clz4_time)
print(" LZ4 HC Parallel %s s" % clz4_time_p)
print(" LZ4 %s s" % lz4_time)
def test_performance_sequential(n, length):
_str = random_string(length)
_strarr = [_str for _ in range(n)]
now = dt.now()
[c.decompress(y) for y in [c.compressHC(x) for x in _strarr]]
clz4_time = (dt.now() - now).total_seconds()
now = dt.now()
c.decompress_array(c.compressHC_array(_strarr))
clz4_time_p = (dt.now() - now).total_seconds()
now = dt.now()
[lz4_decompress(y) for y in [lz4_compress(x) for x in _strarr]]
lz4_time = (dt.now() - now).total_seconds()
print()
print("LZ4 Test %sx len:%s" % (n, length))
print(" LZ4 HC %s s" % clz4_time)
print(" LZ4 HC Parallel %s s" % clz4_time_p)
print(" LZ4 %s s" % lz4_time)
def mozlz4_compress(data):
return b'mozLz40\0' + lz4_compress(data)
"""
Copyright (C) 2020 Man Group
For a list of authors, see README.md; for the license, see file LICENSE in project root directory.
"""
import logging
from multiprocessing.pool import ThreadPool
try:
from lz4.block import compress as lz4_compress, decompress as lz4_decompress
lz4_compressHC = lambda _str: lz4_compress(_str, mode='high_compression')
except ImportError as e:
from lz4 import compress as lz4_compress, compressHC as lz4_compressHC, decompress as lz4_decompress
# ENABLE_PARALLEL mutated in global_scope. Do not remove.
from ._config import ENABLE_PARALLEL, LZ4_HIGH_COMPRESSION, LZ4_WORKERS, LZ4_N_PARALLEL, LZ4_MINSZ_PARALLEL, \
BENCHMARK_MODE # noqa # pylint: disable=unused-import
logger = logging.getLogger(__name__)
_compress_thread_pool = None
def enable_parallel_lz4(mode):
"""
Set the global multithread compression mode
Parameters
----------
mode: `bool`
True: Use parallel compression. False: Use sequential compression
"""
# Deprecated since pandas 0.23.4
from pandas.core.internals.construction import arrays_to_mgr as _arrays_to_mgr
try:
from pandas.api.types import infer_dtype
except ImportError:
from pandas.lib import infer_dtype
from ..date import DateRange, to_pandas_closed_closed, mktz, datetime_to_ms, ms_to_datetime, CLOSED_CLOSED, to_dt, utc_dt_to_local_dt
from ..decorators import mongo_retry
from ..exceptions import OverlappingDataException, NoDataFoundException, UnorderedDataException, UnhandledDtypeException, ArcticException
from .._util import indent
try:
from lz4.block import compress as lz4_compress, decompress as lz4_decompress
lz4_compressHC = lambda _str: lz4_compress(_str, mode='high_compression')
except ImportError as e:
from lz4 import compress as lz4_compress, compressHC as lz4_compressHC, decompress as lz4_decompress
logger = logging.getLogger(__name__)
# Example-Schema:
# --------------
# {ID: ObjectId('52b1d39eed5066ab5e87a56d'),
# SYMBOL: u'symbol'
# INDEX: Binary('...', 0),
# IMAGE_DOC: { IMAGE: {
# 'ASK': 10.
# ...
# }
