def init_lz4():
from lz4 import VERSION, block
import struct
block_compress = block.compress
block_decompress = block.decompress
LZ4_HEADER = struct.Struct(b'<L')
try:
from lz4 import library_version_string
version = library_version_string()
except ImportError:
from lz4.version import version
def lz4_compress(packet, level):
flag = min(15, level) | LZ4_FLAG
if level>=9:
return flag, block_compress(packet, mode="high_compression", compression=level)
if level<=3:
return flag, block_compress(packet, mode="fast", acceleration=8-level*2)
return flag, block_compress(packet)
def lz4_decompress(data):
size = LZ4_HEADER.unpack_from(data[:4])[0]
#it would be better to use the max_size we have in protocol,
#but this hardcoded value will do for now
if size>MAX_DECOMPRESSED_SIZE:
sizemb = size//1024//1024
maxmb = MAX_DECOMPRESSED_SIZE//1024//1024
raise Exception("uncompressed data is too large: %iMB, limit is %iMB" % (sizemb, maxmb))
return block_decompress(data)
return Compression("lz4", version, VERSION.encode("latin1"), lz4_compress, lz4_decompress)
def dump_environment():
"""Dumps information about the environment. Must only be called after
_import_wx and _import_deps."""
import wx as _wx
import lz4
import yaml
fse = bolt.Path.sys_fs_enc
msg = [
u'Using Wrye Bash Version %s%s' % (bass.AppVersion,
u' (Standalone)' if bass.is_standalone else u''),
u'OS info: %s, running on %s' % (
platform.platform(), platform.processor() or u'<unknown>'),
u'Python version: %s' % sys.version,
u'wxPython version: %s' % _wx.version() if _wx is not None else \
u'wxPython not found',
u'python-lz4 version: %s; bundled LZ4 version: %s' % (
lz4.version.version, lz4.library_version_string()),
u'pyyaml version: %s' % yaml.__version__,
# Standalone: stdout will actually be pointing to stderr, which has no
# 'encoding' attribute
u'Input encoding: %s; output encoding: %s' % (
sys.stdin.encoding, getattr(sys.stdout, u'encoding', None)),
u'Filesystem encoding: %s%s' % (fse,
(u' - using %s' % bolt.Path.sys_fs_enc) if not fse else u''),
u'Command line: %s' % sys.argv,
]
if getattr(bolt, u'scandir', None) is not None:
msg.append(u'Using scandir v%s' % bolt.scandir.__version__)
for m in msg:
bolt.deprint(m)
return u'\n'.join(msg)
def dump_environment():
fse = sys.getfilesystemencoding()
msg = [
u'Using Wrye Bash Version %s%s' % (bass.AppVersion,
u' (Standalone)' if bass.is_standalone else u''),
u'OS info: %s, running on %s' % (
platform.platform(), platform.processor()),
u'Python version: %s' % sys.version,
u'wxPython version: %s' % _wx.version() if _wx is not None else \
u'wxPython not found',
u'python-lz4 version: %s; bundled LZ4 version: %s' % (
_lz4.version.version, _lz4.library_version_string()),
u'pyyaml version: %s' % _yaml.__version__,
# Standalone: stdout will actually be pointing to stderr, which has no
# 'encoding' attribute
u'Input encoding: %s; output encoding: %s' % (
sys.stdin.encoding, getattr(sys.stdout, 'encoding', None)),
u'Filesystem encoding: %s%s' % (fse,
(u' - using %s' % bolt.Path.sys_fs_enc) if not fse else u''),
u'Command line: %s' % sys.argv,
]
if getattr(bolt, 'scandir', None) is not None:
msg.append(u'Using scandir v%s' % bolt.scandir.__version__)
for m in msg:
bolt.deprint(m)
return u'\n'.join(msg)
import numpy as np
import pandas as pd
import lz4,lz4.frame
import sys
import os
from tabulate import tabulate as tbl
import timeit
import json
import zstandard as zstd
lz4lver=lz4.library_version_string()
lz4ver=lz4.VERSION
df=pd.read_csv(r"trend.dat")
nd=df.to_numpy()
sp=nd.shape
tb=nd.tobytes()
with open('test2.dat','wb') as fh:
cctx = zstd.ZstdCompressor(level=1,threads=4)
start2 = timeit.default_timer()
for i in range(10**3):
compressed2 = cctx.compress(tb)
stop2 = timeit.default_timer()
time2=(stop2 - start2)/10**3
fh.write(compressed2)
with open('test2.dat','rb') as fh:
cctx = zstd.ZstdDecompressor()
tmp = fh.read()
start3 = timeit.default_timer()
for i in range(10**3):
decompressed2 = cctx.decompress(tmp)
def test_library_version_string():
v = lz4.library_version_string()
assert isinstance(v, str)
assert v.count('.') == 2
r = re.compile(r'^[0-9]*\.[0-9]*\.[0-9]*$')
assert r.match(v) is not None
def test_library_version_string():
v = lz4.library_version_string()
assert isinstance(v, str)
assert v.count('.') == 2
r = re.compile(r'^[0-9]*\.[0-9]*\.[0-9]*$')
assert r.match(v) is not None
