def lz4_compress(packet, level):
flag = min(15, level) | LZ4_FLAG
if level>=7:
return flag, compress(packet, mode="high_compression", compression=level)
elif level<=3:
return flag, compress(packet, mode="fast", acceleration=8-level*2)
return flag, compress(packet)
def compress(cls, bytearr: bytes):
print(cls.DEBUG)
compressed = lb.compress(bytearr)
if cls.DEBUG:
print('debug, do compress', 'orig len', len(bytearr),
'compress len', len(compressed))
return compressed
def arrtolz4string_list(arr):
"""
Converts (multi-dimensional) array to list of lz4 compressed strings.
Parameters
----------
arr : np.array
Returns
-------
list of str
lz4 compressed string
"""
if isinstance(arr, list):
arr = np.array(arr)
if len(arr) == 0:
return [""]
try:
str_lst = [compress(arr.tobytes())]
# catch Value error which is thrown in py3 lz4 version
except (OverflowError, ValueError):
half_ix = len(arr) // 2
str_lst = arrtolz4string_list(arr[:half_ix]) + \
arrtolz4string_list(arr[half_ix:])
return str_lst
def arrtolz4string(arr):
"""
Converts (multi-dimensional) array to lz4 compressed string.
Parameters
----------
arr : np.array
Returns
-------
byte
lz4 compressed string
"""
if isinstance(arr, list):
arr = np.array(arr)
if len(arr) == 0:
return ""
try:
comp_arr = compress(arr.tobytes())
except OverflowError:
log_handler.warning(
OverflowError, "Overflow occurred when compression array."
"Use 'arrtolz4string_list' instead.")
comp_arr = arrtolz4string_list(arr)
return comp_arr
def zlib_compress(packet, level):
level = max(1, level//2)
if isinstance(packet, memoryview):
packet = packet.tobytes()
elif not isinstance(packet, bytes):
packet = bytes(str(packet), 'UTF-8')
return level + ZLIB_FLAG, compress(packet, level)
def render(self):
cpy = []
key_order = [c.column_name for c in self.data.table.columns.values() if not c.is_no_pack]
key_order.extend([c.column_name for c in self.data.table.shadow_columns.values() if not c.is_no_pack])
for row_idx, row in enumerate(self.data.data):
c = []
for key in key_order:
c.append(row[key])
cpy.append(c)
pack = msgpack.packb([cpy])
pack_len = len(pack)
# ブロックフォーマットで圧縮。MsgPackにサイズを付与するため、compressバイナリには含めない
comp = block.compress(pack, mode='default', store_size=False)
# ヘッダ作成
# Pythonでは明示的にExt32が指定できない？ので自前で突っ込む
t = struct.pack('B', 0xc9)
# サイズ情報がタイプ＋4バイト付与されるため、5バイト膨らむ
t += struct.pack('>I', len(comp) + 5)
# msgpack-csharpは99番をLZ4コンテナとして使っているようだ
t += struct.pack('B', 99)
# 解凍後サイズを頭に突っ込む
t += struct.pack('B', 0xd2)
t += struct.pack('>I', pack_len)
# ヘッダとボディを結合
t += comp
return t
def __init__(self, frames: List, compress: bool = False):
if compress:
from lz4.block import compress
frames = [compress(frame) for frame in frames]
self._frames = frames
self.compress = compress
def reaper(signum, _):
for pid, monitor in six.iteritems(monitors):
# LOG.debug("Checking %d", proc.pid)
try:
proc = monitor['process']
proc.wait(timeout=0)
LOG.debug("Terminated %d", proc.pid)
try:
outs, _ = proc.communicate(timeout=1)
except ValueError:
outs = None
output_file = monitor['output']
if running:
start_monitor(pid, monitor)
with open(output_file, 'r') as f:
samples = f.read()
os.remove(output_file)
key = ':'.join([node_name, monitor['hostname'], monitor['name'], monitor['cgroup'], str(monitor['cgroup_pid'])])
LOG.debug("%s:%f: '%s'", key, monitor['timestamp'], len(samples))
with db.pipeline() as pipe:
pipe.zadd(key, monitor['timestamp'], compress(samples.replace('/var/lib/kolla/venv/local/lib/python2.7/site-packages', '')))
# pipe.zadd()
old = monitor['timestamp'] - 3600 * 24
if old > 0:
pipe.zremrangebyscore(key, 0, old)
pipe.execute()
except subprocess.TimeoutExpired, KeyError:
pass
def __init__(self, frames, lz4_compress=False):
if lz4_compress:
from lz4.block import compress
self.shape = frames[0].shape
self.dtype = frames[0].dtype
frames = [compress(frame) for frame in frames]
self._frames = frames
self.lz4_compress = lz4_compress
def brotli_compress(packet, level):
if len(packet)>1024*1024:
level = min(9, level)
else:
level = min(11, level)
if not isinstance(packet, bytes):
packet = bytes(str(packet), 'UTF-8')
return level | BROTLI_FLAG, compress(packet, quality=level)
def write_file(self):
json_dump = json.dumps(self.json_all, separators=(",", ":"))
if self.compressed:
json_dump = b"mozLz40\0" + block.compress(bytes(json_dump, encoding="utf-8"))
self.open_file(write=True)
self.file_handle.write(json_dump)
self.close_file()
def compress(data, attrs, compression = None):
if compression is None:
return data, attrs, False
elif compression == 'lz4':
from lz4.block import compress
compressed_data = bytearray(compress(data, mode='high_compression', compression=12))
compressed_attrs = bytearray(compress(attrs, mode='high_compression', compression=12)) if attrs else bytearray()
else:
raise Exception("unknown compression %s" %args.algorithm)
ctotal, total = len(compressed_data) + len(compressed_attrs), len(data) + len(attrs)
if ctotal >= total:
return data, attrs, False
header.append("//compressed size: %d+%d (%d) of %d+%d (%d) = %d%%\n" %(len(compressed_data), len(compressed_attrs), ctotal, len(data), len(attrs), total, 100 * ctotal // total))
header.append("#define TEXTURE_%s_DATA_CSIZE (%d)" %(args.name.upper(), len(compressed_data)))
header.append("#define TEXTURE_%s_ATTRS_CSIZE (%d)" %(args.name.upper(), len(compressed_attrs)))
return compressed_data, compressed_attrs, True
def __init__(self, frames, lz4_compress=False):
self.frame_shape = tuple(frames[0].shape)
self.shape = (len(frames), ) + self.frame_shape
self.dtype = frames[0].dtype
if lz4_compress:
from lz4.block import compress
frames = [compress(frame) for frame in frames]
self._frames = frames
self.lz4_compress = lz4_compress
def load_data(self, prefix, data):
selector = {'id': {'$in': [user_id for user_id in data.keys()]}}
collection = self.mongo_recsys_storage.get_collection(prefix)
collection.delete_many(selector)
collection.insert_many([{
'id':
int(user_id),
'value':
compress(packb(data[user_id], default=encode))
} for user_id in data])
def get_compressed_data(self, extra_header_size):
rv = BytesIO()
data = self.get_value()
compressed = block.compress(data, store_size=False, mode=self.mode)
header_size = extra_header_size + 4 + 4 # sizes
write_binary_uint32(header_size + len(compressed), rv)
write_binary_uint32(len(data), rv)
rv.write(compressed)
return rv.getvalue()
def load_data(self, prefix, data):
selector = {'id': {'$in': [user_id for user_id in data.keys()]}}
print("Загружены пользователи {} ...".format(list(data.keys())[:20]))
collection = self.mongo_recsys_storage.get_collection(prefix)
collection.delete_many(selector)
collection.insert_many(
[
{
'id': int(user_id),
'value': compress(packb(data[user_id], default=encode))
}
for user_id in data
]
)
def _compress_framez(self):
compressed_data = lz4block.compress(b"".join(self.intermediate_storage), compression=self.compression_level)
record_count = len(self.intermediate_storage)
hdr = CONTAINER_HEADER_STRUCT.pack(record_count, len(compressed_data), self.container_size_uncompressed)
self.set(self.current_offset, compressed_data)
self.set(self.container_header_offset, hdr)
self.container_header_offset = self.current_offset + len(compressed_data)
self.current_offset = self.container_header_offset + CONTAINER_HEADER_STRUCT.size
self.intermediate_storage = []
self.total_record_count += record_count
self.num_containers += 1
self.total_size_uncompressed += self.container_size_uncompressed
self.total_size_compressed += len(compressed_data)
self.container_size_uncompressed = 0
self.container_size_compressed = 0
def __init__(self, frames, lz4_compress=False):
self.frame_shape = tuple(frames[0].shape)
# OLD:
self.shape = (len(frames),) + self.frame_shape
#self.shape = self.frame_shape + (len(frames),)
self.dtype = frames[0].dtype
if lz4_compress:
from lz4.block import compress
frames = [compress(frame) for frame in frames]
# OLD:
self._frames = frames
# self._frames = np.moveaxis(frames, 0, -1)
self.lz4_compress = lz4_compress
def __init__(self, frames: list, lz4_compress: bool = False):
"""Lazyframe for a set of frames and if to apply lz4.
Args:
frames (list): The frames to convert to lazy frames
lz4_compress (bool): Use lz4 to compress the frames internally
"""
self.frame_shape = tuple(frames[0].shape)
self.shape = (len(frames), ) + self.frame_shape
self.dtype = frames[0].dtype
if lz4_compress:
try:
from lz4.block import compress
except ImportError:
raise DependencyNotInstalled(
"lz4 is not installed, run `pip install gym[other]`")
frames = [compress(frame) for frame in frames]
self._frames = frames
self.lz4_compress = lz4_compress
def __init__(self, frames, lz4_compress: bool = False):
"""
This object ensures that common frames between the observations are only stored once.
It exists purely to optimize memory usage which can be huge for DQN's 1M frames replay
buffers.
This object should only be converted to numpy array before being passed to the model.
To further reduce the memory use, it is optionally to turn on lz4 to \
compress the observations.
"""
if lz4_compress:
from lz4.block import compress
self.shape = frames[0].shape
self.dtype = frames[0].dtype
frames = [compress(frame) for frame in frames]
self._frames = frames
self._out = None
self.lz4_compress = lz4_compress
def arrtolz4string(arr: np.ndarray) -> bytes:
"""
Converts (multi-dimensional) array to list of lz4 compressed strings.
Args:
arr: Input array.
Returns:
lz4 compressed string.
"""
if isinstance(arr, list):
arr = np.array(arr)
if len(arr) == 0:
return b""
try:
comp_arr = compress(arr.tobytes())
except OverflowError:
log_handler.warning(OverflowError, "Overflow occurred when compression array."
"Use 'arrtolz4string_list' instead.")
comp_arr = arrtolz4string_list(arr)
return comp_arr
def arrtolz4string_list(arr: np.ndarray) -> List[bytes]:
"""
Converts (multi-dimensional) array to list of lz4 compressed strings.
Args:
arr: Input array.
Returns:
lz4 compressed string.
"""
if isinstance(arr, list):
arr = np.array(arr)
if len(arr) == 0:
return [b""]
try:
str_lst = [compress(arr.tobytes())]
# catch Value error which is thrown in py3 lz4 version
except (OverflowError, ValueError):
half_ix = len(arr) // 2
str_lst = arrtolz4string_list(arr[:half_ix]) + \
arrtolz4string_list(arr[half_ix:])
return str_lst
def lz4_compress(data):
try:
import lz4.block as block
except ImportError:
block = lz4.LZ4_compress
return block.compress(data, 'high_compression', store_size=False)
print label("cdata")
packed_data = bytearray()
for ty in xrange(0, ny):
basey = ty * th
for tx in xrange(0, nx):
basex = tx * tw
for plane in xrange(0, args.planes):
for y in xrange(0, th):
for x in xrange(0, tw / 8):
byte = 0
for bit in xrange(0, 8):
byte |= get_pixel(basex + x * 8 + bit, basey + y, plane) << (7 - bit)
packed_data.append(byte)
if args.algorithm == "lz4":
from lz4.block import compress
compressed_data = bytearray(compress(packed_data, mode='high_compression', compression=12))
elif args.algorithm == "huffman":
compressed_data = huffman(packed_data)
print "#compressed size: %d of %d\n" %(len(compressed_data), len(packed_data))
print " ".join(map(lambda x: "0x%02x" %x, compressed_data))
else:
print label("data"),
for ty in xrange(0, ny):
basey = ty * th
if nx > 1 or ny > 1:
print "\n" + label("row_%d" %ty)
for tx in xrange(0, nx):
basex = tx * tw
if nx > 1 or ny > 1:
def compress(file_obj):
compressed = lz4.compress(file_obj.read())
return b"mozLz40\0" + compressed
def compress_data(self, data):
return block.compress(data, store_size=False, mode=self.mode)
#!/usr/bin/env python3
from sys import stdin, stdout, argv, stderr
import os
try:
import lz4.block as lz4
except ImportError:
import lz4
stdin = os.fdopen(stdin.fileno(), 'rb')
stdout = os.fdopen(stdout.fileno(), 'wb')
if argv[1:] == ['-c']:
stdout.write(b'mozLz40\0' + lz4.compress(stdin.read()))
elif argv[1:] == ['-d']:
assert stdin.read(8) == b'mozLz40\0'
stdout.write(lz4.decompress(stdin.read()))
else:
stderr.write('Usage: %s -c|-d < infile > outfile\n' % argv[0])
stderr.write('Compress or decompress Mozilla-flavor LZ4 files.\n\n')
stderr.write('Examples:\n')
stderr.write('\t%s -d < infile.json.mozlz4 > outfile.json\n' % argv[0])
stderr.write('\t%s -c < infile.json > outfile.json.mozlz4\n' % argv[0])
exit(1)
def __init__(self, frames, lz4_compress=False):
if lz4_compress:
self.shape = frames[0].shape
frames = [compress(frame) for frame in frames]
self._frames = frames
self.lz4_compress = lz4_compress
def lz4_compress(byts):
# write length in big-endian instead of little-endian
return int32_pack(len(byts)) + lz4_block.compress(byts)[4:]
def compress(cls, data):
return block.compress(data.encode())
def lz4_compress(byts):
# write length in big-endian instead of little-endian
return int32_pack(len(byts)) + lz4_block.compress(byts)[4:]
def compress(file_obj):
compressed = lz4.compress(file_obj.read())
return b"mozLz40\0" + compressed
