def lz4_compress(packet, level): flag = min(15, level) | LZ4_FLAG if level>=7: return flag, compress(packet, mode="high_compression", compression=level) elif level<=3: return flag, compress(packet, mode="fast", acceleration=8-level*2) return flag, compress(packet)
def compress(cls, bytearr: bytes): print(cls.DEBUG) compressed = lb.compress(bytearr) if cls.DEBUG: print('debug, do compress', 'orig len', len(bytearr), 'compress len', len(compressed)) return compressed
def arrtolz4string_list(arr): """ Converts (multi-dimensional) array to list of lz4 compressed strings. Parameters ---------- arr : np.array Returns ------- list of str lz4 compressed string """ if isinstance(arr, list): arr = np.array(arr) if len(arr) == 0: return [""] try: str_lst = [compress(arr.tobytes())] # catch Value error which is thrown in py3 lz4 version except (OverflowError, ValueError): half_ix = len(arr) // 2 str_lst = arrtolz4string_list(arr[:half_ix]) + \ arrtolz4string_list(arr[half_ix:]) return str_lst
def arrtolz4string(arr): """ Converts (multi-dimensional) array to lz4 compressed string. Parameters ---------- arr : np.array Returns ------- byte lz4 compressed string """ if isinstance(arr, list): arr = np.array(arr) if len(arr) == 0: return "" try: comp_arr = compress(arr.tobytes()) except OverflowError: log_handler.warning( OverflowError, "Overflow occurred when compression array." "Use 'arrtolz4string_list' instead.") comp_arr = arrtolz4string_list(arr) return comp_arr
def zlib_compress(packet, level): level = max(1, level//2) if isinstance(packet, memoryview): packet = packet.tobytes() elif not isinstance(packet, bytes): packet = bytes(str(packet), 'UTF-8') return level + ZLIB_FLAG, compress(packet, level)
def render(self): cpy = [] key_order = [c.column_name for c in self.data.table.columns.values() if not c.is_no_pack] key_order.extend([c.column_name for c in self.data.table.shadow_columns.values() if not c.is_no_pack]) for row_idx, row in enumerate(self.data.data): c = [] for key in key_order: c.append(row[key]) cpy.append(c) pack = msgpack.packb([cpy]) pack_len = len(pack) # ブロックフォーマットで圧縮。MsgPackにサイズを付与するため、compressバイナリには含めない comp = block.compress(pack, mode='default', store_size=False) # ヘッダ作成 # Pythonでは明示的にExt32が指定できない?ので自前で突っ込む t = struct.pack('B', 0xc9) # サイズ情報がタイプ+4バイト付与されるため、5バイト膨らむ t += struct.pack('>I', len(comp) + 5) # msgpack-csharpは99番をLZ4コンテナとして使っているようだ t += struct.pack('B', 99) # 解凍後サイズを頭に突っ込む t += struct.pack('B', 0xd2) t += struct.pack('>I', pack_len) # ヘッダとボディを結合 t += comp return t
def __init__(self, frames: List, compress: bool = False): if compress: from lz4.block import compress frames = [compress(frame) for frame in frames] self._frames = frames self.compress = compress
def reaper(signum, _): for pid, monitor in six.iteritems(monitors): # LOG.debug("Checking %d", proc.pid) try: proc = monitor['process'] proc.wait(timeout=0) LOG.debug("Terminated %d", proc.pid) try: outs, _ = proc.communicate(timeout=1) except ValueError: outs = None output_file = monitor['output'] if running: start_monitor(pid, monitor) with open(output_file, 'r') as f: samples = f.read() os.remove(output_file) key = ':'.join([node_name, monitor['hostname'], monitor['name'], monitor['cgroup'], str(monitor['cgroup_pid'])]) LOG.debug("%s:%f: '%s'", key, monitor['timestamp'], len(samples)) with db.pipeline() as pipe: pipe.zadd(key, monitor['timestamp'], compress(samples.replace('/var/lib/kolla/venv/local/lib/python2.7/site-packages', ''))) # pipe.zadd() old = monitor['timestamp'] - 3600 * 24 if old > 0: pipe.zremrangebyscore(key, 0, old) pipe.execute() except subprocess.TimeoutExpired, KeyError: pass
def __init__(self, frames, lz4_compress=False): if lz4_compress: from lz4.block import compress self.shape = frames[0].shape self.dtype = frames[0].dtype frames = [compress(frame) for frame in frames] self._frames = frames self.lz4_compress = lz4_compress
def brotli_compress(packet, level): if len(packet)>1024*1024: level = min(9, level) else: level = min(11, level) if not isinstance(packet, bytes): packet = bytes(str(packet), 'UTF-8') return level | BROTLI_FLAG, compress(packet, quality=level)
def write_file(self): json_dump = json.dumps(self.json_all, separators=(",", ":")) if self.compressed: json_dump = b"mozLz40\0" + block.compress(bytes(json_dump, encoding="utf-8")) self.open_file(write=True) self.file_handle.write(json_dump) self.close_file()
def compress(data, attrs, compression = None): if compression is None: return data, attrs, False elif compression == 'lz4': from lz4.block import compress compressed_data = bytearray(compress(data, mode='high_compression', compression=12)) compressed_attrs = bytearray(compress(attrs, mode='high_compression', compression=12)) if attrs else bytearray() else: raise Exception("unknown compression %s" %args.algorithm) ctotal, total = len(compressed_data) + len(compressed_attrs), len(data) + len(attrs) if ctotal >= total: return data, attrs, False header.append("//compressed size: %d+%d (%d) of %d+%d (%d) = %d%%\n" %(len(compressed_data), len(compressed_attrs), ctotal, len(data), len(attrs), total, 100 * ctotal // total)) header.append("#define TEXTURE_%s_DATA_CSIZE (%d)" %(args.name.upper(), len(compressed_data))) header.append("#define TEXTURE_%s_ATTRS_CSIZE (%d)" %(args.name.upper(), len(compressed_attrs))) return compressed_data, compressed_attrs, True
def __init__(self, frames, lz4_compress=False): self.frame_shape = tuple(frames[0].shape) self.shape = (len(frames), ) + self.frame_shape self.dtype = frames[0].dtype if lz4_compress: from lz4.block import compress frames = [compress(frame) for frame in frames] self._frames = frames self.lz4_compress = lz4_compress
def load_data(self, prefix, data): selector = {'id': {'$in': [user_id for user_id in data.keys()]}} collection = self.mongo_recsys_storage.get_collection(prefix) collection.delete_many(selector) collection.insert_many([{ 'id': int(user_id), 'value': compress(packb(data[user_id], default=encode)) } for user_id in data])
def get_compressed_data(self, extra_header_size): rv = BytesIO() data = self.get_value() compressed = block.compress(data, store_size=False, mode=self.mode) header_size = extra_header_size + 4 + 4 # sizes write_binary_uint32(header_size + len(compressed), rv) write_binary_uint32(len(data), rv) rv.write(compressed) return rv.getvalue()
def load_data(self, prefix, data): selector = {'id': {'$in': [user_id for user_id in data.keys()]}} print("Загружены пользователи {} ...".format(list(data.keys())[:20])) collection = self.mongo_recsys_storage.get_collection(prefix) collection.delete_many(selector) collection.insert_many( [ { 'id': int(user_id), 'value': compress(packb(data[user_id], default=encode)) } for user_id in data ] )
def _compress_framez(self): compressed_data = lz4block.compress(b"".join(self.intermediate_storage), compression=self.compression_level) record_count = len(self.intermediate_storage) hdr = CONTAINER_HEADER_STRUCT.pack(record_count, len(compressed_data), self.container_size_uncompressed) self.set(self.current_offset, compressed_data) self.set(self.container_header_offset, hdr) self.container_header_offset = self.current_offset + len(compressed_data) self.current_offset = self.container_header_offset + CONTAINER_HEADER_STRUCT.size self.intermediate_storage = [] self.total_record_count += record_count self.num_containers += 1 self.total_size_uncompressed += self.container_size_uncompressed self.total_size_compressed += len(compressed_data) self.container_size_uncompressed = 0 self.container_size_compressed = 0
def __init__(self, frames, lz4_compress=False): self.frame_shape = tuple(frames[0].shape) # OLD: self.shape = (len(frames),) + self.frame_shape #self.shape = self.frame_shape + (len(frames),) self.dtype = frames[0].dtype if lz4_compress: from lz4.block import compress frames = [compress(frame) for frame in frames] # OLD: self._frames = frames # self._frames = np.moveaxis(frames, 0, -1) self.lz4_compress = lz4_compress
def __init__(self, frames: list, lz4_compress: bool = False): """Lazyframe for a set of frames and if to apply lz4. Args: frames (list): The frames to convert to lazy frames lz4_compress (bool): Use lz4 to compress the frames internally """ self.frame_shape = tuple(frames[0].shape) self.shape = (len(frames), ) + self.frame_shape self.dtype = frames[0].dtype if lz4_compress: try: from lz4.block import compress except ImportError: raise DependencyNotInstalled( "lz4 is not installed, run `pip install gym[other]`") frames = [compress(frame) for frame in frames] self._frames = frames self.lz4_compress = lz4_compress
def __init__(self, frames, lz4_compress: bool = False): """ This object ensures that common frames between the observations are only stored once. It exists purely to optimize memory usage which can be huge for DQN's 1M frames replay buffers. This object should only be converted to numpy array before being passed to the model. To further reduce the memory use, it is optionally to turn on lz4 to \ compress the observations. """ if lz4_compress: from lz4.block import compress self.shape = frames[0].shape self.dtype = frames[0].dtype frames = [compress(frame) for frame in frames] self._frames = frames self._out = None self.lz4_compress = lz4_compress
def arrtolz4string(arr: np.ndarray) -> bytes: """ Converts (multi-dimensional) array to list of lz4 compressed strings. Args: arr: Input array. Returns: lz4 compressed string. """ if isinstance(arr, list): arr = np.array(arr) if len(arr) == 0: return b"" try: comp_arr = compress(arr.tobytes()) except OverflowError: log_handler.warning(OverflowError, "Overflow occurred when compression array." "Use 'arrtolz4string_list' instead.") comp_arr = arrtolz4string_list(arr) return comp_arr
def arrtolz4string_list(arr: np.ndarray) -> List[bytes]: """ Converts (multi-dimensional) array to list of lz4 compressed strings. Args: arr: Input array. Returns: lz4 compressed string. """ if isinstance(arr, list): arr = np.array(arr) if len(arr) == 0: return [b""] try: str_lst = [compress(arr.tobytes())] # catch Value error which is thrown in py3 lz4 version except (OverflowError, ValueError): half_ix = len(arr) // 2 str_lst = arrtolz4string_list(arr[:half_ix]) + \ arrtolz4string_list(arr[half_ix:]) return str_lst
def lz4_compress(data): try: import lz4.block as block except ImportError: block = lz4.LZ4_compress return block.compress(data, 'high_compression', store_size=False)
print label("cdata") packed_data = bytearray() for ty in xrange(0, ny): basey = ty * th for tx in xrange(0, nx): basex = tx * tw for plane in xrange(0, args.planes): for y in xrange(0, th): for x in xrange(0, tw / 8): byte = 0 for bit in xrange(0, 8): byte |= get_pixel(basex + x * 8 + bit, basey + y, plane) << (7 - bit) packed_data.append(byte) if args.algorithm == "lz4": from lz4.block import compress compressed_data = bytearray(compress(packed_data, mode='high_compression', compression=12)) elif args.algorithm == "huffman": compressed_data = huffman(packed_data) print "#compressed size: %d of %d\n" %(len(compressed_data), len(packed_data)) print " ".join(map(lambda x: "0x%02x" %x, compressed_data)) else: print label("data"), for ty in xrange(0, ny): basey = ty * th if nx > 1 or ny > 1: print "\n" + label("row_%d" %ty) for tx in xrange(0, nx): basex = tx * tw if nx > 1 or ny > 1:
def compress(file_obj): compressed = lz4.compress(file_obj.read()) return b"mozLz40\0" + compressed
def compress_data(self, data): return block.compress(data, store_size=False, mode=self.mode)
#!/usr/bin/env python3 from sys import stdin, stdout, argv, stderr import os try: import lz4.block as lz4 except ImportError: import lz4 stdin = os.fdopen(stdin.fileno(), 'rb') stdout = os.fdopen(stdout.fileno(), 'wb') if argv[1:] == ['-c']: stdout.write(b'mozLz40\0' + lz4.compress(stdin.read())) elif argv[1:] == ['-d']: assert stdin.read(8) == b'mozLz40\0' stdout.write(lz4.decompress(stdin.read())) else: stderr.write('Usage: %s -c|-d < infile > outfile\n' % argv[0]) stderr.write('Compress or decompress Mozilla-flavor LZ4 files.\n\n') stderr.write('Examples:\n') stderr.write('\t%s -d < infile.json.mozlz4 > outfile.json\n' % argv[0]) stderr.write('\t%s -c < infile.json > outfile.json.mozlz4\n' % argv[0]) exit(1)
def __init__(self, frames, lz4_compress=False): if lz4_compress: self.shape = frames[0].shape frames = [compress(frame) for frame in frames] self._frames = frames self.lz4_compress = lz4_compress
def lz4_compress(byts): # write length in big-endian instead of little-endian return int32_pack(len(byts)) + lz4_block.compress(byts)[4:]
def compress(cls, data): return block.compress(data.encode())