def compressTarDefault(dirName, overwrite=None, outname=None): """ :type string: dirName - the name of the dir to tar :type bool: overwrite - overwrite destination Generic compress method for creating .tar.lz4 from a dir. ***WARNING*** Currently uses StringIO object until lz4file supports write. Avoid using for large directories, it will consume quite a bit of RAM. """ if not outname: outname = '.'.join([dirName.rstrip('/'), 'tar', 'lz4']) if not os.path.exists(dirName): print('Unable to locate the directory to compress.') return buff = StringIO() tarbuff = Lz4Tar.open(fileobj=buff, mode='w|') tarbuff.add(dirName) tarbuff.close() buff.seek(0) with __builtin__.open(outname, 'wb') as out: out.write(lz4f.compressFrame(buff.read())) out.flush() out.close() buff.close() del tarbuff, buff
def test_lz4f(self): if platform.python_implementation() == "PyPy": pytest.skip("lz4f is currently unsupported with PyPy") encoded = lz4f.compressFrame(self.text) self.assertNotEqual(self.text, encoded) decoded = compression.decode_lz4f(encoded) self.assertEqual(self.text, decoded)
def lz4_encode(payload): data = lz4f.compressFrame(payload) # pylint: disable-msg=no-member # Kafka's LZ4 code has a bug in its header checksum implementation header_size = 7 if isinstance(data[4], int): flg = data[4] else: flg = ord(data[4]) content_size_bit = ((flg >> 3) & 1) if content_size_bit: header_size += 8 # This is the incorrect hc hc = xxhash.xxh32(data[0:header_size - 1]).digest()[-2:-1] # pylint: disable-msg=no-member return b''.join([data[0:header_size - 1], hc, data[header_size:]])
def lz4_encode(payload): data = lz4f.compressFrame(payload) # pylint: disable-msg=no-member # Kafka's LZ4 code has a bug in its header checksum implementation header_size = 7 if isinstance(data[4], int): flg = data[4] else: flg = ord(data[4]) content_size_bit = ((flg >> 3) & 1) if content_size_bit: header_size += 8 # This is the incorrect hc hc = xxhash.xxh32(data[0:header_size-1]).digest()[-2:-1] # pylint: disable-msg=no-member return b''.join([ data[0:header_size-1], hc, data[header_size:] ])
def test_lz4f(self): encoded = lz4f.compressFrame(self.text) self.assertNotEqual(self.text, encoded) decoded = compression.decode_lz4f(encoded) self.assertEqual(self.text, decoded)
def lz4_encode(payload): """Encode payload using interoperable LZ4 framing. Requires Kafka >= 0.10""" # pylint: disable-msg=no-member return lz4f.compressFrame(payload)
def compress_slice(slice_data): compressed_slice_data = lz4f.compressFrame(slice_data) return compressed_slice_data