def compressTarDefault(dirName, overwrite=None, outname=None):
"""
:type string: dirName - the name of the dir to tar
:type bool: overwrite - overwrite destination
Generic compress method for creating .tar.lz4 from a dir.
***WARNING*** Currently uses StringIO object until lz4file supports write.
Avoid using for large directories, it will consume quite a bit of RAM.
"""
if not outname:
outname = '.'.join([dirName.rstrip('/'), 'tar', 'lz4'])
if not os.path.exists(dirName):
print('Unable to locate the directory to compress.')
return
buff = StringIO()
tarbuff = Lz4Tar.open(fileobj=buff, mode='w|')
tarbuff.add(dirName)
tarbuff.close()
buff.seek(0)
with __builtin__.open(outname, 'wb') as out:
out.write(lz4f.compressFrame(buff.read()))
out.flush()
out.close()
buff.close()
del tarbuff, buff
def test_lz4f(self):
if platform.python_implementation() == "PyPy":
pytest.skip("lz4f is currently unsupported with PyPy")
encoded = lz4f.compressFrame(self.text)
self.assertNotEqual(self.text, encoded)
decoded = compression.decode_lz4f(encoded)
self.assertEqual(self.text, decoded)
def lz4_encode(payload):
data = lz4f.compressFrame(payload) # pylint: disable-msg=no-member
# Kafka's LZ4 code has a bug in its header checksum implementation
header_size = 7
if isinstance(data[4], int):
flg = data[4]
else:
flg = ord(data[4])
content_size_bit = ((flg >> 3) & 1)
if content_size_bit:
header_size += 8
# This is the incorrect hc
hc = xxhash.xxh32(data[0:header_size -
1]).digest()[-2:-1] # pylint: disable-msg=no-member
return b''.join([data[0:header_size - 1], hc, data[header_size:]])
def lz4_encode(payload):
data = lz4f.compressFrame(payload) # pylint: disable-msg=no-member
# Kafka's LZ4 code has a bug in its header checksum implementation
header_size = 7
if isinstance(data[4], int):
flg = data[4]
else:
flg = ord(data[4])
content_size_bit = ((flg >> 3) & 1)
if content_size_bit:
header_size += 8
# This is the incorrect hc
hc = xxhash.xxh32(data[0:header_size-1]).digest()[-2:-1] # pylint: disable-msg=no-member
return b''.join([
data[0:header_size-1],
hc,
data[header_size:]
])
def test_lz4f(self):
encoded = lz4f.compressFrame(self.text)
self.assertNotEqual(self.text, encoded)
decoded = compression.decode_lz4f(encoded)
self.assertEqual(self.text, decoded)
def lz4_encode(payload):
"""Encode payload using interoperable LZ4 framing. Requires Kafka >= 0.10"""
# pylint: disable-msg=no-member
return lz4f.compressFrame(payload)
def compress_slice(slice_data):
compressed_slice_data = lz4f.compressFrame(slice_data)
return compressed_slice_data
