def compressFile(cls, name, overwrite=False, outname=None, prefs=None):
"""
This is large file safe. It will now read the input in 64Kb chunks.
"""
if not outname:
outname = '.'.join([name, 'lz4r'])
if os.path.exists(outname):
if not overwrite:
print('File Exists!')
return
print('Overwrite authorized')
if not os.path.exists(name):
print('Unable to locate the original file. Please check filename.')
return
cCtx = lz4f.createCompContext()
header = lz4f.compressBegin(cCtx, prefs)
with open(outname, 'wb') as out:
out.write(header)
with open(name, 'rb') as infile:
while True:
decompData = infile.read((64*(1 << 10)))
if not decompData:
break
compData = lz4f.compressUpdate(decompData, cCtx)
out.write(compData)
out.write(lz4f.compressEnd(cCtx))
out.flush()
out.close()
lz4f.freeCompContext(cCtx)
def test_3_prefs(self):
cCtx = lz4f.createCompContext()
dCtx = lz4f.createDecompContext()
prefs = lz4f.makePrefs(5)
header = lz4f.compressBegin(cCtx, prefs)
frameInfo = lz4f.getFrameInfo(header, dCtx)
self.assertEqual(5, frameInfo.get('blkSize'))
def compressTarDefault(dirName, overwrite=None, outname=None, prefs=None):
"""
:type string: dirName - the name of the dir to tar
:type bool: overwrite - overwrite destination
Generic compress method for creating .tar.lz4 from a dir.
***WARNING*** Currently uses StringIO object until lz4file supports write.
Avoid using for large directories, it will consume quite a bit of RAM.
"""
if not outname:
outname = '.'.join([dirName.rstrip('/'), 'tar', 'lz4'])
if not os.path.exists(dirName):
print('Unable to locate the directory to compress.')
return
buff = StringIO()
tarbuff = Lz4Tar.open(fileobj=buff, mode='w')
tarbuff.add(dirName)
tarbuff.close()
buff.seek(0)
cCtx = lz4f.createCompContext()
header = lz4f.compressBegin(cCtx, prefs)
with __builtin__.open(outname, 'wb') as out:
out.write(header)
while True:
decompData = buff.read((64*(1<<10)))
if not decompData:
break
compData = lz4f.compressUpdate(decompData, cCtx)
out.write(compData)
out.write(lz4f.compressEnd(cCtx))
out.flush()
lz4f.freeCompContext(cCtx)
del tarbuff, buff
def lz4_decode(payload):
# Kafka's LZ4 code has a bug in its header checksum implementation
header_size = 7
if isinstance(payload[4], int):
flg = payload[4]
else:
flg = ord(payload[4])
content_size_bit = ((flg >> 3) & 1)
if content_size_bit:
header_size += 8
# This should be the correct hc
hc = xxhash.xxh32(
payload[4:header_size -
1]).digest()[-2:-1] # pylint: disable-msg=no-member
munged_payload = b''.join(
[payload[0:header_size - 1], hc, payload[header_size:]])
cCtx = lz4f.createCompContext() # pylint: disable-msg=no-member
data = lz4f.decompressFrame(munged_payload,
cCtx) # pylint: disable-msg=no-member
return data['decomp']
def compressFileDefault(name, overwrite=False, outname=None, prefs=None):
"""
:type string: name - name of file to compress
:type bool: overwrite - overwrite destination
:type string: outname - name for compressed file, not required.
Default will be '.'.join([name, 'lz4'])
Generic compress method for a file. Adds .lz4 to original file name for
output, unless outname is provided.
***NOTE*** No longer uses compressFrame. This is now large file safe!
It will now read the input in 64Kb chunks.
"""
if not outname:
outname = '.'.join([name, 'lz4'])
if os.path.exists(outname):
if not overwrite:
print('File Exists!')
return
print('Overwrite authorized')
if not os.path.exists(name):
print('Unable to locate the original file. Please check filename.')
return
cCtx = lz4f.createCompContext()
header = lz4f.compressBegin(cCtx, prefs)
with __builtin__.open(outname, 'wb') as out:
out.write(header)
with __builtin__.open(name, 'rb') as infile:
while True:
decompData = infile.read((64*(1 << 10)))
if not decompData:
break
compData = lz4f.compressUpdate(decompData, cCtx)
out.write(compData)
out.write(lz4f.compressEnd(cCtx))
out.flush()
out.close()
lz4f.freeCompContext(cCtx)
def __init__(p, name, mode):
p.ctx = lz4f.createCompContext()
p.outf = open(name + '.lz4', mode)
p.outf.write(lz4f.compressBegin(p.ctx, None))
