def compressFile(cls, name, overwrite=False, outname=None, prefs=None): """ This is large file safe. It will now read the input in 64Kb chunks. """ if not outname: outname = '.'.join([name, 'lz4r']) if os.path.exists(outname): if not overwrite: print('File Exists!') return print('Overwrite authorized') if not os.path.exists(name): print('Unable to locate the original file. Please check filename.') return cCtx = lz4f.createCompContext() header = lz4f.compressBegin(cCtx, prefs) with open(outname, 'wb') as out: out.write(header) with open(name, 'rb') as infile: while True: decompData = infile.read((64*(1 << 10))) if not decompData: break compData = lz4f.compressUpdate(decompData, cCtx) out.write(compData) out.write(lz4f.compressEnd(cCtx)) out.flush() out.close() lz4f.freeCompContext(cCtx)
def test_3_prefs(self): cCtx = lz4f.createCompContext() dCtx = lz4f.createDecompContext() prefs = lz4f.makePrefs(5) header = lz4f.compressBegin(cCtx, prefs) frameInfo = lz4f.getFrameInfo(header, dCtx) self.assertEqual(5, frameInfo.get('blkSize'))
def compressTarDefault(dirName, overwrite=None, outname=None, prefs=None): """ :type string: dirName - the name of the dir to tar :type bool: overwrite - overwrite destination Generic compress method for creating .tar.lz4 from a dir. ***WARNING*** Currently uses StringIO object until lz4file supports write. Avoid using for large directories, it will consume quite a bit of RAM. """ if not outname: outname = '.'.join([dirName.rstrip('/'), 'tar', 'lz4']) if not os.path.exists(dirName): print('Unable to locate the directory to compress.') return buff = StringIO() tarbuff = Lz4Tar.open(fileobj=buff, mode='w') tarbuff.add(dirName) tarbuff.close() buff.seek(0) cCtx = lz4f.createCompContext() header = lz4f.compressBegin(cCtx, prefs) with __builtin__.open(outname, 'wb') as out: out.write(header) while True: decompData = buff.read((64*(1<<10))) if not decompData: break compData = lz4f.compressUpdate(decompData, cCtx) out.write(compData) out.write(lz4f.compressEnd(cCtx)) out.flush() lz4f.freeCompContext(cCtx) del tarbuff, buff
def lz4_decode(payload): # Kafka's LZ4 code has a bug in its header checksum implementation header_size = 7 if isinstance(payload[4], int): flg = payload[4] else: flg = ord(payload[4]) content_size_bit = ((flg >> 3) & 1) if content_size_bit: header_size += 8 # This should be the correct hc hc = xxhash.xxh32( payload[4:header_size - 1]).digest()[-2:-1] # pylint: disable-msg=no-member munged_payload = b''.join( [payload[0:header_size - 1], hc, payload[header_size:]]) cCtx = lz4f.createCompContext() # pylint: disable-msg=no-member data = lz4f.decompressFrame(munged_payload, cCtx) # pylint: disable-msg=no-member return data['decomp']
def compressFileDefault(name, overwrite=False, outname=None, prefs=None): """ :type string: name - name of file to compress :type bool: overwrite - overwrite destination :type string: outname - name for compressed file, not required. Default will be '.'.join([name, 'lz4']) Generic compress method for a file. Adds .lz4 to original file name for output, unless outname is provided. ***NOTE*** No longer uses compressFrame. This is now large file safe! It will now read the input in 64Kb chunks. """ if not outname: outname = '.'.join([name, 'lz4']) if os.path.exists(outname): if not overwrite: print('File Exists!') return print('Overwrite authorized') if not os.path.exists(name): print('Unable to locate the original file. Please check filename.') return cCtx = lz4f.createCompContext() header = lz4f.compressBegin(cCtx, prefs) with __builtin__.open(outname, 'wb') as out: out.write(header) with __builtin__.open(name, 'rb') as infile: while True: decompData = infile.read((64*(1 << 10))) if not decompData: break compData = lz4f.compressUpdate(decompData, cCtx) out.write(compData) out.write(lz4f.compressEnd(cCtx)) out.flush() out.close() lz4f.freeCompContext(cCtx)
def __init__(p, name, mode): p.ctx = lz4f.createCompContext() p.outf = open(name + '.lz4', mode) p.outf.write(lz4f.compressBegin(p.ctx, None))