def _regenDCTX(self):
"""
Regenerate the decompression context.
"""
try:
lz4f.freeDecompContext(self.dCtx)
del self.dCtx
self.dCtx = lz4f.createDecompContext()
frameInfo = lz4f.getFrameInfo(self.header, self.dCtx)
lz4f.disableChecksum(self.dCtx)
except AttributeError:
self.dCtx = lz4f.createDecompContext()
frameInfo = lz4f.getFrameInfo(self.header, self.dCtx)
pass
del frameInfo
def _regenDCTX(self):
"""
Regenerate the decompression context.
"""
try:
lz4f.freeDecompContext(self.dCtx)
del self.dCtx
self.dCtx = lz4f.createDecompContext()
frameInfo = lz4f.getFrameInfo(self.header, self.dCtx)
lz4f.disableChecksum(self.dCtx)
except AttributeError:
self.dCtx = lz4f.createDecompContext()
frameInfo = lz4f.getFrameInfo(self.header, self.dCtx)
pass
del frameInfo
def __init__(self, name, mode):
self.__bRead = False
self.__bWrite = False
if 'r' in mode:
self.__bRead = True
if 'w' in mode:
self.__bWrite = True
if 'a' in mode:
self.__bWrite = True
if '+' in mode:
raise IOError('Both write and read not supported')
if 'b' not in mode:
mode += 'b'
self.__name = name
self.__mode = mode
self.__file = open(name, mode)
self.__bClose = False
self.__frames = []
self.__iFrame = -1
self.__block = None
self.__pos = 0
self.__ctx = lz4f.createDecompContext()
self.__blockdata = ''
self.__blockpos = 0
def __init__(self, name, fileObj=None, seekable=True):
parseMagic = lambda x: binascii.hexlify(x[:4])
self.name = name
self.pos = 0
self.blkDict = {}
self.curBlk = 0
if fileObj:
self.fileObj = fileObj
self.compEnd = self.tell_end()
else:
return open(name)
self.header = fileObj.read(7)
if parseMagic(self.header) == b'04224d18':
self.dCtx = lz4f.createDecompContext()
self.fileInfo = lz4f.getFrameInfo(self.header, self.dCtx)
self.blkSizeID = self.fileInfo.get('blkSize')
else:
raise IOError
if seekable:
try:
self.load_blocks()
except Exception as e:
logger.error('Exception %s' % e)
logger.info(traceback.format_exc())
print('Unable to load blockDict. Possibly not a lz4 file.')
raise IOError
def test_3_prefs(self):
cCtx = lz4f.createCompContext()
dCtx = lz4f.createDecompContext()
prefs = lz4f.makePrefs(5)
header = lz4f.compressBegin(cCtx, prefs)
frameInfo = lz4f.getFrameInfo(header, dCtx)
self.assertEqual(5, frameInfo.get('blkSize'))
def test_3_prefs(self):
cCtx = lz4f.createCompContext()
dCtx = lz4f.createDecompContext()
prefs = lz4f.makePrefs(5)
header = lz4f.compressBegin(cCtx, prefs)
frameInfo = lz4f.getFrameInfo(header, dCtx)
self.assertEqual(5, frameInfo.get('blkSize'))
def __init__(self, name, mode):
self.__bRead = False
self.__bWrite = False
if 'r' in mode:
self.__bRead = True
if 'w' in mode:
self.__bWrite = True
if 'a' in mode:
self.__bWrite = True
if '+' in mode:
raise IOError('Both write and read not supported')
if 'b' not in mode:
mode += 'b'
self.__name = name
self.__mode = mode
self.__file = open(name, mode)
self.__bClose = False
self.__frames = []
self.__iFrame = -1
self.__block = None
self.__pos = 0
self.__ctx = lz4f.createDecompContext()
self.__blockdata = ''
self.__blockpos = 0
def lz4_decode(payload):
"""Decode payload using interoperable LZ4 framing. Requires Kafka >= 0.10"""
# pylint: disable-msg=no-member
ctx = lz4f.createDecompContext()
data = lz4f.decompressFrame(payload, ctx)
# lz4f python module does not expose how much of the payload was
# actually read if the decompression was only partial.
if data['next'] != 0:
raise RuntimeError('lz4f unable to decompress full payload')
return data['decomp']
def getFileInfo(name):
"""
:type string: name - name of file to examine
Returns a dict object containing the file's header information.
"""
if not os.path.exists(name):
print('Unable to locate the file')
return
dCtx = lz4f.createDecompContext()
with __builtin__.open(name, 'rb') as inFile:
header = inFile.read(7)
return lz4f.getFrameInfo(header, dCtx)
def getFileInfo(name):
"""
:type string: name - name of file to examine
Returns a dict object containing the file's header information.
"""
if not os.path.exists(name):
print('Unable to locate the file')
return
dCtx = lz4f.createDecompContext()
with __builtin__.open(name, 'rb') as inFile:
header = inFile.read(7)
return lz4f.getFrameInfo(header, dCtx)
def __init__(self, name, fileObj=None, seekable=True):
self.name = name
if fileObj:
self.fileObj = fileObj
self.compEnd = self.tell_end()
else:
return open(name)
self.dCtx = lz4f.createDecompContext()
self.header = fileObj.read(7)
self.fileInfo = lz4f.getFrameInfo(self.header, self.dCtx)
self.blkSizeID = self.fileInfo.get('blkSize')
if seekable:
try:
self.load_blocks()
except:
print('Unable to load blockDict. Possibly not a lz4 file.')
raise
def __init__(self, name, fileObj=None, seekable=True):
self.name = name
if fileObj is None:
self.fileObj = open(name, 'rb')
else:
self.fileObj = fileObj
self.compEnd = self.tell_end()
self.header = self.fileObj.read(7)
if binascii.hexlify(self.header[:4]) == b'04224d18':
self.dCtx = lz4f.createDecompContext()
self.fileInfo = lz4f.getFrameInfo(self.header, self.dCtx)
self.blkSizeID = self.fileInfo.get('blkSize')
else:
raise IOError('Bad header or not supported header')
if seekable:
self.load_blocks()
def __init__(self, name, fileObj=None, seekable=True):
self.name = name
if fileObj is None:
self.fileObj = open(name, 'rb')
else:
self.fileObj = fileObj
self.compEnd = self.tell_end()
self.header = self.fileObj.read(7)
if binascii.hexlify(self.header[:4]) == b'04224d18':
self.dCtx = lz4f.createDecompContext()
self.fileInfo = lz4f.getFrameInfo(self.header, self.dCtx)
self.blkSizeID = self.fileInfo.get('blkSize')
else:
raise IOError('Bad header or not supported header')
if seekable:
self.load_blocks()
def __init__(self, name, fileObj=None, seekable=True):
parseMagic = lambda x: binascii.hexlify(x[:4])
self.name = name
if fileObj:
self.fileObj = fileObj
self.compEnd = self.tell_end()
else:
return open(name)
self.header = fileObj.read(7)
if parseMagic(self.header) == b'04224d18':
self.dCtx = lz4f.createDecompContext()
self.fileInfo = lz4f.getFrameInfo(self.header, self.dCtx)
self.blkSizeID = self.fileInfo.get('blkSize')
else:
raise IOError
if seekable:
try:
self.load_blocks()
except:
print('Unable to load blockDict. Possibly not a lz4r file.')
raise IOError
def __init__(self, name, fileObj=None, seekable=True):
parseMagic = lambda x: binascii.hexlify(x[:4])
self.name = name
if fileObj:
self.fileObj = fileObj
self.compEnd = self.tell_end()
else:
self.fileObj = _open(name)
self.compEnd = self.tell_end()
self.header = fileObj.read(7)
if parseMagic(self.header) == b'04224d18':
self.dCtx = lz4f.createDecompContext()
self.fileInfo = lz4f.getFrameInfo(self.header, self.dCtx)
self.blkSizeID = self.fileInfo.get('blkSize')
else:
raise IOError
if seekable:
try:
self.load_blocks()
except:
print('Unable to load blockDict. Possibly not a lz4 file.')
raise IOError
def extract_slice_data(slison_zip, data_filename, value_type):
with slison_zip.open(data_filename) as slice_file:
data = lz4f.decompressFrame(slice_file.read(),
dCtx=lz4f.createDecompContext())
slice_data = np.frombuffer(data['decomp'], dtype=value_type)
return slice_data