def pre_decompress_left_data(self, buf, buf_size):
# in this case there is data left that needs to be processed before the first
# argument can be processed
lzs = self.lzs
addr_input_buffer = int(ffi.cast('uintptr_t', self._input_buffer))
addr_next_in = int(ffi.cast('uintptr_t', lzs.next_in))
avail_now = (addr_input_buffer + self._input_buffer_size) - \
(addr_next_in + lzs.avail_in)
avail_total = self._input_buffer_size - lzs.avail_in
if avail_total < buf_size:
# resize the buffer, it is too small!
offset = addr_next_in - addr_input_buffer
new_size = self._input_buffer_size + buf_size - avail_now
# there is no realloc?
tmp = ffi.cast("uint8_t*", m.malloc(new_size))
if tmp == ffi.NULL:
raise MemoryError
ffi.memmove(tmp, lzs.next_in, lzs.avail_in)
lzs.next_in = tmp
m.free(self._input_buffer)
self._input_buffer = tmp
self._input_buffer_size = new_size
elif avail_now < buf_size:
# the buffer is not too small, but we cannot append it!
# move all data to the front
ffi.memmove(self._input_buffer, lzs.next_in, lzs.avail_in)
lzs.next_in = self._input_buffer
ffi.memmove(lzs.next_in + lzs.avail_in, buf, buf_size)
lzs.avail_in += buf_size
return lzs.next_in, lzs.avail_in
def _compress(self, data, action=m.LZMA_RUN):
# TODO use realloc like in LZMADecompressor
BUFSIZ = 8192
lzs = self.lzs
lzs.next_in = input_ = ffi.new('uint8_t[]', to_bytes(data))
lzs.avail_in = input_len = len(data)
outs = [ffi.new('uint8_t[]', BUFSIZ)]
lzs.next_out, = outs
lzs.avail_out = BUFSIZ
siz = BUFSIZ
while True:
next_out_pos = int(ffi.cast('intptr_t', lzs.next_out))
ret = catch_lzma_error(m.lzma_code, lzs, action,
ignore_buf_error=(input_len==0 and lzs.avail_out > 0))
data_size = int(ffi.cast('intptr_t', lzs.next_out)) - next_out_pos
if (action == m.LZMA_RUN and lzs.avail_in == 0) or \
(action == m.LZMA_FINISH and ret == m.LZMA_STREAM_END):
break
elif lzs.avail_out == 0:
# ran out of space in the output buffer
#siz = (BUFSIZ << 1) + 6
siz = 512
outs.append(ffi.new('uint8_t[]', siz))
lzs.next_out = outs[-1]
lzs.avail_out = siz
last_out = outs.pop()
last_out_len = siz - lzs.avail_out
last_out_piece = ffi.buffer(last_out[0:last_out_len], last_out_len)[:]
return b''.join(ffi.buffer(nn)[:] for nn in outs) + last_out_piece
def _decompress(self, buf, buf_len, max_length):
lzs = self.lzs
lzs.next_in = buf
lzs.avail_in = buf_len
if buf_len == 0:
return b""
bufsiz = self._bufsiz
if not (max_length < 0 or max_length > io.DEFAULT_BUFFER_SIZE):
bufsiz = max_length
lzs.next_out = orig_out = m.malloc(bufsiz)
if orig_out == ffi.NULL:
raise MemoryError
lzs.avail_out = bufsiz
data_size = 0
try:
while True:
ret = catch_lzma_error(m.lzma_code, lzs, m.LZMA_RUN)
data_size = int(ffi.cast('uintptr_t', lzs.next_out)) - int(ffi.cast('uintptr_t', orig_out))
# data_size is the amount lzma_code has already outputted
if ret in (m.LZMA_NO_CHECK, m.LZMA_GET_CHECK):
self.check = m.lzma_get_check(lzs)
if ret == m.LZMA_STREAM_END:
self.eof = True
break
elif lzs.avail_in == 0:
# it ate everything
break
elif lzs.avail_out == 0:
if data_size == max_length:
break
# ran out of space in the output buffer, let's grow it
bufsiz += (bufsiz >> 3) + 6
next_out = m.realloc(orig_out, bufsiz)
if next_out == ffi.NULL:
# realloc unsuccessful
m.free(orig_out)
orig_out = ffi.NULL
raise MemoryError
orig_out = next_out
lzs.next_out = orig_out + data_size
lzs.avail_out = bufsiz - data_size
result = ffi.buffer(orig_out, data_size)[:]
finally:
m.free(orig_out)
return result
def decompress(self, data, max_length=-1):
"""
decompress(data, max_length=-1) -> bytes
Provide data to the decompressor object. Returns a chunk of
decompressed data if possible, or b"" otherwise.
Attempting to decompress data after the end of the stream is
reached raises an EOFError. Any data found after the end of the
stream is ignored, and saved in the unused_data attribute.
"""
if not isinstance(max_length, int):
raise TypeError(
"max_length parameter object cannot be interpreted as an integer"
)
with self.lock:
if self.eof:
raise EOFError("Already at end of stream")
lzs = self.lzs
data = to_bytes(data)
buf = ffi.new('uint8_t[]', data)
buf_size = len(data)
if lzs.next_in:
buf, buf_size = self.pre_decompress_left_data(buf, buf_size)
used__input_buffer = True
else:
lzs.avail_in = buf_size
lzs.next_in = ffi.cast("uint8_t*", buf)
used__input_buffer = False
# actual decompression
result = self._decompress(buf, buf_size, max_length)
if self.eof:
self.needs_input = False
if lzs.avail_in > 0:
self.unused_data = ffi.buffer(lzs.next_in, lzs.avail_in)[:]
self.clear_input_buffer()
elif lzs.avail_in == 0:
# completed successfully!
lzs.next_in = ffi.NULL
if lzs.avail_out == 0:
# (avail_in==0 && avail_out==0)
# Maybe lzs's internal state still have a few bytes can
# be output, try to output them next time.
self.needs_input = False
assert max_length >= 0 # if < 0, lzs.avail_out always > 0
else:
# Input buffer exhausted, output buffer has space.
self.needs_input = True
self.clear_input_buffer()
else:
self.needs_input = False
if not used__input_buffer:
self.post_decompress_avail_data()
return result
def post_decompress_avail_data(self):
lzs = self.lzs
# free buffer it is to small
if self._input_buffer is not ffi.NULL and \
self._input_buffer_size < lzs.avail_in:
m.free(self._input_buffer)
self._input_buffer = ffi.NONE
# allocate if necessary
if self._input_buffer is ffi.NULL:
self._input_buffer = ffi.cast("uint8_t*", m.malloc(lzs.avail_in))
if self._input_buffer == ffi.NULL:
raise MemoryError
self._input_buffer_size = lzs.avail_in
ffi.memmove(self._input_buffer, lzs.next_in, lzs.avail_in)
lzs.next_in = self._input_buffer
def _decompress(self, buf, buf_len, max_length):
lzs = self.lzs
lzs.next_in = buf
lzs.avail_in = buf_len
bufsiz = self._bufsiz
if not (max_length < 0 or max_length > io.DEFAULT_BUFFER_SIZE):
bufsiz = max_length
lzs.next_out = orig_out = m.malloc(bufsiz)
if orig_out == ffi.NULL:
raise MemoryError
lzs.avail_out = bufsiz
data_size = 0
try:
while True:
ret = catch_lzma_error(
m.lzma_code,
lzs,
m.LZMA_RUN,
ignore_buf_error=(lzs.avail_in == 0 and lzs.avail_out > 0))
data_size = int(ffi.cast('uintptr_t', lzs.next_out)) - int(
ffi.cast('uintptr_t', orig_out))
# data_size is the amount lzma_code has already outputted
if ret in (m.LZMA_NO_CHECK, m.LZMA_GET_CHECK):
self.check = m.lzma_get_check(lzs)
if ret == m.LZMA_STREAM_END:
self.eof = True
break
elif lzs.avail_out == 0:
# Need to check lzs->avail_out before lzs->avail_in.
# Maybe lzs's internal state still have a few bytes
# can be output, grow the output buffer and continue
# if max_lengh < 0.
if data_size == max_length:
break
# ran out of space in the output buffer, let's grow it
bufsiz += (bufsiz >> 3) + 6
if max_length > 0 and bufsiz > max_length:
bufsiz = max_length
next_out = m.realloc(orig_out, bufsiz)
if next_out == ffi.NULL:
# realloc unsuccessful
m.free(orig_out)
orig_out = ffi.NULL
raise MemoryError
orig_out = next_out
lzs.next_out = orig_out + data_size
lzs.avail_out = bufsiz - data_size
elif lzs.avail_in == 0:
# it ate everything
break
result = ffi.buffer(orig_out, data_size)[:]
finally:
m.free(orig_out)
return result
def _addr(self, ptr):
return long(ffi.cast('uintptr_t', ptr))
def add_opts(options_type, *opts):
options = ffi.cast('%s*' % (options_type, ), filter.options)
for v in opts:
spec[v] = getattr(options, v)
