def _compress(self, data, action=m.LZMA_RUN):
# TODO use realloc like in LZMADecompressor
BUFSIZ = 8192
lzs = self.lzs
lzs.next_in = input_ = ffi.new('uint8_t[]', to_bytes(data))
lzs.avail_in = input_len = len(data)
outs = [ffi.new('uint8_t[]', BUFSIZ)]
lzs.next_out, = outs
lzs.avail_out = BUFSIZ
siz = BUFSIZ
while True:
next_out_pos = int(ffi.cast('intptr_t', lzs.next_out))
ret = catch_lzma_error(m.lzma_code, lzs, action,
ignore_buf_error=(input_len==0 and lzs.avail_out > 0))
data_size = int(ffi.cast('intptr_t', lzs.next_out)) - next_out_pos
if (action == m.LZMA_RUN and lzs.avail_in == 0) or \
(action == m.LZMA_FINISH and ret == m.LZMA_STREAM_END):
break
elif lzs.avail_out == 0:
# ran out of space in the output buffer
#siz = (BUFSIZ << 1) + 6
siz = 512
outs.append(ffi.new('uint8_t[]', siz))
lzs.next_out = outs[-1]
lzs.avail_out = siz
last_out = outs.pop()
last_out_len = siz - lzs.avail_out
last_out_piece = ffi.buffer(last_out[0:last_out_len], last_out_len)[:]
return b''.join(ffi.buffer(nn)[:] for nn in outs) + last_out_piece
def decompress(self, data, max_length=-1):
"""
decompress(data, max_length=-1) -> bytes
Provide data to the decompressor object. Returns a chunk of
decompressed data if possible, or b"" otherwise.
Attempting to decompress data after the end of the stream is
reached raises an EOFError. Any data found after the end of the
stream is ignored, and saved in the unused_data attribute.
"""
if not isinstance(max_length, int):
raise TypeError(
"max_length parameter object cannot be interpreted as an integer"
)
with self.lock:
if self.eof:
raise EOFError("Already at end of stream")
lzs = self.lzs
data = to_bytes(data)
buf = ffi.new('uint8_t[]', data)
buf_size = len(data)
if lzs.next_in:
buf, buf_size = self.pre_decompress_left_data(buf, buf_size)
used__input_buffer = True
else:
lzs.avail_in = buf_size
lzs.next_in = ffi.cast("uint8_t*", buf)
used__input_buffer = False
# actual decompression
result = self._decompress(buf, buf_size, max_length)
if self.eof:
self.needs_input = False
if lzs.avail_in > 0:
self.unused_data = ffi.buffer(lzs.next_in, lzs.avail_in)[:]
self.clear_input_buffer()
elif lzs.avail_in == 0:
# completed successfully!
lzs.next_in = ffi.NULL
if lzs.avail_out == 0:
# (avail_in==0 && avail_out==0)
# Maybe lzs's internal state still have a few bytes can
# be output, try to output them next time.
self.needs_input = False
assert max_length >= 0 # if < 0, lzs.avail_out always > 0
else:
# Input buffer exhausted, output buffer has space.
self.needs_input = True
self.clear_input_buffer()
else:
self.needs_input = False
if not used__input_buffer:
self.post_decompress_avail_data()
return result
def _decompress(self, buf, buf_len, max_length):
lzs = self.lzs
lzs.next_in = buf
lzs.avail_in = buf_len
if buf_len == 0:
return b""
bufsiz = self._bufsiz
if not (max_length < 0 or max_length > io.DEFAULT_BUFFER_SIZE):
bufsiz = max_length
lzs.next_out = orig_out = m.malloc(bufsiz)
if orig_out == ffi.NULL:
raise MemoryError
lzs.avail_out = bufsiz
data_size = 0
try:
while True:
ret = catch_lzma_error(m.lzma_code, lzs, m.LZMA_RUN)
data_size = int(ffi.cast('uintptr_t', lzs.next_out)) - int(ffi.cast('uintptr_t', orig_out))
# data_size is the amount lzma_code has already outputted
if ret in (m.LZMA_NO_CHECK, m.LZMA_GET_CHECK):
self.check = m.lzma_get_check(lzs)
if ret == m.LZMA_STREAM_END:
self.eof = True
break
elif lzs.avail_in == 0:
# it ate everything
break
elif lzs.avail_out == 0:
if data_size == max_length:
break
# ran out of space in the output buffer, let's grow it
bufsiz += (bufsiz >> 3) + 6
next_out = m.realloc(orig_out, bufsiz)
if next_out == ffi.NULL:
# realloc unsuccessful
m.free(orig_out)
orig_out = ffi.NULL
raise MemoryError
orig_out = next_out
lzs.next_out = orig_out + data_size
lzs.avail_out = bufsiz - data_size
result = ffi.buffer(orig_out, data_size)[:]
finally:
m.free(orig_out)
return result
def _encode_filter_properties(filterspec):
"""_encode_filter_properties(filter) -> bytes
Return a bytes object encoding the options (properties) of the filter
specified by *filter* (a dict).
The result does not include the filter ID itself, only the options."""
filter = parse_filter_spec(filterspec)
size = ffi.new("uint32_t*")
catch_lzma_error(m.lzma_properties_size, size, filter)
result = ffi.new('uint8_t[]', size[0])
catch_lzma_error(m.lzma_properties_encode, filter, result)
return ffi.buffer(result)[:]
def _decompress(self, buf, buf_len, max_length):
lzs = self.lzs
lzs.next_in = buf
lzs.avail_in = buf_len
bufsiz = self._bufsiz
if not (max_length < 0 or max_length > io.DEFAULT_BUFFER_SIZE):
bufsiz = max_length
lzs.next_out = orig_out = m.malloc(bufsiz)
if orig_out == ffi.NULL:
raise MemoryError
lzs.avail_out = bufsiz
data_size = 0
try:
while True:
ret = catch_lzma_error(
m.lzma_code,
lzs,
m.LZMA_RUN,
ignore_buf_error=(lzs.avail_in == 0 and lzs.avail_out > 0))
data_size = int(ffi.cast('uintptr_t', lzs.next_out)) - int(
ffi.cast('uintptr_t', orig_out))
# data_size is the amount lzma_code has already outputted
if ret in (m.LZMA_NO_CHECK, m.LZMA_GET_CHECK):
self.check = m.lzma_get_check(lzs)
if ret == m.LZMA_STREAM_END:
self.eof = True
break
elif lzs.avail_out == 0:
# Need to check lzs->avail_out before lzs->avail_in.
# Maybe lzs's internal state still have a few bytes
# can be output, grow the output buffer and continue
# if max_lengh < 0.
if data_size == max_length:
break
# ran out of space in the output buffer, let's grow it
bufsiz += (bufsiz >> 3) + 6
if max_length > 0 and bufsiz > max_length:
bufsiz = max_length
next_out = m.realloc(orig_out, bufsiz)
if next_out == ffi.NULL:
# realloc unsuccessful
m.free(orig_out)
orig_out = ffi.NULL
raise MemoryError
orig_out = next_out
lzs.next_out = orig_out + data_size
lzs.avail_out = bufsiz - data_size
elif lzs.avail_in == 0:
# it ate everything
break
result = ffi.buffer(orig_out, data_size)[:]
finally:
m.free(orig_out)
return result
