def _compress(self, data, action=m.LZMA_RUN): # TODO use realloc like in LZMADecompressor BUFSIZ = 8192 lzs = self.lzs lzs.next_in = input_ = ffi.new('uint8_t[]', to_bytes(data)) lzs.avail_in = input_len = len(data) outs = [ffi.new('uint8_t[]', BUFSIZ)] lzs.next_out, = outs lzs.avail_out = BUFSIZ siz = BUFSIZ while True: next_out_pos = int(ffi.cast('intptr_t', lzs.next_out)) ret = catch_lzma_error(m.lzma_code, lzs, action, ignore_buf_error=(input_len==0 and lzs.avail_out > 0)) data_size = int(ffi.cast('intptr_t', lzs.next_out)) - next_out_pos if (action == m.LZMA_RUN and lzs.avail_in == 0) or \ (action == m.LZMA_FINISH and ret == m.LZMA_STREAM_END): break elif lzs.avail_out == 0: # ran out of space in the output buffer #siz = (BUFSIZ << 1) + 6 siz = 512 outs.append(ffi.new('uint8_t[]', siz)) lzs.next_out = outs[-1] lzs.avail_out = siz last_out = outs.pop() last_out_len = siz - lzs.avail_out last_out_piece = ffi.buffer(last_out[0:last_out_len], last_out_len)[:] return b''.join(ffi.buffer(nn)[:] for nn in outs) + last_out_piece
def decompress(self, data, max_length=-1): """ decompress(data, max_length=-1) -> bytes Provide data to the decompressor object. Returns a chunk of decompressed data if possible, or b"" otherwise. Attempting to decompress data after the end of the stream is reached raises an EOFError. Any data found after the end of the stream is ignored, and saved in the unused_data attribute. """ if not isinstance(max_length, int): raise TypeError( "max_length parameter object cannot be interpreted as an integer" ) with self.lock: if self.eof: raise EOFError("Already at end of stream") lzs = self.lzs data = to_bytes(data) buf = ffi.new('uint8_t[]', data) buf_size = len(data) if lzs.next_in: buf, buf_size = self.pre_decompress_left_data(buf, buf_size) used__input_buffer = True else: lzs.avail_in = buf_size lzs.next_in = ffi.cast("uint8_t*", buf) used__input_buffer = False # actual decompression result = self._decompress(buf, buf_size, max_length) if self.eof: self.needs_input = False if lzs.avail_in > 0: self.unused_data = ffi.buffer(lzs.next_in, lzs.avail_in)[:] self.clear_input_buffer() elif lzs.avail_in == 0: # completed successfully! lzs.next_in = ffi.NULL if lzs.avail_out == 0: # (avail_in==0 && avail_out==0) # Maybe lzs's internal state still have a few bytes can # be output, try to output them next time. self.needs_input = False assert max_length >= 0 # if < 0, lzs.avail_out always > 0 else: # Input buffer exhausted, output buffer has space. self.needs_input = True self.clear_input_buffer() else: self.needs_input = False if not used__input_buffer: self.post_decompress_avail_data() return result
def _decompress(self, buf, buf_len, max_length): lzs = self.lzs lzs.next_in = buf lzs.avail_in = buf_len if buf_len == 0: return b"" bufsiz = self._bufsiz if not (max_length < 0 or max_length > io.DEFAULT_BUFFER_SIZE): bufsiz = max_length lzs.next_out = orig_out = m.malloc(bufsiz) if orig_out == ffi.NULL: raise MemoryError lzs.avail_out = bufsiz data_size = 0 try: while True: ret = catch_lzma_error(m.lzma_code, lzs, m.LZMA_RUN) data_size = int(ffi.cast('uintptr_t', lzs.next_out)) - int(ffi.cast('uintptr_t', orig_out)) # data_size is the amount lzma_code has already outputted if ret in (m.LZMA_NO_CHECK, m.LZMA_GET_CHECK): self.check = m.lzma_get_check(lzs) if ret == m.LZMA_STREAM_END: self.eof = True break elif lzs.avail_in == 0: # it ate everything break elif lzs.avail_out == 0: if data_size == max_length: break # ran out of space in the output buffer, let's grow it bufsiz += (bufsiz >> 3) + 6 next_out = m.realloc(orig_out, bufsiz) if next_out == ffi.NULL: # realloc unsuccessful m.free(orig_out) orig_out = ffi.NULL raise MemoryError orig_out = next_out lzs.next_out = orig_out + data_size lzs.avail_out = bufsiz - data_size result = ffi.buffer(orig_out, data_size)[:] finally: m.free(orig_out) return result
def _encode_filter_properties(filterspec): """_encode_filter_properties(filter) -> bytes Return a bytes object encoding the options (properties) of the filter specified by *filter* (a dict). The result does not include the filter ID itself, only the options.""" filter = parse_filter_spec(filterspec) size = ffi.new("uint32_t*") catch_lzma_error(m.lzma_properties_size, size, filter) result = ffi.new('uint8_t[]', size[0]) catch_lzma_error(m.lzma_properties_encode, filter, result) return ffi.buffer(result)[:]
def _decompress(self, buf, buf_len, max_length): lzs = self.lzs lzs.next_in = buf lzs.avail_in = buf_len bufsiz = self._bufsiz if not (max_length < 0 or max_length > io.DEFAULT_BUFFER_SIZE): bufsiz = max_length lzs.next_out = orig_out = m.malloc(bufsiz) if orig_out == ffi.NULL: raise MemoryError lzs.avail_out = bufsiz data_size = 0 try: while True: ret = catch_lzma_error( m.lzma_code, lzs, m.LZMA_RUN, ignore_buf_error=(lzs.avail_in == 0 and lzs.avail_out > 0)) data_size = int(ffi.cast('uintptr_t', lzs.next_out)) - int( ffi.cast('uintptr_t', orig_out)) # data_size is the amount lzma_code has already outputted if ret in (m.LZMA_NO_CHECK, m.LZMA_GET_CHECK): self.check = m.lzma_get_check(lzs) if ret == m.LZMA_STREAM_END: self.eof = True break elif lzs.avail_out == 0: # Need to check lzs->avail_out before lzs->avail_in. # Maybe lzs's internal state still have a few bytes # can be output, grow the output buffer and continue # if max_lengh < 0. if data_size == max_length: break # ran out of space in the output buffer, let's grow it bufsiz += (bufsiz >> 3) + 6 if max_length > 0 and bufsiz > max_length: bufsiz = max_length next_out = m.realloc(orig_out, bufsiz) if next_out == ffi.NULL: # realloc unsuccessful m.free(orig_out) orig_out = ffi.NULL raise MemoryError orig_out = next_out lzs.next_out = orig_out + data_size lzs.avail_out = bufsiz - data_size elif lzs.avail_in == 0: # it ate everything break result = ffi.buffer(orig_out, data_size)[:] finally: m.free(orig_out) return result