def read(self, size=-1): # XXX CPython uses a more delicate logic here self._check_closed() ll_file = self._ll_file if size == 0: return "" elif size < 0: # read the entire contents buf = lltype.malloc(rffi.CCHARP.TO, BASE_BUF_SIZE, flavor='raw') try: s = StringBuilder() while True: returned_size = self._fread(buf, BASE_BUF_SIZE, ll_file) returned_size = intmask(returned_size) # is between 0 and BASE_BUF_SIZE if returned_size == 0: if c_feof(ll_file): # ok, finished return s.build() raise _error(ll_file) s.append_charpsize(buf, returned_size) finally: lltype.free(buf, flavor='raw') else: # size > 0 with rffi.scoped_alloc_buffer(size) as buf: returned_size = self._fread(buf.raw, size, ll_file) returned_size = intmask(returned_size) # is between 0 and size if returned_size == 0: if not c_feof(ll_file): raise _error(ll_file) s = buf.str(returned_size) assert s is not None return s
def _operate(stream, data, flush, max_length, cfunc, while_doing): """Common code for compress() and decompress(). """ # Prepare the input buffer for the stream with lltype.scoped_alloc(rffi.CCHARP.TO, len(data)) as inbuf: for i in xrange(len(data)): inbuf[i] = data[i] stream.c_next_in = rffi.cast(Bytefp, inbuf) rffi.setintfield(stream, 'c_avail_in', len(data)) # Prepare the output buffer with lltype.scoped_alloc(rffi.CCHARP.TO, OUTPUT_BUFFER_SIZE) as outbuf: # Strategy: we call deflate() to get as much output data as fits in # the buffer, then accumulate all output into a StringBuffer # 'result'. result = StringBuilder() while True: stream.c_next_out = rffi.cast(Bytefp, outbuf) bufsize = OUTPUT_BUFFER_SIZE if max_length < bufsize: if max_length <= 0: err = Z_OK break bufsize = max_length max_length -= bufsize rffi.setintfield(stream, 'c_avail_out', bufsize) err = cfunc(stream, flush) if err == Z_OK or err == Z_STREAM_END: # accumulate data into 'result' avail_out = rffi.cast(lltype.Signed, stream.c_avail_out) result.append_charpsize(outbuf, bufsize - avail_out) # if the output buffer is full, there might be more data # so we need to try again. Otherwise, we're done. if avail_out > 0: break # We're also done if we got a Z_STREAM_END (which should # only occur when flush == Z_FINISH). if err == Z_STREAM_END: break else: continue elif err == Z_BUF_ERROR: avail_out = rffi.cast(lltype.Signed, stream.c_avail_out) # When compressing, we will only get Z_BUF_ERROR if # the output buffer was full but there wasn't more # output when we tried again, so it is not an error # condition. if avail_out == bufsize: break # fallback case: report this error raise RZlibError.fromstream(stream, err, while_doing) # When decompressing, if the compressed stream of data was truncated, # then the zlib simply returns Z_OK and waits for more. If it is # complete it returns Z_STREAM_END. return (result.build(), err, rffi.cast(lltype.Signed, stream.c_avail_in))
def read(self, size=-1): # XXX CPython uses a more delicate logic here ll_file = self.ll_file if not ll_file: raise ValueError("I/O operation on closed file") if size < 0: # read the entire contents buf = lltype.malloc(rffi.CCHARP.TO, BASE_BUF_SIZE, flavor='raw') try: s = StringBuilder() while True: returned_size = c_fread(buf, 1, BASE_BUF_SIZE, ll_file) returned_size = intmask(returned_size) # is between 0 and BASE_BUF_SIZE if returned_size == 0: if c_feof(ll_file): # ok, finished return s.build() raise _error(ll_file) s.append_charpsize(buf, returned_size) finally: lltype.free(buf, flavor='raw') else: raw_buf, gc_buf = rffi.alloc_buffer(size) try: returned_size = c_fread(raw_buf, 1, size, ll_file) returned_size = intmask(returned_size) # is between 0 and size if returned_size == 0: if not c_feof(ll_file): raise _error(ll_file) s = rffi.str_from_buffer(raw_buf, gc_buf, size, returned_size) finally: rffi.keep_buffer_alive_until_here(raw_buf, gc_buf) return s
def test_deflate_set_dictionary(): text = 'abcabc' zdict = 'abc' stream = rzlib.deflateInit() rzlib.deflateSetDictionary(stream, zdict) bytes = rzlib.compress(stream, text, rzlib.Z_FINISH) rzlib.deflateEnd(stream) stream2 = rzlib.inflateInit() from rpython.rtyper.lltypesystem import lltype, rffi, rstr from rpython.rtyper.annlowlevel import llstr from rpython.rlib.rstring import StringBuilder with lltype.scoped_alloc(rffi.CCHARP.TO, len(bytes)) as inbuf: rstr.copy_string_to_raw(llstr(bytes), inbuf, 0, len(bytes)) stream2.c_next_in = rffi.cast(rzlib.Bytefp, inbuf) rffi.setintfield(stream2, 'c_avail_in', len(bytes)) with lltype.scoped_alloc(rffi.CCHARP.TO, 100) as outbuf: stream2.c_next_out = rffi.cast(rzlib.Bytefp, outbuf) bufsize = 100 rffi.setintfield(stream2, 'c_avail_out', bufsize) err = rzlib._inflate(stream2, rzlib.Z_SYNC_FLUSH) assert err == rzlib.Z_NEED_DICT rzlib.inflateSetDictionary(stream2, zdict) rzlib._inflate(stream2, rzlib.Z_SYNC_FLUSH) avail_out = rffi.cast(lltype.Signed, stream2.c_avail_out) result = StringBuilder() result.append_charpsize(outbuf, bufsize - avail_out) rzlib.inflateEnd(stream2) assert result.build() == text
def _operate(stream, data, flush, max_length, cfunc, while_doing): """Common code for compress() and decompress(). """ # Prepare the input buffer for the stream with lltype.scoped_alloc(rffi.CCHARP.TO, len(data)) as inbuf: # XXX (groggi) should be possible to improve this with pinning by # not performing the 'copy_string_to_raw' if non-movable/pinned copy_string_to_raw(llstr(data), inbuf, 0, len(data)) stream.c_next_in = rffi.cast(Bytefp, inbuf) rffi.setintfield(stream, 'c_avail_in', len(data)) # Prepare the output buffer with lltype.scoped_alloc(rffi.CCHARP.TO, OUTPUT_BUFFER_SIZE) as outbuf: # Strategy: we call deflate() to get as much output data as fits in # the buffer, then accumulate all output into a StringBuffer # 'result'. result = StringBuilder() while True: stream.c_next_out = rffi.cast(Bytefp, outbuf) bufsize = OUTPUT_BUFFER_SIZE if max_length < bufsize: if max_length <= 0: err = Z_OK break bufsize = max_length max_length -= bufsize rffi.setintfield(stream, 'c_avail_out', bufsize) err = cfunc(stream, flush) if err == Z_OK or err == Z_STREAM_END: # accumulate data into 'result' avail_out = rffi.cast(lltype.Signed, stream.c_avail_out) result.append_charpsize(outbuf, bufsize - avail_out) # if the output buffer is full, there might be more data # so we need to try again. Otherwise, we're done. if avail_out > 0: break # We're also done if we got a Z_STREAM_END (which should # only occur when flush == Z_FINISH). if err == Z_STREAM_END: break else: continue elif err == Z_BUF_ERROR: avail_out = rffi.cast(lltype.Signed, stream.c_avail_out) # When compressing, we will only get Z_BUF_ERROR if # the output buffer was full but there wasn't more # output when we tried again, so it is not an error # condition. if avail_out == bufsize: break # fallback case: report this error raise RZlibError.fromstream(stream, err, while_doing) # When decompressing, if the compressed stream of data was truncated, # then the zlib simply returns Z_OK and waits for more. If it is # complete it returns Z_STREAM_END. return (result.build(), err, rffi.cast(lltype.Signed, stream.c_avail_in))
def raw_str(self): value = lltype.malloc(rffi.CArray(lltype.typeOf(self.value)), 1, flavor="raw") value[0] = self.value builder = StringBuilder() builder.append_charpsize(rffi.cast(rffi.CCHARP, value), rffi.sizeof(lltype.typeOf(self.value))) ret = builder.build() lltype.free(value, flavor="raw") return ret
def readline(self): if self.ll_file: raw_buf, gc_buf = rffi.alloc_buffer(BASE_LINE_SIZE) try: c = self._readline1(raw_buf) if c >= 0: return rffi.str_from_buffer(raw_buf, gc_buf, BASE_LINE_SIZE, c) # # this is the rare case: the line is longer than BASE_LINE_SIZE s = StringBuilder() while True: s.append_charpsize(raw_buf, BASE_LINE_SIZE - 1) c = self._readline1(raw_buf) if c >= 0: break # s.append_charpsize(raw_buf, c) return s.build() finally: rffi.keep_buffer_alive_until_here(raw_buf, gc_buf) raise ValueError("I/O operation on closed file")
def readline(self, size=-1): self._check_closed() if size == 0: return "" elif size < 0 and not self._univ_newline: with rffi.scoped_alloc_buffer(BASE_LINE_SIZE) as buf: c = self._readline1(buf.raw) if c >= 0: return buf.str(c) # this is the rare case: the line is longer than BASE_LINE_SIZE s = StringBuilder() while True: s.append_charpsize(buf.raw, BASE_LINE_SIZE - 1) c = self._readline1(buf.raw) if c >= 0: break s.append_charpsize(buf.raw, c) return s.build() else: # size > 0 or self._univ_newline ll_file = self._ll_file c = 0 s = StringBuilder() if self._univ_newline: newlinetypes = self._newlinetypes skipnextlf = self._skipnextlf while size < 0 or s.getlength() < size: c = c_getc(ll_file) if c == EOF: break if skipnextlf: skipnextlf = False if c == ord('\n'): newlinetypes |= NEWLINE_CRLF c = c_getc(ll_file) if c == EOF: break else: newlinetypes |= NEWLINE_CR if c == ord('\r'): skipnextlf = True c = ord('\n') elif c == ord('\n'): newlinetypes |= NEWLINE_LF s.append(chr(c)) if c == ord('\n'): break if c == EOF: if skipnextlf: newlinetypes |= NEWLINE_CR self._newlinetypes = newlinetypes self._skipnextlf = skipnextlf else: while s.getlength() < size: c = c_getc(ll_file) if c == EOF: break s.append(chr(c)) if c == ord('\n'): break if c == EOF: if c_ferror(ll_file): raise _error(ll_file) return s.build()
def func(l): s = StringBuilder() with rffi.scoped_str2charp("hello world") as x: s.append_charpsize(x, l) return s.build()
def _operate(stream, data, flush, max_length, cfunc, while_doing, zdict=None): """Common code for compress() and decompress(). """ # Prepare the input buffer for the stream assert data is not None with rffi.scoped_nonmovingbuffer(data) as inbuf: stream.c_next_in = rffi.cast(Bytefp, inbuf) end_inbuf = rffi.ptradd(stream.c_next_in, len(data)) # Prepare the output buffer with lltype.scoped_alloc(rffi.CCHARP.TO, OUTPUT_BUFFER_SIZE) as outbuf: # Strategy: we call deflate() to get as much output data as fits in # the buffer, then accumulate all output into a StringBuffer # 'result'. result = StringBuilder() while True: avail_in = ptrdiff(end_inbuf, stream.c_next_in) if avail_in > INPUT_BUFFER_MAX: avail_in = INPUT_BUFFER_MAX rffi.setintfield(stream, 'c_avail_in', avail_in) stream.c_next_out = rffi.cast(Bytefp, outbuf) bufsize = OUTPUT_BUFFER_SIZE if max_length < bufsize: if max_length <= 0: err = Z_OK break bufsize = max_length max_length -= bufsize rffi.setintfield(stream, 'c_avail_out', bufsize) err = cfunc(stream, flush) if err == Z_NEED_DICT and zdict is not None: inflateSetDictionary(stream, zdict) # repeat the call to inflate err = cfunc(stream, flush) if err == Z_OK or err == Z_STREAM_END: # accumulate data into 'result' avail_out = rffi.cast(lltype.Signed, stream.c_avail_out) result.append_charpsize(outbuf, bufsize - avail_out) # if the output buffer is full, there might be more data # so we need to try again. Otherwise, we're done. if avail_out > 0: break # We're also done if we got a Z_STREAM_END (which should # only occur when flush == Z_FINISH). if err == Z_STREAM_END: break else: continue elif err == Z_BUF_ERROR: avail_out = rffi.cast(lltype.Signed, stream.c_avail_out) # When compressing, we will only get Z_BUF_ERROR if # the output buffer was full but there wasn't more # output when we tried again, so it is not an error # condition. if avail_out == bufsize: break # fallback case: report this error raise RZlibError.fromstream(stream, err, while_doing) # When decompressing, if the compressed stream of data was truncated, # then the zlib simply returns Z_OK and waits for more. If it is # complete it returns Z_STREAM_END. avail_in = ptrdiff(end_inbuf, stream.c_next_in) return (result.build(), err, avail_in)