def read(self, size=-1):
# XXX CPython uses a more delicate logic here
self._check_closed()
ll_file = self._ll_file
if size == 0:
return ""
elif size < 0:
# read the entire contents
buf = lltype.malloc(rffi.CCHARP.TO, BASE_BUF_SIZE, flavor='raw')
try:
s = StringBuilder()
while True:
returned_size = self._fread(buf, BASE_BUF_SIZE, ll_file)
returned_size = intmask(returned_size) # is between 0 and BASE_BUF_SIZE
if returned_size == 0:
if c_feof(ll_file):
# ok, finished
return s.build()
raise _error(ll_file)
s.append_charpsize(buf, returned_size)
finally:
lltype.free(buf, flavor='raw')
else: # size > 0
with rffi.scoped_alloc_buffer(size) as buf:
returned_size = self._fread(buf.raw, size, ll_file)
returned_size = intmask(returned_size) # is between 0 and size
if returned_size == 0:
if not c_feof(ll_file):
raise _error(ll_file)
s = buf.str(returned_size)
assert s is not None
return s
def _operate(stream, data, flush, max_length, cfunc, while_doing):
"""Common code for compress() and decompress().
"""
# Prepare the input buffer for the stream
with lltype.scoped_alloc(rffi.CCHARP.TO, len(data)) as inbuf:
for i in xrange(len(data)):
inbuf[i] = data[i]
stream.c_next_in = rffi.cast(Bytefp, inbuf)
rffi.setintfield(stream, 'c_avail_in', len(data))
# Prepare the output buffer
with lltype.scoped_alloc(rffi.CCHARP.TO, OUTPUT_BUFFER_SIZE) as outbuf:
# Strategy: we call deflate() to get as much output data as fits in
# the buffer, then accumulate all output into a StringBuffer
# 'result'.
result = StringBuilder()
while True:
stream.c_next_out = rffi.cast(Bytefp, outbuf)
bufsize = OUTPUT_BUFFER_SIZE
if max_length < bufsize:
if max_length <= 0:
err = Z_OK
break
bufsize = max_length
max_length -= bufsize
rffi.setintfield(stream, 'c_avail_out', bufsize)
err = cfunc(stream, flush)
if err == Z_OK or err == Z_STREAM_END:
# accumulate data into 'result'
avail_out = rffi.cast(lltype.Signed, stream.c_avail_out)
result.append_charpsize(outbuf, bufsize - avail_out)
# if the output buffer is full, there might be more data
# so we need to try again. Otherwise, we're done.
if avail_out > 0:
break
# We're also done if we got a Z_STREAM_END (which should
# only occur when flush == Z_FINISH).
if err == Z_STREAM_END:
break
else:
continue
elif err == Z_BUF_ERROR:
avail_out = rffi.cast(lltype.Signed, stream.c_avail_out)
# When compressing, we will only get Z_BUF_ERROR if
# the output buffer was full but there wasn't more
# output when we tried again, so it is not an error
# condition.
if avail_out == bufsize:
break
# fallback case: report this error
raise RZlibError.fromstream(stream, err, while_doing)
# When decompressing, if the compressed stream of data was truncated,
# then the zlib simply returns Z_OK and waits for more. If it is
# complete it returns Z_STREAM_END.
return (result.build(),
err,
rffi.cast(lltype.Signed, stream.c_avail_in))
def read(self, size=-1):
# XXX CPython uses a more delicate logic here
ll_file = self.ll_file
if not ll_file:
raise ValueError("I/O operation on closed file")
if size < 0:
# read the entire contents
buf = lltype.malloc(rffi.CCHARP.TO, BASE_BUF_SIZE, flavor='raw')
try:
s = StringBuilder()
while True:
returned_size = c_fread(buf, 1, BASE_BUF_SIZE, ll_file)
returned_size = intmask(returned_size) # is between 0 and BASE_BUF_SIZE
if returned_size == 0:
if c_feof(ll_file):
# ok, finished
return s.build()
raise _error(ll_file)
s.append_charpsize(buf, returned_size)
finally:
lltype.free(buf, flavor='raw')
else:
raw_buf, gc_buf = rffi.alloc_buffer(size)
try:
returned_size = c_fread(raw_buf, 1, size, ll_file)
returned_size = intmask(returned_size) # is between 0 and size
if returned_size == 0:
if not c_feof(ll_file):
raise _error(ll_file)
s = rffi.str_from_buffer(raw_buf, gc_buf, size, returned_size)
finally:
rffi.keep_buffer_alive_until_here(raw_buf, gc_buf)
return s
def read(self, size=-1):
# XXX CPython uses a more delicate logic here
self._check_closed()
ll_file = self._ll_file
if size == 0:
return ""
elif size < 0:
# read the entire contents
buf = lltype.malloc(rffi.CCHARP.TO, BASE_BUF_SIZE, flavor='raw')
try:
s = StringBuilder()
while True:
returned_size = self._fread(buf, BASE_BUF_SIZE, ll_file)
returned_size = intmask(returned_size) # is between 0 and BASE_BUF_SIZE
if returned_size == 0:
if c_feof(ll_file):
# ok, finished
return s.build()
raise _error(ll_file)
s.append_charpsize(buf, returned_size)
finally:
lltype.free(buf, flavor='raw')
else: # size > 0
with rffi.scoped_alloc_buffer(size) as buf:
returned_size = self._fread(buf.raw, size, ll_file)
returned_size = intmask(returned_size) # is between 0 and size
if returned_size == 0:
if not c_feof(ll_file):
raise _error(ll_file)
s = buf.str(returned_size)
assert s is not None
return s
def test_deflate_set_dictionary():
text = 'abcabc'
zdict = 'abc'
stream = rzlib.deflateInit()
rzlib.deflateSetDictionary(stream, zdict)
bytes = rzlib.compress(stream, text, rzlib.Z_FINISH)
rzlib.deflateEnd(stream)
stream2 = rzlib.inflateInit()
from rpython.rtyper.lltypesystem import lltype, rffi, rstr
from rpython.rtyper.annlowlevel import llstr
from rpython.rlib.rstring import StringBuilder
with lltype.scoped_alloc(rffi.CCHARP.TO, len(bytes)) as inbuf:
rstr.copy_string_to_raw(llstr(bytes), inbuf, 0, len(bytes))
stream2.c_next_in = rffi.cast(rzlib.Bytefp, inbuf)
rffi.setintfield(stream2, 'c_avail_in', len(bytes))
with lltype.scoped_alloc(rffi.CCHARP.TO, 100) as outbuf:
stream2.c_next_out = rffi.cast(rzlib.Bytefp, outbuf)
bufsize = 100
rffi.setintfield(stream2, 'c_avail_out', bufsize)
err = rzlib._inflate(stream2, rzlib.Z_SYNC_FLUSH)
assert err == rzlib.Z_NEED_DICT
rzlib.inflateSetDictionary(stream2, zdict)
rzlib._inflate(stream2, rzlib.Z_SYNC_FLUSH)
avail_out = rffi.cast(lltype.Signed, stream2.c_avail_out)
result = StringBuilder()
result.append_charpsize(outbuf, bufsize - avail_out)
rzlib.inflateEnd(stream2)
assert result.build() == text
def test_deflate_set_dictionary():
text = 'abcabc'
zdict = 'abc'
stream = rzlib.deflateInit()
rzlib.deflateSetDictionary(stream, zdict)
bytes = rzlib.compress(stream, text, rzlib.Z_FINISH)
rzlib.deflateEnd(stream)
stream2 = rzlib.inflateInit()
from rpython.rtyper.lltypesystem import lltype, rffi, rstr
from rpython.rtyper.annlowlevel import llstr
from rpython.rlib.rstring import StringBuilder
with lltype.scoped_alloc(rffi.CCHARP.TO, len(bytes)) as inbuf:
rstr.copy_string_to_raw(llstr(bytes), inbuf, 0, len(bytes))
stream2.c_next_in = rffi.cast(rzlib.Bytefp, inbuf)
rffi.setintfield(stream2, 'c_avail_in', len(bytes))
with lltype.scoped_alloc(rffi.CCHARP.TO, 100) as outbuf:
stream2.c_next_out = rffi.cast(rzlib.Bytefp, outbuf)
bufsize = 100
rffi.setintfield(stream2, 'c_avail_out', bufsize)
err = rzlib._inflate(stream2, rzlib.Z_SYNC_FLUSH)
assert err == rzlib.Z_NEED_DICT
rzlib.inflateSetDictionary(stream2, zdict)
rzlib._inflate(stream2, rzlib.Z_SYNC_FLUSH)
avail_out = rffi.cast(lltype.Signed, stream2.c_avail_out)
result = StringBuilder()
result.append_charpsize(outbuf, bufsize - avail_out)
rzlib.inflateEnd(stream2)
assert result.build() == text
def _operate(stream, data, flush, max_length, cfunc, while_doing):
"""Common code for compress() and decompress().
"""
# Prepare the input buffer for the stream
with lltype.scoped_alloc(rffi.CCHARP.TO, len(data)) as inbuf:
# XXX (groggi) should be possible to improve this with pinning by
# not performing the 'copy_string_to_raw' if non-movable/pinned
copy_string_to_raw(llstr(data), inbuf, 0, len(data))
stream.c_next_in = rffi.cast(Bytefp, inbuf)
rffi.setintfield(stream, 'c_avail_in', len(data))
# Prepare the output buffer
with lltype.scoped_alloc(rffi.CCHARP.TO, OUTPUT_BUFFER_SIZE) as outbuf:
# Strategy: we call deflate() to get as much output data as fits in
# the buffer, then accumulate all output into a StringBuffer
# 'result'.
result = StringBuilder()
while True:
stream.c_next_out = rffi.cast(Bytefp, outbuf)
bufsize = OUTPUT_BUFFER_SIZE
if max_length < bufsize:
if max_length <= 0:
err = Z_OK
break
bufsize = max_length
max_length -= bufsize
rffi.setintfield(stream, 'c_avail_out', bufsize)
err = cfunc(stream, flush)
if err == Z_OK or err == Z_STREAM_END:
# accumulate data into 'result'
avail_out = rffi.cast(lltype.Signed, stream.c_avail_out)
result.append_charpsize(outbuf, bufsize - avail_out)
# if the output buffer is full, there might be more data
# so we need to try again. Otherwise, we're done.
if avail_out > 0:
break
# We're also done if we got a Z_STREAM_END (which should
# only occur when flush == Z_FINISH).
if err == Z_STREAM_END:
break
else:
continue
elif err == Z_BUF_ERROR:
avail_out = rffi.cast(lltype.Signed, stream.c_avail_out)
# When compressing, we will only get Z_BUF_ERROR if
# the output buffer was full but there wasn't more
# output when we tried again, so it is not an error
# condition.
if avail_out == bufsize:
break
# fallback case: report this error
raise RZlibError.fromstream(stream, err, while_doing)
# When decompressing, if the compressed stream of data was truncated,
# then the zlib simply returns Z_OK and waits for more. If it is
# complete it returns Z_STREAM_END.
return (result.build(), err, rffi.cast(lltype.Signed, stream.c_avail_in))
def raw_str(self):
value = lltype.malloc(rffi.CArray(lltype.typeOf(self.value)), 1, flavor="raw")
value[0] = self.value
builder = StringBuilder()
builder.append_charpsize(rffi.cast(rffi.CCHARP, value), rffi.sizeof(lltype.typeOf(self.value)))
ret = builder.build()
lltype.free(value, flavor="raw")
return ret
def raw_str(self):
value = lltype.malloc(rffi.CArray(lltype.typeOf(self.value)),
1,
flavor="raw")
value[0] = self.value
builder = StringBuilder()
builder.append_charpsize(rffi.cast(rffi.CCHARP, value),
rffi.sizeof(lltype.typeOf(self.value)))
ret = builder.build()
lltype.free(value, flavor="raw")
return ret
def readline(self):
if self.ll_file:
raw_buf, gc_buf = rffi.alloc_buffer(BASE_LINE_SIZE)
try:
c = self._readline1(raw_buf)
if c >= 0:
return rffi.str_from_buffer(raw_buf, gc_buf,
BASE_LINE_SIZE, c)
#
# this is the rare case: the line is longer than BASE_LINE_SIZE
s = StringBuilder()
while True:
s.append_charpsize(raw_buf, BASE_LINE_SIZE - 1)
c = self._readline1(raw_buf)
if c >= 0:
break
#
s.append_charpsize(raw_buf, c)
return s.build()
finally:
rffi.keep_buffer_alive_until_here(raw_buf, gc_buf)
raise ValueError("I/O operation on closed file")
def readline(self, size=-1):
self._check_closed()
if size == 0:
return ""
elif size < 0 and not self._univ_newline:
with rffi.scoped_alloc_buffer(BASE_LINE_SIZE) as buf:
c = self._readline1(buf.raw)
if c >= 0:
return buf.str(c)
# this is the rare case: the line is longer than BASE_LINE_SIZE
s = StringBuilder()
while True:
s.append_charpsize(buf.raw, BASE_LINE_SIZE - 1)
c = self._readline1(buf.raw)
if c >= 0:
break
s.append_charpsize(buf.raw, c)
return s.build()
else: # size > 0 or self._univ_newline
ll_file = self._ll_file
c = 0
s = StringBuilder()
if self._univ_newline:
newlinetypes = self._newlinetypes
skipnextlf = self._skipnextlf
while size < 0 or s.getlength() < size:
c = c_getc(ll_file)
if c == EOF:
break
if skipnextlf:
skipnextlf = False
if c == ord('\n'):
newlinetypes |= NEWLINE_CRLF
c = c_getc(ll_file)
if c == EOF:
break
else:
newlinetypes |= NEWLINE_CR
if c == ord('\r'):
skipnextlf = True
c = ord('\n')
elif c == ord('\n'):
newlinetypes |= NEWLINE_LF
s.append(chr(c))
if c == ord('\n'):
break
if c == EOF:
if skipnextlf:
newlinetypes |= NEWLINE_CR
self._newlinetypes = newlinetypes
self._skipnextlf = skipnextlf
else:
while s.getlength() < size:
c = c_getc(ll_file)
if c == EOF:
break
s.append(chr(c))
if c == ord('\n'):
break
if c == EOF:
if c_ferror(ll_file):
raise _error(ll_file)
return s.build()
def func(l):
s = StringBuilder()
with rffi.scoped_str2charp("hello world") as x:
s.append_charpsize(x, l)
return s.build()
def func(l):
s = StringBuilder()
with rffi.scoped_str2charp("hello world") as x:
s.append_charpsize(x, l)
return s.build()
def _operate(stream, data, flush, max_length, cfunc, while_doing, zdict=None):
"""Common code for compress() and decompress().
"""
# Prepare the input buffer for the stream
assert data is not None
with rffi.scoped_nonmovingbuffer(data) as inbuf:
stream.c_next_in = rffi.cast(Bytefp, inbuf)
end_inbuf = rffi.ptradd(stream.c_next_in, len(data))
# Prepare the output buffer
with lltype.scoped_alloc(rffi.CCHARP.TO, OUTPUT_BUFFER_SIZE) as outbuf:
# Strategy: we call deflate() to get as much output data as fits in
# the buffer, then accumulate all output into a StringBuffer
# 'result'.
result = StringBuilder()
while True:
avail_in = ptrdiff(end_inbuf, stream.c_next_in)
if avail_in > INPUT_BUFFER_MAX:
avail_in = INPUT_BUFFER_MAX
rffi.setintfield(stream, 'c_avail_in', avail_in)
stream.c_next_out = rffi.cast(Bytefp, outbuf)
bufsize = OUTPUT_BUFFER_SIZE
if max_length < bufsize:
if max_length <= 0:
err = Z_OK
break
bufsize = max_length
max_length -= bufsize
rffi.setintfield(stream, 'c_avail_out', bufsize)
err = cfunc(stream, flush)
if err == Z_NEED_DICT and zdict is not None:
inflateSetDictionary(stream, zdict)
# repeat the call to inflate
err = cfunc(stream, flush)
if err == Z_OK or err == Z_STREAM_END:
# accumulate data into 'result'
avail_out = rffi.cast(lltype.Signed, stream.c_avail_out)
result.append_charpsize(outbuf, bufsize - avail_out)
# if the output buffer is full, there might be more data
# so we need to try again. Otherwise, we're done.
if avail_out > 0:
break
# We're also done if we got a Z_STREAM_END (which should
# only occur when flush == Z_FINISH).
if err == Z_STREAM_END:
break
else:
continue
elif err == Z_BUF_ERROR:
avail_out = rffi.cast(lltype.Signed, stream.c_avail_out)
# When compressing, we will only get Z_BUF_ERROR if
# the output buffer was full but there wasn't more
# output when we tried again, so it is not an error
# condition.
if avail_out == bufsize:
break
# fallback case: report this error
raise RZlibError.fromstream(stream, err, while_doing)
# When decompressing, if the compressed stream of data was truncated,
# then the zlib simply returns Z_OK and waits for more. If it is
# complete it returns Z_STREAM_END.
avail_in = ptrdiff(end_inbuf, stream.c_next_in)
return (result.build(), err, avail_in)
def readline(self, size=-1):
self._check_closed()
if size == 0:
return ""
elif size < 0 and not self._univ_newline:
with rffi.scoped_alloc_buffer(BASE_LINE_SIZE) as buf:
c = self._readline1(buf.raw)
if c >= 0:
return buf.str(c)
# this is the rare case: the line is longer than BASE_LINE_SIZE
s = StringBuilder()
while True:
s.append_charpsize(buf.raw, BASE_LINE_SIZE - 1)
c = self._readline1(buf.raw)
if c >= 0:
break
s.append_charpsize(buf.raw, c)
return s.build()
else: # size > 0 or self._univ_newline
ll_file = self._ll_file
c = 0
s = StringBuilder()
if self._univ_newline:
newlinetypes = self._newlinetypes
skipnextlf = self._skipnextlf
while size < 0 or s.getlength() < size:
c = c_getc(ll_file)
if c == EOF:
break
if skipnextlf:
skipnextlf = False
if c == ord('\n'):
newlinetypes |= NEWLINE_CRLF
c = c_getc(ll_file)
if c == EOF:
break
else:
newlinetypes |= NEWLINE_CR
if c == ord('\r'):
skipnextlf = True
c = ord('\n')
elif c == ord('\n'):
newlinetypes |= NEWLINE_LF
s.append(chr(c))
if c == ord('\n'):
break
if c == EOF:
if skipnextlf:
newlinetypes |= NEWLINE_CR
self._newlinetypes = newlinetypes
self._skipnextlf = skipnextlf
else:
while s.getlength() < size:
c = c_getc(ll_file)
if c == EOF:
break
s.append(chr(c))
if c == ord('\n'):
break
if c == EOF:
if c_ferror(ll_file):
raise _error(ll_file)
return s.build()
