def direct_readline(self, size=-1):
stream = self.getstream()
self.check_readable()
if size < 0:
return stream.readline()
else:
# very inefficient unless there is a peek()
result = StringBuilder()
while size > 0:
# "peeks" on the underlying stream to see how many chars
# we can safely read without reading past an end-of-line
startindex, peeked = stream.peek()
assert 0 <= startindex <= len(peeked)
endindex = startindex + size
pn = peeked.find("\n", startindex, endindex)
if pn < 0:
pn = min(endindex - 1, len(peeked))
c = stream.read(pn - startindex + 1)
if not c:
break
result.append(c)
if c.endswith('\n'):
break
size -= len(c)
return result.build()
def unwrap(self):
# note: always overriden so far
length = self.strlen()
builder = StringBuilder(length)
for i in range(length):
builder.append(self.character(i))
return builder.build()
def _charp2str_to_null(cp, index):
index = rffi.cast(lltype.Signed, index)
string = StringBuilder()
while cp[index] != '\x00':
string.append(cp[index])
index += 1
return string.build()
def readall_w(self, space):
self._check_closed(space)
self._check_readable(space)
total = 0
builder = StringBuilder()
while True:
newsize = int(new_buffersize(self.fd, total))
try:
chunk = os.read(self.fd, newsize - total)
except OSError, e:
if e.errno == errno.EINTR:
space.getexecutioncontext().checksignals()
continue
if total > 0:
# return what we've got so far
break
if e.errno == errno.EAGAIN:
return space.w_None
raise wrap_oserror(space, e,
exception_name='w_IOError')
if not chunk:
break
builder.append(chunk)
total += len(chunk)
def serialize(self, w_obj):
from hippy.module.serialize import SerializerMemo
assert not isinstance(w_obj, W_Reference)
builder = StringBuilder()
w_obj.serialize(self, builder, SerializerMemo())
return builder.build()
def readbuf_w(self, space):
from rpython.rlib.rstruct.unichar import pack_unichar, UNICODE_SIZE
builder = StringBuilder(len(self._value) * UNICODE_SIZE)
for unich in self._value:
pack_unichar(unich, builder)
return StringBuffer(builder.build())
def read(self, size=-1):
# XXX CPython uses a more delicate logic here
ll_file = self.ll_file
if not ll_file:
raise ValueError("I/O operation on closed file")
if size < 0:
# read the entire contents
buf = lltype.malloc(rffi.CCHARP.TO, BASE_BUF_SIZE, flavor='raw')
try:
s = StringBuilder()
while True:
returned_size = c_fread(buf, 1, BASE_BUF_SIZE, ll_file)
returned_size = intmask(returned_size) # is between 0 and BASE_BUF_SIZE
if returned_size == 0:
if c_feof(ll_file):
# ok, finished
return s.build()
raise _error(ll_file)
s.append_charpsize(buf, returned_size)
finally:
lltype.free(buf, flavor='raw')
else:
raw_buf, gc_buf = rffi.alloc_buffer(size)
try:
returned_size = c_fread(raw_buf, 1, size, ll_file)
returned_size = intmask(returned_size) # is between 0 and size
if returned_size == 0:
if not c_feof(ll_file):
raise _error(ll_file)
s = rffi.str_from_buffer(raw_buf, gc_buf, size, returned_size)
finally:
rffi.keep_buffer_alive_until_here(raw_buf, gc_buf)
return s
def string_append(args):
if jit.isconstant(len(args)):
return string_append_fastpath(args)
if not args:
return W_String.fromascii("")
builder = StringBuilder(len(args))
unibuilder = None
ascii_idx = 0
try:
for ascii_idx in range(len(args)):
arg = args[ascii_idx]
if not isinstance(arg, W_String):
raise SchemeException("string-append: expected a string")
builder.append(arg.as_str_ascii())
except ValueError:
unibuilder = UnicodeBuilder(len(args))
unibuilder.append(unicode(builder.build()))
builder = None
for i in range(ascii_idx, len(args)):
arg = args[i]
if not isinstance(arg, W_String):
raise SchemeException("string-append: expected a string")
unibuilder.append(arg.as_unicode())
if unibuilder is None:
assert builder is not None
return W_String.fromascii(builder.build())
else:
assert unibuilder is not None
return W_String.fromunicode(unibuilder.build())
def my_replace(string):
from rpython.rlib.rstring import StringBuilder
result = StringBuilder()
for char in string:
if not char==' ':
result.append(char)
return result.build()
def test_deflate_set_dictionary():
text = 'abcabc'
zdict = 'abc'
stream = rzlib.deflateInit()
rzlib.deflateSetDictionary(stream, zdict)
bytes = rzlib.compress(stream, text, rzlib.Z_FINISH)
rzlib.deflateEnd(stream)
stream2 = rzlib.inflateInit()
from rpython.rtyper.lltypesystem import lltype, rffi, rstr
from rpython.rtyper.annlowlevel import llstr
from rpython.rlib.rstring import StringBuilder
with lltype.scoped_alloc(rffi.CCHARP.TO, len(bytes)) as inbuf:
rstr.copy_string_to_raw(llstr(bytes), inbuf, 0, len(bytes))
stream2.c_next_in = rffi.cast(rzlib.Bytefp, inbuf)
rffi.setintfield(stream2, 'c_avail_in', len(bytes))
with lltype.scoped_alloc(rffi.CCHARP.TO, 100) as outbuf:
stream2.c_next_out = rffi.cast(rzlib.Bytefp, outbuf)
bufsize = 100
rffi.setintfield(stream2, 'c_avail_out', bufsize)
err = rzlib._inflate(stream2, rzlib.Z_SYNC_FLUSH)
assert err == rzlib.Z_NEED_DICT
rzlib.inflateSetDictionary(stream2, zdict)
rzlib._inflate(stream2, rzlib.Z_SYNC_FLUSH)
avail_out = rffi.cast(lltype.Signed, stream2.c_avail_out)
result = StringBuilder()
result.append_charpsize(outbuf, bufsize - avail_out)
rzlib.inflateEnd(stream2)
assert result.build() == text
def bitwise_not(self, space):
length = self.strlen()
builder = StringBuilder(length)
for i in range(length):
c = ord(self.character(i))
builder.append(chr(c ^ 0xff))
return W_ConstStringObject(builder.build())
def _operate(stream, data, flush, max_length, cfunc, while_doing):
"""Common code for compress() and decompress().
"""
# Prepare the input buffer for the stream
with lltype.scoped_alloc(rffi.CCHARP.TO, len(data)) as inbuf:
for i in xrange(len(data)):
inbuf[i] = data[i]
stream.c_next_in = rffi.cast(Bytefp, inbuf)
rffi.setintfield(stream, 'c_avail_in', len(data))
# Prepare the output buffer
with lltype.scoped_alloc(rffi.CCHARP.TO, OUTPUT_BUFFER_SIZE) as outbuf:
# Strategy: we call deflate() to get as much output data as fits in
# the buffer, then accumulate all output into a StringBuffer
# 'result'.
result = StringBuilder()
while True:
stream.c_next_out = rffi.cast(Bytefp, outbuf)
bufsize = OUTPUT_BUFFER_SIZE
if max_length < bufsize:
if max_length <= 0:
err = Z_OK
break
bufsize = max_length
max_length -= bufsize
rffi.setintfield(stream, 'c_avail_out', bufsize)
err = cfunc(stream, flush)
if err == Z_OK or err == Z_STREAM_END:
# accumulate data into 'result'
avail_out = rffi.cast(lltype.Signed, stream.c_avail_out)
result.append_charpsize(outbuf, bufsize - avail_out)
# if the output buffer is full, there might be more data
# so we need to try again. Otherwise, we're done.
if avail_out > 0:
break
# We're also done if we got a Z_STREAM_END (which should
# only occur when flush == Z_FINISH).
if err == Z_STREAM_END:
break
else:
continue
elif err == Z_BUF_ERROR:
avail_out = rffi.cast(lltype.Signed, stream.c_avail_out)
# When compressing, we will only get Z_BUF_ERROR if
# the output buffer was full but there wasn't more
# output when we tried again, so it is not an error
# condition.
if avail_out == bufsize:
break
# fallback case: report this error
raise RZlibError.fromstream(stream, err, while_doing)
# When decompressing, if the compressed stream of data was truncated,
# then the zlib simply returns Z_OK and waits for more. If it is
# complete it returns Z_STREAM_END.
return (result.build(),
err,
rffi.cast(lltype.Signed, stream.c_avail_in))
def func():
s = StringBuilder()
s.append("a")
s.append("abc")
s.append_slice("abc", 1, 2)
s.append_multiple_char('d', 4)
return s.build()
def read(self, size=-1):
# XXX CPython uses a more delicate logic here
self._check_closed()
ll_file = self._ll_file
if size == 0:
return ""
elif size < 0:
# read the entire contents
buf = lltype.malloc(rffi.CCHARP.TO, BASE_BUF_SIZE, flavor='raw')
try:
s = StringBuilder()
while True:
returned_size = self._fread(buf, BASE_BUF_SIZE, ll_file)
returned_size = intmask(returned_size) # is between 0 and BASE_BUF_SIZE
if returned_size == 0:
if c_feof(ll_file):
# ok, finished
return s.build()
raise _error(ll_file)
s.append_charpsize(buf, returned_size)
finally:
lltype.free(buf, flavor='raw')
else: # size > 0
with rffi.scoped_alloc_buffer(size) as buf:
returned_size = self._fread(buf.raw, size, ll_file)
returned_size = intmask(returned_size) # is between 0 and size
if returned_size == 0:
if not c_feof(ll_file):
raise _error(ll_file)
s = buf.str(returned_size)
assert s is not None
return s
def rlecode_hqx(space, data):
"Binhex RLE-code binary data."
# that's a guesstimation of the resulting length
res = StringBuilder(len(data))
i = 0
end = len(data)
while i < end:
c = data[i]
res.append(c)
if c == '\x90':
# Escape it, and ignore repetitions (*).
res.append('\x00')
else:
# Check how many following are the same
inend = i + 1
while inend < end and data[inend] == c and inend < i + 255:
inend += 1
if inend - i > 3:
# More than 3 in a row. Output RLE. For the case of more
# than 255, see (*) below.
res.append('\x90')
res.append(chr(inend - i))
i = inend
continue
i += 1
# (*) Note that we put simplicity before compatness here, like CPython.
# I am sure that if we tried harder to produce the smallest possible
# string that rledecode_hqx() would expand back to 'data', there are
# some programs somewhere that would start failing obscurely in rare
# cases.
return space.newbytes(res.build())
def descr_buffer__new__(space, w_subtype, w_object, offset=0, size=-1):
# w_subtype can only be exactly 'buffer' for now
if not space.is_w(w_subtype, space.gettypefor(Buffer)):
raise OperationError(space.w_TypeError,
space.wrap("argument 1 must be 'buffer'"))
if space.isinstance_w(w_object, space.w_unicode):
# unicode objects support the old buffer interface
# but not the new buffer interface (change in python 2.7)
from rpython.rlib.rstruct.unichar import pack_unichar, UNICODE_SIZE
unistr = space.unicode_w(w_object)
builder = StringBuilder(len(unistr) * UNICODE_SIZE)
for unich in unistr:
pack_unichar(unich, builder)
from pypy.interpreter.buffer import StringBuffer
w_buffer = space.wrap(StringBuffer(builder.build()))
else:
w_buffer = space.buffer(w_object)
buffer = space.interp_w(Buffer, w_buffer) # type-check
if offset == 0 and size == -1:
return w_buffer
# handle buffer slices
if offset < 0:
raise OperationError(space.w_ValueError,
space.wrap("offset must be zero or positive"))
if size < -1:
raise OperationError(space.w_ValueError,
space.wrap("size must be zero or positive"))
if isinstance(buffer, RWBuffer):
buffer = RWSubBuffer(buffer, offset, size)
else:
buffer = SubBuffer(buffer, offset, size)
return space.wrap(buffer)
def text_build(self, space, frame, bytecode, no):
items = [None] * no
for i in range(no - 1, -1, -1):
items[i] = space.str(frame.pop())
sb = StringBuilder()
for item in items:
sb.append(item)
frame.push(space.newtext(sb.build()))
def fn(_):
s = StringBuilder(4)
got = []
for i in range(50):
s.append(chr(33+i))
got.append(s.build())
gc.collect()
return ' '.join(got)
def add__StringBuffer_String(space, w_self, w_other):
if w_self.builder.getlength() != w_self.length:
builder = StringBuilder()
builder.append(w_self.force())
else:
builder = w_self.builder
builder.append(w_other._value)
return W_StringBufferObject(builder)
def _parse_plain_flags(source):
b = StringBuilder(4)
while True:
ch = source.get()
if ch == ":":
break
else:
b.append(ch)
return b.build()
def string_func(space, w_left, w_right):
left = w_left.unwrap()
right = w_right.unwrap()
n = min(len(left), len(right))
s = StringBuilder(n)
for i in range(n):
char = chr(bitwise_op(ord(left[i]), ord(right[i])))
s.append(char)
return space.newstr(s.build())
def hexdigest(self, space):
"Return the digest value as a string of hexadecimal digits."
digest = self._digest(space)
hexdigits = '0123456789abcdef'
result = StringBuilder(self.digest_size * 2)
for c in digest:
result.append(hexdigits[(ord(c) >> 4) & 0xf])
result.append(hexdigits[ ord(c) & 0xf])
return space.wrap(result.build())
def test_prebuilt_string_builder(self):
s = StringBuilder(100)
s.append("abc")
def f():
return len(s.build())
res = self.interpret(f, [])
assert res == 3
def str2hexstr(arr, size):
HEXCHARS = ['0', '1', '2', '3', '4', '5', '6', '7',
'8', '9', 'a', 'b', 'c', 'd', 'e', 'f']
s = StringBuilder(size)
for i in range(size):
s.append(HEXCHARS[(ord(arr[i]) >> 4)])
s.append(HEXCHARS[(ord(arr[i])) & 15])
return s.build()
def fn(_):
got = []
for j in range(3, 76, 5):
s = StringBuilder()
for i in range(j):
s.append(chr(33+i))
gc.collect()
got.append(s.build())
return ' '.join(got)
def readline_w(self, space, w_limit=None):
# For backwards compatibility, a (slowish) readline().
limit = convert_size(space, w_limit)
has_peek = space.findattr(self, space.wrap("peek"))
builder = StringBuilder()
size = 0
while limit < 0 or size < limit:
nreadahead = 1
if has_peek:
w_readahead = space.call_method(self, "peek", space.wrap(1))
if not space.isinstance_w(w_readahead, space.w_str):
raise operationerrfmt(
space.w_IOError,
"peek() should have returned a bytes object, not '%T'",
w_readahead)
length = space.len_w(w_readahead)
if length > 0:
n = 0
buf = space.str_w(w_readahead)
if limit >= 0:
while True:
if n >= length or n >= limit:
break
n += 1
if buf[n-1] == '\n':
break
else:
while True:
if n >= length:
break
n += 1
if buf[n-1] == '\n':
break
nreadahead = n
w_read = space.call_method(self, "read", space.wrap(nreadahead))
if not space.isinstance_w(w_read, space.w_str):
raise operationerrfmt(
space.w_IOError,
"peek() should have returned a bytes object, not '%T'",
w_read)
read = space.str_w(w_read)
if not read:
break
size += len(read)
builder.append(read)
if read[-1] == '\n':
break
return space.wrap(builder.build())
def raw_str(self):
value = lltype.malloc(rffi.CArray(lltype.typeOf(self.value)), 1, flavor="raw")
value[0] = self.value
builder = StringBuilder()
builder.append_charpsize(rffi.cast(rffi.CCHARP, value), rffi.sizeof(lltype.typeOf(self.value)))
ret = builder.build()
lltype.free(value, flavor="raw")
return ret
def print_r(space, w_expression, returns=False):
""" Prints human-readable information about a variable"""
builder = StringBuilder()
_print_r(space, w_expression, '', {}, builder)
result = builder.build()
if returns:
return space.newstr(result)
else:
space.ec.interpreter.writestr(result)
return space.w_True
class W_StringBuilder(W_Root):
def __init__(self, length):
self._s = StringBuilder(length)
@unwrap_spec(txt='utf8')
def append(self, txt):
self._s.append(txt)
def build(self, space):
return space.newtext(self._s.build())
class StringBuilderWithOneCharCancellable(object):
def __init__(self, crlf, initial):
self.crlf = crlf
self.builder = StringBuilder(initial)
self.pending = -1
def _flush(self):
if self.pending >= 0:
self.builder.append(chr(self.pending))
self.pending = -1
_flush._always_inline_ = True
def append(self, c):
self._flush()
self.pending = ord(c)
def newline(self):
self._flush()
if self.crlf: self.builder.append('\r')
self.pending = ord('\n')
def to_hex(self, c):
self._flush()
uvalue = ord(c)
self.builder.append("0123456789ABCDEF"[uvalue >> 4])
self.builder.append("0123456789ABCDEF"[uvalue & 0xf])
def build(self):
self._flush()
return self.builder.build()
def descr_getitem(self, w_index):
self.check_valid()
space = self.space
start, stop, step, length = space.decode_index4(
w_index, self.mmap.size)
if step == 0: # index only
return space.wrap(ord(self.mmap.getitem(start)))
elif step == 1:
if stop - start < 0:
return space.wrapbytes("")
return space.wrapbytes(self.mmap.getslice(start, length))
else:
b = StringBuilder(length)
for i in range(start, stop, step):
b.append(self.mmap.getitem(i))
return space.wrapbytes(b.build())
def _read_all(self, space):
"Read all the file, don't update the cache"
# Must run with the lock held!
builder = StringBuilder()
# First copy what we have in the current buffer
current_size = self._readahead()
data = None
if current_size:
data = self.buffer[self.pos:self.pos + current_size]
builder.append(data)
self.pos += current_size
# We're going past the buffer's bounds, flush it
if self.writable:
self._flush_and_rewind_unlocked(space)
self._reader_reset_buf()
while True:
# Read until EOF or until read() would block
w_data = space.call_method(self.w_raw, "read")
if space.is_w(w_data, space.w_None):
if current_size == 0:
return w_data
break
data = space.bytes_w(w_data)
size = len(data)
if size == 0:
break
builder.append(data)
current_size += size
if self.abs_pos != -1:
self.abs_pos += size
return space.newbytes(builder.build())
def http_build_query(interp, w_data, num_prefix="", arg_sep=None, enctype=1):
space = interp.space
if arg_sep is None:
arg_sep = interp.config.get_ini_str("arg_separator.output")
w_data = w_data.deref()
out = StringBuilder()
if not w_data.tp in [space.tp_array, space.tp_object]:
interp.space.ec.warn("http_build_query(): Parameter 1 "
"expected to be Array or Object. "
"Incorrect value given")
if w_data.tp == space.tp_array:
with space.iter(w_data) as itr:
while not itr.done():
w_key, w_value = itr.next_item(space)
key = _get_key(space, num_prefix, w_key)
res = _build_query(space, [], key, w_value, num_prefix,
arg_sep, enctype)
out.append(''.join(res))
if w_data.tp == space.tp_object:
for key, w_value in w_data.get_instance_attrs(interp).iteritems():
_, prop = demangle_property(key)
if prop:
continue
res = _build_query(space, [], key, w_value, num_prefix, arg_sep,
enctype)
out.append(''.join(res))
outstr = out.build()
if outstr.endswith(arg_sep):
outstr = outstr.rstrip(arg_sep)
return interp.space.newstr(outstr)
def read_number_or_id(f, init):
sofar = StringBuilder(64)
sofar.append(init)
while True:
c = f.peek()
if c == "":
break
if idchar(c):
v = f.read(1)
assert v == c
sofar.append(v)
else:
break
got = sofar.build()
try:
val = string_to_int(got)
return values.W_Fixnum.make_or_interned(val)
except ParseStringOverflowError:
val = rbigint.fromdecimalstr(got)
return values.W_Bignum(val)
except ParseStringError:
try:
return values.W_Flonum(float(got))
except:
return values.W_Symbol.make(got)
class Buffer(object):
def __init__(self, space, callback, chunk_size, prev):
self.space = space
self.callback = callback
self.chunk_size = chunk_size
self.buffer = StringBuilder(chunk_size)
self.buffer_len = 0
self.prev = prev
def reset(self):
self.buffer = StringBuilder(self.chunk_size)
self.buffer_len = 0
def getlength(self):
if self.prev is None:
return 1
return self.prev.getlength() + 1
def write(self, str):
self.buffer.append(str)
if self.chunk_size and len(str) + self.buffer_len >= self.chunk_size:
self.flush()
self.reset()
else:
self.buffer_len += len(str)
def flush(self):
val = self.buffer.build()
if self.callback is not None:
flags = FLUSH
w_buffer = self.space.call_args(self.callback,
[self.space.wrap(val),
self.space.wrap(flags)])
val = self.space.str_w(w_buffer)
if self.prev is None:
self.space.ec.interpreter.writestr(val, buffer=False)
else:
self.prev.write(val)
self.reset()
def clean(self):
self.reset()
def get_contents(self):
return self.space.newstr(self.buffer.build())
def decode_string_escaped(self, start):
i = self.pos
builder = StringBuilder((i - start) * 2) # just an estimate
assert start >= 0
assert i >= 0
builder.append_slice(self.s, start, i)
while True:
ch = self.ll_chars[i]
i += 1
if ch == '"':
content_utf8 = builder.build()
content_unicode = unicodehelper.decode_utf8(
self.space, content_utf8)
self.last_type = TYPE_STRING
self.pos = i
return self.space.newunicode(content_unicode)
elif ch == '\\':
i = self.decode_escape_sequence(i, builder)
elif ch < '\x20':
if ch == '\0':
self._raise("Unterminated string starting at char %d",
start - 1)
else:
self._raise("Invalid control character at char %d", i - 1)
else:
builder.append(ch)
def var_dump(self, space, indent, recursion):
if self in recursion:
return '%s*RECURSION*\n' % indent
s = StringBuilder()
recursion[self] = None
header = 'object(%s)#%d ' % (self.getclass().name,
self.get_instance_number())
orig_indent = indent
if indent.endswith('&'):
indent = indent[:-1]
subindent = indent + ' '
counter = 0
all_names = []
all_values_w = []
self.enum_properties(space.ec.interpreter, all_names, all_values_w)
properties = OrderedDict()
for i in range(len(all_names)):
name, access = demangle_property(all_names[i])
key = dump_property(name, access)
properties[key] = '%s[%s]=>\n%s' % (
subindent, key, all_values_w[i].var_dump(
space, subindent, recursion))
for part in properties.itervalues():
counter += 1
s.append(part)
s.append('%s}\n' % indent)
del recursion[self]
return '%s%s(%d) {\n' % (orig_indent, header, counter) + s.build()
def build(self):
self.fmt_interpreted = self.interpret()
self.result = StringBuilder()
for fmtdesc, repetitions in self.fmt_interpreted:
if repetitions == -1 and fmtdesc.many_args:
repetitions = len(self.arg_w) - self.arg_index
try:
fmtdesc.pack(self, fmtdesc, repetitions)
except FormatException as e:
self.space.ec.warn(
"pack(): Type %s: %s" % (fmtdesc.fmtchar, e.message))
if self.arg_index < len(self.arg_w):
self.space.ec.warn(
"pack(): %s "
"arguments unused" % (len(self.arg_w) - self.arg_index))
return self.result.build()
def str_swapcase__String(space, w_self):
self = w_self._value
builder = StringBuilder(len(self))
for i in range(len(self)):
ch = self[i]
if ch.isupper():
o = ord(ch) + 32
builder.append(chr(o))
elif ch.islower():
o = ord(ch) - 32
builder.append(chr(o))
else:
builder.append(ch)
return space.wrap(builder.build())
def sanitize(self, s):
res = StringBuilder(len(s))
for c in s:
if c in CONTROL_CHARS:
res.append('_')
else:
res.append(c)
return res.build()
def readline(self):
# mostly inefficient, but not as laughably bad as with the default
# readline() from Stream
result = StringBuilder()
while True:
try:
c = os.read(self.fd, 1)
except OSError, e:
if e.errno != errno.EINTR:
raise
if self.signal_checker is not None:
self.signal_checker()
continue # try again
if not c:
break
c = c[0]
result.append(c)
if c == '\n':
break
def unicode_encode_utf_16_helper(s,
size,
errors,
errorhandler=None,
allow_surrogates=True,
byteorder='little',
public_encoding_name='utf16'):
if errorhandler is None:
errorhandler = default_unicode_error_encode
if size == 0:
if byteorder == 'native':
result = StringBuilder(2)
_STORECHAR(result, 0xFEFF, BYTEORDER)
return result.build()
return ""
result = StringBuilder(size * 2 + 2)
if byteorder == 'native':
_STORECHAR(result, 0xFEFF, BYTEORDER)
byteorder = BYTEORDER
pos = 0
while pos < size:
ch = ord(s[pos])
pos += 1
if ch < 0xD800:
_STORECHAR(result, ch, byteorder)
elif ch >= 0x10000:
_STORECHAR(result, 0xD800 | ((ch - 0x10000) >> 10), byteorder)
_STORECHAR(result, 0xDC00 | ((ch - 0x10000) & 0x3FF), byteorder)
elif ch >= 0xE000 or allow_surrogates:
_STORECHAR(result, ch, byteorder)
else:
ru, rs, pos = errorhandler(errors, public_encoding_name,
'surrogates not allowed', s, pos - 1,
pos)
if rs is not None:
# py3k only
if len(rs) % 2 != 0:
errorhandler('strict', public_encoding_name,
'surrogates not allowed', s, pos - 1, pos)
result.append(rs)
continue
for ch in ru:
if ord(ch) < 0xD800:
_STORECHAR(result, ord(ch), byteorder)
else:
errorhandler('strict', public_encoding_name,
'surrogates not allowed', s, pos - 1, pos)
continue
return result.build()
def format(self):
lgt = len(self.fmt) + 4 * len(self.values_w) + 10
result = StringBuilder(lgt)
self.result = result
while True:
# fast path: consume as many characters as possible
fmt = self.fmt
i = i0 = self.fmtpos
while i < len(fmt):
if fmt[i] == '%':
break
i += 1
else:
result.append_slice(fmt, i0, len(fmt))
break # end of 'fmt' string
result.append_slice(fmt, i0, i)
self.fmtpos = i + 1
c = self.peekchr()
if c == '%':
self.forward()
self.result.append('%')
continue
# interpret the next formatter
w_value = self.parse_fmt()
c = self.peekchr()
self.forward()
if c == '%':
# if we get here there were extra characters between the
# two %, forbidden now
self.two_percent_error(i + 1)
continue
# first check whether it's a invalid char, *then* call
# nextinputvalue, otherwise the error generated by
# nextinputvalue can cover that of unknown_fmtchar
for c1 in FORMATTER_CHARS:
if c == c1:
break
else:
self.unknown_fmtchar()
if w_value is None:
w_value = self.nextinputvalue()
# dispatch on the formatter
# (this turns into a switch after translation)
for c1 in FORMATTER_CHARS:
if c == c1:
# 'c1' is an annotation constant here,
# so this getattr() is ok
do_fmt = getattr(self, 'fmt_' + c1)
do_fmt(w_value)
break
self.checkconsumed()
return result.build()
def hexdigest(self, space):
"Return the digest value as a string of hexadecimal digits."
digest = self._digest(space)
hexdigits = '0123456789abcdef'
result = StringBuilder(self.digest_size * 2)
for c in digest:
result.append(hexdigits[(ord(c) >> 4) & 0xf])
result.append(hexdigits[ord(c) & 0xf])
return space.newtext(result.build())
def unicode_encode_utf_32_helper(s,
size,
errors,
errorhandler=None,
allow_surrogates=True,
byteorder='little',
public_encoding_name='utf32'):
if errorhandler is None:
errorhandler = default_unicode_error_encode
if size == 0:
if byteorder == 'native':
result = StringBuilder(4)
_STORECHAR32(result, 0xFEFF, BYTEORDER)
return result.build()
return ""
result = StringBuilder(size * 4 + 4)
if byteorder == 'native':
_STORECHAR32(result, 0xFEFF, BYTEORDER)
byteorder = BYTEORDER
pos = 0
while pos < size:
ch = ord(s[pos])
pos += 1
ch2 = 0
if not allow_surrogates and 0xD800 <= ch < 0xE000:
ru, rs, pos = errorhandler(errors, public_encoding_name,
'surrogates not allowed', s, pos - 1,
pos)
if rs is not None:
# py3k only
if len(rs) % 4 != 0:
errorhandler('strict', public_encoding_name,
'surrogates not allowed', s, pos - 1, pos)
result.append(rs)
continue
for ch in ru:
if ord(ch) < 0xD800:
_STORECHAR32(result, ord(ch), byteorder)
else:
errorhandler('strict', public_encoding_name,
'surrogates not allowed', s, pos - 1, pos)
continue
if 0xD800 <= ch < 0xDC00 and MAXUNICODE < 65536 and pos < size:
ch2 = ord(s[pos])
if 0xDC00 <= ch2 < 0xE000:
ch = (((ch & 0x3FF) << 10) | (ch2 & 0x3FF)) + 0x10000
pos += 1
_STORECHAR32(result, ch, byteorder)
return result.build()
def test_string_builder_union(self):
s = StringBuilder()
def f(i):
if i % 2:
s2 = StringBuilder()
else:
s2 = s
return s2.build()
self.interpret(f, [3])
def strhex(argbuf, arglen):
assert arglen >= 0
if arglen > sys.maxint / 2:
raise MemoryError
builder = StringBuilder(arglen * 2)
for i in range(arglen):
b = ord(argbuf[i])
builder.append(hexdigits[(b >> 4) & 0xf])
builder.append(hexdigits[b & 0xf])
return builder.build()
def b2a_uu(space, bin, __kwonly__, backtick=False):
"Uuencode a line of data."
length = len(bin)
if length > 45:
raise_Error(space, 'At most 45 bytes at once')
res = StringBuilder(2 + ((length + 2) // 3) * 4)
_b2a_write(res, length, backtick)
for i in range(0, length, 3):
A = _b2a_read(bin, i)
B = _b2a_read(bin, i + 1)
C = _b2a_read(bin, i + 2)
#
_b2a_write(res, A >> 2, backtick)
_b2a_write(res, (A & 0x3) << 4 | B >> 4, backtick)
_b2a_write(res, (B & 0xF) << 2 | C >> 6, backtick)
_b2a_write(res, C & 0x3F, backtick)
res.append('\n')
return space.newbytes(res.build())
def _parse_property(source, info, positive, in_set):
here = source.pos
if source.match("{"):
negate = source.match("^")
b = StringBuilder(5)
found = False
while True:
ch = source.get()
if ch == "}":
found = True
break
elif not ch:
break
else:
b.append(ch)
if found:
name = b.build()
if name in PROPERTIES:
return Property(PROPERTIES[name], positive != negate)
source.pos = here
return make_character(info, ord("p" if positive else "P"), in_set)
def readall_w(self, space):
if self.handle == rwin32.INVALID_HANDLE_VALUE:
raise err_closed(space)
# Read the wstr 16-bit data from the console as 8-byte bytes
result = StringBuilder()
while True:
wbuf = read_console_wide(space, self.handle, BUFSIZ)
if len(wbuf) == 0:
break
result.append(wbuf)
wbuf = result.build()
state = space.fromcache(CodecState)
errh = state.decode_error_handler
utf8, lgt, pos = str_decode_utf_16(wbuf,
'strict',
final=True,
errorhandler=errh)
return space.newtext(utf8, lgt)
def _flush_codes(self, space):
b = StringBuilder()
for code in self.current_codes:
name = code._get_full_name()
b.append('\x02')
write_long_to_string_builder(code._unique_id, b)
write_long_to_string_builder(len(name), b)
b.append(name)
os.write(self.fileno, b.build())
self.current_codes = []
def string_append(args):
if not args:
return W_String.fromascii("")
builder = StringBuilder()
unibuilder = None
ascii_idx = 0
try:
for ascii_idx in range(len(args)):
arg = args[ascii_idx]
if not isinstance(arg, W_String):
raise SchemeException("string-append: expected a string")
builder.append(arg.as_str_ascii())
except ValueError:
unibuilder = UnicodeBuilder()
unibuilder.append(unicode(builder.build()))
builder = None
for i in range(ascii_idx, len(args)):
arg = args[i]
if not isinstance(arg, W_String):
raise SchemeException("string-append: expected a string")
unibuilder.append(arg.as_unicode())
if unibuilder is None:
assert builder is not None
return W_String.fromascii(builder.build())
else:
assert unibuilder is not None
return W_String.fromunicode(unibuilder.build())
class W_BytesBuilder(W_Root):
def __init__(self, space, size):
if size < 0:
self.builder = StringBuilder()
else:
self.builder = StringBuilder(size)
@unwrap_spec(size=int)
def descr__new__(space, w_subtype, size=-1):
return W_BytesBuilder(space, size)
@unwrap_spec(s='bytes')
def descr_append(self, space, s):
self.builder.append(s)
@unwrap_spec(s='bytes', start=int, end=int)
def descr_append_slice(self, space, s, start, end):
if not 0 <= start <= end <= len(s):
raise oefmt(space.w_ValueError, "bad start/stop")
self.builder.append_slice(s, start, end)
def descr_build(self, space):
w_s = space.newbytes(self.builder.build())
# after build(), we can continue to append more strings
# to the same builder. This is supported since
# 2ff5087aca28 in RPython.
return w_s
def descr_len(self, space):
if self.builder is None:
raise oefmt(space.w_ValueError, "no length of built builder")
return space.newint(self.builder.getlength())
def read(self, space, length):
""" read from the console up to `length` utf-16 chars
If mode is 'u', return `length` codepoints. If mode is `b`,
return `length` bytes.`
"""
if self.handle == rwin32.INVALID_HANDLE_VALUE:
raise err_closed(space)
if not self.readable:
raise err_mode(space, "reading")
if length <= 0:
return '', 0
if length > BUFMAX:
raise oefmt(space.w_ValueError, "cannot read more than %d bytes",
BUFMAX)
# first copy any remaining buffered utf16 data
builder = StringBuilder(length)
wbuf = self._getbuffer(length * 2)
state = space.fromcache(CodecState)
errh = state.decode_error_handler
outlen = 0
if len(wbuf) > 0:
utf8, lgt, pos = str_decode_utf_16(wbuf,
'strict',
final=True,
errorhandler=errh)
if self.mode == 'u':
length -= lgt
outlen += lgt
else:
length -= len(utf8)
outlen += len(utf8)
builder.append(utf8)
if length > 0:
wbuf = read_console_wide(space, self.handle, length)
utf8, lgt, pos = str_decode_utf_16(wbuf,
'strict',
final=True,
errorhandler=errh)
if 1 or self.mode == 'u':
length -= lgt
outlen += lgt
else:
length -= len(utf8)
outlen += len(utf8)
builder.append(utf8)
res = builder.build()
return res, outlen
def _urldecode(url):
res = StringBuilder(len(url))
l = len(url)
i = 0
while i < l:
c = url[i]
if (c == '%' and (i < l - 2) and _ishexdigit(url[i + 1])
and _ishexdigit(url[i + 2])):
k = (_decode(url[i + 1]) << 4) + _decode(url[i + 2])
res.append(chr(k))
i += 3
elif c == '+':
res.append(' ')
i += 1
else:
res.append(url[i])
i += 1
return res.build()
def readline_w(self, space, w_limit=None):
# For backwards compatibility, a (slowish) readline().
limit = convert_size(space, w_limit)
has_peek = space.findattr(self, space.wrap("peek"))
builder = StringBuilder()
size = 0
while limit < 0 or size < limit:
nreadahead = 1
if has_peek:
try:
w_readahead = space.call_method(self, "peek",
space.wrap(1))
except OperationError, e:
if trap_eintr(space, e):
continue
raise
if not space.isinstance_w(w_readahead, space.w_str):
raise oefmt(
space.w_IOError,
"peek() should have returned a bytes object, "
"not '%T'", w_readahead)
length = space.len_w(w_readahead)
if length > 0:
n = 0
buf = space.str_w(w_readahead)
if limit >= 0:
while True:
if n >= length or n >= limit:
break
n += 1
if buf[n - 1] == '\n':
break
else:
while True:
if n >= length:
break
n += 1
if buf[n - 1] == '\n':
break
nreadahead = n
try:
w_read = space.call_method(self, "read",
space.wrap(nreadahead))
except OperationError, e:
if trap_eintr(space, e):
continue
raise
def bindec(space, w_obj):
""" bindec - Binary to decimal"""
s = space.str_w(space.as_string(w_obj))
binstr = StringBuilder(len(s))
i = 0
while i < len(s) and s[i] != '1':
i += 1
while i < len(s):
c = s[i]
if c == '0' or c == '1':
binstr.append(c)
i += 1
binstr = binstr.build()
if len(binstr) == 0:
return space.newint(0)
elif len(binstr) <= MAX_BITS:
return space.newint(int(binstr, 2))
else:
fnum = float(int(binstr[:MAX_BITS], 2))
for i in range(MAX_BITS, len(binstr)):
fnum = 2 * fnum + int(binstr[i])
return space.newfloat(fnum)
def readall_w(self, space):
builder = StringBuilder()
while True:
try:
w_data = space.call_method(self, "read",
space.wrap(DEFAULT_BUFFER_SIZE))
except OperationError, e:
if trap_eintr(space, e):
continue
raise
if space.is_w(w_data, space.w_None):
if not builder.getlength():
return w_data
break
if not space.isinstance_w(w_data, space.w_str):
raise OperationError(space.w_TypeError,
space.wrap("read() should return bytes"))
data = space.str_w(w_data)
if not data:
break
builder.append(data)
def var_dump(self, space, indent, recursion):
if self in recursion:
return '%s*RECURSION*\n' % indent
s = StringBuilder()
recursion[self] = None
header = 'object(%s)#%d ' % (self.getclass().name,
self.get_instance_number())
orig_indent = indent
if indent.endswith('&'):
indent = indent[:-1]
subindent = indent + ' '
counter = 0
for name, w_value in self.iterproperties(space.ec.interpreter):
counter += 1
name, access = demangle_property(name)
key = dump_property(name, access)
s.append('%s[%s]=>\n' % (subindent, key))
s.append(w_value.var_dump(space, subindent, recursion))
s.append('%s}\n' % indent)
del recursion[self]
return '%s%s(%d) {\n' % (orig_indent, header, counter) + s.build()
return s.build()
def hexlify(space, data):
'''Hexadecimal representation of binary data.
This function is also available as "hexlify()".'''
try:
newlength = ovfcheck(len(data) * 2)
except OverflowError:
raise OperationError(space.w_MemoryError, space.w_None)
res = StringBuilder(newlength)
for c in data:
res.append(_value2char(ord(c) >> 4))
res.append(_value2char(ord(c) & 0xf))
return space.newbytes(res.build())