def str_translate__String_ANY_ANY(space, w_string, w_table, w_deletechars=''):
"""charfilter - unicode handling is not implemented
Return a copy of the string where all characters occurring
in the optional argument deletechars are removed, and the
remaining characters have been mapped through the given translation table,
which must be a string of length 256"""
if space.is_w(w_table, space.w_None):
table = DEFAULT_NOOP_TABLE
else:
table = space.bufferstr_w(w_table)
if len(table) != 256:
raise OperationError(
space.w_ValueError,
space.wrap("translation table must be 256 characters long"))
string = w_string._value
deletechars = space.str_w(w_deletechars)
if len(deletechars) == 0:
buf = StringBuilder(len(string))
for char in string:
buf.append(table[ord(char)])
else:
buf = StringBuilder()
deletion_table = [False] * 256
for c in deletechars:
deletion_table[ord(c)] = True
for char in string:
if not deletion_table[ord(char)]:
buf.append(table[ord(char)])
return W_StringObject(buf.build())
def here_doc(self):
ch = self.read()
indent = ch == "-"
expand = True
regexp = False
if indent:
ch = self.read()
if ch in "'\"`":
term = ch
if term == "'":
expand = False
elif term == "`":
regexp = True
marker = StringBuilder()
while True:
ch = self.read()
if ch == self.EOF:
self.unread()
break
elif ch == term:
break
else:
marker.append(ch)
else:
if not (ch.isalnum() or ch == "_"):
self.unread()
if indent:
self.unread()
return
marker = StringBuilder()
marker.append(ch)
while True:
ch = self.read()
if ch == self.EOF or not (ch.isalnum() or ch == "_"):
self.unread()
break
marker.append(ch)
last_line = StringBuilder()
while True:
ch = self.read()
if ch == "\n":
break
elif ch == self.EOF:
self.unread()
break
last_line.append(ch)
self.str_term = HeredocTerm(self, marker.build(), last_line.build(), indent=indent, expand=expand)
if regexp:
yield self.emit("XSTRING_BEG")
else:
yield self.emit("STRING_BEG")
def _string_replace(space, input, sub, by, maxsplit):
if maxsplit == 0:
return space.wrap(input)
if not sub:
upper = len(input)
if maxsplit > 0 and maxsplit < upper + 2:
upper = maxsplit - 1
assert upper >= 0
try:
result_size = ovfcheck(upper * len(by))
result_size = ovfcheck(result_size + upper)
result_size = ovfcheck(result_size + len(by))
remaining_size = len(input) - upper
result_size = ovfcheck(result_size + remaining_size)
except OverflowError:
raise OperationError(space.w_OverflowError,
space.wrap("replace string is too long"))
builder = StringBuilder(result_size)
for i in range(upper):
builder.append(by)
builder.append(input[i])
builder.append(by)
builder.append_slice(input, upper, len(input))
else:
# First compute the exact result size
count = input.count(sub)
if count > maxsplit and maxsplit > 0:
count = maxsplit
diff_len = len(by) - len(sub)
try:
result_size = ovfcheck(diff_len * count)
result_size = ovfcheck(result_size + len(input))
except OverflowError:
raise OperationError(space.w_OverflowError,
space.wrap("replace string is too long"))
builder = StringBuilder(result_size)
start = 0
sublen = len(sub)
while maxsplit != 0:
next = input.find(sub, start)
if next < 0:
break
builder.append_slice(input, start, next)
builder.append(by)
start = next + sublen
maxsplit -= 1 # NB. if it's already < 0, it stays < 0
builder.append_slice(input, start, len(input))
return space.wrap(builder.build())
def a2b_hqx(space, ascii):
"""Decode .hqx coding. Returns (bin, done)."""
# overestimate the resulting length
res = StringBuilder(len(ascii))
done = 0
pending_value = 0
pending_bits = 0
for c in ascii:
n = ord(table_a2b_hqx[ord(c)])
if n <= 0x3F:
pending_value = (pending_value << 6) | n
pending_bits += 6
if pending_bits == 24:
# flush
res.append(chr(pending_value >> 16))
res.append(chr((pending_value >> 8) & 0xff))
res.append(chr(pending_value & 0xff))
pending_value = 0
pending_bits = 0
elif n == FAIL:
raise_Error(space, 'Illegal character')
elif n == DONE:
if pending_bits >= 8:
res.append(chr(pending_value >> (pending_bits - 8)))
if pending_bits >= 16:
res.append(chr((pending_value >> (pending_bits - 16)) & 0xff))
done = 1
break
#elif n == SKIP: pass
else:
if pending_bits > 0:
raise_Incomplete(space, 'String has incomplete number of bytes')
return space.newtuple([space.wrap(res.build()), space.wrap(done)])
def repr__Bytearray(space, w_bytearray):
s = w_bytearray.data
# Good default if there are no replacements.
buf = StringBuilder(len("bytearray(b'')") + len(s))
buf.append("bytearray(b'")
for i in range(len(s)):
c = s[i]
if c == '\\' or c == "'":
buf.append('\\')
buf.append(c)
elif c == '\t':
buf.append('\\t')
elif c == '\r':
buf.append('\\r')
elif c == '\n':
buf.append('\\n')
elif not '\x20' <= c < '\x7f':
n = ord(c)
buf.append('\\x')
buf.append("0123456789abcdef"[n >> 4])
buf.append("0123456789abcdef"[n & 0xF])
else:
buf.append(c)
buf.append("')")
return space.wrap(buf.build())
def descr_buffer__new__(space, w_subtype, w_object, offset=0, size=-1):
# w_subtype can only be exactly 'buffer' for now
if not space.is_w(w_subtype, space.gettypefor(Buffer)):
raise OperationError(space.w_TypeError,
space.wrap("argument 1 must be 'buffer'"))
if space.isinstance_w(w_object, space.w_unicode):
# unicode objects support the old buffer interface
# but not the new buffer interface (change in python 2.7)
from pypy.rlib.rstruct.unichar import pack_unichar, UNICODE_SIZE
unistr = space.unicode_w(w_object)
builder = StringBuilder(len(unistr) * UNICODE_SIZE)
for unich in unistr:
pack_unichar(unich, builder)
from pypy.interpreter.buffer import StringBuffer
w_buffer = space.wrap(StringBuffer(builder.build()))
else:
w_buffer = space.buffer(w_object)
buffer = space.interp_w(Buffer, w_buffer) # type-check
if offset == 0 and size == -1:
return w_buffer
# handle buffer slices
if offset < 0:
raise OperationError(space.w_ValueError,
space.wrap("offset must be zero or positive"))
if size < -1:
raise OperationError(space.w_ValueError,
space.wrap("size must be zero or positive"))
if isinstance(buffer, RWBuffer):
buffer = RWSubBuffer(buffer, offset, size)
else:
buffer = SubBuffer(buffer, offset, size)
return space.wrap(buffer)
def readline_w(self, space, w_limit=None):
# For backwards compatibility, a (slowish) readline().
limit = convert_size(space, w_limit)
old_size = -1
has_peek = space.findattr(self, space.wrap("peek"))
builder = StringBuilder()
size = 0
while limit < 0 or size < limit:
nreadahead = 1
if has_peek:
w_readahead = space.call_method(self, "peek", space.wrap(1))
if not space.isinstance_w(w_readahead, space.w_str):
raise operationerrfmt(
space.w_IOError,
"peek() should have returned a bytes object, "
"not '%s'",
space.type(w_readahead).getname(space))
length = space.len_w(w_readahead)
if length > 0:
n = 0
buf = space.str_w(w_readahead)
if limit >= 0:
while True:
if n >= length or n >= limit:
break
n += 1
if buf[n - 1] == '\n':
break
else:
while True:
if n >= length:
break
n += 1
if buf[n - 1] == '\n':
break
nreadahead = n
w_read = space.call_method(self, "read", space.wrap(nreadahead))
if not space.isinstance_w(w_read, space.w_str):
raise operationerrfmt(
space.w_IOError,
"peek() should have returned a bytes object, "
"not '%s'",
space.type(w_read).getname(space))
read = space.str_w(w_read)
if not read:
break
size += len(read)
builder.append(read)
if read[-1] == '\n':
break
return space.wrap(builder.build())
def func():
s = StringBuilder()
s.append("a")
s.append("abc")
s.append_slice("abc", 1, 2)
s.append_multiple_char('d', 4)
return s.build()
def b2a_base64(space, bin):
"Base64-code line of data."
newlength = (len(bin) + 2) // 3
try:
newlength = ovfcheck(newlength * 4)
except OverflowError:
raise OperationError(space.w_MemoryError, space.w_None)
newlength += 1
res = StringBuilder(newlength)
leftchar = 0
leftbits = 0
for c in bin:
# Shift into our buffer, and output any 6bits ready
leftchar = (leftchar << 8) | ord(c)
leftbits += 8
res.append(table_b2a_base64[(leftchar >> (leftbits - 6)) & 0x3f])
leftbits -= 6
if leftbits >= 6:
res.append(table_b2a_base64[(leftchar >> (leftbits - 6)) & 0x3f])
leftbits -= 6
#
if leftbits == 2:
res.append(table_b2a_base64[(leftchar & 3) << 4])
res.append(PAD)
res.append(PAD)
elif leftbits == 4:
res.append(table_b2a_base64[(leftchar & 0xf) << 2])
res.append(PAD)
res.append('\n')
return space.wrap(res.build())
def _read_all(self, space):
"Read all the file, don't update the cache"
builder = StringBuilder()
# First copy what we have in the current buffer
current_size = self._readahead()
data = None
if current_size:
data = ''.join(self.buffer[self.pos:self.pos + current_size])
builder.append(data)
self._reader_reset_buf()
# We're going past the buffer's bounds, flush it
if self.writable:
self._writer_flush_unlocked(space, restore_pos=True)
while True:
# Read until EOF or until read() would block
w_data = space.call_method(self.w_raw, "read")
if space.is_w(w_data, space.w_None):
if current_size == 0:
return w_data
break
data = space.str_w(w_data)
size = len(data)
if size == 0:
break
builder.append(data)
current_size += size
if self.abs_pos != -1:
self.abs_pos += size
return space.wrap(builder.build())
def rledecode_hqx(space, hexbin):
"Decode hexbin RLE-coded string."
# that's a guesstimation of the resulting length
res = StringBuilder(len(hexbin))
end = len(hexbin)
i = 0
lastpushed = -1
while i < end:
c = hexbin[i]
i += 1
if c != '\x90':
res.append(c)
lastpushed = ord(c)
else:
if i == end:
raise_Incomplete(space, 'String ends with the RLE code \x90')
count = ord(hexbin[i]) - 1
i += 1
if count < 0:
res.append('\x90')
lastpushed = 0x90
else:
if lastpushed < 0:
raise_Error(space, 'String starts with the RLE code \x90')
res.append_multiple_char(chr(lastpushed), count)
return space.wrap(res.build())
def rlecode_hqx(space, data):
"Binhex RLE-code binary data."
# that's a guesstimation of the resulting length
res = StringBuilder(len(data))
i = 0
end = len(data)
while i < end:
c = data[i]
res.append(c)
if c == '\x90':
# Escape it, and ignore repetitions (*).
res.append('\x00')
else:
# Check how many following are the same
inend = i + 1
while inend < end and data[inend] == c and inend < i + 255:
inend += 1
if inend - i > 3:
# More than 3 in a row. Output RLE. For the case of more
# than 255, see (*) below.
res.append('\x90')
res.append(chr(inend - i))
i = inend
continue
i += 1
# (*) Note that we put simplicity before compatness here, like CPython.
# I am sure that if we tried harder to produce the smallest possible
# string that rledecode_hqx() would expand back to 'data', there are
# some programs somewhere that would start failing obscurely in rare
# cases.
return space.wrap(res.build())
def str_join__String_ANY(space, w_self, w_list):
list_w = space.listview(w_list)
if list_w:
self = w_self._value
reslen = 0
for i in range(len(list_w)):
w_s = list_w[i]
if not space.is_true(space.isinstance(w_s, space.w_str)):
if space.is_true(space.isinstance(w_s, space.w_unicode)):
# we need to rebuild w_list here, because the original
# w_list might be an iterable which we already consumed
w_list = space.newlist(list_w)
w_u = space.call_function(space.w_unicode, w_self)
return space.call_method(w_u, "join", w_list)
raise operationerrfmt(
space.w_TypeError,
"sequence item %d: expected string, %s "
"found", i, space.type(w_s).getname(space, '?'))
reslen += len(space.str_w(w_s))
reslen += len(self) * (len(list_w) - 1)
sb = StringBuilder(reslen)
for i in range(len(list_w)):
if self and i != 0:
sb.append(self)
sb.append(space.str_w(list_w[i]))
return space.wrap(sb.build())
else:
return W_StringObject.EMPTY
def add__StringBuffer_String(space, w_self, w_other):
if w_self.builder.getlength() != w_self.length:
builder = StringBuilder()
builder.append(w_self.force())
else:
builder = w_self.builder
builder.append(w_other._value)
return W_StringBufferObject(builder)
def fn(_):
s = StringBuilder(4)
got = []
for i in range(50):
s.append(chr(33+i))
got.append(s.build())
gc.collect()
return ' '.join(got)
def test_string_builder():
s = StringBuilder()
s.append("a")
s.append("abc")
s.append("a")
s.append_slice("abc", 1, 2)
s.append_multiple_char('d', 4)
assert s.build() == "aabcabdddd"
def _operate(stream, data, flush, max_length, cfunc, while_doing):
"""Common code for compress() and decompress().
"""
# Prepare the input buffer for the stream
with lltype.scoped_alloc(rffi.CCHARP.TO, len(data)) as inbuf:
for i in xrange(len(data)):
inbuf[i] = data[i]
stream.c_next_in = rffi.cast(Bytefp, inbuf)
rffi.setintfield(stream, 'c_avail_in', len(data))
# Prepare the output buffer
with lltype.scoped_alloc(rffi.CCHARP.TO, OUTPUT_BUFFER_SIZE) as outbuf:
# Strategy: we call deflate() to get as much output data as fits in
# the buffer, then accumulate all output into a StringBuffer
# 'result'.
result = StringBuilder()
while True:
stream.c_next_out = rffi.cast(Bytefp, outbuf)
bufsize = OUTPUT_BUFFER_SIZE
if max_length < bufsize:
if max_length <= 0:
err = Z_OK
break
bufsize = max_length
max_length -= bufsize
rffi.setintfield(stream, 'c_avail_out', bufsize)
err = cfunc(stream, flush)
if err == Z_OK or err == Z_STREAM_END:
# accumulate data into 'result'
avail_out = rffi.cast(lltype.Signed, stream.c_avail_out)
result.append_charpsize(outbuf, bufsize - avail_out)
# if the output buffer is full, there might be more data
# so we need to try again. Otherwise, we're done.
if avail_out > 0:
break
# We're also done if we got a Z_STREAM_END (which should
# only occur when flush == Z_FINISH).
if err == Z_STREAM_END:
break
else:
continue
elif err == Z_BUF_ERROR:
avail_out = rffi.cast(lltype.Signed, stream.c_avail_out)
# When compressing, we will only get Z_BUF_ERROR if
# the output buffer was full but there wasn't more
# output when we tried again, so it is not an error
# condition.
if avail_out == bufsize:
break
# fallback case: report this error
raise RZlibError.fromstream(stream, err, while_doing)
# When decompressing, if the compressed stream of data was truncated,
# then the zlib simply returns Z_OK and waits for more. If it is
# complete it returns Z_STREAM_END.
return (result.build(), err, rffi.cast(lltype.Signed, stream.c_avail_in))
def hexdigest(self, space):
"Return the digest value as a string of hexadecimal digits."
digest = self._digest(space)
hexdigits = '0123456789abcdef'
result = StringBuilder(self.digest_size * 2)
for c in digest:
result.append(hexdigits[(ord(c) >> 4) & 0xf])
result.append(hexdigits[ord(c) & 0xf])
return space.wrap(result.build())
def main(n):
result = 0
while n >= 0:
jitdriver.jit_merge_point(n=n, result=result)
b = StringBuilder(6)
b.append("Hello!")
result += ord((b.build() + "xyz")[0])
n -= 1
return result
def fn():
s = StringBuilder(4)
s.append("abcd")
s.append("defg")
s.append("rty")
s.append_multiple_char('y', 1000)
gc.collect()
s.append_multiple_char('y', 1000)
res = s.build()[1000]
gc.collect()
return ord(res)
def buffer__RopeUnicode(space, w_unicode):
from pypy.rlib.rstruct.unichar import pack_unichar, UNICODE_SIZE
node = w_unicode._node
iter = rope.ItemIterator(node)
length = node.length()
builder = StringBuilder(length * UNICODE_SIZE)
for idx in range(length):
unich = unichr(iter.nextint())
pack_unichar(unich, builder)
from pypy.interpreter.buffer import StringBuffer
return space.wrap(StringBuffer(builder.build()))
def hexlify(space, data):
'''Hexadecimal representation of binary data.
This function is also available as "hexlify()".'''
try:
newlength = ovfcheck(len(data) * 2)
except OverflowError:
raise OperationError(space.w_MemoryError, space.w_None)
res = StringBuilder(newlength)
for c in data:
res.append(_value2char(ord(c) >> 4))
res.append(_value2char(ord(c) & 0xf))
return space.wrap(res.build())
def direct_read(self, n=-1):
stream = self.getstream()
if n < 0:
return stream.readall()
else:
result = StringBuilder(n)
while n > 0:
data = stream.read(n)
if not data:
break
n -= len(data)
result.append(data)
return result.build()
def unhexlify(space, hexstr):
'''Binary data of hexadecimal representation.
hexstr must contain an even number of hex digits (upper or lower case).
This function is also available as "unhexlify()".'''
if len(hexstr) & 1:
raise OperationError(space.w_TypeError,
space.wrap('Odd-length string'))
res = StringBuilder(len(hexstr) >> 1)
for i in range(0, len(hexstr), 2):
a = _char2value(space, hexstr[i])
b = _char2value(space, hexstr[i+1])
res.append(chr((a << 4) | b))
return space.wrap(res.build())
def readall_w(self, space):
builder = StringBuilder()
while True:
w_data = space.call_method(self, "read",
space.wrap(DEFAULT_BUFFER_SIZE))
if not space.isinstance_w(w_data, space.w_str):
raise OperationError(space.w_TypeError, space.wrap(
"read() should return bytes"))
data = space.str_w(w_data)
if not data:
break
builder.append(data)
return space.wrap(builder.build())
def repr__String(space, w_str):
s = w_str._value
buf = StringBuilder(50)
quote = "'"
if quote in s and '"' not in s:
quote = '"'
buf.append(quote)
startslice = 0
for i in range(len(s)):
c = s[i]
use_bs_char = False # character quoted by backspace
if c == '\\' or c == quote:
bs_char = c
use_bs_char = True
elif c == '\t':
bs_char = 't'
use_bs_char = True
elif c == '\r':
bs_char = 'r'
use_bs_char = True
elif c == '\n':
bs_char = 'n'
use_bs_char = True
elif not '\x20' <= c < '\x7f':
n = ord(c)
if i != startslice:
buf.append_slice(s, startslice, i)
startslice = i + 1
buf.append('\\x')
buf.append("0123456789abcdef"[n>>4])
buf.append("0123456789abcdef"[n&0xF])
if use_bs_char:
if i != startslice:
buf.append_slice(s, startslice, i)
startslice = i + 1
buf.append('\\')
buf.append(bs_char)
if len(s) != startslice:
buf.append_slice(s, startslice, len(s))
buf.append(quote)
return space.wrap(buf.build())
def str_swapcase__String(space, w_self):
self = w_self._value
builder = StringBuilder(len(self))
for i in range(len(self)):
ch = self[i]
if ch.isupper():
o = ord(ch) + 32
builder.append(chr(o))
elif ch.islower():
o = ord(ch) - 32
builder.append(chr(o))
else:
builder.append(ch)
return space.wrap(builder.build())
def str_title__String(space, w_self):
input = w_self._value
builder = StringBuilder(len(input))
prev_letter = ' '
for pos in range(len(input)):
ch = input[pos]
if not prev_letter.isalpha():
ch = _upper(ch)
builder.append(ch)
else:
ch = _lower(ch)
builder.append(ch)
prev_letter = ch
return space.wrap(builder.build())
def format(self):
lgt = len(self.fmt) + 4 * len(self.values_w) + 10
if do_unicode:
result = UnicodeBuilder(lgt)
else:
result = StringBuilder(lgt)
self.result = result
while True:
# fast path: consume as many characters as possible
fmt = self.fmt
i = i0 = self.fmtpos
while i < len(fmt):
if fmt[i] == '%':
break
i += 1
else:
result.append_slice(fmt, i0, len(fmt))
break # end of 'fmt' string
result.append_slice(fmt, i0, i)
self.fmtpos = i + 1
# interpret the next formatter
w_value = self.parse_fmt()
c = self.peekchr()
self.forward()
if c == '%':
self.std_wp(const('%'))
continue
if w_value is None:
w_value = self.nextinputvalue()
# dispatch on the formatter
# (this turns into a switch after translation)
for c1 in FORMATTER_CHARS:
if c == c1:
# 'c1' is an annotation constant here,
# so this getattr() is ok
do_fmt = getattr(self, 'fmt_' + c1)
do_fmt(w_value)
break
else:
self.unknown_fmtchar()
self.checkconsumed()
return result.build()
def string_escape_encode(s, quote):
buf = StringBuilder(len(s) + 2)
buf.append(quote)
startslice = 0
for i in range(len(s)):
c = s[i]
use_bs_char = False # character quoted by backspace
if c == '\\' or c == quote:
bs_char = c
use_bs_char = True
elif c == '\t':
bs_char = 't'
use_bs_char = True
elif c == '\r':
bs_char = 'r'
use_bs_char = True
elif c == '\n':
bs_char = 'n'
use_bs_char = True
elif not '\x20' <= c < '\x7f':
n = ord(c)
if i != startslice:
buf.append_slice(s, startslice, i)
startslice = i + 1
buf.append('\\x')
buf.append("0123456789abcdef"[n >> 4])
buf.append("0123456789abcdef"[n & 0xF])
if use_bs_char:
if i != startslice:
buf.append_slice(s, startslice, i)
startslice = i + 1
buf.append('\\')
buf.append(bs_char)
if len(s) != startslice:
buf.append_slice(s, startslice, len(s))
buf.append(quote)
return buf.build()
