def a2b_uu(space, ascii):
"Decode a line of uuencoded data."
if len(ascii) == 0: # obscure case, for compability with CPython
length = (-0x20) & 0x3f
else:
length = (ord(ascii[0]) - 0x20) & 0x3f
res = StringBuilder(length)
for i in range(1, len(ascii), 4):
A = _a2b_read(space, ascii, i)
B = _a2b_read(space, ascii, i + 1)
C = _a2b_read(space, ascii, i + 2)
D = _a2b_read(space, ascii, i + 3)
#
if res.getlength() < length:
res.append(chr(A << 2 | B >> 4))
elif A != 0 or B != 0:
raise_Error(space, "Trailing garbage")
#
if res.getlength() < length:
res.append(chr((B & 0xf) << 4 | C >> 2))
elif C != 0:
raise_Error(space, "Trailing garbage")
#
if res.getlength() < length:
res.append(chr((C & 0x3) << 6 | D))
elif D != 0:
raise_Error(space, "Trailing garbage")
remaining = length - res.getlength()
if remaining > 0:
res.append_multiple_char('\x00', remaining)
return space.wrap(res.build())
def a2b_uu(space, ascii):
"Decode a line of uuencoded data."
if len(ascii) == 0: # obscure case, for compability with CPython
length = (-0x20) & 0x3F
else:
length = (ord(ascii[0]) - 0x20) & 0x3F
res = StringBuilder(length)
for i in range(1, len(ascii), 4):
A = _a2b_read(space, ascii, i)
B = _a2b_read(space, ascii, i + 1)
C = _a2b_read(space, ascii, i + 2)
D = _a2b_read(space, ascii, i + 3)
#
if res.getlength() < length:
res.append(chr(A << 2 | B >> 4))
elif A != 0 or B != 0:
raise_Error(space, "Trailing garbage")
#
if res.getlength() < length:
res.append(chr((B & 0xF) << 4 | C >> 2))
elif C != 0:
raise_Error(space, "Trailing garbage")
#
if res.getlength() < length:
res.append(chr((C & 0x3) << 6 | D))
elif D != 0:
raise_Error(space, "Trailing garbage")
remaining = length - res.getlength()
if remaining > 0:
res.append_multiple_char("\x00", remaining)
return space.wrap(res.build())
class W_StringOutputPort(W_OutputPort):
errorname = "output-port"
def __init__(self):
self.closed = False
self.str = StringBuilder()
def write(self, s):
self.str.append(s)
def contents(self):
return self.str.build()
def seek(self, offset, end=False):
if end or offset == self.str.getlength():
return
if offset > self.str.getlength():
self.str.append("\0" * (self.str.getlength() - offset))
else:
# FIXME: this is potentially slow.
content = self.contents()
self.str = StringBuilder(offset)
self.str.append_slice(content, 0, offset)
def tell(self):
return self.str.getlength()
class W_StringOutputPort(W_OutputPort):
errorname = "output-port"
_attrs_ = ["closed", "str"]
def __init__(self):
self.closed = False
self.str = StringBuilder()
def write(self, s):
self.str.append(s)
def contents(self):
return self.str.build()
def seek(self, offset, end=False):
if end or offset == self.str.getlength():
return
if offset > self.str.getlength():
self.str.append("\0" * (self.str.getlength() - offset))
else:
# FIXME: this is potentially slow.
content = self.contents()
self.str = StringBuilder(offset)
self.str.append_slice(content, 0, offset)
def tell(self):
return self.str.getlength()
def readline_w(self, space, w_limit=None):
self._check_attached(space)
self._check_closed(space)
self._writeflush(space)
limit = convert_size(space, w_limit)
remnant = None
builder = StringBuilder()
# XXX maybe use Utf8StringBuilder instead?
while True:
# First, get some data if necessary
has_data = self._ensure_data(space)
if not has_data:
# end of file
if remnant:
builder.append(remnant)
break
if remnant:
assert not self.readtranslate and self.readnl == '\r\n'
assert self.decoded.pos == 0
if remnant == '\r' and self.decoded.text[0] == '\n':
builder.append('\r\n')
self.decoded.pos = 1
remnant = None
break
else:
builder.append(remnant)
remnant = None
continue
if limit >= 0:
remaining = limit - builder.getlength()
assert remaining >= 0
else:
remaining = -1
start = self.decoded.pos
assert start >= 0
found = self._scan_line_ending(remaining)
end_scan = self.decoded.pos
if end_scan > start:
s = self.decoded.text[start:end_scan]
builder.append(s)
if found or (limit >= 0 and builder.getlength() >= limit):
break
# There may be some remaining chars we'll have to prepend to the
# next chunk of data
if not self.decoded.exhausted():
remnant = self.decoded.get_chars(-1)
# We have consumed the buffer
self.decoded.reset()
result = builder.build()
lgt = get_utf8_length(result)
return space.newutf8(result, lgt)
class W_BytesBuilder(W_Root):
def __init__(self, space, size):
if size < 0:
self.builder = StringBuilder()
else:
self.builder = StringBuilder(size)
@unwrap_spec(size=int)
def descr__new__(space, w_subtype, size=-1):
return W_BytesBuilder(space, size)
@unwrap_spec(s='bytes')
def descr_append(self, space, s):
self.builder.append(s)
@unwrap_spec(s='bytes', start=int, end=int)
def descr_append_slice(self, space, s, start, end):
if not 0 <= start <= end <= len(s):
raise oefmt(space.w_ValueError, "bad start/stop")
self.builder.append_slice(s, start, end)
def descr_build(self, space):
w_s = space.newbytes(self.builder.build())
# after build(), we can continue to append more strings
# to the same builder. This is supported since
# 2ff5087aca28 in RPython.
return w_s
def descr_len(self, space):
if self.builder is None:
raise oefmt(space.w_ValueError, "no length of built builder")
return space.newint(self.builder.getlength())
def test_string_builder():
s = StringBuilder()
s.append("a")
s.append("abc")
assert s.getlength() == len('aabc')
s.append("a")
s.append_slice("abc", 1, 2)
s.append_multiple_char('d', 4)
assert s.build() == "aabcabdddd"
def test_string_builder():
s = StringBuilder()
s.append("a")
s.append("abc")
assert s.getlength() == len("aabc")
s.append("a")
s.append_slice("abc", 1, 2)
s.append_multiple_char("d", 4)
result = s.build()
assert result == "aabcabdddd"
assert result == s.build()
s.append("x")
assert s.build() == result + "x"
def test_string_builder():
s = StringBuilder()
s.append("a")
s.append("abc")
assert s.getlength() == len('aabc')
s.append("a")
s.append_slice("abc", 1, 2)
s.append_multiple_char('d', 4)
result = s.build()
assert result == "aabcabdddd"
assert result == s.build()
s.append("x")
assert s.build() == result + "x"
def readall_w(self, space):
builder = StringBuilder()
while True:
w_data = space.call_method(self, "read",
space.wrap(DEFAULT_BUFFER_SIZE))
if space.is_w(w_data, space.w_None):
if not builder.getlength():
return w_data
break
if not space.isinstance_w(w_data, space.w_str):
raise OperationError(space.w_TypeError, space.wrap(
"read() should return bytes"))
data = space.str_w(w_data)
if not data:
break
builder.append(data)
return space.wrap(builder.build())
def readall_w(self, space):
builder = StringBuilder()
while True:
try:
w_data = space.call_method(self, "read",
space.wrap(DEFAULT_BUFFER_SIZE))
except OperationError as e:
if trap_eintr(space, e):
continue
raise
if space.is_w(w_data, space.w_None):
if not builder.getlength():
return w_data
break
if not space.isinstance_w(w_data, space.w_str):
raise oefmt(space.w_TypeError, "read() should return bytes")
data = space.bytes_w(w_data)
if not data:
break
builder.append(data)
return space.newbytes(builder.build())
def readall_w(self, space):
builder = StringBuilder()
while True:
try:
w_data = space.call_method(self, "read",
space.wrap(DEFAULT_BUFFER_SIZE))
except OperationError, e:
if trap_eintr(space, e):
continue
raise
if space.is_w(w_data, space.w_None):
if not builder.getlength():
return w_data
break
if not space.isinstance_w(w_data, space.w_str):
raise OperationError(space.w_TypeError,
space.wrap("read() should return bytes"))
data = space.str_w(w_data)
if not data:
break
builder.append(data)
def a2b_uu(space, ascii):
"Decode a line of uuencoded data."
if len(ascii) == 0: # obscure case, for compability with CPython
length = (-0x20) & 0x3F
else:
length = (ord(ascii[0]) - 0x20) & 0x3F
res = StringBuilder(length)
for i in range(1, len(ascii), 4):
A = _a2b_read(space, ascii, i)
B = _a2b_read(space, ascii, i + 1)
C = _a2b_read(space, ascii, i + 2)
D = _a2b_read(space, ascii, i + 3)
#
_a2b_write(space, res, length, A << 2 | B >> 4)
_a2b_write(space, res, length, (B & 0xF) << 4 | C >> 2)
_a2b_write(space, res, length, (C & 0x3) << 6 | D)
remaining = length - res.getlength()
if remaining > 0:
res.append_multiple_char("\x00", remaining)
return space.wrap(res.build())
def readall(self):
pos = self.pos
assert pos >= 0
builder = StringBuilder()
if self.buf:
builder.append_slice(self.buf, pos, len(self.buf))
self.buf = ""
self.pos = 0
bufsize = self.bufsize
while 1:
try:
data = self.do_read(bufsize)
except OSError as o:
# like CPython < 3.4, partial results followed by an error
# are returned as data
if not builder.getlength():
raise
break
if not data:
break
builder.append(data)
bufsize = min(bufsize * 2, self.bigsize)
return builder.build()
def a2b_uu(space, ascii):
"Decode a line of uuencoded data."
if len(ascii) == 0: # obscure case, for compability with CPython
length = (-0x20) & 0x3f
else:
length = (ord(ascii[0]) - 0x20) & 0x3f
res = StringBuilder(length)
for i in range(1, len(ascii), 4):
A = _a2b_read(space, ascii, i)
B = _a2b_read(space, ascii, i + 1)
C = _a2b_read(space, ascii, i + 2)
D = _a2b_read(space, ascii, i + 3)
#
_a2b_write(space, res, length, A << 2 | B >> 4)
_a2b_write(space, res, length, (B & 0xf) << 4 | C >> 2)
_a2b_write(space, res, length, (C & 0x3) << 6 | D)
remaining = length - res.getlength()
if remaining > 0:
res.append_multiple_char('\x00', remaining)
return space.wrap(res.build())
def readline(self, size=-1):
self._check_closed()
if size == 0:
return ""
elif size < 0 and not self._univ_newline:
with rffi.scoped_alloc_buffer(BASE_LINE_SIZE) as buf:
c = self._readline1(buf.raw)
if c >= 0:
return buf.str(c)
# this is the rare case: the line is longer than BASE_LINE_SIZE
s = StringBuilder()
while True:
s.append_charpsize(buf.raw, BASE_LINE_SIZE - 1)
c = self._readline1(buf.raw)
if c >= 0:
break
s.append_charpsize(buf.raw, c)
return s.build()
else: # size > 0 or self._univ_newline
ll_file = self._ll_file
c = 0
s = StringBuilder()
if self._univ_newline:
newlinetypes = self._newlinetypes
skipnextlf = self._skipnextlf
while size < 0 or s.getlength() < size:
c = c_getc(ll_file)
if c == EOF:
break
if skipnextlf:
skipnextlf = False
if c == ord('\n'):
newlinetypes |= NEWLINE_CRLF
c = c_getc(ll_file)
if c == EOF:
break
else:
newlinetypes |= NEWLINE_CR
if c == ord('\r'):
skipnextlf = True
c = ord('\n')
elif c == ord('\n'):
newlinetypes |= NEWLINE_LF
s.append(chr(c))
if c == ord('\n'):
break
if c == EOF:
if skipnextlf:
newlinetypes |= NEWLINE_CR
self._newlinetypes = newlinetypes
self._skipnextlf = skipnextlf
else:
while s.getlength() < size:
c = c_getc(ll_file)
if c == EOF:
break
s.append(chr(c))
if c == ord('\n'):
break
if c == EOF:
if c_ferror(ll_file):
raise _error(ll_file)
return s.build()
def func():
s = StringBuilder()
s.append("a")
s.append("abc")
return s.getlength()
class PackFormatIterator(FormatIterator):
def __init__(self, space, args_w, size):
self.space = space
self.args_w = args_w
self.args_index = 0
self.result = StringBuilder(size)
# This *should* be always unroll safe, the only way to get here is by
# unroll the interpret function, which means the fmt is const, and thus
# this should be const (in theory ;)
@jit.unroll_safe
@specialize.arg(1)
def operate(self, fmtdesc, repetitions):
if fmtdesc.needcount:
fmtdesc.pack(self, repetitions)
else:
for i in range(repetitions):
fmtdesc.pack(self)
_operate_is_specialized_ = True
@jit.unroll_safe
def align(self, mask):
pad = (-self.result.getlength()) & mask
self.result.append_multiple_char('\x00', pad)
def finished(self):
if self.args_index != len(self.args_w):
raise StructError("too many arguments for struct format")
def accept_obj_arg(self):
try:
w_obj = self.args_w[self.args_index]
except IndexError:
raise StructError("struct format requires more arguments")
self.args_index += 1
return w_obj
def accept_int_arg(self):
return self._accept_integral("int_w")
def accept_uint_arg(self):
return self._accept_integral("uint_w")
def accept_longlong_arg(self):
return self._accept_integral("r_longlong_w")
def accept_ulonglong_arg(self):
return self._accept_integral("r_ulonglong_w")
@specialize.arg(1)
def _accept_integral(self, meth):
space = self.space
w_obj = self.accept_obj_arg()
if (space.isinstance_w(w_obj, space.w_int) or
space.isinstance_w(w_obj, space.w_long)):
w_index = w_obj
else:
w_index = None
w_index_method = space.lookup(w_obj, "__index__")
if w_index_method is not None:
try:
w_index = space.index(w_obj)
except OperationError, e:
if not e.match(space, space.w_TypeError):
raise
pass
if w_index is None:
w_index = self._maybe_float(w_obj)
return getattr(space, meth)(w_index)
class PackFormatIterator(FormatIterator):
def __init__(self, space, args_w, size):
self.space = space
self.args_w = args_w
self.args_index = 0
self.result = StringBuilder(size)
# This *should* be always unroll safe, the only way to get here is by
# unroll the interpret function, which means the fmt is const, and thus
# this should be const (in theory ;)
@jit.unroll_safe
@specialize.arg(1)
def operate(self, fmtdesc, repetitions):
if fmtdesc.needcount:
fmtdesc.pack(self, repetitions)
else:
for i in range(repetitions):
fmtdesc.pack(self)
_operate_is_specialized_ = True
@jit.unroll_safe
def align(self, mask):
pad = (-self.result.getlength()) & mask
self.result.append_multiple_char('\x00', pad)
def finished(self):
if self.args_index != len(self.args_w):
raise StructError("too many arguments for struct format")
def accept_obj_arg(self):
try:
w_obj = self.args_w[self.args_index]
except IndexError:
raise StructError("struct format requires more arguments")
self.args_index += 1
return w_obj
def accept_int_arg(self):
return self._accept_integral("int_w")
def accept_uint_arg(self):
return self._accept_integral("uint_w")
def accept_longlong_arg(self):
return self._accept_integral("r_longlong_w")
def accept_ulonglong_arg(self):
return self._accept_integral("r_ulonglong_w")
@specialize.arg(1)
def _accept_integral(self, meth):
space = self.space
w_obj = self.accept_obj_arg()
if (space.isinstance_w(w_obj, space.w_int)
or space.isinstance_w(w_obj, space.w_long)):
w_index = w_obj
else:
w_index = None
if space.lookup(w_obj, '__index__'):
try:
w_index = space.index(w_obj)
except OperationError, e:
if not e.match(space, space.w_TypeError):
raise
pass
if w_index is None and space.lookup(w_obj, '__int__'):
if space.isinstance_w(w_obj, space.w_float):
msg = "integer argument expected, got float"
else:
msg = "integer argument expected, got non-integer" \
" (implicit conversion using __int__ is deprecated)"
space.warn(space.wrap(msg), space.w_DeprecationWarning)
w_index = space.int(
w_obj) # wrapped float -> wrapped int or long
if w_index is None:
raise StructError("cannot convert argument to integer")
return getattr(space, meth)(w_index)
def func():
s = StringBuilder()
s.append("a")
s.append("abc")
return s.getlength()
class PackFormatIterator(FormatIterator):
def __init__(self, space, args_w, size):
self.space = space
self.args_w = args_w
self.args_index = 0
self.result = StringBuilder(size)
# This *should* be always unroll safe, the only way to get here is by
# unroll the interpret function, which means the fmt is const, and thus
# this should be const (in theory ;)
@jit.unroll_safe
@specialize.arg(1)
def operate(self, fmtdesc, repetitions):
if fmtdesc.needcount:
fmtdesc.pack(self, repetitions)
else:
for i in range(repetitions):
fmtdesc.pack(self)
_operate_is_specialized_ = True
@jit.unroll_safe
def align(self, mask):
pad = (-self.result.getlength()) & mask
self.result.append_multiple_char('\x00', pad)
def finished(self):
if self.args_index != len(self.args_w):
raise StructError("too many arguments for struct format")
def accept_obj_arg(self):
try:
w_obj = self.args_w[self.args_index]
except IndexError:
raise StructError("struct format requires more arguments")
self.args_index += 1
return w_obj
def accept_int_arg(self):
return self._accept_integral("int_w")
def accept_uint_arg(self):
return self._accept_integral("uint_w")
def accept_longlong_arg(self):
return self._accept_integral("r_longlong_w")
def accept_ulonglong_arg(self):
return self._accept_integral("r_ulonglong_w")
@specialize.arg(1)
def _accept_integral(self, meth):
space = self.space
w_obj = self.accept_obj_arg()
if (space.isinstance_w(w_obj, space.w_int) or
space.isinstance_w(w_obj, space.w_long)):
w_index = w_obj
else:
w_index = None
if space.lookup(w_obj, '__index__'):
try:
w_index = space.index(w_obj)
except OperationError, e:
if not e.match(space, space.w_TypeError):
raise
pass
if w_index is None and space.lookup(w_obj, '__int__'):
if space.isinstance_w(w_obj, space.w_float):
msg = "integer argument expected, got float"
else:
msg = "integer argument expected, got non-integer" \
" (implicit conversion using __int__ is deprecated)"
space.warn(space.wrap(msg), space.w_DeprecationWarning)
w_index = space.int(w_obj) # wrapped float -> wrapped int or long
if w_index is None:
raise StructError("cannot convert argument to integer")
method = getattr(space, meth)
try:
return method(w_index)
except OperationError as e:
if e.match(self.space, self.space.w_OverflowError):
raise StructError("argument out of range")
raise
class PackFormatIterator(FormatIterator):
def __init__(self, space, args_w, size):
self.space = space
self.args_w = args_w
self.args_index = 0
self.result = StringBuilder(size)
# This *should* be always unroll safe, the only way to get here is by
# unroll the interpret function, which means the fmt is const, and thus
# this should be const (in theory ;)
@jit.unroll_safe
@specialize.arg(1)
def operate(self, fmtdesc, repetitions):
if fmtdesc.needcount:
fmtdesc.pack(self, repetitions)
else:
for i in range(repetitions):
fmtdesc.pack(self)
_operate_is_specialized_ = True
@jit.unroll_safe
def align(self, mask):
pad = (-self.result.getlength()) & mask
self.result.append_multiple_char('\x00', pad)
def finished(self):
if self.args_index != len(self.args_w):
raise StructError("too many arguments for struct format")
def accept_obj_arg(self):
try:
w_obj = self.args_w[self.args_index]
except IndexError:
raise StructError("struct format requires more arguments")
self.args_index += 1
return w_obj
def accept_int_arg(self):
return self._accept_integral("int_w")
def accept_uint_arg(self):
return self._accept_integral("uint_w")
def accept_longlong_arg(self):
return self._accept_integral("r_longlong_w")
def accept_ulonglong_arg(self):
return self._accept_integral("r_ulonglong_w")
@specialize.arg(1)
def _accept_integral(self, meth):
space = self.space
w_obj = self.accept_obj_arg()
if space.isinstance_w(w_obj, space.w_int):
w_index = w_obj
else:
w_index = None
if space.lookup(w_obj, '__index__'):
try:
w_index = space.index(w_obj)
except OperationError, e:
if not e.match(space, space.w_TypeError):
raise
pass
if w_index is None:
raise StructError("required argument is not an integer")
method = getattr(space, meth)
try:
return method(w_index)
except OperationError as e:
if e.match(self.space, self.space.w_OverflowError):
raise StructError("argument out of range")
raise
def readline(self, size=-1):
self._check_closed()
if size == 0:
return ""
elif size < 0 and not self._univ_newline:
with rffi.scoped_alloc_buffer(BASE_LINE_SIZE) as buf:
c = self._readline1(buf.raw)
if c >= 0:
return buf.str(c)
# this is the rare case: the line is longer than BASE_LINE_SIZE
s = StringBuilder()
while True:
s.append_charpsize(buf.raw, BASE_LINE_SIZE - 1)
c = self._readline1(buf.raw)
if c >= 0:
break
s.append_charpsize(buf.raw, c)
return s.build()
else: # size > 0 or self._univ_newline
ll_file = self._ll_file
c = 0
s = StringBuilder()
if self._univ_newline:
newlinetypes = self._newlinetypes
skipnextlf = self._skipnextlf
while size < 0 or s.getlength() < size:
c = c_getc(ll_file)
if c == EOF:
break
if skipnextlf:
skipnextlf = False
if c == ord('\n'):
newlinetypes |= NEWLINE_CRLF
c = c_getc(ll_file)
if c == EOF:
break
else:
newlinetypes |= NEWLINE_CR
if c == ord('\r'):
skipnextlf = True
c = ord('\n')
elif c == ord('\n'):
newlinetypes |= NEWLINE_LF
s.append(chr(c))
if c == ord('\n'):
break
if c == EOF:
if skipnextlf:
newlinetypes |= NEWLINE_CR
self._newlinetypes = newlinetypes
self._skipnextlf = skipnextlf
else:
while s.getlength() < size:
c = c_getc(ll_file)
if c == EOF:
break
s.append(chr(c))
if c == ord('\n'):
break
if c == EOF:
if c_ferror(ll_file):
raise _error(ll_file)
return s.build()
