class Utf8StringBuilder(object):
@always_inline
def __init__(self, size=0):
self._s = StringBuilder(size)
self._lgt = 0
@always_inline
def append(self, s):
# for strings
self._s.append(s)
newlgt = codepoints_in_utf8(s)
self._lgt += newlgt
@always_inline
def append_slice(self, s, start, end):
self._s.append_slice(s, start, end)
newlgt = codepoints_in_utf8(s, start, end)
self._lgt += newlgt
@signature(types.self(), char(), returns=none())
@always_inline
def append_char(self, s):
# for characters, ascii
self._s.append(s)
self._lgt += 1
@try_inline
def append_code(self, code):
unichr_as_utf8_append(self._s, code, True)
self._lgt += 1
@always_inline
def append_utf8(self, utf8, length):
self._s.append(utf8)
self._lgt += length
@always_inline
def append_utf8_slice(self, utf8, start, end, slicelength):
self._s.append_slice(utf8, start, end)
self._lgt += slicelength
if not we_are_translated():
assert len(utf8[start:end].decode("utf-8")) == slicelength
@always_inline
def append_multiple_char(self, utf8, times):
self._s.append(utf8 * times)
self._lgt += times
@always_inline
def build(self):
return self._s.build()
@always_inline
def getlength(self):
return self._lgt
def test_ptr():
policy = LowLevelAnnotatorPolicy()
@signature(types.ptr(rstr.STR), returns=types.none())
def f(buf):
pass
argtype = getsig(f, policy=policy)[0]
assert isinstance(argtype, SomePtr)
assert argtype.ll_ptrtype.TO == rstr.STR
def g():
f(rstr.mallocstr(10))
getsig(g, policy=policy)
def test_ptr():
policy = LowLevelAnnotatorPolicy()
@signature(types.ptr(rstr.STR), returns=types.none())
def f(buf):
pass
argtype = getsig(f, policy=policy)[0]
assert isinstance(argtype, model.SomePtr)
assert argtype.ll_ptrtype.TO == rstr.STR
def g():
f(rstr.mallocstr(10))
getsig(g, policy=policy)
class Utf8StringBuilder(object):
@always_inline
def __init__(self, size=0):
self._s = StringBuilder(size)
self._lgt = 0
@always_inline
def append(self, s):
# for strings
self._s.append(s)
newlgt = get_utf8_length(s)
self._lgt += newlgt
@always_inline
def append_slice(self, s, start, end):
self._s.append_slice(s, start, end)
newlgt = get_utf8_length(s, start, end)
self._lgt += newlgt
@signature(types.self(), char(), returns=none())
@always_inline
def append_char(self, s):
# for characters, ascii
self._s.append(s)
self._lgt += 1
@try_inline
def append_code(self, code):
unichr_as_utf8_append(self._s, code, True)
self._lgt += 1
@always_inline
def append_utf8(self, utf8, length):
self._s.append(utf8)
self._lgt += length
@always_inline
def append_multiple_char(self, utf8, times):
self._s.append(utf8 * times)
self._lgt += times
@always_inline
def build(self):
return self._s.build()
@always_inline
def getlength(self):
return self._lgt
def _new_copy_contents_fun(SRC_TP, DST_TP, CHAR_TP, name):
@specialize.arg(0)
def _str_ofs(TP, item):
return (llmemory.offsetof(TP, 'chars') +
llmemory.itemoffsetof(TP.chars, 0) +
llmemory.sizeof(CHAR_TP) * item)
@signature(types.any(), types.any(), types.int(), returns=types.any())
@specialize.arg(0)
def _get_raw_buf(TP, src, ofs):
"""
WARNING: dragons ahead.
Return the address of the internal char* buffer of the low level
string. The return value is valid as long as no GC operation occur, so
you must ensure that it will be used inside a "GC safe" section, for
example by marking your function with @rgc.no_collect
"""
assert typeOf(src).TO == TP
assert ofs >= 0
return llmemory.cast_ptr_to_adr(src) + _str_ofs(TP, ofs)
_get_raw_buf._always_inline_ = True
@jit.oopspec('stroruni.copy_contents(src, dst, srcstart, dststart, length)'
)
@signature(types.any(),
types.any(),
types.int(),
types.int(),
types.int(),
returns=types.none())
def copy_string_contents(src, dst, srcstart, dststart, length):
"""Copies 'length' characters from the 'src' string to the 'dst'
string, starting at position 'srcstart' and 'dststart'."""
# xxx Warning: don't try to do this at home. It relies on a lot
# of details to be sure that it works correctly in all cases.
# Notably: no GC operation at all from the first cast_ptr_to_adr()
# because it might move the strings. The keepalive_until_here()
# are obscurely essential to make sure that the strings stay alive
# longer than the raw_memcopy().
assert length >= 0
ll_assert(srcstart >= 0, "copystrc: negative srcstart")
ll_assert(srcstart + length <= len(src.chars), "copystrc: src ovf")
ll_assert(dststart >= 0, "copystrc: negative dststart")
ll_assert(dststart + length <= len(dst.chars), "copystrc: dst ovf")
# from here, no GC operations can happen
asrc = _get_raw_buf(SRC_TP, src, srcstart)
adst = _get_raw_buf(DST_TP, dst, dststart)
llmemory.raw_memcopy(asrc, adst, llmemory.sizeof(CHAR_TP) * length)
# end of "no GC" section
keepalive_until_here(src)
keepalive_until_here(dst)
copy_string_contents._always_inline_ = True
copy_string_contents = func_with_new_name(copy_string_contents,
'copy_%s_contents' % name)
@jit.oopspec('stroruni.copy_string_to_raw(src, ptrdst, srcstart, length)')
def copy_string_to_raw(src, ptrdst, srcstart, length):
"""
Copies 'length' characters from the 'src' string to the 'ptrdst'
buffer, starting at position 'srcstart'.
'ptrdst' must be a non-gc Array of Char.
"""
# xxx Warning: same note as above apply: don't do this at home
assert length >= 0
# from here, no GC operations can happen
asrc = _get_raw_buf(SRC_TP, src, srcstart)
adst = llmemory.cast_ptr_to_adr(ptrdst)
adst = adst + llmemory.itemoffsetof(typeOf(ptrdst).TO, 0)
llmemory.raw_memcopy(asrc, adst, llmemory.sizeof(CHAR_TP) * length)
# end of "no GC" section
keepalive_until_here(src)
copy_string_to_raw._always_inline_ = True
copy_string_to_raw = func_with_new_name(copy_string_to_raw,
'copy_%s_to_raw' % name)
@jit.dont_look_inside
@signature(types.any(),
types.any(),
types.int(),
types.int(),
returns=types.none())
def copy_raw_to_string(ptrsrc, dst, dststart, length):
# xxx Warning: same note as above apply: don't do this at home
assert length >= 0
# from here, no GC operations can happen
adst = _get_raw_buf(SRC_TP, dst, dststart)
asrc = llmemory.cast_ptr_to_adr(ptrsrc)
asrc = asrc + llmemory.itemoffsetof(typeOf(ptrsrc).TO, 0)
llmemory.raw_memcopy(asrc, adst, llmemory.sizeof(CHAR_TP) * length)
# end of "no GC" section
keepalive_until_here(dst)
copy_raw_to_string._always_inline_ = True
copy_raw_to_string = func_with_new_name(copy_raw_to_string,
'copy_raw_to_%s' % name)
return (copy_string_to_raw, copy_raw_to_string, copy_string_contents,
_get_raw_buf)
def test_none():
@signature(returns=types.none())
def f():
pass
assert getsig(f) == [model.s_None]
# by the sandboxing mechanism
ll_read_not_sandboxed = rposix.external('read',
[rffi.INT, rffi.CCHARP, rffi.SIZE_T],
rffi.SIZE_T,
sandboxsafe=True)
ll_write_not_sandboxed = rposix.external('write',
[rffi.INT, rffi.CCHARP, rffi.SIZE_T],
rffi.SIZE_T,
sandboxsafe=True)
@signature(types.int(),
types.ptr(rffi.CCHARP.TO),
types.int(),
returns=types.none())
def writeall_not_sandboxed(fd, buf, length):
while length > 0:
size = rffi.cast(rffi.SIZE_T, length)
count = rffi.cast(lltype.Signed, ll_write_not_sandboxed(fd, buf, size))
if count <= 0:
raise IOError
length -= count
buf = lltype.direct_ptradd(lltype.direct_arrayitems(buf), count)
buf = rffi.cast(rffi.CCHARP, buf)
class FdLoader(rmarshal.Loader):
def __init__(self, fd):
rmarshal.Loader.__init__(self, "")
self.fd = fd
def prepare_const(self, n):
result = malloc(self.LIST, n, immortal=True)
return result
# ____________________________________________________________
#
# Low-level methods. These can be run for testing, but are meant to
# be direct_call'ed from rtyped flow graphs, which means that they will
# get flowed and annotated, mostly with SomePtr.
# adapted C code
@jit.look_inside_iff(lambda l, newsize, overallocate: jit.isconstant(len(l.items)) and jit.isconstant(newsize))
@signature(types.any(), types.int(), types.bool(), returns=types.none())
def _ll_list_resize_hint_really(l, newsize, overallocate):
"""
Ensure l.items has room for at least newsize elements. Note that
l.items may change, and even if newsize is less than l.length on
entry.
"""
# This over-allocates proportional to the list size, making room
# for additional growth. The over-allocation is mild, but is
# enough to give linear-time amortized behavior over a long
# sequence of appends() in the presence of a poorly-performing
# system malloc().
# The growth pattern is: 0, 4, 8, 16, 25, 35, 46, 58, 72, 88, ...
if newsize <= 0:
ll_assert(newsize == 0, "negative list length")
l.length = 0
class SubBuffer(Buffer):
_attrs_ = ['buffer', 'offset', 'size', 'readonly']
_immutable_ = True
@signature(types.any(), types.instance(Buffer), types.int(), types.int(),
returns=types.none())
def __init__(self, buffer, offset, size):
self.readonly = buffer.readonly
if isinstance(buffer, SubBuffer): # don't nest them
# we want a view (offset, size) over a view
# (buffer.offset, buffer.size) over buffer.buffer.
# Note that either '.size' can be -1 to mean 'up to the end'.
at_most = buffer.getlength() - offset
if size > at_most or size < 0:
if at_most < 0:
at_most = 0
size = at_most
offset += buffer.offset
buffer = buffer.buffer
#
self.buffer = buffer
self.offset = offset
self.size = size
def getlength(self):
at_most = self.buffer.getlength() - self.offset
if 0 <= self.size <= at_most:
return self.size
elif at_most >= 0:
return at_most
else:
return 0
def getitem(self, index):
return self.buffer.getitem(self.offset + index)
def getslice(self, start, step, size):
if size == 0:
# otherwise, adding self.offset might make them out of bounds
return ''
return self.buffer.getslice(self.offset + start, step, size)
def setitem(self, index, char):
self.buffer.setitem(self.offset + index, char)
def setslice(self, start, string):
if len(string) == 0:
# otherwise, adding self.offset might make 'start' out of bounds
return
self.buffer.setslice(self.offset + start, string)
def get_raw_address(self):
from rpython.rtyper.lltypesystem import rffi
ptr = self.buffer.get_raw_address()
return rffi.ptradd(ptr, self.offset)
@specialize.ll_and_arg(1)
def typed_read(self, TP, byte_offset):
return self.buffer.typed_read(TP, byte_offset + self.offset)
@specialize.ll_and_arg(1)
def typed_write(self, TP, byte_offset, value):
return self.buffer.typed_write(TP, byte_offset + self.offset, value)
def _new_copy_contents_fun(SRC_TP, DST_TP, CHAR_TP, name):
@specialize.arg(0)
def _str_ofs(TP, item):
return (llmemory.offsetof(TP, 'chars') +
llmemory.itemoffsetof(TP.chars, 0) +
llmemory.sizeof(CHAR_TP) * item)
@signature(types.any(), types.any(), types.int(), returns=types.any())
@specialize.arg(0)
def _get_raw_buf(TP, src, ofs):
"""
WARNING: dragons ahead.
Return the address of the internal char* buffer of the low level
string. The return value is valid as long as no GC operation occur, so
you must ensure that it will be used inside a "GC safe" section, for
example by marking your function with @rgc.no_collect
"""
assert typeOf(src).TO == TP
assert ofs >= 0
return llmemory.cast_ptr_to_adr(src) + _str_ofs(TP, ofs)
_get_raw_buf._always_inline_ = True
@jit.oopspec('stroruni.copy_contents(src, dst, srcstart, dststart, length)')
@signature(types.any(), types.any(), types.int(), types.int(), types.int(), returns=types.none())
def copy_string_contents(src, dst, srcstart, dststart, length):
"""Copies 'length' characters from the 'src' string to the 'dst'
string, starting at position 'srcstart' and 'dststart'."""
# xxx Warning: don't try to do this at home. It relies on a lot
# of details to be sure that it works correctly in all cases.
# Notably: no GC operation at all from the first cast_ptr_to_adr()
# because it might move the strings. The keepalive_until_here()
# are obscurely essential to make sure that the strings stay alive
# longer than the raw_memcopy().
assert length >= 0
ll_assert(srcstart >= 0, "copystrc: negative srcstart")
ll_assert(srcstart + length <= len(src.chars), "copystrc: src ovf")
ll_assert(dststart >= 0, "copystrc: negative dststart")
ll_assert(dststart + length <= len(dst.chars), "copystrc: dst ovf")
# from here, no GC operations can happen
asrc = _get_raw_buf(SRC_TP, src, srcstart)
adst = _get_raw_buf(DST_TP, dst, dststart)
llmemory.raw_memcopy(asrc, adst, llmemory.sizeof(CHAR_TP) * length)
# end of "no GC" section
keepalive_until_here(src)
keepalive_until_here(dst)
copy_string_contents._always_inline_ = True
copy_string_contents = func_with_new_name(copy_string_contents,
'copy_%s_contents' % name)
@jit.oopspec('stroruni.copy_string_to_raw(src, ptrdst, srcstart, length)')
def copy_string_to_raw(src, ptrdst, srcstart, length):
"""
Copies 'length' characters from the 'src' string to the 'ptrdst'
buffer, starting at position 'srcstart'.
'ptrdst' must be a non-gc Array of Char.
"""
# xxx Warning: same note as above apply: don't do this at home
assert length >= 0
# from here, no GC operations can happen
asrc = _get_raw_buf(SRC_TP, src, srcstart)
adst = llmemory.cast_ptr_to_adr(ptrdst)
adst = adst + llmemory.itemoffsetof(typeOf(ptrdst).TO, 0)
llmemory.raw_memcopy(asrc, adst, llmemory.sizeof(CHAR_TP) * length)
# end of "no GC" section
keepalive_until_here(src)
copy_string_to_raw._always_inline_ = True
copy_string_to_raw = func_with_new_name(copy_string_to_raw, 'copy_%s_to_raw' % name)
@jit.dont_look_inside
@signature(types.any(), types.any(), types.int(), types.int(),
returns=types.none())
def copy_raw_to_string(ptrsrc, dst, dststart, length):
# xxx Warning: same note as above apply: don't do this at home
assert length >= 0
# from here, no GC operations can happen
adst = _get_raw_buf(SRC_TP, dst, dststart)
asrc = llmemory.cast_ptr_to_adr(ptrsrc)
asrc = asrc + llmemory.itemoffsetof(typeOf(ptrsrc).TO, 0)
llmemory.raw_memcopy(asrc, adst, llmemory.sizeof(CHAR_TP) * length)
# end of "no GC" section
keepalive_until_here(dst)
copy_raw_to_string._always_inline_ = True
copy_raw_to_string = func_with_new_name(copy_raw_to_string,
'copy_raw_to_%s' % name)
return copy_string_to_raw, copy_raw_to_string, copy_string_contents
# a version of os.read() and os.write() that are not mangled
# by the sandboxing mechanism
ll_read_not_sandboxed = rffi.llexternal('read',
[rffi.INT, rffi.CCHARP, rffi.SIZE_T],
rffi.SIZE_T,
sandboxsafe=True)
ll_write_not_sandboxed = rffi.llexternal('write',
[rffi.INT, rffi.CCHARP, rffi.SIZE_T],
rffi.SIZE_T,
sandboxsafe=True)
@signature(types.int(), types.ptr(rffi.CCHARP.TO), types.int(), returns=types.none())
def writeall_not_sandboxed(fd, buf, length):
while length > 0:
size = rffi.cast(rffi.SIZE_T, length)
count = rffi.cast(lltype.Signed, ll_write_not_sandboxed(fd, buf, size))
if count <= 0:
raise IOError
length -= count
buf = lltype.direct_ptradd(lltype.direct_arrayitems(buf), count)
buf = rffi.cast(rffi.CCHARP, buf)
class FdLoader(rmarshal.Loader):
def __init__(self, fd):
rmarshal.Loader.__init__(self, "")
self.fd = fd
def _new_copy_contents_fun(SRC_TP, DST_TP, CHAR_TP, name):
@specialize.arg(0)
def _str_ofs(TP, item):
return (llmemory.offsetof(TP, 'chars') +
llmemory.itemoffsetof(TP.chars, 0) +
llmemory.sizeof(CHAR_TP) * item)
@signature(types.any(), types.any(), types.int(), returns=types.any())
@specialize.arg(0)
def _get_raw_buf(TP, src, ofs):
assert typeOf(src).TO == TP
assert ofs >= 0
return llmemory.cast_ptr_to_adr(src) + _str_ofs(TP, ofs)
_get_raw_buf._always_inline_ = True
@jit.oopspec('stroruni.copy_contents(src, dst, srcstart, dststart, length)')
@signature(types.any(), types.any(), types.int(), types.int(), types.int(), returns=types.none())
def copy_string_contents(src, dst, srcstart, dststart, length):
"""Copies 'length' characters from the 'src' string to the 'dst'
string, starting at position 'srcstart' and 'dststart'."""
# xxx Warning: don't try to do this at home. It relies on a lot
# of details to be sure that it works correctly in all cases.
# Notably: no GC operation at all from the first cast_ptr_to_adr()
# because it might move the strings. The keepalive_until_here()
# are obscurely essential to make sure that the strings stay alive
# longer than the raw_memcopy().
assert length >= 0
# from here, no GC operations can happen
src = _get_raw_buf(SRC_TP, src, srcstart)
dst = _get_raw_buf(DST_TP, dst, dststart)
llmemory.raw_memcopy(src, dst, llmemory.sizeof(CHAR_TP) * length)
# end of "no GC" section
keepalive_until_here(src)
keepalive_until_here(dst)
copy_string_contents._always_inline_ = True
copy_string_contents = func_with_new_name(copy_string_contents,
'copy_%s_contents' % name)
@jit.oopspec('stroruni.copy_string_to_raw(src, ptrdst, srcstart, length)')
def copy_string_to_raw(src, ptrdst, srcstart, length):
"""
Copies 'length' characters from the 'src' string to the 'ptrdst'
buffer, starting at position 'srcstart'.
'ptrdst' must be a non-gc Array of Char.
"""
# xxx Warning: same note as above apply: don't do this at home
assert length >= 0
# from here, no GC operations can happen
src = _get_raw_buf(SRC_TP, src, srcstart)
adr = llmemory.cast_ptr_to_adr(ptrdst)
dstbuf = adr + llmemory.itemoffsetof(typeOf(ptrdst).TO, 0)
llmemory.raw_memcopy(src, dstbuf, llmemory.sizeof(CHAR_TP) * length)
# end of "no GC" section
keepalive_until_here(src)
copy_string_to_raw._always_inline_ = True
copy_string_to_raw = func_with_new_name(copy_string_to_raw, 'copy_%s_to_raw' % name)
@jit.dont_look_inside
@signature(types.any(), types.any(), types.int(), types.int(),
returns=types.none())
def copy_raw_to_string(ptrsrc, dst, dststart, length):
# xxx Warning: same note as above apply: don't do this at home
assert length >= 0
# from here, no GC operations can happen
dst = _get_raw_buf(SRC_TP, dst, dststart)
adr = llmemory.cast_ptr_to_adr(ptrsrc)
srcbuf = adr + llmemory.itemoffsetof(typeOf(ptrsrc).TO, 0)
llmemory.raw_memcopy(srcbuf, dst, llmemory.sizeof(CHAR_TP) * length)
# end of "no GC" section
keepalive_until_here(dst)
copy_raw_to_string._always_inline_ = True
copy_raw_to_string = func_with_new_name(copy_raw_to_string,
'copy_raw_to_%s' % name)
return copy_string_to_raw, copy_raw_to_string, copy_string_contents
def test_none():
@signature(returns=types.none())
def f():
pass
assert getsig(f) == [model.s_None]
class C(object):
@signature(types.self(), returns=types.none())
def incomplete_sig_meth(self):
pass
class C(object):
@signature(types.self(), types.self(), returns=types.none())
def f(self, other):
pass
# a version of os.read() and os.write() that are not mangled
# by the sandboxing mechanism
ll_read_not_sandboxed = rposix.external('read',
[rffi.INT, rffi.CCHARP, rffi.SIZE_T],
rffi.SIZE_T,
sandboxsafe=True)
ll_write_not_sandboxed = rposix.external('write',
[rffi.INT, rffi.CCHARP, rffi.SIZE_T],
rffi.SIZE_T,
sandboxsafe=True)
@signature(types.int(), types.ptr(rffi.CCHARP.TO), types.int(),
returns=types.none())
def writeall_not_sandboxed(fd, buf, length):
while length > 0:
size = rffi.cast(rffi.SIZE_T, length)
count = rffi.cast(lltype.Signed, ll_write_not_sandboxed(fd, buf, size))
if count <= 0:
raise IOError
length -= count
buf = lltype.direct_ptradd(lltype.direct_arrayitems(buf), count)
buf = rffi.cast(rffi.CCHARP, buf)
class FdLoader(rmarshal.Loader):
def __init__(self, fd):
rmarshal.Loader.__init__(self, "")
self.fd = fd