def ll_dict_getitem(d, key):
i = ll_dict_lookup(d, key, d.keyhash(key))
if not i & HIGHEST_BIT:
return ll_get_value(d, i)
else:
raise KeyError
def ll_dict_setitem(d, key, value):
hash = d.keyhash(key)
i = ll_dict_lookup(d, key, hash)
return _ll_dict_setitem_lookup_done(d, key, value, hash, i)
# It may be safe to look inside always, it has a few branches though, and their
# frequencies needs to be investigated.
@jit.look_inside_iff(lambda d, key, value, hash, i: jit.isvirtual(d) and jit.isconstant(key))
def _ll_dict_setitem_lookup_done(d, key, value, hash, i):
valid = (i & HIGHEST_BIT) == 0
i = i & MASK
everused = d.entries.everused(i)
# set up the new entry
ENTRY = lltype.typeOf(d.entries).TO.OF
entry = d.entries[i]
entry.value = value
if valid:
return
entry.key = key
if hasattr(ENTRY, 'f_hash'): entry.f_hash = hash
if hasattr(ENTRY, 'f_valid'): entry.f_valid = True
d.num_items += 1
if not everused:
class LLHelpers(AbstractLLHelpers):
@jit.elidable
def ll_str_mul(s, times):
if times < 0:
times = 0
try:
size = ovfcheck(len(s.chars) * times)
except OverflowError:
raise MemoryError
newstr = s.malloc(size)
i = 0
if i < size:
s.copy_contents(s, newstr, 0, 0, len(s.chars))
i += len(s.chars)
while i < size:
if i <= size - i:
j = i
else:
j = size - i
s.copy_contents(newstr, newstr, 0, i, j)
i += j
return newstr
@jit.elidable
def ll_char_mul(ch, times):
if typeOf(ch) is Char:
malloc = mallocstr
else:
malloc = mallocunicode
if times < 0:
times = 0
newstr = malloc(times)
j = 0
# XXX we can use memset here, not sure how useful this is
while j < times:
newstr.chars[j] = ch
j += 1
return newstr
def ll_strlen(s):
return len(s.chars)
def ll_stritem_nonneg(s, i):
chars = s.chars
ll_assert(i >= 0, "negative str getitem index")
ll_assert(i < len(chars), "str getitem index out of bound")
return chars[i]
ll_stritem_nonneg._annenforceargs_ = [None, int]
def ll_chr2str(ch):
if typeOf(ch) is Char:
malloc = mallocstr
else:
malloc = mallocunicode
s = malloc(1)
s.chars[0] = ch
return s
# @jit.look_inside_iff(lambda str: jit.isconstant(len(str.chars)) and len(str.chars) == 1)
@jit.oopspec("str.str2unicode(str)")
def ll_str2unicode(str):
lgt = len(str.chars)
s = mallocunicode(lgt)
for i in range(lgt):
if ord(str.chars[i]) > 127:
raise UnicodeDecodeError
s.chars[i] = cast_primitive(UniChar, str.chars[i])
return s
@jit.elidable
def ll_strhash(s):
# unlike CPython, there is no reason to avoid to return -1
# but our malloc initializes the memory to zero, so we use zero as the
# special non-computed-yet value.
if not s:
return 0
x = s.hash
if x == 0:
x = _hash_string(s.chars)
if x == 0:
x = 29872897
s.hash = x
return x
def ll_strfasthash(s):
return s.hash # assumes that the hash is already computed
@jit.elidable
def ll_strconcat(s1, s2):
len1 = len(s1.chars)
len2 = len(s2.chars)
# a single '+' like this is allowed to overflow: it gets
# a negative result, and the gc will complain
newstr = s1.malloc(len1 + len2)
s1.copy_contents(s1, newstr, 0, 0, len1)
s1.copy_contents(s2, newstr, 0, len1, len2)
return newstr
ll_strconcat.oopspec = 'stroruni.concat(s1, s2)'
@jit.elidable
def ll_strip(s, ch, left, right):
s_len = len(s.chars)
if s_len == 0:
return s.empty()
lpos = 0
rpos = s_len - 1
if left:
while lpos < rpos and s.chars[lpos] == ch:
lpos += 1
if right:
while lpos < rpos + 1 and s.chars[rpos] == ch:
rpos -= 1
if rpos < lpos:
return s.empty()
r_len = rpos - lpos + 1
result = s.malloc(r_len)
s.copy_contents(s, result, lpos, 0, r_len)
return result
@jit.elidable
def ll_upper(s):
s_chars = s.chars
s_len = len(s_chars)
if s_len == 0:
return s.empty()
i = 0
result = mallocstr(s_len)
# ^^^^^^^^^ specifically to explode on unicode
while i < s_len:
ch = s_chars[i]
if 'a' <= ch <= 'z':
ch = chr(ord(ch) - 32)
result.chars[i] = ch
i += 1
return result
@jit.elidable
def ll_lower(s):
s_chars = s.chars
s_len = len(s_chars)
if s_len == 0:
return s.empty()
i = 0
result = mallocstr(s_len)
# ^^^^^^^^^ specifically to explode on unicode
while i < s_len:
ch = s_chars[i]
if 'A' <= ch <= 'Z':
ch = chr(ord(ch) + 32)
result.chars[i] = ch
i += 1
return result
def ll_join(s, length, items):
s_chars = s.chars
s_len = len(s_chars)
num_items = length
if num_items == 0:
return s.empty()
itemslen = 0
i = 0
while i < num_items:
try:
itemslen = ovfcheck(itemslen + len(items[i].chars))
except OverflowError:
raise MemoryError
i += 1
try:
seplen = ovfcheck(s_len * (num_items - 1))
except OverflowError:
raise MemoryError
# a single '+' at the end is allowed to overflow: it gets
# a negative result, and the gc will complain
result = s.malloc(itemslen + seplen)
res_index = len(items[0].chars)
s.copy_contents(items[0], result, 0, 0, res_index)
i = 1
while i < num_items:
s.copy_contents(s, result, 0, res_index, s_len)
res_index += s_len
lgt = len(items[i].chars)
s.copy_contents(items[i], result, 0, res_index, lgt)
res_index += lgt
i += 1
return result
@jit.elidable
def ll_strcmp(s1, s2):
if not s1 and not s2:
return True
if not s1 or not s2:
return False
chars1 = s1.chars
chars2 = s2.chars
len1 = len(chars1)
len2 = len(chars2)
if len1 < len2:
cmplen = len1
else:
cmplen = len2
i = 0
while i < cmplen:
diff = ord(chars1[i]) - ord(chars2[i])
if diff != 0:
return diff
i += 1
return len1 - len2
@jit.elidable
def ll_streq(s1, s2):
if s1 == s2: # also if both are NULLs
return True
if not s1 or not s2:
return False
len1 = len(s1.chars)
len2 = len(s2.chars)
if len1 != len2:
return False
j = 0
chars1 = s1.chars
chars2 = s2.chars
while j < len1:
if chars1[j] != chars2[j]:
return False
j += 1
return True
ll_streq.oopspec = 'stroruni.equal(s1, s2)'
@jit.elidable
def ll_startswith(s1, s2):
len1 = len(s1.chars)
len2 = len(s2.chars)
if len1 < len2:
return False
j = 0
chars1 = s1.chars
chars2 = s2.chars
while j < len2:
if chars1[j] != chars2[j]:
return False
j += 1
return True
def ll_startswith_char(s, ch):
if not len(s.chars):
return False
return s.chars[0] == ch
@jit.elidable
def ll_endswith(s1, s2):
len1 = len(s1.chars)
len2 = len(s2.chars)
if len1 < len2:
return False
j = 0
chars1 = s1.chars
chars2 = s2.chars
offset = len1 - len2
while j < len2:
if chars1[offset + j] != chars2[j]:
return False
j += 1
return True
def ll_endswith_char(s, ch):
if not len(s.chars):
return False
return s.chars[len(s.chars) - 1] == ch
@jit.elidable
def ll_find_char(s, ch, start, end):
i = start
if end > len(s.chars):
end = len(s.chars)
while i < end:
if s.chars[i] == ch:
return i
i += 1
return -1
ll_find_char._annenforceargs_ = [None, None, int, int]
@jit.elidable
def ll_rfind_char(s, ch, start, end):
if end > len(s.chars):
end = len(s.chars)
i = end
while i > start:
i -= 1
if s.chars[i] == ch:
return i
return -1
@jit.elidable
def ll_count_char(s, ch, start, end):
count = 0
i = start
if end > len(s.chars):
end = len(s.chars)
while i < end:
if s.chars[i] == ch:
count += 1
i += 1
return count
@classmethod
def ll_find(cls, s1, s2, start, end):
if start < 0:
start = 0
if end > len(s1.chars):
end = len(s1.chars)
if end - start < 0:
return -1
m = len(s2.chars)
if m == 0:
return start
elif m == 1:
return cls.ll_find_char(s1, s2.chars[0], start, end)
return cls.ll_search(s1, s2, start, end, FAST_FIND)
@classmethod
def ll_rfind(cls, s1, s2, start, end):
if start < 0:
start = 0
if end > len(s1.chars):
end = len(s1.chars)
if end - start < 0:
return -1
m = len(s2.chars)
if m == 0:
return end
elif m == 1:
return cls.ll_rfind_char(s1, s2.chars[0], start, end)
return cls.ll_search(s1, s2, start, end, FAST_RFIND)
@classmethod
def ll_count(cls, s1, s2, start, end):
if start < 0:
start = 0
if end > len(s1.chars):
end = len(s1.chars)
if end - start < 0:
return 0
m = len(s2.chars)
if m == 0:
return end - start + 1
elif m == 1:
return cls.ll_count_char(s1, s2.chars[0], start, end)
res = cls.ll_search(s1, s2, start, end, FAST_COUNT)
# For a few cases ll_search can return -1 to indicate an "impossible"
# condition for a string match, count just returns 0 in these cases.
if res < 0:
res = 0
return res
@jit.elidable
def ll_search(s1, s2, start, end, mode):
count = 0
n = end - start
m = len(s2.chars)
w = n - m
if w < 0:
return -1
mlast = m - 1
skip = mlast - 1
mask = 0
if mode != FAST_RFIND:
for i in range(mlast):
mask = bloom_add(mask, s2.chars[i])
if s2.chars[i] == s2.chars[mlast]:
skip = mlast - i - 1
mask = bloom_add(mask, s2.chars[mlast])
i = start - 1
while i + 1 <= start + w:
i += 1
if s1.chars[i + m - 1] == s2.chars[m - 1]:
for j in range(mlast):
if s1.chars[i + j] != s2.chars[j]:
break
else:
if mode != FAST_COUNT:
return i
count += 1
i += mlast
continue
if i + m < len(s1.chars):
c = s1.chars[i + m]
else:
c = '\0'
if not bloom(mask, c):
i += m
else:
i += skip
else:
if i + m < len(s1.chars):
c = s1.chars[i + m]
else:
c = '\0'
if not bloom(mask, c):
i += m
else:
mask = bloom_add(mask, s2.chars[0])
for i in range(mlast, 0, -1):
mask = bloom_add(mask, s2.chars[i])
if s2.chars[i] == s2.chars[0]:
skip = i - 1
i = start + w + 1
while i - 1 >= start:
i -= 1
if s1.chars[i] == s2.chars[0]:
for j in xrange(mlast, 0, -1):
if s1.chars[i + j] != s2.chars[j]:
break
else:
return i
if i - 1 >= 0 and not bloom(mask, s1.chars[i - 1]):
i -= m
else:
i -= skip
else:
if i - 1 >= 0 and not bloom(mask, s1.chars[i - 1]):
i -= m
if mode != FAST_COUNT:
return -1
return count
@enforceargs(int, None)
@jit.look_inside_iff(
lambda length, items: jit.loop_unrolling_heuristic(items, length))
def ll_join_strs(length, items):
# Special case for length 1 items, helps both the JIT and other code
if length == 1:
return items[0]
num_items = length
itemslen = 0
i = 0
while i < num_items:
try:
itemslen = ovfcheck(itemslen + len(items[i].chars))
except OverflowError:
raise MemoryError
i += 1
if typeOf(items).TO.OF.TO == STR:
malloc = mallocstr
copy_contents = copy_string_contents
else:
malloc = mallocunicode
copy_contents = copy_unicode_contents
result = malloc(itemslen)
res_chars = result.chars
res_index = 0
i = 0
while i < num_items:
item_chars = items[i].chars
item_len = len(item_chars)
copy_contents(items[i], result, 0, res_index, item_len)
res_index += item_len
i += 1
return result
@jit.look_inside_iff(lambda length, chars, RES: jit.isconstant(length) and
jit.isvirtual(chars))
def ll_join_chars(length, chars, RES):
# no need to optimize this, will be replaced by string builder
# at some point soon
num_chars = length
if RES is StringRepr.lowleveltype:
target = Char
malloc = mallocstr
else:
target = UniChar
malloc = mallocunicode
result = malloc(num_chars)
res_chars = result.chars
i = 0
while i < num_chars:
res_chars[i] = cast_primitive(target, chars[i])
i += 1
return result
@jit.elidable
def _ll_stringslice(s1, start, stop):
lgt = stop - start
assert start >= 0
# If start > stop, return a empty string. This can happen if the start
# is greater than the length of the string. Use < instead of <= to avoid
# creating another path for the JIT when start == stop.
if lgt < 0:
return s1.empty()
newstr = s1.malloc(lgt)
s1.copy_contents(s1, newstr, start, 0, lgt)
return newstr
_ll_stringslice.oopspec = 'stroruni.slice(s1, start, stop)'
_ll_stringslice._annenforceargs_ = [None, int, int]
def ll_stringslice_startonly(s1, start):
return LLHelpers._ll_stringslice(s1, start, len(s1.chars))
def ll_stringslice_startstop(s1, start, stop):
if jit.we_are_jitted():
if stop > len(s1.chars):
stop = len(s1.chars)
else:
if stop >= len(s1.chars):
if start == 0:
return s1
stop = len(s1.chars)
return LLHelpers._ll_stringslice(s1, start, stop)
def ll_stringslice_minusone(s1):
newlen = len(s1.chars) - 1
return LLHelpers._ll_stringslice(s1, 0, newlen)
def ll_split_chr(LIST, s, c, max):
chars = s.chars
strlen = len(chars)
count = 1
i = 0
if max == 0:
i = strlen
while i < strlen:
if chars[i] == c:
count += 1
if max >= 0 and count > max:
break
i += 1
res = LIST.ll_newlist(count)
items = res.ll_items()
i = 0
j = 0
resindex = 0
if max == 0:
j = strlen
while j < strlen:
if chars[j] == c:
item = items[resindex] = s.malloc(j - i)
item.copy_contents(s, item, i, 0, j - i)
resindex += 1
i = j + 1
if max >= 0 and resindex >= max:
j = strlen
break
j += 1
item = items[resindex] = s.malloc(j - i)
item.copy_contents(s, item, i, 0, j - i)
return res
def ll_rsplit_chr(LIST, s, c, max):
chars = s.chars
strlen = len(chars)
count = 1
i = 0
if max == 0:
i = strlen
while i < strlen:
if chars[i] == c:
count += 1
if max >= 0 and count > max:
break
i += 1
res = LIST.ll_newlist(count)
items = res.ll_items()
i = strlen
j = strlen
resindex = count - 1
assert resindex >= 0
if max == 0:
j = 0
while j > 0:
j -= 1
if chars[j] == c:
item = items[resindex] = s.malloc(i - j - 1)
item.copy_contents(s, item, j + 1, 0, i - j - 1)
resindex -= 1
i = j
if resindex == 0:
j = 0
break
item = items[resindex] = s.malloc(i - j)
item.copy_contents(s, item, j, 0, i - j)
return res
@jit.elidable
def ll_replace_chr_chr(s, c1, c2):
length = len(s.chars)
newstr = s.malloc(length)
src = s.chars
dst = newstr.chars
j = 0
while j < length:
c = src[j]
if c == c1:
c = c2
dst[j] = c
j += 1
return newstr
@jit.elidable
def ll_contains(s, c):
chars = s.chars
strlen = len(chars)
i = 0
while i < strlen:
if chars[i] == c:
return True
i += 1
return False
@jit.elidable
def ll_int(s, base):
if not 2 <= base <= 36:
raise ValueError
chars = s.chars
strlen = len(chars)
i = 0
#XXX: only space is allowed as white space for now
while i < strlen and chars[i] == ' ':
i += 1
if not i < strlen:
raise ValueError
#check sign
sign = 1
if chars[i] == '-':
sign = -1
i += 1
elif chars[i] == '+':
i += 1
# skip whitespaces between sign and digits
while i < strlen and chars[i] == ' ':
i += 1
#now get digits
val = 0
oldpos = i
while i < strlen:
c = ord(chars[i])
if ord('a') <= c <= ord('z'):
digit = c - ord('a') + 10
elif ord('A') <= c <= ord('Z'):
digit = c - ord('A') + 10
elif ord('0') <= c <= ord('9'):
digit = c - ord('0')
else:
break
if digit >= base:
break
val = val * base + digit
i += 1
if i == oldpos:
raise ValueError # catch strings like '+' and '+ '
#skip trailing whitespace
while i < strlen and chars[i] == ' ':
i += 1
if not i == strlen:
raise ValueError
return sign * val
# interface to build strings:
# x = ll_build_start(n)
# ll_build_push(x, next_string, 0)
# ll_build_push(x, next_string, 1)
# ...
# ll_build_push(x, next_string, n-1)
# s = ll_build_finish(x)
def ll_build_start(parts_count):
return malloc(TEMP, parts_count)
def ll_build_push(builder, next_string, index):
builder[index] = next_string
def ll_build_finish(builder):
return LLHelpers.ll_join_strs(len(builder), builder)
def ll_constant(s):
return string_repr.convert_const(s)
ll_constant._annspecialcase_ = 'specialize:memo'
def do_stringformat(cls, hop, sourcevarsrepr):
s_str = hop.args_s[0]
assert s_str.is_constant()
s = s_str.const
things = cls.parse_fmt_string(s)
size = inputconst(Signed, len(things)) # could be unsigned?
cTEMP = inputconst(Void, TEMP)
cflags = inputconst(Void, {'flavor': 'gc'})
vtemp = hop.genop("malloc_varsize", [cTEMP, cflags, size],
resulttype=Ptr(TEMP))
argsiter = iter(sourcevarsrepr)
InstanceRepr = hop.rtyper.type_system.rclass.InstanceRepr
for i, thing in enumerate(things):
if isinstance(thing, tuple):
code = thing[0]
vitem, r_arg = argsiter.next()
if not hasattr(r_arg, 'll_str'):
raise TyperError("ll_str unsupported for: %r" % r_arg)
if code == 's' or (code == 'r'
and isinstance(r_arg, InstanceRepr)):
vchunk = hop.gendirectcall(r_arg.ll_str, vitem)
elif code == 'd':
assert isinstance(r_arg, IntegerRepr)
#vchunk = hop.gendirectcall(r_arg.ll_str, vitem)
vchunk = hop.gendirectcall(ll_str.ll_int2dec, vitem)
elif code == 'f':
#assert isinstance(r_arg, FloatRepr)
vchunk = hop.gendirectcall(r_arg.ll_str, vitem)
elif code == 'x':
assert isinstance(r_arg, IntegerRepr)
vchunk = hop.gendirectcall(ll_str.ll_int2hex, vitem,
inputconst(Bool, False))
elif code == 'o':
assert isinstance(r_arg, IntegerRepr)
vchunk = hop.gendirectcall(ll_str.ll_int2oct, vitem,
inputconst(Bool, False))
else:
raise TyperError, "%%%s is not RPython" % (code, )
else:
from pypy.rpython.lltypesystem.rstr import string_repr
vchunk = inputconst(string_repr, thing)
i = inputconst(Signed, i)
hop.genop('setarrayitem', [vtemp, i, vchunk])
hop.exception_cannot_occur() # to ignore the ZeroDivisionError of '%'
return hop.gendirectcall(cls.ll_join_strs, size, vtemp)
do_stringformat = classmethod(do_stringformat)
def ll_pop(func, l, index):
length = l.ll_length()
if index < 0:
index += length
if func is dum_checkidx:
if index < 0 or index >= length:
raise IndexError
else:
ll_assert(index >= 0, "negative list pop index out of bound")
ll_assert(index < length, "list pop index out of bound")
res = l.ll_getitem_fast(index)
ll_delitem_nonneg(dum_nocheck, l, index)
return res
@jit.look_inside_iff(lambda l: jit.isvirtual(l))
def ll_reverse(l):
length = l.ll_length()
i = 0
length_1_i = length - 1 - i
while i < length_1_i:
tmp = l.ll_getitem_fast(i)
l.ll_setitem_fast(i, l.ll_getitem_fast(length_1_i))
l.ll_setitem_fast(length_1_i, tmp)
i += 1
length_1_i -= 1
def ll_getitem_nonneg(func, l, index):
ll_assert(index >= 0, "unexpectedly negative list getitem index")
if func is dum_checkidx:
def ll_pop(func, l, index):
length = l.ll_length()
if index < 0:
index += length
if func is dum_checkidx:
if index < 0 or index >= length:
raise IndexError
else:
ll_assert(index >= 0, "negative list pop index out of bound")
ll_assert(index < length, "list pop index out of bound")
res = l.ll_getitem_fast(index)
ll_delitem_nonneg(dum_nocheck, l, index)
return res
@jit.look_inside_iff(lambda l: jit.isvirtual(l))
def ll_reverse(l):
length = l.ll_length()
i = 0
length_1_i = length-1-i
while i < length_1_i:
tmp = l.ll_getitem_fast(i)
l.ll_setitem_fast(i, l.ll_getitem_fast(length_1_i))
l.ll_setitem_fast(length_1_i, tmp)
i += 1
length_1_i -= 1
def ll_getitem_nonneg(func, l, index):
ll_assert(index >= 0, "unexpectedly negative list getitem index")
if func is dum_checkidx:
if index >= l.ll_length():
if not i & HIGHEST_BIT:
return ll_get_value(d, i)
else:
raise KeyError
def ll_dict_setitem(d, key, value):
hash = d.keyhash(key)
i = ll_dict_lookup(d, key, hash)
return _ll_dict_setitem_lookup_done(d, key, value, hash, i)
# It may be safe to look inside always, it has a few branches though, and their
# frequencies needs to be investigated.
@jit.look_inside_iff(
lambda d, key, value, hash, i: jit.isvirtual(d) and jit.isconstant(key))
def _ll_dict_setitem_lookup_done(d, key, value, hash, i):
valid = (i & HIGHEST_BIT) == 0
i = i & MASK
everused = d.entries.everused(i)
# set up the new entry
ENTRY = lltype.typeOf(d.entries).TO.OF
entry = d.entries[i]
entry.value = value
if valid:
return
entry.key = key
if hasattr(ENTRY, 'f_hash'): entry.f_hash = hash
if hasattr(ENTRY, 'f_valid'): entry.f_valid = True
d.num_items += 1
if not everused:
