def pow_small_int(self, n): if n >= 0: if jit.isconstant(n) and n == 2: return self.mul(self) return self.pow_positive_int(n) else: return w_one.div(self.pow_positive_int(-n))
def _combine_starstarargs_wrapped(self, w_starstararg): # unpack the ** arguments space = self.space keywords, values_w = space.view_as_kwargs(w_starstararg) if keywords is not None: # this path also taken for empty dicts if self.keywords is None: self.keywords = keywords[:] # copy to make non-resizable self.keywords_w = values_w[:] else: self._check_not_duplicate_kwargs(keywords, values_w) self.keywords = self.keywords + keywords self.keywords_w = self.keywords_w + values_w return not jit.isconstant(len(self.keywords)) if space.isinstance_w(w_starstararg, space.w_dict): keys_w = space.unpackiterable(w_starstararg) else: try: w_keys = space.call_method(w_starstararg, "keys") except OperationError, e: if e.match(space, space.w_AttributeError): w_type = space.type(w_starstararg) typename = w_type.getname(space) raise OperationError( space.w_TypeError, space.wrap("argument after ** must be " "a mapping, not %s" % (typename,))) raise keys_w = space.unpackiterable(w_keys)
def _combine_starstarargs_wrapped(self, w_starstararg): # unpack the ** arguments space = self.space keywords, values_w = space.view_as_kwargs(w_starstararg) if keywords is not None: # this path also taken for empty dicts if self.keywords is None: self.keywords = keywords[:] # copy to make non-resizable self.keywords_w = values_w[:] else: self._check_not_duplicate_kwargs(keywords, values_w) self.keywords = self.keywords + keywords self.keywords_w = self.keywords_w + values_w return not jit.isconstant(len(self.keywords)) if space.isinstance_w(w_starstararg, space.w_dict): keys_w = space.unpackiterable(w_starstararg) else: try: w_keys = space.call_method(w_starstararg, "keys") except OperationError, e: if e.match(space, space.w_AttributeError): w_type = space.type(w_starstararg) typename = w_type.getname(space) raise OperationError( space.w_TypeError, space.wrap("argument after ** must be " "a mapping, not %s" % (typename, ))) raise keys_w = space.unpackiterable(w_keys)
def decrement_ticker(self, by): value = self._ticker if self.has_bytecode_counter: # this 'if' is constant-folded if jit.isconstant(by) and by == 0: pass # normally constant-folded too else: value -= by self._ticker = value return value
def decrement_ticker(self, by): p = pypysig_getaddr_occurred() value = p.c_value if self.has_bytecode_counter: # this 'if' is constant-folded if jit.isconstant(by) and by == 0: pass # normally constant-folded too else: value -= by p.c_value = value return value
def pack(space, format, args_w): if jit.isconstant(format): size = _calcsize(space, format) else: size = 8 fmtiter = PackFormatIterator(space, args_w, size) try: fmtiter.interpret(format) except StructError, e: raise e.at_applevel(space)
def repr__Tuple(space, w_tuple): items = w_tuple.wrappeditems # XXX this is quite innefficient, still better than calling # it via applevel if len(items) == 1: return space.wrap("(" + space.str_w(space.repr(items[0])) + ",)") return space.wrap( "(" + (", ".join([space.str_w(space.repr(item)) for item in items])) + ")") def hash__Tuple(space, w_tuple): return space.wrap(hash_tuple(space, w_tuple.wrappeditems)) @jit.look_inside_iff(lambda space, wrappeditems: jit.isconstant( len(wrappeditems)) and len(wrappeditems) < UNROLL_TUPLE_LIMIT) def hash_tuple(space, wrappeditems): # this is the CPython 2.4 algorithm (changed from 2.3) mult = 1000003 x = 0x345678 z = len(wrappeditems) for w_item in wrappeditems: y = space.hash_w(w_item) x = (x ^ y) * mult z -= 1 mult += 82520 + z + z x += 97531 return intmask(x) def getnewargs__Tuple(space, w_tuple):
z = ovfcheck(x // y) except ZeroDivisionError: raise OperationError(space.w_ZeroDivisionError, space.wrap("integer divmod by zero")) except OverflowError: raise FailedToImplementArgs(space.w_OverflowError, space.wrap("integer modulo")) # no overflow possible m = x % y w = space.wrap return space.newtuple([w(z), w(m)]) # helper for pow() @jit.look_inside_iff( lambda space, iv, iw, iz: jit.isconstant(iw) and jit.isconstant(iz)) def _impl_int_int_pow(space, iv, iw, iz): if iw < 0: if iz != 0: raise OperationError( space.w_TypeError, space.wrap("pow() 2nd argument " "cannot be negative when 3rd argument specified")) ## bounce it, since it always returns float raise FailedToImplementArgs(space.w_ValueError, space.wrap("integer exponentiation")) temp = iv ix = 1 try: while iw > 0: if iw & 1:
raise OperationError(space.w_OverflowError, space.wrap("result has too many items")) else: howmany = 0 res_w = [None] * howmany v = start for idx in range(howmany): res_w[idx] = space.newlong_from_rbigint(v) v = v.add(step) return space.newlist(res_w) @specialize.arg(2) @jit.look_inside_iff(lambda space, args, implementation_of: jit.isconstant(len(args.arguments_w)) and len(args.arguments_w) == 2 ) def min_max(space, args, implementation_of): if implementation_of == "max": compare = space.gt else: compare = space.lt args_w = args.arguments_w if len(args_w) > 1: w_sequence = space.newtuple(args_w) elif len(args_w): w_sequence = args_w[0] else: msg = "%s() expects at least one argument" % (implementation_of,) raise OperationError(space.w_TypeError, space.wrap(msg))
"""The unicode/str format() method""" import string from pypy.interpreter.error import OperationError from pypy.rlib import rstring, runicode, rlocale, rarithmetic, rfloat, jit from pypy.rlib.objectmodel import specialize from pypy.rlib.rfloat import copysign, formatd from pypy.tool import sourcetools @specialize.argtype(1) @jit.look_inside_iff(lambda space, s, start, end: jit.isconstant(s) and jit.isconstant(start) and jit.isconstant(end)) def _parse_int(space, s, start, end): """Parse a number and check for overflows""" result = 0 i = start while i < end: c = ord(s[i]) if ord("0") <= c <= ord("9"): try: result = rarithmetic.ovfcheck(result * 10) except OverflowError: msg = "too many decimal digits in format string" raise OperationError(space.w_ValueError, space.wrap(msg)) result += c - ord("0") else: break
def make_range_list(space, start, step, length): if length <= 0: strategy = space.fromcache(EmptyListStrategy) storage = strategy.erase(None) else: strategy = space.fromcache(RangeListStrategy) storage = strategy.erase((start, step, length)) return W_ListObject.from_storage_and_strategy(space, storage, strategy) def make_empty_list(space): strategy = space.fromcache(EmptyListStrategy) storage = strategy.erase(None) return W_ListObject.from_storage_and_strategy(space, storage, strategy) @jit.look_inside_iff(lambda space, list_w: jit.isconstant(len(list_w)) and len(list_w) < UNROLL_CUTOFF) def get_strategy_from_list_objects(space, list_w): if not list_w: return space.fromcache(EmptyListStrategy) # check for ints for w_obj in list_w: if not is_W_IntObject(w_obj): break else: return space.fromcache(IntegerListStrategy) # check for strings for w_obj in list_w: if not is_W_StringObject(w_obj): break
def repr__Tuple(space, w_tuple): items = w_tuple.wrappeditems # XXX this is quite innefficient, still better than calling # it via applevel if len(items) == 1: return space.wrap("(" + space.str_w(space.repr(items[0])) + ",)") return space.wrap("(" + (", ".join([space.str_w(space.repr(item)) for item in items])) + ")") def hash__Tuple(space, w_tuple): return space.wrap(hash_tuple(space, w_tuple.wrappeditems)) @jit.look_inside_iff(lambda space, wrappeditems: jit.isconstant(len(wrappeditems)) and len(wrappeditems) < UNROLL_TUPLE_LIMIT) def hash_tuple(space, wrappeditems): # this is the CPython 2.4 algorithm (changed from 2.3) mult = 1000003 x = 0x345678 z = len(wrappeditems) for w_item in wrappeditems: y = space.hash_w(w_item) x = (x ^ y) * mult z -= 1 mult += 82520 + z + z x += 97531 return intmask(x) def getnewargs__Tuple(space, w_tuple):
class BaseStringBuilderRepr(AbstractStringBuilderRepr): def empty(self): return nullptr(self.lowleveltype.TO) @classmethod def ll_new(cls, init_size): if init_size < 0 or init_size > MAX: init_size = MAX ll_builder = lltype.malloc(cls.lowleveltype.TO) ll_builder.allocated = init_size ll_builder.used = 0 ll_builder.buf = cls.mallocfn(init_size) return ll_builder @staticmethod def ll_append(ll_builder, ll_str): used = ll_builder.used lgt = len(ll_str.chars) needed = lgt + used if needed > ll_builder.allocated: ll_builder.grow(ll_builder, lgt) ll_str.copy_contents(ll_str, ll_builder.buf, 0, used, lgt) ll_builder.used = needed @staticmethod def ll_append_char(ll_builder, char): if ll_builder.used == ll_builder.allocated: ll_builder.grow(ll_builder, 1) ll_builder.buf.chars[ll_builder.used] = char ll_builder.used += 1 @staticmethod def ll_append_slice(ll_builder, ll_str, start, end): needed = end - start used = ll_builder.used if needed + used > ll_builder.allocated: ll_builder.grow(ll_builder, needed) assert needed >= 0 ll_str.copy_contents(ll_str, ll_builder.buf, start, used, needed) ll_builder.used = needed + used @staticmethod @jit.look_inside_iff( lambda ll_builder, char, times: jit.isconstant(times) and times <= 4) def ll_append_multiple_char(ll_builder, char, times): used = ll_builder.used if times + used > ll_builder.allocated: ll_builder.grow(ll_builder, times) for i in range(times): ll_builder.buf.chars[used] = char used += 1 ll_builder.used = used @staticmethod def ll_append_charpsize(ll_builder, charp, size): used = ll_builder.used if used + size > ll_builder.allocated: ll_builder.grow(ll_builder, size) for i in xrange(size): ll_builder.buf.chars[used] = charp[i] used += 1 ll_builder.used = used @staticmethod def ll_getlength(ll_builder): return ll_builder.used @staticmethod def ll_build(ll_builder): final_size = ll_builder.used assert final_size >= 0 if final_size < ll_builder.allocated: ll_builder.allocated = final_size ll_builder.buf = rgc.ll_shrink_array(ll_builder.buf, final_size) return ll_builder.buf @classmethod def ll_is_true(cls, ll_builder): return ll_builder != nullptr(cls.lowleveltype.TO)
class AbstractAttribute(object): _immutable_fields_ = ['terminator'] cache_attrs = None _size_estimate = 0 def __init__(self, space, terminator): self.space = space assert isinstance(terminator, Terminator) self.terminator = terminator def read(self, obj, selector): index = self.index(selector) if index < 0: return self.terminator._read_terminator(obj, selector) return obj._mapdict_read_storage(index) def write(self, obj, selector, w_value): index = self.index(selector) if index < 0: return self.terminator._write_terminator(obj, selector, w_value) obj._mapdict_write_storage(index, w_value) return True def delete(self, obj, selector): return None def index(self, selector): if jit.we_are_jitted(): # hack for the jit: # the _index method is pure too, but its argument is never # constant, because it is always a new tuple return self._index_jit_pure(selector[0], selector[1]) else: return self._index_indirection(selector) @jit.elidable def _index_jit_pure(self, name, index): return self._index_indirection((name, index)) @jit.dont_look_inside def _index_indirection(self, selector): if (self.space.config.objspace.std.withmethodcache): return self._index_cache(selector) return self._index(selector) @jit.dont_look_inside def _index_cache(self, selector): space = self.space cache = space.fromcache(IndexCache) SHIFT2 = r_uint.BITS - space.config.objspace.std.methodcachesizeexp SHIFT1 = SHIFT2 - 5 attrs_as_int = objectmodel.current_object_addr_as_int(self) # ^^^Note: see comment in typeobject.py for # _pure_lookup_where_with_method_cache() hash_selector = objectmodel.compute_hash(selector) product = intmask(attrs_as_int * hash_selector) index_hash = (r_uint(product) ^ (r_uint(product) << SHIFT1)) >> SHIFT2 # ^^^Note2: same comment too cached_attr = cache.attrs[index_hash] if cached_attr is self: cached_selector = cache.selectors[index_hash] if cached_selector == selector: index = cache.indices[index_hash] if space.config.objspace.std.withmethodcachecounter: name = selector[0] cache.hits[name] = cache.hits.get(name, 0) + 1 return index index = self._index(selector) cache.attrs[index_hash] = self cache.selectors[index_hash] = selector cache.indices[index_hash] = index if space.config.objspace.std.withmethodcachecounter: name = selector[0] cache.misses[name] = cache.misses.get(name, 0) + 1 return index def _index(self, selector): while isinstance(self, PlainAttribute): if selector == self.selector: return self.position self = self.back return -1 def copy(self, obj): raise NotImplementedError("abstract base class") def length(self): raise NotImplementedError("abstract base class") def get_terminator(self): return self.terminator def set_terminator(self, obj, terminator): raise NotImplementedError("abstract base class") @jit.elidable def size_estimate(self): return self._size_estimate >> NUM_DIGITS def search(self, attrtype): return None @jit.elidable def _get_new_attr(self, name, index): selector = name, index cache = self.cache_attrs if cache is None: cache = self.cache_attrs = {} attr = cache.get(selector, None) if attr is None: attr = PlainAttribute(selector, self) cache[selector] = attr return attr @jit.look_inside_iff(lambda self, obj, selector, w_value: jit.isconstant( self) and jit.isconstant(selector[0]) and jit.isconstant(selector[1])) def add_attr(self, obj, selector, w_value): # grumble, jit needs this attr = self._get_new_attr(selector[0], selector[1]) oldattr = obj._get_mapdict_map() if not jit.we_are_jitted(): size_est = (oldattr._size_estimate + attr.size_estimate() - oldattr.size_estimate()) assert size_est >= (oldattr.length() * NUM_DIGITS_POW2) oldattr._size_estimate = size_est if attr.length() > obj._mapdict_storage_length(): # note that attr.size_estimate() is always at least attr.length() new_storage = [None] * attr.size_estimate() for i in range(obj._mapdict_storage_length()): new_storage[i] = obj._mapdict_read_storage(i) obj._set_mapdict_storage_and_map(new_storage, attr) # the order is important here: first change the map, then the storage, # for the benefit of the special subclasses obj._set_mapdict_map(attr) obj._mapdict_write_storage(attr.position, w_value) def materialize_r_dict(self, space, obj, dict_w): raise NotImplementedError("abstract base class") def remove_dict_entries(self, obj): raise NotImplementedError("abstract base class") def __repr__(self): return "<%s>" % (self.__class__.__name__, )
# no oopspec -- the function is inlined by the JIT def ll_listslice_minusone(RESLIST, l1): newlength = l1.ll_length() - 1 ll_assert(newlength >= 0, "empty list is sliced with [:-1]") l = RESLIST.ll_newlist(newlength) ll_arraycopy(l1, l, 0, 0, newlength) return l # no oopspec -- the function is inlined by the JIT @jit.look_inside_iff( lambda l, start: jit.isconstant(start) and jit.isvirtual(l)) @jit.oopspec('list.delslice_startonly(l, start)') def ll_listdelslice_startonly(l, start): ll_assert(start >= 0, "del l[start:] with unexpectedly negative start") ll_assert(start <= l.ll_length(), "del l[start:] with start > len(l)") newlength = start null = ll_null_item(l) if null is not None: j = l.ll_length() - 1 while j >= newlength: l.ll_setitem_fast(j, null) j -= 1 l._ll_resize_le(newlength) def ll_listdelslice_startstop(l, start, stop):
if not i & HIGHEST_BIT: return ll_get_value(d, i) else: raise KeyError def ll_dict_setitem(d, key, value): hash = d.keyhash(key) i = ll_dict_lookup(d, key, hash) return _ll_dict_setitem_lookup_done(d, key, value, hash, i) # It may be safe to look inside always, it has a few branches though, and their # frequencies needs to be investigated. @jit.look_inside_iff( lambda d, key, value, hash, i: jit.isvirtual(d) and jit.isconstant(key)) def _ll_dict_setitem_lookup_done(d, key, value, hash, i): valid = (i & HIGHEST_BIT) == 0 i = i & MASK everused = d.entries.everused(i) # set up the new entry ENTRY = lltype.typeOf(d.entries).TO.OF entry = d.entries[i] entry.value = value if valid: return entry.key = key if hasattr(ENTRY, 'f_hash'): entry.f_hash = hash if hasattr(ENTRY, 'f_valid'): entry.f_valid = True d.num_items += 1 if not everused:
def repr__Tuple(space, w_tuple): items = w_tuple.wrappeditems # XXX this is quite innefficient, still better than calling # it via applevel if len(items) == 1: return space.wrap("(" + space.str_w(space.repr(items[0])) + ",)") return space.wrap("(" + (", ".join([space.str_w(space.repr(item)) for item in items])) + ")") def hash__Tuple(space, w_tuple): return space.wrap(hash_tuple(space, w_tuple.wrappeditems)) @jit.look_inside_iff( lambda space, wrappeditems: jit.isconstant(len(wrappeditems)) and len(wrappeditems) < UNROLL_TUPLE_LIMIT ) def hash_tuple(space, wrappeditems): # this is the CPython 2.4 algorithm (changed from 2.3) mult = 1000003 x = 0x345678 z = len(wrappeditems) for w_item in wrappeditems: y = space.hash_w(w_item) x = (x ^ y) * mult z -= 1 mult += 82520 + z + z x += 97531 return intmask(x)
def tuple_unroll_condition(space, w_tuple1, w_tuple2): lgt1 = len(w_tuple1.wrappeditems) lgt2 = len(w_tuple2.wrappeditems) return ((jit.isconstant(lgt1) and lgt1 <= UNROLL_TUPLE_LIMIT) or (jit.isconstant(lgt2) and lgt2 <= UNROLL_TUPLE_LIMIT))
from pypy.rlib import jit from pypy.interpreter.error import OperationError @jit.look_inside_iff(lambda shape, start, strides, backstrides, chunks: jit.isconstant(len(chunks)) ) def calculate_slice_strides(shape, start, strides, backstrides, chunks): rstrides = [] rbackstrides = [] rstart = start rshape = [] i = -1 for i, chunk in enumerate(chunks): if chunk.step != 0: rstrides.append(strides[i] * chunk.step) rbackstrides.append(strides[i] * (chunk.lgt - 1) * chunk.step) rshape.append(chunk.lgt) rstart += strides[i] * chunk.start # add a reminder s = i + 1 assert s >= 0 rstrides += strides[s:] rbackstrides += backstrides[s:] rshape += shape[s:] return rshape, rstart, rstrides, rbackstrides def calculate_broadcast_strides(strides, backstrides, orig_shape, res_shape): rstrides = [] rbackstrides = [] for i in range(len(orig_shape)): if orig_shape[i] == 1:
raise operationerrfmt( self.space.w_TypeError, "got multiple values " "for keyword argument " "'%s'", key) keywords[i] = key keywords_w[i] = space.getitem(w_starstararg, w_key) i += 1 if self.keywords is None: self.keywords = keywords self.keywords_w = keywords_w else: self.keywords = self.keywords + keywords self.keywords_w = self.keywords_w + keywords_w self.keyword_names_w = keys_w @jit.look_inside_iff(lambda self, keywords, keywords_w: jit.isconstant( len(keywords) and jit.isconstant(self.keywords))) def _check_not_duplicate_kwargs(self, keywords, keywords_w): # looks quadratic, but the JIT should remove all of it nicely. # Also, all the lists should be small for key in keywords: for otherkey in self.keywords: if otherkey == key: raise operationerrfmt( self.space.w_TypeError, "got multiple values " "for keyword argument " "'%s'", key) def fixedunpack(self, argcount): """The simplest argument parsing: get the 'argcount' arguments, or raise a real ValueError if the length is wrong.""" if self.keywords:
def f(n): assert isconstant(n) is False l = [] l.append(n) return len(l)
l.items = newitems # this common case was factored out of _ll_list_resize # to see if inlining it gives some speed-up. def _ll_list_resize(l, newsize): # Bypass realloc() when a previous overallocation is large enough # to accommodate the newsize. If the newsize falls lower than half # the allocated size, then proceed with the realloc() to shrink the list. allocated = len(l.items) if allocated >= newsize and newsize >= ((allocated >> 1) - 5): l.length = newsize else: _ll_list_resize_really(l, newsize) @jit.look_inside_iff(lambda l, newsize: jit.isconstant(len(l.items)) and jit.isconstant(newsize)) @jit.oopspec("list._resize_ge(l, newsize)") def _ll_list_resize_ge(l, newsize): if len(l.items) >= newsize: l.length = newsize else: _ll_list_resize_really(l, newsize) @jit.look_inside_iff(lambda l, newsize: jit.isconstant(len(l.items)) and jit.isconstant(newsize)) @jit.oopspec("list._resize_le(l, newsize)") def _ll_list_resize_le(l, newsize): if newsize >= (len(l.items) >> 1) - 5: l.length = newsize else: _ll_list_resize_really(l, newsize)
class AbstractUnwrappedStrategy(object): _mixin_ = True def wrap(self, unwrapped): raise NotImplementedError def unwrap(self, wrapped): raise NotImplementedError @staticmethod def unerase(storage): raise NotImplementedError("abstract base class") @staticmethod def erase(obj): raise NotImplementedError("abstract base class") def is_correct_type(self, w_obj): raise NotImplementedError("abstract base class") def list_is_correct_type(self, w_list): raise NotImplementedError("abstract base class") @jit.look_inside_iff(lambda space, w_list, list_w: jit.isconstant( len(list_w)) and len(list_w) < UNROLL_CUTOFF) def init_from_list_w(self, w_list, list_w): l = [self.unwrap(w_item) for w_item in list_w] w_list.lstorage = self.erase(l) def get_empty_storage(self): return self.erase([]) def clone(self, w_list): l = self.unerase(w_list.lstorage) storage = self.erase(l[:]) w_clone = W_ListObject.from_storage_and_strategy( self.space, storage, self) return w_clone def copy_into(self, w_list, w_other): w_other.strategy = self items = self.unerase(w_list.lstorage)[:] w_other.lstorage = self.erase(items) def contains(self, w_list, w_obj): if self.is_correct_type(w_obj): obj = self.unwrap(w_obj) l = self.unerase(w_list.lstorage) for i in l: if i == obj: return True return ListStrategy.contains(self, w_list, w_obj) def length(self, w_list): return len(self.unerase(w_list.lstorage)) def getitem(self, w_list, index): l = self.unerase(w_list.lstorage) try: r = l[index] except IndexError: # make RPython raise the exception raise return self.wrap(r) @jit.look_inside_iff(lambda self, w_list: jit.isconstant(w_list.length()) and w_list.length() < UNROLL_CUTOFF) def getitems_copy(self, w_list): return [self.wrap(item) for item in self.unerase(w_list.lstorage)] @jit.unroll_safe def getitems_unroll(self, w_list): return [self.wrap(item) for item in self.unerase(w_list.lstorage)] @jit.look_inside_iff(lambda self, w_list: jit.isconstant(w_list.length()) and w_list.length() < UNROLL_CUTOFF) def getitems_fixedsize(self, w_list): return self.getitems_unroll(w_list) def getstorage_copy(self, w_list): items = self.unerase(w_list.lstorage)[:] return self.erase(items) def getslice(self, w_list, start, stop, step, length): if step == 1 and 0 <= start <= stop: l = self.unerase(w_list.lstorage) assert start >= 0 assert stop >= 0 sublist = l[start:stop] storage = self.erase(sublist) return W_ListObject.from_storage_and_strategy( self.space, storage, self) else: subitems_w = [self._none_value] * length l = self.unerase(w_list.lstorage) for i in range(length): try: subitems_w[i] = l[start] start += step except IndexError: raise storage = self.erase(subitems_w) return W_ListObject.from_storage_and_strategy( self.space, storage, self) def append(self, w_list, w_item): if self.is_correct_type(w_item): self.unerase(w_list.lstorage).append(self.unwrap(w_item)) return w_list.switch_to_object_strategy() w_list.append(w_item) def insert(self, w_list, index, w_item): l = self.unerase(w_list.lstorage) if self.is_correct_type(w_item): l.insert(index, self.unwrap(w_item)) return w_list.switch_to_object_strategy() w_list.insert(index, w_item) def extend(self, w_list, w_other): l = self.unerase(w_list.lstorage) if self.list_is_correct_type(w_other): l += self.unerase(w_other.lstorage) return elif w_other.strategy is self.space.fromcache(EmptyListStrategy): return w_other = w_other._temporarily_as_objects() w_list.switch_to_object_strategy() w_list.extend(w_other) def setitem(self, w_list, index, w_item): l = self.unerase(w_list.lstorage) if self.is_correct_type(w_item): try: l[index] = self.unwrap(w_item) except IndexError: raise return w_list.switch_to_object_strategy() w_list.setitem(index, w_item) def setslice(self, w_list, start, step, slicelength, w_other): assert slicelength >= 0 items = self.unerase(w_list.lstorage) if self is self.space.fromcache(ObjectListStrategy): w_other = w_other._temporarily_as_objects() elif (not self.list_is_correct_type(w_other) and w_other.length() != 0): w_list.switch_to_object_strategy() w_other_as_object = w_other._temporarily_as_objects() assert w_other_as_object.strategy is self.space.fromcache( ObjectListStrategy) w_list.setslice(start, step, slicelength, w_other_as_object) return oldsize = len(items) len2 = w_other.length() if step == 1: # Support list resizing for non-extended slices delta = slicelength - len2 if delta < 0: delta = -delta newsize = oldsize + delta # XXX support this in rlist! items += [self._none_value] * delta lim = start + len2 i = newsize - 1 while i >= lim: items[i] = items[i - delta] i -= 1 elif start >= 0: del items[start:start + delta] else: assert delta == 0 # start<0 is only possible with slicelength==0 elif len2 != slicelength: # No resize for extended slices raise operationerrfmt( self.space.w_ValueError, "attempt to " "assign sequence of size %d to extended slice of size %d", len2, slicelength) if w_other.strategy is self.space.fromcache(EmptyListStrategy): other_items = [] else: # at this point both w_list and w_other have the same type, so # self.unerase is valid for both of them other_items = self.unerase(w_other.lstorage) if other_items is items: if step > 0: # Always copy starting from the right to avoid # having to make a shallow copy in the case where # the source and destination lists are the same list. i = len2 - 1 start += i * step while i >= 0: items[start] = other_items[i] start -= step i -= 1 return else: # Make a shallow copy to more easily handle the reversal case w_list.reverse() return #other_items = list(other_items) for i in range(len2): items[start] = other_items[i] start += step def deleteslice(self, w_list, start, step, slicelength): items = self.unerase(w_list.lstorage) if slicelength == 0: return if step < 0: start = start + step * (slicelength - 1) step = -step if step == 1: assert start >= 0 assert slicelength >= 0 del items[start:start + slicelength] else: n = len(items) i = start for discard in range(1, slicelength): j = i + 1 i += step while j < i: items[j - discard] = items[j] j += 1 j = i + 1 while j < n: items[j - slicelength] = items[j] j += 1 start = n - slicelength assert start >= 0 # annotator hint del items[start:] def pop_end(self, w_list): l = self.unerase(w_list.lstorage) return self.wrap(l.pop()) def pop(self, w_list, index): l = self.unerase(w_list.lstorage) # not sure if RPython raises IndexError on pop # so check again here if index < 0: raise IndexError try: item = l.pop(index) except IndexError: raise w_item = self.wrap(item) return w_item def inplace_mul(self, w_list, times): l = self.unerase(w_list.lstorage) l *= times def reverse(self, w_list): self.unerase(w_list.lstorage).reverse()
def ll_dict_getitem(d, key): i = ll_dict_lookup(d, key, d.keyhash(key)) if not i & HIGHEST_BIT: return ll_get_value(d, i) else: raise KeyError def ll_dict_setitem(d, key, value): hash = d.keyhash(key) i = ll_dict_lookup(d, key, hash) return _ll_dict_setitem_lookup_done(d, key, value, hash, i) # It may be safe to look inside always, it has a few branches though, and their # frequencies needs to be investigated. @jit.look_inside_iff(lambda d, key, value, hash, i: jit.isvirtual(d) and jit.isconstant(key)) def _ll_dict_setitem_lookup_done(d, key, value, hash, i): valid = (i & HIGHEST_BIT) == 0 i = i & MASK everused = d.entries.everused(i) # set up the new entry ENTRY = lltype.typeOf(d.entries).TO.OF entry = d.entries[i] entry.value = value if valid: return entry.key = key if hasattr(ENTRY, 'f_hash'): entry.f_hash = hash if hasattr(ENTRY, 'f_valid'): entry.f_valid = True d.num_items += 1 if not everused:
try: z = ovfcheck(x // y) except ZeroDivisionError: raise OperationError(space.w_ZeroDivisionError, space.wrap("integer divmod by zero")) except OverflowError: raise FailedToImplementArgs(space.w_OverflowError, space.wrap("integer modulo")) # no overflow possible m = x % y w = space.wrap return space.newtuple([w(z), w(m)]) # helper for pow() @jit.look_inside_iff(lambda space, iv, iw, iz: jit.isconstant(iw) and jit.isconstant(iz)) def _impl_int_int_pow(space, iv, iw, iz): if iw < 0: if iz != 0: raise OperationError(space.w_TypeError, space.wrap("pow() 2nd argument " "cannot be negative when 3rd argument specified")) ## bounce it, since it always returns float raise FailedToImplementArgs(space.w_ValueError, space.wrap("integer exponentiation")) temp = iv ix = 1 try: while iw > 0: if iw & 1: ix = ovfcheck(ix*temp)
class LLHelpers(AbstractLLHelpers): @jit.elidable def ll_str_mul(s, times): if times < 0: times = 0 try: size = ovfcheck(len(s.chars) * times) except OverflowError: raise MemoryError newstr = s.malloc(size) i = 0 if i < size: s.copy_contents(s, newstr, 0, 0, len(s.chars)) i += len(s.chars) while i < size: if i <= size - i: j = i else: j = size - i s.copy_contents(newstr, newstr, 0, i, j) i += j return newstr @jit.elidable def ll_char_mul(ch, times): if typeOf(ch) is Char: malloc = mallocstr else: malloc = mallocunicode if times < 0: times = 0 newstr = malloc(times) j = 0 # XXX we can use memset here, not sure how useful this is while j < times: newstr.chars[j] = ch j += 1 return newstr def ll_strlen(s): return len(s.chars) def ll_stritem_nonneg(s, i): chars = s.chars ll_assert(i >= 0, "negative str getitem index") ll_assert(i < len(chars), "str getitem index out of bound") return chars[i] ll_stritem_nonneg._annenforceargs_ = [None, int] def ll_chr2str(ch): if typeOf(ch) is Char: malloc = mallocstr else: malloc = mallocunicode s = malloc(1) s.chars[0] = ch return s # @jit.look_inside_iff(lambda str: jit.isconstant(len(str.chars)) and len(str.chars) == 1) @jit.oopspec("str.str2unicode(str)") def ll_str2unicode(str): lgt = len(str.chars) s = mallocunicode(lgt) for i in range(lgt): if ord(str.chars[i]) > 127: raise UnicodeDecodeError s.chars[i] = cast_primitive(UniChar, str.chars[i]) return s @jit.elidable def ll_strhash(s): # unlike CPython, there is no reason to avoid to return -1 # but our malloc initializes the memory to zero, so we use zero as the # special non-computed-yet value. if not s: return 0 x = s.hash if x == 0: x = _hash_string(s.chars) if x == 0: x = 29872897 s.hash = x return x def ll_strfasthash(s): return s.hash # assumes that the hash is already computed @jit.elidable def ll_strconcat(s1, s2): len1 = len(s1.chars) len2 = len(s2.chars) # a single '+' like this is allowed to overflow: it gets # a negative result, and the gc will complain newstr = s1.malloc(len1 + len2) s1.copy_contents(s1, newstr, 0, 0, len1) s1.copy_contents(s2, newstr, 0, len1, len2) return newstr ll_strconcat.oopspec = 'stroruni.concat(s1, s2)' @jit.elidable def ll_strip(s, ch, left, right): s_len = len(s.chars) if s_len == 0: return s.empty() lpos = 0 rpos = s_len - 1 if left: while lpos < rpos and s.chars[lpos] == ch: lpos += 1 if right: while lpos < rpos + 1 and s.chars[rpos] == ch: rpos -= 1 if rpos < lpos: return s.empty() r_len = rpos - lpos + 1 result = s.malloc(r_len) s.copy_contents(s, result, lpos, 0, r_len) return result @jit.elidable def ll_upper(s): s_chars = s.chars s_len = len(s_chars) if s_len == 0: return s.empty() i = 0 result = mallocstr(s_len) # ^^^^^^^^^ specifically to explode on unicode while i < s_len: ch = s_chars[i] if 'a' <= ch <= 'z': ch = chr(ord(ch) - 32) result.chars[i] = ch i += 1 return result @jit.elidable def ll_lower(s): s_chars = s.chars s_len = len(s_chars) if s_len == 0: return s.empty() i = 0 result = mallocstr(s_len) # ^^^^^^^^^ specifically to explode on unicode while i < s_len: ch = s_chars[i] if 'A' <= ch <= 'Z': ch = chr(ord(ch) + 32) result.chars[i] = ch i += 1 return result def ll_join(s, length, items): s_chars = s.chars s_len = len(s_chars) num_items = length if num_items == 0: return s.empty() itemslen = 0 i = 0 while i < num_items: try: itemslen = ovfcheck(itemslen + len(items[i].chars)) except OverflowError: raise MemoryError i += 1 try: seplen = ovfcheck(s_len * (num_items - 1)) except OverflowError: raise MemoryError # a single '+' at the end is allowed to overflow: it gets # a negative result, and the gc will complain result = s.malloc(itemslen + seplen) res_index = len(items[0].chars) s.copy_contents(items[0], result, 0, 0, res_index) i = 1 while i < num_items: s.copy_contents(s, result, 0, res_index, s_len) res_index += s_len lgt = len(items[i].chars) s.copy_contents(items[i], result, 0, res_index, lgt) res_index += lgt i += 1 return result @jit.elidable def ll_strcmp(s1, s2): if not s1 and not s2: return True if not s1 or not s2: return False chars1 = s1.chars chars2 = s2.chars len1 = len(chars1) len2 = len(chars2) if len1 < len2: cmplen = len1 else: cmplen = len2 i = 0 while i < cmplen: diff = ord(chars1[i]) - ord(chars2[i]) if diff != 0: return diff i += 1 return len1 - len2 @jit.elidable def ll_streq(s1, s2): if s1 == s2: # also if both are NULLs return True if not s1 or not s2: return False len1 = len(s1.chars) len2 = len(s2.chars) if len1 != len2: return False j = 0 chars1 = s1.chars chars2 = s2.chars while j < len1: if chars1[j] != chars2[j]: return False j += 1 return True ll_streq.oopspec = 'stroruni.equal(s1, s2)' @jit.elidable def ll_startswith(s1, s2): len1 = len(s1.chars) len2 = len(s2.chars) if len1 < len2: return False j = 0 chars1 = s1.chars chars2 = s2.chars while j < len2: if chars1[j] != chars2[j]: return False j += 1 return True def ll_startswith_char(s, ch): if not len(s.chars): return False return s.chars[0] == ch @jit.elidable def ll_endswith(s1, s2): len1 = len(s1.chars) len2 = len(s2.chars) if len1 < len2: return False j = 0 chars1 = s1.chars chars2 = s2.chars offset = len1 - len2 while j < len2: if chars1[offset + j] != chars2[j]: return False j += 1 return True def ll_endswith_char(s, ch): if not len(s.chars): return False return s.chars[len(s.chars) - 1] == ch @jit.elidable def ll_find_char(s, ch, start, end): i = start if end > len(s.chars): end = len(s.chars) while i < end: if s.chars[i] == ch: return i i += 1 return -1 ll_find_char._annenforceargs_ = [None, None, int, int] @jit.elidable def ll_rfind_char(s, ch, start, end): if end > len(s.chars): end = len(s.chars) i = end while i > start: i -= 1 if s.chars[i] == ch: return i return -1 @jit.elidable def ll_count_char(s, ch, start, end): count = 0 i = start if end > len(s.chars): end = len(s.chars) while i < end: if s.chars[i] == ch: count += 1 i += 1 return count @classmethod def ll_find(cls, s1, s2, start, end): if start < 0: start = 0 if end > len(s1.chars): end = len(s1.chars) if end - start < 0: return -1 m = len(s2.chars) if m == 0: return start elif m == 1: return cls.ll_find_char(s1, s2.chars[0], start, end) return cls.ll_search(s1, s2, start, end, FAST_FIND) @classmethod def ll_rfind(cls, s1, s2, start, end): if start < 0: start = 0 if end > len(s1.chars): end = len(s1.chars) if end - start < 0: return -1 m = len(s2.chars) if m == 0: return end elif m == 1: return cls.ll_rfind_char(s1, s2.chars[0], start, end) return cls.ll_search(s1, s2, start, end, FAST_RFIND) @classmethod def ll_count(cls, s1, s2, start, end): if start < 0: start = 0 if end > len(s1.chars): end = len(s1.chars) if end - start < 0: return 0 m = len(s2.chars) if m == 0: return end - start + 1 elif m == 1: return cls.ll_count_char(s1, s2.chars[0], start, end) res = cls.ll_search(s1, s2, start, end, FAST_COUNT) # For a few cases ll_search can return -1 to indicate an "impossible" # condition for a string match, count just returns 0 in these cases. if res < 0: res = 0 return res @jit.elidable def ll_search(s1, s2, start, end, mode): count = 0 n = end - start m = len(s2.chars) w = n - m if w < 0: return -1 mlast = m - 1 skip = mlast - 1 mask = 0 if mode != FAST_RFIND: for i in range(mlast): mask = bloom_add(mask, s2.chars[i]) if s2.chars[i] == s2.chars[mlast]: skip = mlast - i - 1 mask = bloom_add(mask, s2.chars[mlast]) i = start - 1 while i + 1 <= start + w: i += 1 if s1.chars[i + m - 1] == s2.chars[m - 1]: for j in range(mlast): if s1.chars[i + j] != s2.chars[j]: break else: if mode != FAST_COUNT: return i count += 1 i += mlast continue if i + m < len(s1.chars): c = s1.chars[i + m] else: c = '\0' if not bloom(mask, c): i += m else: i += skip else: if i + m < len(s1.chars): c = s1.chars[i + m] else: c = '\0' if not bloom(mask, c): i += m else: mask = bloom_add(mask, s2.chars[0]) for i in range(mlast, 0, -1): mask = bloom_add(mask, s2.chars[i]) if s2.chars[i] == s2.chars[0]: skip = i - 1 i = start + w + 1 while i - 1 >= start: i -= 1 if s1.chars[i] == s2.chars[0]: for j in xrange(mlast, 0, -1): if s1.chars[i + j] != s2.chars[j]: break else: return i if i - 1 >= 0 and not bloom(mask, s1.chars[i - 1]): i -= m else: i -= skip else: if i - 1 >= 0 and not bloom(mask, s1.chars[i - 1]): i -= m if mode != FAST_COUNT: return -1 return count @enforceargs(int, None) @jit.look_inside_iff( lambda length, items: jit.loop_unrolling_heuristic(items, length)) def ll_join_strs(length, items): # Special case for length 1 items, helps both the JIT and other code if length == 1: return items[0] num_items = length itemslen = 0 i = 0 while i < num_items: try: itemslen = ovfcheck(itemslen + len(items[i].chars)) except OverflowError: raise MemoryError i += 1 if typeOf(items).TO.OF.TO == STR: malloc = mallocstr copy_contents = copy_string_contents else: malloc = mallocunicode copy_contents = copy_unicode_contents result = malloc(itemslen) res_chars = result.chars res_index = 0 i = 0 while i < num_items: item_chars = items[i].chars item_len = len(item_chars) copy_contents(items[i], result, 0, res_index, item_len) res_index += item_len i += 1 return result @jit.look_inside_iff(lambda length, chars, RES: jit.isconstant(length) and jit.isvirtual(chars)) def ll_join_chars(length, chars, RES): # no need to optimize this, will be replaced by string builder # at some point soon num_chars = length if RES is StringRepr.lowleveltype: target = Char malloc = mallocstr else: target = UniChar malloc = mallocunicode result = malloc(num_chars) res_chars = result.chars i = 0 while i < num_chars: res_chars[i] = cast_primitive(target, chars[i]) i += 1 return result @jit.elidable def _ll_stringslice(s1, start, stop): lgt = stop - start assert start >= 0 # If start > stop, return a empty string. This can happen if the start # is greater than the length of the string. Use < instead of <= to avoid # creating another path for the JIT when start == stop. if lgt < 0: return s1.empty() newstr = s1.malloc(lgt) s1.copy_contents(s1, newstr, start, 0, lgt) return newstr _ll_stringslice.oopspec = 'stroruni.slice(s1, start, stop)' _ll_stringslice._annenforceargs_ = [None, int, int] def ll_stringslice_startonly(s1, start): return LLHelpers._ll_stringslice(s1, start, len(s1.chars)) def ll_stringslice_startstop(s1, start, stop): if jit.we_are_jitted(): if stop > len(s1.chars): stop = len(s1.chars) else: if stop >= len(s1.chars): if start == 0: return s1 stop = len(s1.chars) return LLHelpers._ll_stringslice(s1, start, stop) def ll_stringslice_minusone(s1): newlen = len(s1.chars) - 1 return LLHelpers._ll_stringslice(s1, 0, newlen) def ll_split_chr(LIST, s, c, max): chars = s.chars strlen = len(chars) count = 1 i = 0 if max == 0: i = strlen while i < strlen: if chars[i] == c: count += 1 if max >= 0 and count > max: break i += 1 res = LIST.ll_newlist(count) items = res.ll_items() i = 0 j = 0 resindex = 0 if max == 0: j = strlen while j < strlen: if chars[j] == c: item = items[resindex] = s.malloc(j - i) item.copy_contents(s, item, i, 0, j - i) resindex += 1 i = j + 1 if max >= 0 and resindex >= max: j = strlen break j += 1 item = items[resindex] = s.malloc(j - i) item.copy_contents(s, item, i, 0, j - i) return res def ll_rsplit_chr(LIST, s, c, max): chars = s.chars strlen = len(chars) count = 1 i = 0 if max == 0: i = strlen while i < strlen: if chars[i] == c: count += 1 if max >= 0 and count > max: break i += 1 res = LIST.ll_newlist(count) items = res.ll_items() i = strlen j = strlen resindex = count - 1 assert resindex >= 0 if max == 0: j = 0 while j > 0: j -= 1 if chars[j] == c: item = items[resindex] = s.malloc(i - j - 1) item.copy_contents(s, item, j + 1, 0, i - j - 1) resindex -= 1 i = j if resindex == 0: j = 0 break item = items[resindex] = s.malloc(i - j) item.copy_contents(s, item, j, 0, i - j) return res @jit.elidable def ll_replace_chr_chr(s, c1, c2): length = len(s.chars) newstr = s.malloc(length) src = s.chars dst = newstr.chars j = 0 while j < length: c = src[j] if c == c1: c = c2 dst[j] = c j += 1 return newstr @jit.elidable def ll_contains(s, c): chars = s.chars strlen = len(chars) i = 0 while i < strlen: if chars[i] == c: return True i += 1 return False @jit.elidable def ll_int(s, base): if not 2 <= base <= 36: raise ValueError chars = s.chars strlen = len(chars) i = 0 #XXX: only space is allowed as white space for now while i < strlen and chars[i] == ' ': i += 1 if not i < strlen: raise ValueError #check sign sign = 1 if chars[i] == '-': sign = -1 i += 1 elif chars[i] == '+': i += 1 # skip whitespaces between sign and digits while i < strlen and chars[i] == ' ': i += 1 #now get digits val = 0 oldpos = i while i < strlen: c = ord(chars[i]) if ord('a') <= c <= ord('z'): digit = c - ord('a') + 10 elif ord('A') <= c <= ord('Z'): digit = c - ord('A') + 10 elif ord('0') <= c <= ord('9'): digit = c - ord('0') else: break if digit >= base: break val = val * base + digit i += 1 if i == oldpos: raise ValueError # catch strings like '+' and '+ ' #skip trailing whitespace while i < strlen and chars[i] == ' ': i += 1 if not i == strlen: raise ValueError return sign * val # interface to build strings: # x = ll_build_start(n) # ll_build_push(x, next_string, 0) # ll_build_push(x, next_string, 1) # ... # ll_build_push(x, next_string, n-1) # s = ll_build_finish(x) def ll_build_start(parts_count): return malloc(TEMP, parts_count) def ll_build_push(builder, next_string, index): builder[index] = next_string def ll_build_finish(builder): return LLHelpers.ll_join_strs(len(builder), builder) def ll_constant(s): return string_repr.convert_const(s) ll_constant._annspecialcase_ = 'specialize:memo' def do_stringformat(cls, hop, sourcevarsrepr): s_str = hop.args_s[0] assert s_str.is_constant() s = s_str.const things = cls.parse_fmt_string(s) size = inputconst(Signed, len(things)) # could be unsigned? cTEMP = inputconst(Void, TEMP) cflags = inputconst(Void, {'flavor': 'gc'}) vtemp = hop.genop("malloc_varsize", [cTEMP, cflags, size], resulttype=Ptr(TEMP)) argsiter = iter(sourcevarsrepr) InstanceRepr = hop.rtyper.type_system.rclass.InstanceRepr for i, thing in enumerate(things): if isinstance(thing, tuple): code = thing[0] vitem, r_arg = argsiter.next() if not hasattr(r_arg, 'll_str'): raise TyperError("ll_str unsupported for: %r" % r_arg) if code == 's' or (code == 'r' and isinstance(r_arg, InstanceRepr)): vchunk = hop.gendirectcall(r_arg.ll_str, vitem) elif code == 'd': assert isinstance(r_arg, IntegerRepr) #vchunk = hop.gendirectcall(r_arg.ll_str, vitem) vchunk = hop.gendirectcall(ll_str.ll_int2dec, vitem) elif code == 'f': #assert isinstance(r_arg, FloatRepr) vchunk = hop.gendirectcall(r_arg.ll_str, vitem) elif code == 'x': assert isinstance(r_arg, IntegerRepr) vchunk = hop.gendirectcall(ll_str.ll_int2hex, vitem, inputconst(Bool, False)) elif code == 'o': assert isinstance(r_arg, IntegerRepr) vchunk = hop.gendirectcall(ll_str.ll_int2oct, vitem, inputconst(Bool, False)) else: raise TyperError, "%%%s is not RPython" % (code, ) else: from pypy.rpython.lltypesystem.rstr import string_repr vchunk = inputconst(string_repr, thing) i = inputconst(Signed, i) hop.genop('setarrayitem', [vtemp, i, vchunk]) hop.exception_cannot_occur() # to ignore the ZeroDivisionError of '%' return hop.gendirectcall(cls.ll_join_strs, size, vtemp) do_stringformat = classmethod(do_stringformat)
"got multiple values " "for keyword argument " "'%s'", key) keywords[i] = key keywords_w[i] = space.getitem(w_starstararg, w_key) i += 1 if self.keywords is None: self.keywords = keywords self.keywords_w = keywords_w else: self.keywords = self.keywords + keywords self.keywords_w = self.keywords_w + keywords_w self.keyword_names_w = keys_w @jit.look_inside_iff(lambda self, keywords, keywords_w: jit.isconstant(len(keywords) and jit.isconstant(self.keywords))) def _add_keywordargs_no_unwrapping(self, keywords, keywords_w): if self.keywords is None: self.keywords = keywords[:] # copy to make non-resizable self.keywords_w = keywords_w[:] else: # looks quadratic, but the JIT should remove all of it nicely. # Also, all the lists should be small for key in keywords: for otherkey in self.keywords: if otherkey == key: raise operationerrfmt(self.space.w_TypeError, "got multiple values " "for keyword argument " "'%s'", key) self.keywords = self.keywords + keywords
class TemplateFormatter(object): parser_list_w = None def __init__(self, space, is_unicode, template): self.space = space self.is_unicode = is_unicode self.empty = u"" if is_unicode else "" self.template = template def build(self, args): self.args, self.kwargs = args.unpack() self.auto_numbering = 0 self.auto_numbering_state = ANS_INIT return self._build_string(0, len(self.template), 2) def _build_string(self, start, end, level): space = self.space if self.is_unicode: out = rstring.UnicodeBuilder() else: out = rstring.StringBuilder() if not level: raise OperationError(space.w_ValueError, space.wrap("Recursion depth exceeded")) level -= 1 s = self.template return self._do_build_string(start, end, level, out, s) @jit.look_inside_iff(lambda self, start, end, level, out, s: jit.isconstant(s)) def _do_build_string(self, start, end, level, out, s): space = self.space last_literal = i = start while i < end: c = s[i] i += 1 if c == "{" or c == "}": at_end = i == end # Find escaped "{" and "}" markup_follows = True if c == "}": if at_end or s[i] != "}": raise OperationError(space.w_ValueError, space.wrap("Single '}'")) i += 1 markup_follows = False if c == "{": if at_end: raise OperationError(space.w_ValueError, space.wrap("Single '{'")) if s[i] == "{": i += 1 markup_follows = False # Attach literal data, ending with { or } out.append_slice(s, last_literal, i - 1) if not markup_follows: if self.parser_list_w is not None: end_literal = i - 1 assert end_literal > last_literal literal = self.template[last_literal:end_literal] w_entry = space.newtuple([ space.wrap(literal), space.w_None, space.w_None, space.w_None]) self.parser_list_w.append(w_entry) self.last_end = i last_literal = i continue nested = 1 field_start = i recursive = False while i < end: c = s[i] if c == "{": recursive = True nested += 1 elif c == "}": nested -= 1 if not nested: break i += 1 if nested: raise OperationError(space.w_ValueError, space.wrap("Unmatched '{'")) rendered = self._render_field(field_start, i, recursive, level) out.append(rendered) i += 1 last_literal = i out.append_slice(s, last_literal, end) return out.build() # This is only ever called if we're already unrolling _do_build_string @jit.unroll_safe def _parse_field(self, start, end): s = self.template # Find ":" or "!" i = start while i < end: c = s[i] if c == ":" or c == "!": end_name = i if c == "!": i += 1 if i == end: w_msg = self.space.wrap("expected conversion") raise OperationError(self.space.w_ValueError, w_msg) conversion = s[i] i += 1 if i < end: if s[i] != ':': w_msg = self.space.wrap("expected ':' after" " format specifier") raise OperationError(self.space.w_ValueError, w_msg) i += 1 else: conversion = None i += 1 return s[start:end_name], conversion, i i += 1 return s[start:end], None, end @jit.unroll_safe def _get_argument(self, name): # First, find the argument. space = self.space i = 0 end = len(name) while i < end: c = name[i] if c == "[" or c == ".": break i += 1 empty = not i if empty: index = -1 else: index, stop = _parse_int(self.space, name, 0, i) if stop != i: index = -1 use_numeric = empty or index != -1 if self.auto_numbering_state == ANS_INIT and use_numeric: if empty: self.auto_numbering_state = ANS_AUTO else: self.auto_numbering_state = ANS_MANUAL if use_numeric: if self.auto_numbering_state == ANS_MANUAL: if empty: msg = "switching from manual to automatic numbering" raise OperationError(space.w_ValueError, space.wrap(msg)) elif not empty: msg = "switching from automatic to manual numbering" raise OperationError(space.w_ValueError, space.wrap(msg)) if empty: index = self.auto_numbering self.auto_numbering += 1 if index == -1: kwarg = name[:i] if self.is_unicode: try: arg_key = kwarg.encode("latin-1") except UnicodeEncodeError: # Not going to be found in a dict of strings. raise OperationError(space.w_KeyError, space.wrap(kwarg)) else: arg_key = kwarg try: w_arg = self.kwargs[arg_key] except KeyError: raise OperationError(space.w_KeyError, space.wrap(arg_key)) else: try: w_arg = self.args[index] except IndexError: w_msg = space.wrap("index out of range") raise OperationError(space.w_IndexError, w_msg) return self._resolve_lookups(w_arg, name, i, end) @jit.unroll_safe def _resolve_lookups(self, w_obj, name, start, end): # Resolve attribute and item lookups. space = self.space i = start while i < end: c = name[i] if c == ".": i += 1 start = i while i < end: c = name[i] if c == "[" or c == ".": break i += 1 if start == i: w_msg = space.wrap("Empty attribute in format string") raise OperationError(space.w_ValueError, w_msg) w_attr = space.wrap(name[start:i]) if w_obj is not None: w_obj = space.getattr(w_obj, w_attr) else: self.parser_list_w.append(space.newtuple([ space.w_True, w_attr])) elif c == "[": got_bracket = False i += 1 start = i while i < end: c = name[i] if c == "]": got_bracket = True break i += 1 if not got_bracket: raise OperationError(space.w_ValueError, space.wrap("Missing ']'")) index, reached = _parse_int(self.space, name, start, i) if index != -1 and reached == i: w_item = space.wrap(index) else: w_item = space.wrap(name[start:i]) i += 1 # Skip "]" if w_obj is not None: w_obj = space.getitem(w_obj, w_item) else: self.parser_list_w.append(space.newtuple([ space.w_False, w_item])) else: msg = "Only '[' and '.' may follow ']'" raise OperationError(space.w_ValueError, space.wrap(msg)) return w_obj def formatter_field_name_split(self): space = self.space name = self.template i = 0 end = len(name) while i < end: c = name[i] if c == "[" or c == ".": break i += 1 if i == 0: index = -1 else: index, stop = _parse_int(self.space, name, 0, i) if stop != i: index = -1 if index >= 0: w_first = space.wrap(index) else: w_first = space.wrap(name[:i]) # self.parser_list_w = [] self._resolve_lookups(None, name, i, end) # return space.newtuple([w_first, space.iter(space.newlist(self.parser_list_w))]) def _convert(self, w_obj, conversion): space = self.space conv = conversion[0] if conv == "r": return space.repr(w_obj) elif conv == "s": if self.is_unicode: return space.call_function(space.w_unicode, w_obj) return space.str(w_obj) else: raise OperationError(self.space.w_ValueError, self.space.wrap("invalid conversion")) def _render_field(self, start, end, recursive, level): name, conversion, spec_start = self._parse_field(start, end) spec = self.template[spec_start:end] # if self.parser_list_w is not None: # used from formatter_parser() if level == 1: # ignore recursive calls space = self.space startm1 = start - 1 assert startm1 >= self.last_end w_entry = space.newtuple([ space.wrap(self.template[self.last_end:startm1]), space.wrap(name), space.wrap(spec), space.wrap(conversion)]) self.parser_list_w.append(w_entry) self.last_end = end + 1 return self.empty # w_obj = self._get_argument(name) if conversion is not None: w_obj = self._convert(w_obj, conversion) if recursive: spec = self._build_string(spec_start, end, level) w_rendered = self.space.format(w_obj, self.space.wrap(spec)) unwrapper = "unicode_w" if self.is_unicode else "str_w" to_interp = getattr(self.space, unwrapper) return to_interp(w_rendered) def formatter_parser(self): self.parser_list_w = [] self.last_end = 0 self._build_string(0, len(self.template), 2) # space = self.space if self.last_end < len(self.template): w_lastentry = space.newtuple([ space.wrap(self.template[self.last_end:]), space.w_None, space.w_None, space.w_None]) self.parser_list_w.append(w_lastentry) return space.iter(space.newlist(self.parser_list_w))
if length <= 0: strategy = space.fromcache(EmptyListStrategy) storage = strategy.erase(None) else: strategy = space.fromcache(RangeListStrategy) storage = strategy.erase((start, step, length)) return W_ListObject.from_storage_and_strategy(space, storage, strategy) def make_empty_list(space): strategy = space.fromcache(EmptyListStrategy) storage = strategy.erase(None) return W_ListObject.from_storage_and_strategy(space, storage, strategy) @jit.look_inside_iff(lambda space, list_w: jit.isconstant(len(list_w)) and len( list_w) < UNROLL_CUTOFF) def get_strategy_from_list_objects(space, list_w): if not list_w: return space.fromcache(EmptyListStrategy) # check for ints for w_obj in list_w: if not is_W_IntObject(w_obj): break else: return space.fromcache(IntegerListStrategy) # check for strings for w_obj in list_w: if not is_W_StringObject(w_obj):
class KwargsDictStrategy(DictStrategy): erase, unerase = rerased.new_erasing_pair("kwargsdict") erase = staticmethod(erase) unerase = staticmethod(unerase) def wrap(self, key): return self.space.wrap(key) def unwrap(self, wrapped): return self.space.str_w(wrapped) def get_empty_storage(self): d = ([], []) return self.erase(d) def is_correct_type(self, w_obj): space = self.space return space.is_w(space.type(w_obj), space.w_str) def _never_equal_to(self, w_lookup_type): return False def iter(self, w_dict): return KwargsDictIterator(self.space, self, w_dict) def w_keys(self, w_dict): return self.space.newlist( [self.space.wrap(key) for key in self.unerase(w_dict.dstorage)[0]]) def setitem(self, w_dict, w_key, w_value): space = self.space if self.is_correct_type(w_key): self.setitem_str(w_dict, self.unwrap(w_key), w_value) return else: self.switch_to_object_strategy(w_dict) w_dict.setitem(w_key, w_value) def setitem_str(self, w_dict, key, w_value): self._setitem_str_indirection(w_dict, key, w_value) @jit.look_inside_iff(lambda self, w_dict, key, w_value: jit.isconstant( self.length(w_dict)) and jit.isconstant(key)) def _setitem_str_indirection(self, w_dict, key, w_value): keys, values_w = self.unerase(w_dict.dstorage) result = [] for i in range(len(keys)): if keys[i] == key: values_w[i] = w_value break else: # limit the size so that the linear searches don't become too long if len(keys) >= 16: self.switch_to_string_strategy(w_dict) w_dict.setitem_str(key, w_value) else: keys.append(key) values_w.append(w_value) def setdefault(self, w_dict, w_key, w_default): # XXX could do better, but is it worth it? self.switch_to_object_strategy(w_dict) return w_dict.setdefault(w_key, w_default) def delitem(self, w_dict, w_key): # XXX could do better, but is it worth it? self.switch_to_object_strategy(w_dict) return w_dict.delitem(w_key) def length(self, w_dict): return len(self.unerase(w_dict.dstorage)[0]) def getitem_str(self, w_dict, key): return self._getitem_str_indirection(w_dict, key) @jit.look_inside_iff(lambda self, w_dict, key: jit.isconstant( self.length(w_dict)) and jit.isconstant(key)) def _getitem_str_indirection(self, w_dict, key): keys, values_w = self.unerase(w_dict.dstorage) result = [] for i in range(len(keys)): if keys[i] == key: return values_w[i] return None def getitem(self, w_dict, w_key): space = self.space if self.is_correct_type(w_key): return self.getitem_str(w_dict, self.unwrap(w_key)) elif self._never_equal_to(space.type(w_key)): return None else: self.switch_to_object_strategy(w_dict) return w_dict.getitem(w_key) def w_keys(self, w_dict): l = self.unerase(w_dict.dstorage)[0] return self.space.newlist_str(l[:]) def values(self, w_dict): return self.unerase(w_dict.dstorage)[1][:] # to make non-resizable def items(self, w_dict): space = self.space keys, values_w = self.unerase(w_dict.dstorage) result = [] for i in range(len(keys)): result.append(space.newtuple([self.wrap(keys[i]), values_w[i]])) return result def popitem(self, w_dict): keys, values_w = self.unerase(w_dict.dstorage) key = keys.pop() w_value = values_w.pop() return (self.wrap(key), w_value) def clear(self, w_dict): w_dict.dstorage = self.get_empty_storage() def switch_to_object_strategy(self, w_dict): strategy = self.space.fromcache(ObjectDictStrategy) keys, values_w = self.unerase(w_dict.dstorage) d_new = strategy.unerase(strategy.get_empty_storage()) for i in range(len(keys)): d_new[self.wrap(keys[i])] = values_w[i] w_dict.strategy = strategy w_dict.dstorage = strategy.erase(d_new) def switch_to_string_strategy(self, w_dict): strategy = self.space.fromcache(StringDictStrategy) keys, values_w = self.unerase(w_dict.dstorage) storage = strategy.get_empty_storage() d_new = strategy.unerase(storage) for i in range(len(keys)): d_new[keys[i]] = values_w[i] w_dict.strategy = strategy w_dict.dstorage = storage def view_as_kwargs(self, w_dict): return self.unerase(w_dict.dstorage)
def tuple_unroll_condition(space, w_tuple1, w_tuple2): lgt1 = len(w_tuple1.wrappeditems) lgt2 = len(w_tuple2.wrappeditems) return (jit.isconstant(lgt1) and lgt1 <= UNROLL_TUPLE_LIMIT) or ( jit.isconstant(lgt2) and lgt2 <= UNROLL_TUPLE_LIMIT )
# to see if inlining it gives some speed-up. def _ll_list_resize(l, newsize): # Bypass realloc() when a previous overallocation is large enough # to accommodate the newsize. If the newsize falls lower than half # the allocated size, then proceed with the realloc() to shrink the list. allocated = len(l.items) if allocated >= newsize and newsize >= ((allocated >> 1) - 5): l.length = newsize else: _ll_list_resize_really(l, newsize) @jit.look_inside_iff( lambda l, newsize: jit.isconstant(len(l.items)) and jit.isconstant(newsize) ) @jit.oopspec("list._resize_ge(l, newsize)") def _ll_list_resize_ge(l, newsize): if len(l.items) >= newsize: l.length = newsize else: _ll_list_resize_really(l, newsize) @jit.look_inside_iff( lambda l, newsize: jit.isconstant(len(l.items)) and jit.isconstant(newsize) ) @jit.oopspec("list._resize_le(l, newsize)") def _ll_list_resize_le(l, newsize): if newsize >= (len(l.items) >> 1) - 5:
stop = length newlength = stop - start l = RESLIST.ll_newlist(newlength) ll_arraycopy(l1, l, start, 0, newlength) return l # no oopspec -- the function is inlined by the JIT def ll_listslice_minusone(RESLIST, l1): newlength = l1.ll_length() - 1 ll_assert(newlength >= 0, "empty list is sliced with [:-1]") l = RESLIST.ll_newlist(newlength) ll_arraycopy(l1, l, 0, 0, newlength) return l # no oopspec -- the function is inlined by the JIT @jit.look_inside_iff(lambda l, start: jit.isconstant(start) and jit.isvirtual(l)) @jit.oopspec('list.delslice_startonly(l, start)') def ll_listdelslice_startonly(l, start): ll_assert(start >= 0, "del l[start:] with unexpectedly negative start") ll_assert(start <= l.ll_length(), "del l[start:] with start > len(l)") newlength = start null = ll_null_item(l) if null is not None: j = l.ll_length() - 1 while j >= newlength: l.ll_setitem_fast(j, null) j -= 1 l._ll_resize_le(newlength) def ll_listdelslice_startstop(l, start, stop): length = l.ll_length()
from pypy.rlib import jit from pypy.interpreter.error import OperationError @jit.look_inside_iff(lambda chunks: jit.isconstant(len(chunks))) def enumerate_chunks(chunks): result = [] i = -1 for chunk in chunks: i += chunk.axis_step result.append((i, chunk)) return result @jit.look_inside_iff(lambda shape, start, strides, backstrides, chunks: jit.isconstant(len(chunks)) ) def calculate_slice_strides(shape, start, strides, backstrides, chunks): rstrides = [] rbackstrides = [] rstart = start rshape = [] i = -1 for i, chunk in enumerate_chunks(chunks): if chunk.step != 0: rstrides.append(strides[i] * chunk.step) rbackstrides.append(strides[i] * (chunk.lgt - 1) * chunk.step) rshape.append(chunk.lgt) rstart += strides[i] * chunk.start # add a reminder s = i + 1 assert s >= 0 rstrides += strides[s:]
class StringFormatter(BaseStringFormatter): def __init__(self, space, fmt, values_w, w_valuedict): BaseStringFormatter.__init__(self, space, values_w, w_valuedict) self.fmt = fmt # either a string or a unicode def peekchr(self): # return the 'current' character try: return self.fmt[self.fmtpos] except IndexError: space = self.space raise OperationError(space.w_ValueError, space.wrap("incomplete format")) # Only shows up if we've already started inlining format(), so just # unconditionally unroll this. @jit.unroll_safe def getmappingkey(self): # return the mapping key in a '%(key)s' specifier fmt = self.fmt i = self.fmtpos + 1 # first character after '(' i0 = i pcount = 1 while 1: try: c = fmt[i] except IndexError: space = self.space raise OperationError(space.w_ValueError, space.wrap("incomplete format key")) if c == ')': pcount -= 1 if pcount == 0: break elif c == '(': pcount += 1 i += 1 self.fmtpos = i + 1 # first character after ')' return fmt[i0:i] def getmappingvalue(self, key): # return the value corresponding to a key in the input dict space = self.space if self.w_valuedict is None: raise OperationError(space.w_TypeError, space.wrap("format requires a mapping")) w_key = space.wrap(key) return space.getitem(self.w_valuedict, w_key) def parse_fmt(self): if self.peekchr() == '(': w_value = self.getmappingvalue(self.getmappingkey()) else: w_value = None self.peel_flags() self.width = self.peel_num() if self.width < 0: # this can happen: '%*s' % (-5, "hi") self.f_ljust = True self.width = -self.width if self.peekchr() == '.': self.forward() self.prec = self.peel_num() if self.prec < 0: self.prec = 0 # this can happen: '%.*f' % (-5, 3) else: self.prec = -1 c = self.peekchr() if c == 'h' or c == 'l' or c == 'L': self.forward() return w_value # Same as getmappingkey @jit.unroll_safe def peel_flags(self): self.f_ljust = False self.f_sign = False self.f_blank = False self.f_alt = False self.f_zero = False while True: c = self.peekchr() if c == '-': self.f_ljust = True elif c == '+': self.f_sign = True elif c == ' ': self.f_blank = True elif c == '#': self.f_alt = True elif c == '0': self.f_zero = True else: break self.forward() # Same as getmappingkey @jit.unroll_safe def peel_num(self): space = self.space c = self.peekchr() if c == '*': self.forward() w_value = self.nextinputvalue() return space.int_w(maybe_int(space, w_value)) result = 0 while True: n = ord(c) - ord('0') if not (0 <= n < 10): break try: result = ovfcheck(ovfcheck(result * 10) + n) except OverflowError: raise OperationError(space.w_OverflowError, space.wrap("precision too large")) self.forward() c = self.peekchr() return result @jit.look_inside_iff(lambda self: jit.isconstant(self.fmt)) def format(self): lgt = len(self.fmt) + 4 * len(self.values_w) + 10 if do_unicode: result = UnicodeBuilder(lgt) else: result = StringBuilder(lgt) self.result = result while True: # fast path: consume as many characters as possible fmt = self.fmt i = i0 = self.fmtpos while i < len(fmt): if fmt[i] == '%': break i += 1 else: result.append_slice(fmt, i0, len(fmt)) break # end of 'fmt' string result.append_slice(fmt, i0, i) self.fmtpos = i + 1 # interpret the next formatter w_value = self.parse_fmt() c = self.peekchr() self.forward() if c == '%': self.std_wp(const('%')) continue if w_value is None: w_value = self.nextinputvalue() # dispatch on the formatter # (this turns into a switch after translation) for c1 in FORMATTER_CHARS: if c == c1: # 'c1' is an annotation constant here, # so this getattr() is ok do_fmt = getattr(self, 'fmt_' + c1) do_fmt(w_value) break else: self.unknown_fmtchar() self.checkconsumed() return result.build() def unknown_fmtchar(self): space = self.space c = self.fmt[self.fmtpos - 1] if do_unicode: w_defaultencoding = space.call_function( space.sys.get('getdefaultencoding')) w_s = space.call_method(space.wrap(c), "encode", w_defaultencoding, space.wrap('replace')) s = space.str_w(w_s) else: s = c msg = "unsupported format character '%s' (0x%x) at index %d" % ( s, ord(c), self.fmtpos - 1) raise OperationError(space.w_ValueError, space.wrap(msg)) def std_wp(self, r): length = len(r) if do_unicode and isinstance(r, str): # convert string to unicode explicitely here from pypy.objspace.std.unicodetype import plain_str2unicode r = plain_str2unicode(self.space, r) prec = self.prec if prec == -1 and self.width == 0: # fast path self.result.append(const(r)) return if prec >= 0 and prec < length: length = prec # ignore the end of the string if too long result = self.result padding = self.width - length if padding < 0: padding = 0 assert padding >= 0 if not self.f_ljust and padding > 0: result.append_multiple_char(const(' '), padding) # add any padding at the left of 'r' padding = 0 result.append_slice(r, 0, length) # add 'r' itself if padding > 0: result.append_multiple_char(const(' '), padding) # add any remaining padding at the right std_wp._annspecialcase_ = 'specialize:argtype(1)' def std_wp_number(self, r, prefix=''): # add a '+' or ' ' sign if necessary sign = r.startswith('-') if not sign: if self.f_sign: r = '+' + r sign = True elif self.f_blank: r = ' ' + r sign = True # do the padding requested by self.width and the flags, # without building yet another RPython string but directly # by pushing the pad character into self.result result = self.result padding = self.width - len(r) - len(prefix) if padding <= 0: padding = 0 if self.f_ljust: padnumber = '<' elif self.f_zero: padnumber = '0' else: padnumber = '>' assert padding >= 0 if padnumber == '>': result.append_multiple_char(const(' '), padding) # pad with spaces on the left if sign: result.append(const(r[0])) # the sign result.append(const(prefix)) # the prefix if padnumber == '0': result.append_multiple_char(const('0'), padding) # pad with zeroes result.append_slice(const(r), int(sign), len(r)) # the rest of the number if padnumber == '<': # spaces on the right result.append_multiple_char(const(' '), padding) def string_formatting(self, w_value): space = self.space w_impl = space.lookup(w_value, '__str__') if w_impl is None: raise OperationError( space.w_TypeError, space.wrap("operand does not support " "unary str")) w_result = space.get_and_call_function(w_impl, w_value) if space.isinstance_w(w_result, space.w_unicode): raise NeedUnicodeFormattingError return space.str_w(w_result) def fmt_s(self, w_value): space = self.space got_unicode = space.isinstance_w(w_value, space.w_unicode) if not do_unicode: if got_unicode: raise NeedUnicodeFormattingError s = self.string_formatting(w_value) else: if not got_unicode: w_value = space.call_function(space.w_unicode, w_value) else: w_value = unicode_from_object(space, w_value) s = space.unicode_w(w_value) self.std_wp(s) def fmt_r(self, w_value): self.std_wp(self.space.str_w(self.space.repr(w_value))) def fmt_c(self, w_value): self.prec = -1 # just because space = self.space if space.isinstance_w(w_value, space.w_str): s = space.str_w(w_value) if len(s) != 1: raise OperationError(space.w_TypeError, space.wrap("%c requires int or char")) self.std_wp(s) elif space.isinstance_w(w_value, space.w_unicode): if not do_unicode: raise NeedUnicodeFormattingError ustr = space.unicode_w(w_value) if len(ustr) != 1: raise OperationError( space.w_TypeError, space.wrap("%c requires int or unichar")) self.std_wp(ustr) else: n = space.int_w(w_value) if do_unicode: try: c = unichr(n) except ValueError: raise OperationError( space.w_OverflowError, space.wrap("unicode character code out of range")) self.std_wp(c) else: try: s = chr(n) except ValueError: # chr(out-of-range) raise OperationError( space.w_OverflowError, space.wrap("character code not in range(256)")) self.std_wp(s)
class FormatIterator(object): """ An iterator-like object that follows format strings step by step. It provides input to the packer/unpacker and accumulates their output. The subclasses are specialized for either packing, unpacking, or just computing the size. """ _mixin_ = True _operate_is_specialized_ = False @jit.look_inside_iff(lambda self, fmt: jit.isconstant(fmt)) def interpret(self, fmt): # decode the byte order, size and alignment based on the 1st char table = unroll_native_fmtdescs self.bigendian = native_is_bigendian index = 0 if len(fmt) > 0: c = fmt[0] index = 1 if c == '@': pass elif c == '=': table = unroll_standard_fmtdescs elif c == '<': table = unroll_standard_fmtdescs self.bigendian = False elif c == '>' or c == '!': table = unroll_standard_fmtdescs self.bigendian = True else: index = 0 # interpret the format string, # calling self.operate() for each format unit while index < len(fmt): c = fmt[index] index += 1 if c.isspace(): continue if c.isdigit(): repetitions = ord(c) - ord('0') while True: if index == len(fmt): raise StructError("incomplete struct format") c = fmt[index] index += 1 if not c.isdigit(): break try: repetitions = ovfcheck(repetitions * 10) repetitions = ovfcheck(repetitions + (ord(c) - ord('0'))) except OverflowError: raise StructError("overflow in item count") assert repetitions >= 0 else: repetitions = 1 for fmtdesc in table: if c == fmtdesc.fmtchar: if self._operate_is_specialized_: if fmtdesc.alignment > 1: self.align(fmtdesc.mask) self.operate(fmtdesc, repetitions) break else: raise StructError("bad char in struct format") if not self._operate_is_specialized_: if fmtdesc.alignment > 1: self.align(fmtdesc.mask) self.operate(fmtdesc, repetitions) self.finished() def finished(self): pass