def test_unroll_issue_3(self): py.test.skip("decide") from rpython.rlib.rerased import new_erasing_pair b_erase, b_unerase = new_erasing_pair("B") # list of ints c_erase, c_unerase = new_erasing_pair("C") # list of Nones @elidable def unpack_b(a): return b_unerase(a) jitdriver = JitDriver(greens=[], reds='auto') def f(a, flag): i = 0 total = 0 while i < 10: jitdriver.jit_merge_point() if flag: total += unpack_b(a)[0] flag += 1 i += 1 return total def run(n): res = f(b_erase([n]), 1) f(c_erase([None]), 0) return res assert run(42) == 420 res = self.meta_interp(run, [42], backendopt=True) assert res == 420
def test_boxed_unerased_pointers_in_short_preamble(self): from rpython.rlib.rerased import new_erasing_pair from rpython.rtyper.lltypesystem import lltype class A(object): def __init__(self, val): self.val = val def tst(self): return self.val class Box(object): def __init__(self, val): self.val = val erase_A, unerase_A = new_erasing_pair('A') erase_TP, unerase_TP = new_erasing_pair('TP') TP = lltype.GcArray(lltype.Signed) myjitdriver = JitDriver(greens = [], reds = ['n', 'm', 'i', 'sa', 'p']) def f(n, m): i = sa = 0 p = Box(erase_A(A(7))) while i < n: myjitdriver.jit_merge_point(n=n, m=m, i=i, sa=sa, p=p) if i < m: sa += unerase_A(p.val).tst() elif i == m: a = lltype.malloc(TP, 5) a[0] = 42 p = Box(erase_TP(a)) else: sa += unerase_TP(p.val)[0] sa -= A(i).val i += 1 return sa res = self.meta_interp(f, [20, 10]) assert res == f(20, 10)
def test_unerased_pointers_in_short_preamble(self): from rpython.rlib.rerased import new_erasing_pair from rpython.rtyper.lltypesystem import lltype class A(object): def __init__(self, val): self.val = val erase_A, unerase_A = new_erasing_pair('A') erase_TP, unerase_TP = new_erasing_pair('TP') TP = lltype.GcArray(lltype.Signed) myjitdriver = JitDriver(greens=[], reds=['n', 'm', 'i', 'j', 'sa', 'p']) def f(n, m, j): i = sa = 0 p = erase_A(A(7)) while i < n: myjitdriver.jit_merge_point(n=n, m=m, i=i, j=j, sa=sa, p=p) if i < m: sa += unerase_A(p).val elif i == m: a = lltype.malloc(TP, 5) a[0] = 42 p = erase_TP(a) else: sa += unerase_TP(p)[0] sa += A(i).val assert n > 0 and m > 0 i += j return sa res = self.meta_interp(f, [20, 10, 1]) assert res == f(20, 10, 1)
def define_erased(cls): from rpython.rlib import rerased erase, unerase = rerased.new_erasing_pair("test") class Unrelated(object): pass u = Unrelated() u.tagged = True u.x = rerased.erase_int(41) class A(object): pass def fn(): n = 1 while n >= 0: if u.tagged: n = rerased.unerase_int(u.x) a = A() a.n = n - 1 u.x = erase(a) u.tagged = False else: n = unerase(u.x).n u.x = rerased.erase_int(n - 1) u.tagged = True def func(): rgc.collect() # check that a prebuilt erased integer doesn't explode u.x = rerased.erase_int(1000) u.tagged = True fn() return 1 return func
class FlonumVectorStrategy(VectorStrategy): import_from_mixin(UnwrappedVectorStrategyMixin) erase, unerase = rerased.new_erasing_pair("flonum-vector-strategy") erase = staticmethod(erase) unerase = staticmethod(unerase) def is_correct_type(self, w_vector, w_obj): return isinstance(w_obj, W_Flonum) def wrap(self, val): assert isinstance(val, float) return W_Flonum(val) def unwrap(self, w_val): assert isinstance(w_val, W_Flonum) return w_val.value def immutable_variant(self): return FlonumImmutableVectorStrategy.singleton def dehomogenize(self, w_vector, hint): if type(hint) is W_Fixnum and can_encode_int32(hint.value): new_strategy = FlonumTaggedVectorStrategy.singleton w_vector.set_strategy(new_strategy) w_vector.set_len(0) else: VectorStrategy.dehomogenize(self, w_vector, hint)
class DoubleFormStrategy(FormStrategy): import_from_mixin(CommonFormStrategy) def __init__(self): self.name = String(u"double") erase, unerase = rerased.new_erasing_pair("double_form") erase = staticmethod(erase) unerase = staticmethod(unerase) def wrap(self, raw): return Float(raw) def unwrap(self, obj): return cast(obj, Float, u"numeric (double form strategy)").number def add(self, a_n, b_n): a = self.unerase(a_n.storage) b = self.unerase(b_n.storage) if len(a) != len(b): raise unwind(LError(u"incompatible numerics for arithmetic")) c = [] for i in range(len(a)): c.append(a[i] + b[i]) return Numeric(self, self.erase(c[:]))
def test_opaque_list(self): from rpython.rlib.rerased import new_erasing_pair erase, unerase = new_erasing_pair("test_opaque_list") def fn(n, ca, cb): l1 = [n] l2 = [n] a1 = erase(l1) a2 = erase(l1) a = a1 if ca: a = a2 if n < -100: unerase(a).append(5) b = a1 if cb: b = a return unerase(a)[0] + unerase(b)[0] res = self.interp_operations(fn, [7, 0, 1]) assert res == 7 * 2 self.check_operations_history(getarrayitem_gc_i=0, getfield_gc_i=0, getfield_gc_r=0) res = self.interp_operations(fn, [-7, 1, 1]) assert res == -7 * 2 self.check_operations_history(getarrayitem_gc_i=0, getfield_gc_i=0, getfield_gc_r=0)
class ObjectFormStrategy(FormStrategy): import_from_mixin(CommonFormStrategy) # I am planning to add shape of the numeric into the strategy and # store a table of used strategies with their associated shape. # That should allow JIT to optimize common shapes encountered? def __init__(self): self.name = String(u"object") erase, unerase = rerased.new_erasing_pair("object_form") erase = staticmethod(erase) unerase = staticmethod(unerase) def wrap(self, raw): return raw def unwrap(self, obj): return obj def add(self, a_n, b_n): a = self.unerase(a_n.storage) b = self.unerase(b_n.storage) if len(a) != len(b): raise unwind(LError(u"incompatible numerics for arithmetic")) c = [] for i in range(len(a)): c.append(operators.add.call([a[i], b[i]])) return Numeric(self, self.erase(c[:]))
class ConstantVectorStrategy(VectorStrategy): # Strategy desribing a vector whose contents are all the same object. import_from_mixin(UnwrappedVectorStrategyMixin) erase, unerase = rerased.new_erasing_pair("constant-vector-strategy") erase = staticmethod(erase) unerase = staticmethod(unerase) def is_correct_type(self, w_vector, w_obj): from pycket.prims.equal import eqp_logic val = self._storage(w_vector)[0] return eqp_logic(val, w_obj) def create_storage_for_element(self, element, times): return self.erase([element]) def _ref(self, w_vector, i): return self._storage(w_vector)[0] def _set(self, w_vector, i, w_val): if not we_are_translated(): from pycket.prims.equal import eqp_logic self.indexcheck(w_vector, i) assert eqp_logic(w_val, self._storage(w_vector)[0]) def immutable_variant(self): return ConstantImmutableVectorStrategy.singleton def ref_all(self, w_vector): val = self._storage(w_vector)[0] return [val] * w_vector.length() def dehomogenize(self, w_vector, hint): val = self._storage(w_vector)[0] len = w_vector.length() hinttype = type(hint) valtype = type(val) if not len: newstrategy = ObjectVectorStrategy.singleton elif hinttype is valtype: if valtype is W_Fixnum: newstrategy = FixnumVectorStrategy.singleton elif valtype is W_Flonum: newstrategy = FlonumVectorStrategy.singleton else: newstrategy = ObjectVectorStrategy.singleton elif hinttype is W_Flonum and valtype is W_Fixnum and can_encode_int32( val.value): newstrategy = FlonumTaggedVectorStrategy.singleton w_vector.set_len(0) else: newstrategy = ObjectVectorStrategy.singleton storage = newstrategy.create_storage_for_element(val, len) w_vector.set_strategy(newstrategy) w_vector.set_storage(storage)
def test_unroll_issue_2(self): py.test.skip("decide") class B(object): def __init__(self, b_value): self.b_value = b_value class C(object): pass from rpython.rlib.rerased import new_erasing_pair b_erase, b_unerase = new_erasing_pair("B") c_erase, c_unerase = new_erasing_pair("C") @elidable def unpack_b(a): return b_unerase(a) jitdriver = JitDriver(greens=[], reds='auto') def f(a, flag): i = 0 total = 0 while i < 10: jitdriver.jit_merge_point() if flag: total += unpack_b(a).b_value flag += 1 i += 1 return total def run(n): res = f(b_erase(B(n)), 1) f(c_erase(C()), 0) return res assert run(42) == 420 res = self.meta_interp(run, [42], backendopt=True) assert res == 420
def __new__(self, name, bases, attrs): attrs['_is_strategy'] = False attrs['_is_singleton'] = False attrs['_specializations'] = [] # Not every strategy uses rerased-pairs, but they won't hurt erase, unerase = rerased.new_erasing_pair(name) def get_storage(self, w_self): erased = self.strategy_factory().get_storage(w_self) return unerase(erased) def set_storage(self, w_self, storage): erased = erase(storage) self.strategy_factory().set_storage(w_self, erased) attrs['get_storage'] = get_storage attrs['set_storage'] = set_storage return type.__new__(self, name, bases, attrs)
class W_Immutable_PointersObject(W_AbstractImmutable_PointersObject): """`W_PointersObject` subclass with immutable storage of variable size.""" _immutable_fields_ = ['_storage[*]'] repr_classname = '%s_Immutable_N' % W_PointersObject.repr_classname erase, unerase = map(staticmethod, rerased.new_erasing_pair('storage_eraser')) def __init__(self, space, w_cls, pointers_w): W_AbstractImmutable_PointersObject.__init__(self, space, w_cls) self._storage = self.erase(pointers_w) def size(self): return len(self.unerase(self._storage)) def fetch(self, space, n0): return self.unerase(self._storage)[n0]
class CharacterVectorStrategy(VectorStrategy): import_from_mixin(UnwrappedVectorStrategyMixin) erase, unerase = rerased.new_erasing_pair("character-vector-strategy") erase = staticmethod(erase) unerase = staticmethod(unerase) def is_correct_type(self, w_obj): return isinstance(w_obj, W_Character) def wrap(self, val): return W_Character(val) def unwrap(self, w_val): assert isinstance(w_val, W_Character) return w_val.value
class FlonumVectorStrategy(VectorStrategy): import_from_mixin(UnwrappedVectorStrategyMixin) erase, unerase = rerased.new_erasing_pair("flonum-vector-strategry") erase = staticmethod(erase) unerase = staticmethod(unerase) def is_correct_type(self, w_obj): return isinstance(w_obj, W_Flonum) def wrap(self, val): assert isinstance(val, float) return W_Flonum(val) def unwrap(self, w_val): assert isinstance(w_val, W_Flonum) return w_val.value
class DictProxyStrategy(DictStrategy): erase, unerase = rerased.new_erasing_pair("dictproxy") erase = staticmethod(erase) unerase = staticmethod(unerase) def __init__(w_self, space): DictStrategy.__init__(w_self, space) def getitem(self, w_dict, w_key): space = self.space w_lookup_type = space.type(w_key) if space.is_true(space.issubtype(w_lookup_type, space.w_unicode)): return self.getitem_str(w_dict, space.str_w(w_key)) else: return None def getitem_str(self, w_dict, key): return self.unerase(w_dict.dstorage).getdictvalue(self.space, key) def setitem(self, w_dict, w_key, w_value): space = self.space if space.is_w(space.type(w_key), space.w_unicode): self.setitem_str(w_dict, self.space.str_w(w_key), w_value) else: raise OperationError( space.w_TypeError, space.wrap("cannot add non-string keys to dict of a type")) def setitem_str(self, w_dict, key, w_value): w_type = self.unerase(w_dict.dstorage) try: w_type.setdictvalue(self.space, key, w_value) except OperationError, e: if not e.match(self.space, self.space.w_TypeError): raise if not w_type.is_cpytype(): raise # Allow cpyext to write to type->tp_dict even in the case # of a builtin type. # Like CPython, we assume that this is only done early # after the type is created, and we don't invalidate any # cache. User code shoud call PyType_Modified(). w_type.dict_w[key] = w_value
class ObjectVectorStrategy(VectorStrategy): import_from_mixin(UnwrappedVectorStrategyMixin) erase, unerase = rerased.new_erasing_pair("object-vector-strategry") erase = staticmethod(erase) unerase = staticmethod(unerase) def wrap(self, obj): return obj def unwrap(self, w_obj): return w_obj def is_correct_type(self, w_obj): return True def create_storage_for_elements(self, elements_w): return self.erase(elements_w) def dehomogenize(self, w_vector): assert 0 # should be unreachable because is_correct_type is always True
class DictProxyStrategy(DictStrategy): erase, unerase = rerased.new_erasing_pair("dictproxy") erase = staticmethod(erase) unerase = staticmethod(unerase) def __init__(w_self, space): DictStrategy.__init__(w_self, space) def getitem(self, w_dict, w_key): space = self.space w_lookup_type = space.type(w_key) if (space.is_w(w_lookup_type, space.w_str) or # Most common path first space.abstract_issubclass_w(w_lookup_type, space.w_str)): return self.getitem_str(w_dict, space.str_w(w_key)) elif space.abstract_issubclass_w(w_lookup_type, space.w_unicode): try: w_key = space.str(w_key) except OperationError, e: if not e.match(space, space.w_UnicodeEncodeError): raise # non-ascii unicode is never equal to a byte string return None return self.getitem_str(w_dict, space.str_w(w_key)) else:
class FixnumVectorStrategy(VectorStrategy): import_from_mixin(UnwrappedVectorStrategyMixin) erase, unerase = rerased.new_erasing_pair("fixnum-vector-strategy") erase = staticmethod(erase) unerase = staticmethod(unerase) def is_correct_type(self, w_vector, w_obj): return isinstance(w_obj, W_Fixnum) def wrap(self, val): assert isinstance(val, int) return W_Fixnum(val) def unwrap(self, w_val): assert isinstance(w_val, W_Fixnum) return w_val.value def immutable_variant(self): return FixnumImmutableVectorStrategy.singleton def ref_all(self, w_vector): unwrapped = self._storage(w_vector) return [W_Fixnum.make_or_interned(i) for i in unwrapped]
class ClassDictStrategy(DictStrategy): erase, unerase = rerased.new_erasing_pair("dictproxy") erase = staticmethod(erase) unerase = staticmethod(unerase) def __init__(self, space): DictStrategy.__init__(self, space) def getitem(self, w_dict, w_key): space = self.space w_lookup_type = space.type(w_key) if (space.is_w(w_lookup_type, space.w_text) or # Most common path first space.abstract_issubclass_w(w_lookup_type, space.w_text)): return self.getitem_str(w_dict, space.text_w(w_key)) elif space.abstract_issubclass_w(w_lookup_type, space.w_unicode): try: w_key = space.str(w_key) except OperationError as e: if not e.match(space, space.w_UnicodeEncodeError): raise # non-ascii unicode is never equal to a byte string return None return self.getitem_str(w_dict, space.text_w(w_key)) else: return None def getitem_str(self, w_dict, key): return self.unerase(w_dict.dstorage).getdictvalue(self.space, key) def setitem(self, w_dict, w_key, w_value): space = self.space if space.is_w(space.type(w_key), space.w_text): self.setitem_str(w_dict, self.space.text_w(w_key), w_value) else: raise oefmt(space.w_TypeError, "cannot add non-string keys to dict of a type") def setitem_str(self, w_dict, key, w_value): w_type = self.unerase(w_dict.dstorage) try: w_type.setdictvalue(self.space, key, w_value) except OperationError as e: if not e.match(self.space, self.space.w_TypeError): raise if not w_type.is_cpytype(): raise # Allow cpyext to write to type->tp_dict even in the case # of a builtin type. # Like CPython, we assume that this is only done early # after the type is created, and we don't invalidate any # cache. User code shoud call PyType_Modified(). w_type.dict_w[key] = w_value def setdefault(self, w_dict, w_key, w_default): w_result = self.getitem(w_dict, w_key) if w_result is not None: return w_result self.setitem(w_dict, w_key, w_default) return w_default def delitem(self, w_dict, w_key): space = self.space w_key_type = space.type(w_key) if space.is_w(w_key_type, space.w_text): key = self.space.text_w(w_key) if not self.unerase(w_dict.dstorage).deldictvalue(space, key): raise KeyError else: raise KeyError def length(self, w_dict): return len(self.unerase(w_dict.dstorage).dict_w) def w_keys(self, w_dict): space = self.space return space.newlist_text(self.unerase(w_dict.dstorage).dict_w.keys()) def values(self, w_dict): return [ unwrap_cell(self.space, w_value) for w_value in self.unerase(w_dict.dstorage).dict_w.itervalues() ] def items(self, w_dict): space = self.space return [ space.newtuple2(space.newtext(key), unwrap_cell(self.space, w_value)) for (key, w_value) in self.unerase(w_dict.dstorage).dict_w.iteritems() ] def clear(self, w_dict): space = self.space w_type = self.unerase(w_dict.dstorage) if not w_type.is_heaptype(): raise oefmt(space.w_TypeError, "can't clear dictionary of type '%N'", w_type) w_type.dict_w.clear() w_type.mutated(None) def getiterkeys(self, w_dict): return self.unerase(w_dict.dstorage).dict_w.iterkeys() def getitervalues(self, w_dict): return self.unerase(w_dict.dstorage).dict_w.itervalues() def getiteritems_with_hash(self, w_dict): return iteritems_with_hash(self.unerase(w_dict.dstorage).dict_w) def wrapkey(space, key): return space.newtext(key) def wrapvalue(space, value): return unwrap_cell(space, value)
class _ObjectStrategy(_ArrayStrategy): _erase, _unerase = rerased.new_erasing_pair("obj_list") _erase = staticmethod(_erase) _unerase = staticmethod(_unerase) def get_idx(self, storage, idx): store = self._unerase(storage) return store[idx] def set_idx(self, array, idx, value): assert isinstance(array, Array) store = self._unerase(array._storage) store[idx] = value def set_all(self, array, value): assert isinstance(array, Array) assert isinstance(value, AbstractObject) store = self._unerase(array._storage) # TODO: we could avoid an allocation here if value isn't something to specialize for... self._set_all_with_value(array, value, len(store)) def set_all_with_block(self, array, block): assert isinstance(array, Array) store = self._unerase(array._storage) # TODO: perhaps we can sometimes avoid the extra allocation of the underlying storage self._set_all_with_block(array, block, len(store)) def as_arguments_array(self, storage): return self._unerase(storage) def get_size(self, storage): return len(self._unerase(storage)) @staticmethod def new_storage_for(size): return _ObjectStrategy._erase([nilObject] * size) @staticmethod def new_storage_with_values(values): assert isinstance(values, list) make_sure_not_resized(values) return _ObjectStrategy._erase(values) def copy(self, storage): store = self._unerase(storage) return Array._from_storage_and_strategy(self._erase(store[:]), _obj_strategy) def copy_and_extend_with(self, storage, value): store = self._unerase(storage) old_size = len(store) new_size = old_size + 1 new = [None] * new_size for i, _ in enumerate(store): new[i] = store[i] new[old_size] = value return Array._from_storage_and_strategy(self._erase(new), _obj_strategy)
import weakref, sys from rpython.rlib import jit, objectmodel, debug, rerased from rpython.rlib.rarithmetic import intmask, r_uint from pypy.interpreter.baseobjspace import W_Root from pypy.objspace.std.dictmultiobject import ( W_DictMultiObject, DictStrategy, ObjectDictStrategy, BaseKeyIterator, BaseValueIterator, BaseItemIterator, _never_equal_to_string, W_DictObject, ) from pypy.objspace.std.typeobject import MutableCell erase_item, unerase_item = rerased.new_erasing_pair("mapdict storage item") erase_map, unerase_map = rerased.new_erasing_pair("map") erase_list, unerase_list = rerased.new_erasing_pair("mapdict storage list") # ____________________________________________________________ # attribute shapes NUM_DIGITS = 4 NUM_DIGITS_POW2 = 1 << NUM_DIGITS # note: we use "x * NUM_DIGITS_POW2" instead of "x << NUM_DIGITS" because # we want to propagate knowledge that the result cannot be negative class AbstractAttribute(object): _immutable_fields_ = ['terminator'] cache_attrs = None
@autohelp class LoopHandle(Object): """ A handle. """ def __init__(self, handle): self.handle = handle def walkCB(handle, erased): l = uneraseList(erased) l.append(handle) eraseList, uneraseList = new_erasing_pair("handleList") @autohelp class LoopStats(Object): """ Information about the event loop. This object is unsafe because it refers directly to the (read-only) vital statistics of the runtime's reactor. """ def __init__(self, loop): self.loop = loop @method("List") def getHandles(self):
class _LongStrategy(_ArrayStrategy): _erase, _unerase = rerased.new_erasing_pair("int_list") _erase = staticmethod(_erase) _unerase = staticmethod(_unerase) def get_idx(self, storage, idx): store = self._unerase(storage) assert isinstance(store, list) assert isinstance(store[idx], int) return Integer(store[idx]) def set_idx(self, array, idx, value): assert isinstance(array, Array) if isinstance(value, Integer): store = self._unerase(array._storage) store[idx] = value.get_embedded_integer() else: self._transition_to_object_array(array, idx, value) def _transition_to_object_array(self, array, idx, value): store = self._unerase(array._storage) new_store = [None] * len(store) for i, v in enumerate(store): new_store[i] = Integer(v) new_store[idx] = value array._storage = _ObjectStrategy.new_storage_with_values(new_store) array._strategy = _obj_strategy def set_all(self, array, value): assert isinstance(array, Array) store = self._unerase(array._storage) self._set_all_with_value(array, value, len(store)) # we could avoid the allocation of the new array if value is an Integer # for i, _ in enumerate(store): # store[i] = value.get_embedded_integer() def set_all_with_block(self, array, block): assert isinstance(array, Array) store = self._unerase(array._storage) # TODO: perhaps we can sometimes avoid the extra allocation of the underlying storage self._set_all_with_block(array, block, len(store)) def as_arguments_array(self, storage): store = self._unerase(storage) return [Integer(v) for v in store] def get_size(self, storage): return len(self._unerase(storage)) @staticmethod def new_storage_for(size): return _LongStrategy._erase([0] * size) @staticmethod def new_storage_with_values(values): assert isinstance(values, list) make_sure_not_resized(values) # TODO: do we guarantee this externally? new = [v.get_embedded_integer() for v in values] return _LongStrategy._erase(new) def copy(self, storage): store = self._unerase(storage) return Array._from_storage_and_strategy(self._erase(store[:]), _long_strategy) def copy_and_extend_with(self, storage, value): assert isinstance(value, Integer) store = self._unerase(storage) old_size = len(store) new_size = old_size + 1 new = [0] * new_size for i, v in enumerate(store): new[i] = v new[old_size] = value.get_embedded_integer() return Array._from_storage_and_strategy(self._erase(new), _long_strategy)
class ModuleDictStrategy(DictStrategy): erase, unerase = rerased.new_erasing_pair("modulecell") erase = staticmethod(erase) unerase = staticmethod(unerase) _immutable_fields_ = ["version?"] def __init__(self, space): self.space = space self.version = VersionTag() def get_empty_storage(self): return self.erase({}) def mutated(self): self.version = VersionTag() def getdictvalue_no_unwrapping(self, w_dict, key): # NB: it's important to promote self here, so that self.version is a # no-op due to the quasi-immutable field self = jit.promote(self) return self._getdictvalue_no_unwrapping_pure(self.version, w_dict, key) @jit.elidable_promote('0,1,2') def _getdictvalue_no_unwrapping_pure(self, version, w_dict, key): return self.unerase(w_dict.dstorage).get(key, None) def setitem(self, w_dict, w_key, w_value): space = self.space if space.is_w(space.type(w_key), space.w_text): self.setitem_str(w_dict, space.text_w(w_key), w_value) else: self.switch_to_object_strategy(w_dict) w_dict.setitem(w_key, w_value) def setitem_str(self, w_dict, key, w_value): cell = self.getdictvalue_no_unwrapping(w_dict, key) return self._setitem_str_cell_known(cell, w_dict, key, w_value) def _setitem_str_cell_known(self, cell, w_dict, key, w_value): w_value = write_cell(self.space, cell, w_value) if w_value is None: return self.mutated() self.unerase(w_dict.dstorage)[key] = w_value def setdefault(self, w_dict, w_key, w_default): space = self.space if space.is_w(space.type(w_key), space.w_text): key = space.text_w(w_key) cell = self.getdictvalue_no_unwrapping(w_dict, key) w_result = unwrap_cell(self.space, cell) if w_result is not None: return w_result self._setitem_str_cell_known(cell, w_dict, key, w_default) return w_default else: self.switch_to_object_strategy(w_dict) return w_dict.setdefault(w_key, w_default) def delitem(self, w_dict, w_key): space = self.space w_key_type = space.type(w_key) if space.is_w(w_key_type, space.w_text): key = space.text_w(w_key) dict_w = self.unerase(w_dict.dstorage) try: del dict_w[key] except KeyError: raise else: self.mutated() elif _never_equal_to_string(space, w_key_type): raise KeyError else: self.switch_to_object_strategy(w_dict) w_dict.delitem(w_key) def length(self, w_dict): return len(self.unerase(w_dict.dstorage)) def getitem(self, w_dict, w_key): space = self.space w_lookup_type = space.type(w_key) if space.is_w(w_lookup_type, space.w_text): return self.getitem_str(w_dict, space.text_w(w_key)) elif _never_equal_to_string(space, w_lookup_type): return None else: self.switch_to_object_strategy(w_dict) return w_dict.getitem(w_key) def getitem_str(self, w_dict, key): cell = self.getdictvalue_no_unwrapping(w_dict, key) return unwrap_cell(self.space, cell) def w_keys(self, w_dict): space = self.space l = self.unerase(w_dict.dstorage).keys() return space.newlist_text(l) def values(self, w_dict): iterator = self.unerase(w_dict.dstorage).itervalues return [unwrap_cell(self.space, cell) for cell in iterator()] def items(self, w_dict): space = self.space iterator = self.unerase(w_dict.dstorage).iteritems return [space.newtuple([_wrapkey(space, key), unwrap_cell(self.space, cell)]) for key, cell in iterator()] def clear(self, w_dict): self.unerase(w_dict.dstorage).clear() self.mutated() def popitem(self, w_dict): space = self.space d = self.unerase(w_dict.dstorage) key, cell = d.popitem() self.mutated() return _wrapkey(space, key), unwrap_cell(self.space, cell) def switch_to_object_strategy(self, w_dict): space = self.space d = self.unerase(w_dict.dstorage) strategy = space.fromcache(ObjectDictStrategy) d_new = strategy.unerase(strategy.get_empty_storage()) for key, cell in d.iteritems(): d_new[_wrapkey(space, key)] = unwrap_cell(self.space, cell) w_dict.set_strategy(strategy) w_dict.dstorage = strategy.erase(d_new) def getiterkeys(self, w_dict): return self.unerase(w_dict.dstorage).iterkeys() def getitervalues(self, w_dict): return self.unerase(w_dict.dstorage).itervalues() def getiteritems_with_hash(self, w_dict): return objectmodel.iteritems_with_hash(self.unerase(w_dict.dstorage)) wrapkey = _wrapkey def wrapvalue(space, value): return unwrap_cell(space, value)
class _BoolStrategy(_ArrayStrategy): _erase, _unerase = rerased.new_erasing_pair("bool_list") _erase = staticmethod(_erase) _unerase = staticmethod(_unerase) def get_idx(self, storage, idx): store = self._unerase(storage) assert isinstance(store, list) assert isinstance(store[idx], bool) return trueObject if store[idx] else falseObject def set_idx(self, array, idx, value): assert isinstance(array, Array) assert value is trueObject or value is falseObject store = self._unerase(array._storage) store[idx] = value is trueObject def set_all(self, array, value): assert isinstance(array, Array) store = self._unerase(array._storage) self._set_all_with_value(array, value, len(store)) # we could avoid the allocation of the new array if value is an Integer # for i, _ in enumerate(store): # store[i] = value.get_embedded_integer() def set_all_with_block(self, array, block): assert isinstance(array, Array) store = self._unerase(array._storage) # TODO: perhaps we can sometimes avoid the extra allocation of the underlying storage self._set_all_with_block(array, block, len(store)) def as_arguments_array(self, storage): store = self._unerase(storage) return [trueObject if v else falseObject for v in store] def get_size(self, storage): return len(self._unerase(storage)) @staticmethod def new_storage_for(size): return _BoolStrategy._erase([False] * size) @staticmethod def new_storage_with_values(values): assert isinstance(values, list) make_sure_not_resized(values) # TODO: do we guarantee this externally? new = [v is trueObject for v in values] return _BoolStrategy._erase(new) def copy(self, storage): store = self._unerase(storage) return Array._from_storage_and_strategy(self._erase(store[:]), _bool_strategy) def copy_and_extend_with(self, storage, value): assert value is trueObject or value is falseObject store = self._unerase(storage) old_size = len(store) new_size = old_size + 1 new = [False] * new_size for i, v in enumerate(store): new[i] = v new[old_size] = value is trueObject return Array._from_storage_and_strategy(self._erase(new), _bool_strategy)
class MapDictStrategy(DictStrategy): erase, unerase = rerased.new_erasing_pair("map") erase = staticmethod(erase) unerase = staticmethod(unerase) def __init__(self, space): self.space = space def get_empty_storage(self): w_result = Object() terminator = self.space.fromcache(get_terminator_for_dicts) w_result._init_empty(terminator) return self.erase(w_result) def switch_to_object_strategy(self, w_dict): w_obj = self.unerase(w_dict.dstorage) strategy = self.space.fromcache(ObjectDictStrategy) dict_w = strategy.unerase(strategy.get_empty_storage()) w_dict.set_strategy(strategy) w_dict.dstorage = strategy.erase(dict_w) assert w_obj.getdict(self.space) is w_dict or w_obj._get_mapdict_map( ).terminator.w_cls is None materialize_r_dict(self.space, w_obj, dict_w) def getitem(self, w_dict, w_key): space = self.space w_lookup_type = space.type(w_key) if space.is_w(w_lookup_type, space.w_str): return self.getitem_str(w_dict, space.str_w(w_key)) elif _never_equal_to_string(space, w_lookup_type): return None else: self.switch_to_object_strategy(w_dict) return w_dict.getitem(w_key) def getitem_str(self, w_dict, key): w_obj = self.unerase(w_dict.dstorage) return w_obj.getdictvalue(self.space, key) def setitem_str(self, w_dict, key, w_value): w_obj = self.unerase(w_dict.dstorage) flag = w_obj.setdictvalue(self.space, key, w_value) assert flag def setitem(self, w_dict, w_key, w_value): space = self.space if space.is_w(space.type(w_key), space.w_str): self.setitem_str(w_dict, self.space.str_w(w_key), w_value) else: self.switch_to_object_strategy(w_dict) w_dict.setitem(w_key, w_value) def setdefault(self, w_dict, w_key, w_default): space = self.space if space.is_w(space.type(w_key), space.w_str): key = space.str_w(w_key) w_result = self.getitem_str(w_dict, key) if w_result is not None: return w_result self.setitem_str(w_dict, key, w_default) return w_default else: self.switch_to_object_strategy(w_dict) return w_dict.setdefault(w_key, w_default) def delitem(self, w_dict, w_key): space = self.space w_key_type = space.type(w_key) w_obj = self.unerase(w_dict.dstorage) if space.is_w(w_key_type, space.w_str): key = self.space.str_w(w_key) flag = w_obj.deldictvalue(space, key) if not flag: raise KeyError elif _never_equal_to_string(space, w_key_type): raise KeyError else: self.switch_to_object_strategy(w_dict) w_dict.delitem(w_key) def length(self, w_dict): res = 0 curr = self.unerase(w_dict.dstorage)._get_mapdict_map().search(DICT) while curr is not None: curr = curr.back curr = curr.search(DICT) res += 1 return res def clear(self, w_dict): w_obj = self.unerase(w_dict.dstorage) new_obj = w_obj._get_mapdict_map().remove_dict_entries(w_obj) _become(w_obj, new_obj) def popitem(self, w_dict): curr = self.unerase(w_dict.dstorage)._get_mapdict_map().search(DICT) if curr is None: raise KeyError key = curr.name w_value = self.getitem_str(w_dict, key) w_key = self.space.wrap(key) self.delitem(w_dict, w_key) return (w_key, w_value) # XXX could implement a more efficient w_keys based on space.newlist_bytes def iterkeys(self, w_dict): return MapDictIteratorKeys(self.space, self, w_dict) def itervalues(self, w_dict): return MapDictIteratorValues(self.space, self, w_dict) def iteritems(self, w_dict): return MapDictIteratorItems(self.space, self, w_dict)
@autohelp def LoopHandle(Object): """ A handle. """ def __init__(self, handle): self.handle = handle def walkCB(handle, erased): l = uneraseList(erased) l.append(handle) eraseList, uneraseList = new_erasing_pair("handleList") @autohelp class LoopStats(Object): """ Information about the event loop. This object is unsafe because it refers directly to the (read-only) vital statistics of the runtime's reactor. """ def __init__(self, loop): self.loop = loop def recv(self, atom, args): if atom is GETHANDLES_0:
class ModuleDictStrategy(DictStrategy): erase, unerase = rerased.new_erasing_pair("modulecell") erase = staticmethod(erase) unerase = staticmethod(unerase) _immutable_fields_ = ["version?"] def __init__(self, space): self.space = space self.version = VersionTag() self.caches = None def get_empty_storage(self): return self.erase({}) def mutated(self): self.version = VersionTag() def getdictvalue_no_unwrapping(self, w_dict, key): # NB: it's important to promote self here, so that self.version is a # no-op due to the quasi-immutable field self = jit.promote(self) return self._getdictvalue_no_unwrapping_pure(self.version, w_dict, key) @jit.elidable_promote('0,1,2') def _getdictvalue_no_unwrapping_pure(self, version, w_dict, key): return self.unerase(w_dict.dstorage).get(key, None) def setitem(self, w_dict, w_key, w_value): space = self.space if space.is_w(space.type(w_key), space.w_text): self.setitem_str(w_dict, space.text_w(w_key), w_value) else: self.switch_to_object_strategy(w_dict) w_dict.setitem(w_key, w_value) def setitem_str(self, w_dict, key, w_value): cell = self.getdictvalue_no_unwrapping(w_dict, key) return self._setitem_str_cell_known(cell, w_dict, key, w_value) def _setitem_str_cell_known(self, cell, w_dict, key, w_value): w_value = write_cell(self.space, cell, w_value) if w_value is None: return self.mutated() self.unerase(w_dict.dstorage)[key] = w_value if self.caches is None: return cache = self.caches.get(key, None) if cache: cache.cell = w_value def setdefault(self, w_dict, w_key, w_default): space = self.space if space.is_w(space.type(w_key), space.w_text): key = space.text_w(w_key) cell = self.getdictvalue_no_unwrapping(w_dict, key) w_result = unwrap_cell(self.space, cell) if w_result is not None: return w_result self._setitem_str_cell_known(cell, w_dict, key, w_default) return w_default else: self.switch_to_object_strategy(w_dict) return w_dict.setdefault(w_key, w_default) def delitem(self, w_dict, w_key): space = self.space w_key_type = space.type(w_key) if space.is_w(w_key_type, space.w_text): key = space.text_w(w_key) dict_w = self.unerase(w_dict.dstorage) try: del dict_w[key] except KeyError: raise else: if self.caches: cache = self.caches.get(key, None) if cache: cache.cell = None self.mutated() elif _never_equal_to_string(space, w_key_type): raise KeyError else: self.switch_to_object_strategy(w_dict) w_dict.delitem(w_key) def length(self, w_dict): return len(self.unerase(w_dict.dstorage)) def getitem(self, w_dict, w_key): space = self.space w_lookup_type = space.type(w_key) if space.is_w(w_lookup_type, space.w_text): return self.getitem_str(w_dict, space.text_w(w_key)) elif _never_equal_to_string(space, w_lookup_type): return None else: self.switch_to_object_strategy(w_dict) return w_dict.getitem(w_key) def getitem_str(self, w_dict, key): cell = self.getdictvalue_no_unwrapping(w_dict, key) return unwrap_cell(self.space, cell) def w_keys(self, w_dict): space = self.space l = self.unerase(w_dict.dstorage).keys() return space.newlist_text(l) def values(self, w_dict): iterator = self.unerase(w_dict.dstorage).itervalues return [unwrap_cell(self.space, cell) for cell in iterator()] def items(self, w_dict): space = self.space iterator = self.unerase(w_dict.dstorage).iteritems return [space.newtuple2(_wrapkey(space, key), unwrap_cell(self.space, cell)) for key, cell in iterator()] def clear(self, w_dict): self.unerase(w_dict.dstorage).clear() self.mutated() def popitem(self, w_dict): space = self.space d = self.unerase(w_dict.dstorage) key, cell = d.popitem() self.mutated() return _wrapkey(space, key), unwrap_cell(self.space, cell) def switch_to_object_strategy(self, w_dict): space = self.space d = self.unerase(w_dict.dstorage) strategy = space.fromcache(ObjectDictStrategy) d_new = strategy.unerase(strategy.get_empty_storage()) for key, cell in d.iteritems(): d_new[_wrapkey(space, key)] = unwrap_cell(self.space, cell) if self.caches is not None: for cache in self.caches.itervalues(): cache.cell = None cache.valid = False self.caches = None w_dict.set_strategy(strategy) w_dict.dstorage = strategy.erase(d_new) def getiterkeys(self, w_dict): return self.unerase(w_dict.dstorage).iterkeys() def getitervalues(self, w_dict): return self.unerase(w_dict.dstorage).itervalues() def getiteritems_with_hash(self, w_dict): return objectmodel.iteritems_with_hash(self.unerase(w_dict.dstorage)) wrapkey = _wrapkey def wrapvalue(space, value): return unwrap_cell(space, value) def copy(self, w_dict): strategy = self.space.fromcache(BytesDictStrategy) str_dict = strategy.unerase(strategy.get_empty_storage()) d = self.unerase(w_dict.dstorage) for key, cell in d.iteritems(): str_dict[key] = unwrap_cell(self.space, cell) return W_DictObject(strategy.space, strategy, strategy.erase(str_dict)) def get_global_cache(self, w_dict, key): space = w_dict.space if self.caches is None: cache = None self.caches = {} else: cache = self.caches.get(key, None) if cache is None: cell = self.getdictvalue_no_unwrapping(w_dict, key) cache = GlobalCache(cell) if (not space.config.objspace.honor__builtins__ and cell is None and w_dict is not space.builtin.w_dict): w_builtin_dict = space.builtin.w_dict assert isinstance(w_builtin_dict, W_ModuleDictObject) builtin_strategy = w_builtin_dict.mstrategy if isinstance(builtin_strategy, ModuleDictStrategy): cell = builtin_strategy.getdictvalue_no_unwrapping( w_builtin_dict, key) # logic: if the global is not defined but the builtin is, # cache it. otherwise don't cache the builtin ever if cell is not None: builtincache = builtin_strategy.get_global_cache( w_builtin_dict, key) cache.builtincache = builtincache self.caches[key] = cache return cache
class _EmptyStrategy(_ArrayStrategy): # We have these basic erase/unerase methods, and then the once to be used, which # do also the wrapping with Integer objects of the integer value __erase, __unerase = rerased.new_erasing_pair("Integer") __erase = staticmethod(__erase) __unerase = staticmethod(__unerase) def _erase(self, anInt): assert isinstance(anInt, int) return self.__erase(Integer(anInt)) def _unerase(self, storage): return self.__unerase(storage).get_embedded_integer() def get_idx(self, storage, idx): size = self._unerase(storage) if 0 <= idx < size: return nilObject else: raise IndexError() def set_idx(self, array, idx, value): size = self._unerase(array._storage) if 0 <= idx < size: if value is nilObject: return # everything is nil already, avoids transition... assert isinstance(value, AbstractObject) # We need to transition to the _PartiallyEmpty strategy, because # we are not guaranteed to set all elements of the array. array._storage = _partially_empty_strategy.new_storage_with_values([nilObject] * size) array._strategy = _partially_empty_strategy array._strategy.set_idx(array, idx, value) else: raise IndexError() def set_all(self, array, value): if value is nilObject: return # easy short cut size = self._unerase(array._storage) if size > 0: self._set_all_with_value(array, value, size) def set_all_with_block(self, array, block): size = self._unerase(array._storage) self._set_all_with_block(array, block, size) def as_arguments_array(self, storage): size = self._unerase(storage) return [nilObject] * size def get_size(self, storage): size = self._unerase(storage) return size @staticmethod def new_storage_for(size): return _empty_strategy._erase(size) @staticmethod def new_storage_with_values(values): return _empty_strategy._erase(len(values)) def copy(self, storage): return Array._from_storage_and_strategy(storage, _empty_strategy) def copy_and_extend_with(self, storage, value): size = self._unerase(storage) if value is nilObject: return Array.from_size(size + 1) else: new = [nilObject] * (size + 1) new[-1] = value return Array._from_storage_and_strategy(_ObjectStrategy._erase(new), _obj_strategy)
class _PartiallyEmptyStrategy(_ArrayStrategy): # This is an array that we expect to be slowly filled, and we have the hope # that it will turn out to be homogeneous # Thus, we track the number of empty slots left, and we track whether it # is homogeneous in one type _erase, _unerase = rerased.new_erasing_pair("partial_storage") _erase = staticmethod(_erase) _unerase = staticmethod(_unerase) def get_idx(self, storage, idx): store = self._unerase(storage) return store.storage[idx] def set_idx(self, array, idx, value): assert isinstance(array, Array) assert isinstance(value, AbstractObject) store = self._unerase(array._storage) if value is nilObject: if store.storage[idx] is nilObject: return else: store.storage[idx] = nilObject store.empty_elements += 1 return if store.storage[idx] is nilObject: store.empty_elements -= 1 store.storage[idx] = value if isinstance(value, Integer): if store.type is None: store.type = _long_strategy elif store.type is not _long_strategy: store.type = _obj_strategy elif isinstance(value, Double): if store.type is None: store.type = _double_strategy elif store.type is not _double_strategy: store.type = _obj_strategy elif value is trueObject or value is falseObject: if store.type is None: store.type = _bool_strategy elif store.type is not _bool_strategy: store.type = _obj_strategy else: store.type = _obj_strategy if store.empty_elements == 0: array._strategy = store.type array._storage = array._strategy.new_storage_with_values(store.storage) def set_all(self, array, value): assert isinstance(array, Array) assert isinstance(value, AbstractObject) store = self._unerase(array._storage) self._set_all_with_value(array, value, store.size) def set_all_with_block(self, array, block): assert isinstance(array, Array) store = self._unerase(array._storage) # TODO: perhaps we can sometimes avoid the extra allocation of the underlying storage self._set_all_with_block(array, block, store.size) def as_arguments_array(self, storage): return self._unerase(storage).storage def get_size(self, storage): return self._unerase(storage).size @staticmethod def new_storage_for(size): return _PartiallyEmptyStrategy._erase( _PartiallyEmptyStrategy.from_size(size)) @staticmethod def new_storage_with_values(values): assert isinstance(values, list) make_sure_not_resized(values) return _PartiallyEmptyStrategy._erase( _PartialStorage.from_obj_values(values)) def copy(self, storage): store = self._unerase(storage) return Array._from_storage_and_strategy( self._erase(_PartialStorage.from_obj_values(store.storage[:])), _partially_empty_strategy) def copy_and_extend_with(self, storage, value): store = self._unerase(storage) old_size = store.size new_size = old_size + 1 new = [nilObject] * new_size for i, _ in enumerate(store.storage): new[i] = store.storage[i] new_store = instantiate(_PartiallyEmptyStrategy) new_store.storage = new new_store.size = new_size new_store.empty_elements = store.empty_elements + 1 new_store.type = store.type new_arr = Array._from_storage_and_strategy(self._erase(new_store), _partially_empty_strategy) new_arr.set_indexable_field(old_size, value) return new_arr
from rpython.rlib.rarithmetic import intmask, r_uint from pypy.interpreter.baseobjspace import W_Root from pypy.objspace.std.dictmultiobject import ( W_DictMultiObject, DictStrategy, ObjectDictStrategy, BaseKeyIterator, BaseValueIterator, BaseItemIterator, _never_equal_to_string, W_DictObject, ) from pypy.objspace.std.typeobject import MutableCell erase_item, unerase_item = rerased.new_erasing_pair("mapdict storage item") erase_map, unerase_map = rerased.new_erasing_pair("map") erase_list, unerase_list = rerased.new_erasing_pair("mapdict storage list") # ____________________________________________________________ # attribute shapes NUM_DIGITS = 4 NUM_DIGITS_POW2 = 1 << NUM_DIGITS # note: we use "x * NUM_DIGITS_POW2" instead of "x << NUM_DIGITS" because # we want to propagate knowledge that the result cannot be negative class AbstractAttribute(object): _immutable_fields_ = ['terminator'] cache_attrs = None
class CPyListStrategy(ListStrategy): erase, unerase = rerased.new_erasing_pair("empty") erase = staticmethod(erase) unerase = staticmethod(unerase) def _check_index(self, index, length): if index < 0: index = length + index if index < 0 or index >= length: raise IndexError return index def getitem(self, w_list, index): storage = self.unerase(w_list.lstorage) index = self._check_index(index, storage._length) return from_ref(w_list.space, storage._elems[index]) def setitem(self, w_list, index, w_obj): storage = self.unerase(w_list.lstorage) index = self._check_index(index, storage._length) py_old = storage._elems[index] storage._elems[index] = make_ref(w_list.space, w_obj) decref(w_list.space, py_old) def length(self, w_list): storage = self.unerase(w_list.lstorage) return storage._length def get_raw_items(self, w_list): storage = self.unerase(w_list.lstorage) return storage._elems def getslice(self, w_list, start, stop, step, length): w_list.switch_to_object_strategy() return w_list.strategy.getslice(w_list, start, stop, step, length) def getitems(self, w_list): # called when switching list strategy, so convert storage storage = self.unerase(w_list.lstorage) retval = [None] * storage._length for i in range(storage._length): retval[i] = from_ref(w_list.space, storage._elems[i]) return retval @jit.unroll_safe def getitems_unroll(self, w_list): storage = self.unerase(w_list.lstorage) retval = [None] * storage._length for i in range(storage._length): retval[i] = from_ref(w_list.space, storage._elems[i]) return retval @jit.look_inside_iff(lambda self, w_list: jit.loop_unrolling_heuristic(w_list, w_list.length(), UNROLL_CUTOFF)) def getitems_fixedsize(self, w_list): return self.getitems_unroll(w_list) def copy_into(self, w_list, w_other): w_other.strategy = self w_other.lstorage = self.getstorage_copy(w_list) def clone(self, w_list): storage = self.getstorage_copy(w_list) w_clone = W_ListObject.from_storage_and_strategy(self.space, storage, self) return w_clone def getitems_copy(self, w_list): return self.getitems(w_list) # getitems copies anyway def getstorage_copy(self, w_list): lst = self.getitems(w_list) return self.erase(CPyListStorage(w_list.space, lst)) #------------------------------------------ # all these methods fail or switch strategy and then call ListObjectStrategy's method def setslice(self, w_list, start, stop, step, length): w_list.switch_to_object_strategy() w_list.strategy.setslice(w_list, start, stop, step, length) def init_from_list_w(self, w_list, list_w): raise NotImplementedError def _resize_hint(self, w_list, hint): pass def find(self, w_list, w_item, start, stop): w_list.switch_to_object_strategy() return w_list.strategy.find(w_list, w_item, start, stop) def append(self, w_list, w_item): w_list.switch_to_object_strategy() w_list.strategy.append(w_list, w_item) def inplace_mul(self, w_list, times): w_list.switch_to_object_strategy() w_list.strategy.inplace_mul(w_list, times) def deleteslice(self, w_list, start, step, slicelength): w_list.switch_to_object_strategy() w_list.strategy.deleteslice(w_list, start, step, slicelength) def pop(self, w_list, index): w_list.switch_to_object_strategy() return w_list.strategy.pop(w_list, index) def pop_end(self, w_list): w_list.switch_to_object_strategy() return w_list.strategy.pop_end(w_list) def insert(self, w_list, index, w_item): w_list.switch_to_object_strategy() w_list.strategy.insert(w_list, index, w_item) def extend(self, w_list, w_any): w_list.switch_to_object_strategy() w_list.strategy.extend(w_list, w_any) def _extend_from_list(self, w_list, w_other): w_list.switch_to_object_strategy() w_list.strategy._extend_from_list(w_list, w_other) def _extend_from_iterable(self, w_list, w_iterable): w_list.switch_to_object_strategy() w_list.strategy._extend_from_iterable(w_list, w_iterable) def reverse(self, w_list): w_list.switch_to_object_strategy() w_list.strategy.reverse(w_list) def sort(self, w_list, reverse): w_list.switch_to_object_strategy() w_list.descr_sort(w_list.space, reverse=reverse) def is_empty_strategy(self): return False
rffi.setintfield(buf, "c_len", size) return buf def freeBuf(buf): free_raw_storage(buf.c_base) free(buf) def allocCB(handle, size, buf): # This is almost certainly the right thing to pass to alloc_cb buf.c_base = alloc_raw_storage(size) rffi.setintfield(buf, "c_len", size) streamReadStart_erase, streamReadStart_unerase = new_erasing_pair( "streamReadStart") class StreamReadStartFutureCallback(object): pass stashStream2, unstashStream2, unstashingStream2 = stashFor( "stream", stream_tp, initial=streamReadStart_erase(StreamReadStartFutureCallback())) def readStreamCB(stream, status, buf): status = intmask(status) # We only restash in the success case, not the error cases. state, v = unstashStream2(stream)
class JsonDictStrategy(DictStrategy): erase, unerase = rerased.new_erasing_pair("jsondict") erase = staticmethod(erase) unerase = staticmethod(unerase) _immutable_fields_ = ['jsonmap'] def __init__(self, space, jsonmap): DictStrategy.__init__(self, space) self.jsonmap = jsonmap def wrap(self, w_key): return w_key def wrapkey(space, key): return key def get_empty_storage(self): raise NotImplementedError("should not be reachable") def is_correct_type(self, w_obj): space = self.space return space.is_w(space.type(w_obj), space.w_unicode) def _never_equal_to(self, w_lookup_type): return False def length(self, w_dict): return len(self.unerase(w_dict.dstorage)) def getitem(self, w_dict, w_key): if self.is_correct_type(w_key): return self.getitem_unicode(w_dict, w_key) else: self.switch_to_unicode_strategy(w_dict) return w_dict.getitem(w_key) def getitem_unicode(self, w_dict, w_key): storage_w = self.unerase(w_dict.dstorage) if jit.isconstant(w_key): jit.promote(self) index = self.jsonmap.get_index(w_key) if index == -1: return None return storage_w[index] def setitem(self, w_dict, w_key, w_value): if self.is_correct_type(w_key): storage_w = self.unerase(w_dict.dstorage) index = self.jsonmap.get_index(w_key) if index != -1: storage_w[index] = w_value return self.switch_to_unicode_strategy(w_dict) w_dict.setitem(w_key, w_value) # for setitem_str use the default implementation # because the jsonmap needs a wrapped key anyway def setdefault(self, w_dict, w_key, w_default): if self.is_correct_type(w_key): w_result = self.getitem_unicode(w_dict, w_key) if w_result is not None: return w_result self.switch_to_unicode_strategy(w_dict) return w_dict.setdefault(w_key, w_default) def delitem(self, w_dict, w_key): self.switch_to_unicode_strategy(w_dict) return w_dict.delitem(w_key) def popitem(self, w_dict): self.switch_to_unicode_strategy(w_dict) return w_dict.popitem() def switch_to_unicode_strategy(self, w_dict): storage = self._make_unicode_dict(w_dict) strategy = self.space.fromcache(UnicodeDictStrategy) w_dict.set_strategy(strategy) w_dict.dstorage = storage def _make_unicode_dict(self, w_dict): strategy = self.space.fromcache(UnicodeDictStrategy) values_w = self.unerase(w_dict.dstorage) storage = strategy.get_empty_storage() d_new = strategy.unerase(storage) keys_in_order = self.jsonmap.get_keys_in_order() assert len(keys_in_order) == len(values_w) for index, w_key in enumerate(keys_in_order): assert w_key is not None assert type(w_key) is self.space.UnicodeObjectCls d_new[w_key] = values_w[index] return storage def copy(self, w_dict): storage = self._make_unicode_dict(w_dict) strategy = self.space.fromcache(UnicodeDictStrategy) return W_DictObject(strategy.space, strategy, storage) def w_keys(self, w_dict): return self.space.newlist(self.jsonmap.get_keys_in_order()) def values(self, w_dict): return self.unerase(w_dict.dstorage)[:] # to make resizable def items(self, w_dict): space = self.space storage_w = self.unerase(w_dict.dstorage) res = [None] * len(storage_w) for index, w_key in enumerate(self.jsonmap.get_keys_in_order()): res[index] = space.newtuple2(w_key, storage_w[index]) return res def getiterkeys(self, w_dict): return iter(self.jsonmap.get_keys_in_order()) def getitervalues(self, w_dict): storage_w = self.unerase(w_dict.dstorage) return iter(storage_w) def getiteritems_with_hash(self, w_dict): storage_w = self.unerase(w_dict.dstorage) return ZipItemsWithHash(self.jsonmap.get_keys_in_order(), storage_w)
class KwargsDictStrategy(DictStrategy): erase, unerase = rerased.new_erasing_pair("kwargsdict") erase = staticmethod(erase) unerase = staticmethod(unerase) def wrap(self, key): return _wrapkey(self.space, key) def unwrap(self, wrapped): return self.space.str_w(wrapped) def get_empty_storage(self): d = ([], []) return self.erase(d) def is_correct_type(self, w_obj): space = self.space return space.is_w(space.type(w_obj), space.w_str) def _never_equal_to(self, w_lookup_type): return False def setitem(self, w_dict, w_key, w_value): if self.is_correct_type(w_key): self.setitem_str(w_dict, self.unwrap(w_key), w_value) return else: self.switch_to_object_strategy(w_dict) w_dict.setitem(w_key, w_value) def setitem_str(self, w_dict, key, w_value): self._setitem_str_indirection(w_dict, key, w_value) @jit.look_inside_iff(lambda self, w_dict, key, w_value: jit.isconstant( self.length(w_dict)) and jit.isconstant(key)) def _setitem_str_indirection(self, w_dict, key, w_value): keys, values_w = self.unerase(w_dict.dstorage) for i in range(len(keys)): if keys[i] == key: values_w[i] = w_value break else: # limit the size so that the linear searches don't become too long if len(keys) >= 16: self.switch_to_bytes_strategy(w_dict) w_dict.setitem_str(key, w_value) else: keys.append(key) values_w.append(w_value) def setdefault(self, w_dict, w_key, w_default): if self.is_correct_type(w_key): key = self.unwrap(w_key) w_result = self.getitem_str(w_dict, key) if w_result is not None: return w_result self.setitem_str(w_dict, key, w_default) return w_default else: self.switch_to_object_strategy(w_dict) return w_dict.setdefault(w_key, w_default) def delitem(self, w_dict, w_key): # XXX could do better, but is it worth it? self.switch_to_object_strategy(w_dict) return w_dict.delitem(w_key) def length(self, w_dict): return len(self.unerase(w_dict.dstorage)[0]) def getitem_str(self, w_dict, key): return self._getitem_str_indirection(w_dict, key) @jit.look_inside_iff(lambda self, w_dict, key: jit.isconstant( self.length(w_dict)) and jit.isconstant(key)) def _getitem_str_indirection(self, w_dict, key): keys, values_w = self.unerase(w_dict.dstorage) for i in range(len(keys)): if keys[i] == key: return values_w[i] return None def getitem(self, w_dict, w_key): space = self.space if self.is_correct_type(w_key): return self.getitem_str(w_dict, self.unwrap(w_key)) elif self._never_equal_to(space.type(w_key)): return None else: self.switch_to_object_strategy(w_dict) return w_dict.getitem(w_key) def w_keys(self, w_dict): l = self.unerase(w_dict.dstorage)[0] return self.space.newlist_bytes(l[:]) def values(self, w_dict): return self.unerase(w_dict.dstorage)[1][:] # to make non-resizable def items(self, w_dict): space = self.space keys, values_w = self.unerase(w_dict.dstorage) return [ space.newtuple([self.wrap(keys[i]), values_w[i]]) for i in range(len(keys)) ] def popitem(self, w_dict): keys, values_w = self.unerase(w_dict.dstorage) key = keys.pop() w_value = values_w.pop() return self.wrap(key), w_value def clear(self, w_dict): w_dict.dstorage = self.get_empty_storage() def switch_to_object_strategy(self, w_dict): strategy = self.space.fromcache(ObjectDictStrategy) keys, values_w = self.unerase(w_dict.dstorage) d_new = strategy.unerase(strategy.get_empty_storage()) for i in range(len(keys)): d_new[self.wrap(keys[i])] = values_w[i] w_dict.strategy = strategy w_dict.dstorage = strategy.erase(d_new) def switch_to_bytes_strategy(self, w_dict): strategy = self.space.fromcache(BytesDictStrategy) keys, values_w = self.unerase(w_dict.dstorage) storage = strategy.get_empty_storage() d_new = strategy.unerase(storage) for i in range(len(keys)): d_new[keys[i]] = values_w[i] w_dict.strategy = strategy w_dict.dstorage = storage def view_as_kwargs(self, w_dict): keys, values_w = self.unerase(w_dict.dstorage) return keys[:], values_w[:] # copy to make non-resizable def getiterkeys(self, w_dict): return iter(self.unerase(w_dict.dstorage)[0]) def getitervalues(self, w_dict): return iter(self.unerase(w_dict.dstorage)[1]) def getiteritems(self, w_dict): keys = self.unerase(w_dict.dstorage)[0] return iter(range(len(keys))) wrapkey = _wrapkey
def makeFingerTreeClass(zero, add, measure): """ Produce a finger tree class. `zero` is the monoidal zero of the monoid for this class, and `add` is the monoidal binary operation; `measure` is a function which turns values into monoidal values. """ class Node(object): _immutable_ = True class Node2(Node): def __init__(self, x, y, depth): self.x = x self.y = y self.depth = depth if depth: x = uneraseNode(x) y = uneraseNode(y) self.measure = add(x.measure, y.measure) else: x = uneraseValue(x) y = uneraseValue(y) self.measure = add(measure(x), measure(y)) def __repr__(self): return "N2(measure=%d, %r, %r)" % (self.measure, self.x, self.y) def asDigits(self): return Two(self.x, self.y, self.depth) class Node3(Node): def __init__(self, x, y, z, depth): self.x = x self.y = y self.z = z self.depth = depth if depth: x = uneraseNode(x) y = uneraseNode(y) z = uneraseNode(z) self.measure = add(add(x.measure, y.measure), z.measure) else: x = uneraseValue(x) y = uneraseValue(y) z = uneraseValue(z) self.measure = add(add(measure(x), measure(y)), measure(z)) def __repr__(self): return "N3(measure=%d, %r, %r, %r)" % (self.measure, self.x, self.y, self.z) def asDigits(self): return Three(self.x, self.y, self.z, self.depth) eraseNode, uneraseNode = new_erasing_pair("Node") eraseValue, uneraseValue = new_erasing_pair("Value") def gatherNodes(l, depth): nodes = [] while l: if len(l) == 2: node = eraseNode(Node2(l[0], l[1], depth)) nodes.append(node) l = l[2:] elif len(l) == 4: node = eraseNode(Node2(l[0], l[1], depth)) nodes.append(node) node = eraseNode(Node2(l[2], l[3], depth)) nodes.append(node) l = l[4:] else: node = eraseNode(Node3(l[0], l[1], l[2], depth)) nodes.append(node) l = l[3:] return nodes class Digit(object): _immutable_ = True def split(self, predicate, i): right = self.asList() left = [] while right: if len(right) == 1: return left, right[0], [] item = right.pop(0) if self.depth: m = uneraseNode(item).measure else: m = measure(uneraseValue(item)) i = add(i, m) if predicate(i): return left, item, right else: left.append(item) class One(Digit): def __init__(self, a, depth): self.a = a self.depth = depth if depth: a = uneraseNode(a) self.measure = a.measure else: a = uneraseValue(a) self.measure = measure(a) def __repr__(self): return "1(%r)" % self.a def pushLeft(self, value): return Two(value, self.a, self.depth) def pushRight(self, value): return Two(self.a, value, self.depth) def popLeft(self): assert False, "popcorn" popRight = popLeft def asTree(self): return Single(self.a, self.depth) def asList(self): return [self.a] class Two(Digit): def __init__(self, a, b, depth): self.a = a self.b = b self.depth = depth if depth: a = uneraseNode(a) b = uneraseNode(b) self.measure = add(a.measure, b.measure) else: a = uneraseValue(a) b = uneraseValue(b) self.measure = add(measure(a), measure(b)) def __repr__(self): return "2(%r, %r)" % (self.a, self.b) def pushLeft(self, value): return Three(value, self.a, self.b, self.depth) def pushRight(self, value): return Three(self.a, self.b, value, self.depth) def popLeft(self): return self.a, One(self.b, self.depth) def popRight(self): return self.b, One(self.a, self.depth) def asTree(self): return Deep(One(self.a, self.depth), Empty(self.depth + 1), One(self.b, self.depth), self.depth) def asList(self): return [self.a, self.b] class Three(Digit): def __init__(self, a, b, c, depth): self.a = a self.b = b self.c = c self.depth = depth if depth: a = uneraseNode(a) b = uneraseNode(b) c = uneraseNode(c) self.measure = add(add(a.measure, b.measure), c.measure) else: a = uneraseValue(a) b = uneraseValue(b) c = uneraseValue(c) self.measure = add(add(measure(a), measure(b)), measure(c)) def __repr__(self): return "3(%r, %r, %r)" % (self.a, self.b, self.c) def pushLeft(self, value): return Four(value, self.a, self.b, self.c, self.depth) def pushRight(self, value): return Four(self.a, self.b, self.c, value, self.depth) def popLeft(self): return self.a, Two(self.b, self.c, self.depth) def popRight(self): return self.c, Two(self.a, self.b, self.depth) def asTree(self): # We must pick a bias here; we cannot use a Single instance to # keep the tree centered. I choose left, because I am left-handed. # ~ C. return Deep(Two(self.a, self.b, self.depth), Empty(self.depth + 1), One(self.c, self.depth), self.depth) def asList(self): return [self.a, self.b, self.c] class Four(Digit): def __init__(self, a, b, c, d, depth): self.a = a self.b = b self.c = c self.d = d self.depth = depth if depth: a = uneraseNode(a) b = uneraseNode(b) c = uneraseNode(c) d = uneraseNode(d) self.measure = add(add(a.measure, b.measure), add(c.measure, d.measure)) else: a = uneraseValue(a) b = uneraseValue(b) c = uneraseValue(c) d = uneraseValue(d) self.measure = add(add(measure(a), measure(b)), add(measure(c), measure(d))) def __repr__(self): return "4(%r, %r, %r, %r)" % (self.a, self.b, self.c, self.d) def pushLeft(self, value): assert False, "golfball" pushRight = pushLeft def popLeft(self): return self.a, Three(self.b, self.c, self.d, self.depth) def popRight(self): return self.d, Three(self.a, self.b, self.c, self.depth) def asTree(self): return Deep(Two(self.a, self.b, self.depth), Empty(self.depth + 1), Two(self.c, self.d, self.depth), self.depth) def asList(self): return [self.a, self.b, self.c, self.d] def listToDigit(l, depth): size = len(l) if size == 1: return One(l[0], depth) elif size == 2: return Two(l[0], l[1], depth) elif size == 3: return Three(l[0], l[1], l[2], depth) else: assert size == 4, "divot" return Four(l[0], l[1], l[2], l[3], depth) class FingerTree(object): _immutable_ = True def pushLeft(self, value): return self._pushLeft(eraseValue(value)) def pushRight(self, value): return self._pushRight(eraseValue(value)) def popLeft(self): value, ft = self._viewLeft() return uneraseValue(value), ft def popRight(self): value, ft = self._viewRight() return uneraseValue(value), ft def add(self, other): return self._concat([], other) def split(self, predicate): if isinstance(self, Empty): return self, self elif predicate(self.measure): left, item, right = self._split(predicate, zero) return left, right._pushRight(item) else: return self, Empty(self.depth) class Empty(FingerTree): measure = zero def __init__(self, depth=0): self.depth = depth def __repr__(self): return "Empty" def _pushLeft(self, value): return Single(value, self.depth) _pushRight = _pushLeft def _viewLeft(self): return None, None _viewRight = _viewLeft def isEmpty(self): return True def _concat(self, middle, other): for item in middle: other = other._pushLeft(item) return other def _split(self, predicate, i): assert False, "egghead" class Single(FingerTree): _immutable_ = True def __init__(self, value, depth): self.value = value self.depth = depth if depth: self.measure = uneraseNode(value).measure else: self.measure = measure(uneraseValue(value)) def __repr__(self): return "Single(%r)" % self.value def _pushLeft(self, value): return Deep(One(value, self.depth), Empty(self.depth + 1), One(self.value, self.depth), self.depth) def _pushRight(self, value): return Deep(One(self.value, self.depth), Empty(self.depth + 1), One(value, self.depth), self.depth) def _viewLeft(self): return self.value, Empty(self.depth) _viewRight = _viewLeft def isEmpty(self): return False def _concat(self, middle, other): for item in middle: other = other._pushLeft(item) return other._pushLeft(self.value) def _split(self, predicate, i): return Empty(self.depth), self.value, Empty(self.depth) class Deep(FingerTree): _immutable_ = True def __init__(self, left, tree, right, depth): assert tree.depth == depth + 1, "birch" assert left.depth == depth, "pine" assert right.depth == depth, "redwood" assert isinstance(left, Digit), "sinister" assert isinstance(right, Digit), "dexter" self.left = left self.tree = tree self.right = right self.depth = depth self.measure = add(add(left.measure, tree.measure), right.measure) def __repr__(self): return "Deep%d(%r, %r, %r)" % (self.depth, self.left, self.tree, self.right) def _pushLeft(self, value): if isinstance(self.left, Four): node = eraseNode(Node3(self.left.b, self.left.c, self.left.d, self.depth)) return Deep(Two(value, self.left.a, self.depth), self.tree._pushLeft(node), self.right, self.depth) else: left = self.left.pushLeft(value) return Deep(left, self.tree, self.right, self.depth) def _pushRight(self, value): if isinstance(self.right, Four): node = eraseNode(Node3(self.right.a, self.right.b, self.right.c, self.depth)) return Deep(self.left, self.tree._pushRight(node), Two(self.right.d, value, self.depth), self.depth) else: right = self.right.pushRight(value) return Deep(self.left, self.tree, right, self.depth) def _viewLeft(self): if isinstance(self.left, One): value = self.left.a if self.tree.isEmpty(): return value, self.right.asTree() else: n, tree = self.tree._viewLeft() left = uneraseNode(n).asDigits() return value, Deep(left, tree, self.right, self.depth) else: value, left = self.left.popLeft() return value, Deep(left, self.tree, self.right, self.depth) def _viewRight(self): if isinstance(self.right, One): value = self.right.a if self.tree.isEmpty(): return value, self.left.asTree() else: n, tree = self.tree._viewRight() right = uneraseNode(n).asDigits() return value, Deep(self.left, tree, right, self.depth) else: value, right = self.right.popRight() return value, Deep(self.left, self.tree, right, self.depth) def isEmpty(self): return False def _concat(self, middle, other): if isinstance(other, Empty): for item in middle: self = self._pushRight(item) return self elif isinstance(other, Single): for item in middle: self = self._pushRight(item) return self._pushRight(other.value) elif isinstance(other, Deep): newLeft = self.left newRight = self.right l = gatherNodes(self.right.asList() + other.left.asList(), self.depth) newTree = self.tree._concat(l, other.tree) return Deep(newLeft, newTree, newRight, self.depth) else: assert False, "willow" def _split(self, predicate, i): j = add(i, self.left.measure) if predicate(j): left, item, right = self.left.split(predicate, i) if left: leftSplit = listToDigit(left, self.depth).asTree() else: leftSplit = Empty(self.depth) if right: rightSplit = Deep(listToDigit(right, self.depth), self.tree, self.right, self.depth) elif self.tree.isEmpty(): rightSplit = self.right.asTree() else: value, tree = self.tree._viewLeft() rightSplit = Deep(uneraseNode(value).asDigits(), tree, self.right, self.depth) return leftSplit, item, rightSplit k = add(j, self.tree.measure) if predicate(k): leftTree, itemTree, rightTree = self.tree._split(predicate, j) digits = uneraseNode(itemTree).asDigits() leftList, item, rightList = digits.split(predicate, add(j, leftTree.measure)) if leftList: leftSplit = Deep(self.left, leftTree, listToDigit(leftList, self.depth), self.depth) elif leftTree.isEmpty(): leftSplit = self.left.asTree() else: value, leftTree = leftTree._viewRight() leftSplit = Deep(self.left, leftTree, uneraseNode(value).asDigits(), self.depth) if rightList: rightSplit = Deep(listToDigit(rightList, self.depth), rightTree, self.right, self.depth) elif rightTree.isEmpty(): rightSplit = self.right.asTree() else: value, rightTree = rightTree._viewLeft() rightSplit = Deep(uneraseNode(value).asDigits(), rightTree, self.right, self.depth) return leftSplit, item, rightSplit else: left, item, right = self.right.split(predicate, k) if left: leftSplit = Deep(self.left, self.tree, listToDigit(left, self.depth), self.depth) elif self.tree.isEmpty(): leftSplit = self.left.asTree() else: value, tree = self.tree._viewRight() leftSplit = Deep(self.left, tree, uneraseNode(value).asDigits(), self.depth) if right: rightSplit = listToDigit(right, self.depth).asTree() else: rightSplit = Empty(self.depth) return leftSplit, item, rightSplit return Empty