def _resize(self, newsize):
# Note that resize does *not* set self._size. That needs to be done by
# the caller such that that the we never expose invalid item cells.
# The size field is covered by a snapshot done here, though.
mm = self._p_mm
allocated = self._allocated
# Only realloc if we don't have enough space already.
if (allocated >= newsize and newsize >= allocated >> 1):
assert self._items != None or newsize == 0
with mm.transaction():
ob = ffi.cast('PVarObject *', self._body)
mm.snapshot_range(ffi.addressof(ob, 'ob_size'),
ffi.sizeof('size_t'))
ob.ob_size = newsize
return
# We use CPython's overallocation algorithm.
new_allocated = (newsize >> 3) + (3 if newsize < 9 else 6) + newsize
if newsize == 0:
new_allocated = 0
items = self._items
with mm.transaction():
if items is None:
items = mm.zalloc(new_allocated * ffi.sizeof('PObjPtr'),
type_num=LIST_POBJPTR_ARRAY_TYPE_NUM)
else:
items = mm.zrealloc(self._body.ob_items,
new_allocated * ffi.sizeof('PObjPtr'),
LIST_POBJPTR_ARRAY_TYPE_NUM)
mm.snapshot_range(self._body, ffi.sizeof('PListObject'))
self._body.ob_items = items
self._body.allocated = new_allocated
def _resize(self, newsize):
# Note that resize does *not* set self._size. That needs to be done by
# the caller such that that the we never expose invalid item cells.
# The size field is covered by a snapshot done here, though.
mm = self._p_mm
allocated = self._allocated
# Only realloc if we don't have enough space already.
if (allocated >= newsize and newsize >= allocated >> 1):
assert self._items != None or newsize == 0
with mm.transaction():
ob = ffi.cast('PVarObject *', self._body)
mm.snapshot_range(ffi.addressof(ob, 'ob_size'),
ffi.sizeof('size_t'))
ob.ob_size = newsize
return
# We use CPython's overallocation algorithm.
new_allocated = (newsize >> 3) + (3 if newsize < 9 else 6) + newsize
if newsize == 0:
new_allocated = 0
items = self._items
with mm.transaction():
if items is None:
items = mm.zalloc(new_allocated * ffi.sizeof('PObjPtr'),
type_num=LIST_POBJPTR_ARRAY_TYPE_NUM)
else:
items = mm.zrealloc(self._body.ob_items,
new_allocated * ffi.sizeof('PObjPtr'),
LIST_POBJPTR_ARRAY_TYPE_NUM)
mm.snapshot_range(self._body, ffi.sizeof('PListObject'))
self._body.ob_items = items
self._body.allocated = new_allocated
def _add(self, key):
mm = self._p_mm
khash = fixed_hash(key)
result = ADD_RESULT_RESTART
with mm.transaction():
while result == ADD_RESULT_RESTART:
index, result = self._get_available_entry_slot(key, khash)
if result == ADD_RESULT_FOUND_UNUSED or \
result == ADD_RESULT_FOUND_DUMMY:
table_data = ffi.cast('PSetEntry *',
mm.direct(self._body.table))
mm.snapshot_range(ffi.addressof(table_data, index),
ffi.sizeof('PSetEntry'))
oid = mm.persist(key)
mm.incref(oid)
p_obj = ffi.cast('PObject *', mm.direct(oid))
table_data[index].key = oid
table_data[index].hash = khash
mm.snapshot_range(
ffi.addressof(self._body, 'fill'),
ffi.sizeof('PSetObject') - ffi.sizeof('PObject'))
self._body.used += 1
if result == ADD_RESULT_FOUND_UNUSED:
self._body.fill += 1
if self._body.fill * 3 >= self._body.mask * 2:
self._table_resize(self._body.used)
def _resize(self, newsize):
mm = self.__manager__
allocated = self._allocated
# Only realloc if we don't have enough space already.
if (allocated >= newsize and newsize >= allocated >> 1):
assert self._items != None or newsize == 0
with mm.transaction():
ob = ffi.cast('PVarObject *', self._body)
mm.snapshot_range(ffi.addressof(ob, 'ob_size'),
ffi.sizeof('size_t'))
ob.ob_size = newsize
return
# We use CPython's overallocation algorithm.
new_allocated = (newsize >> 3) + (3 if newsize < 9 else 6) + newsize
if newsize == 0:
new_allocated = 0
items = self._items
with mm.transaction():
if items is None:
items = mm.malloc(new_allocated * ffi.sizeof('PObjPtr'),
type_num=LIST_POBJPTR_ARRAY_TYPE_NUM)
else:
items = mm.realloc(self._body.ob_items,
new_allocated * ffi.sizeof('PObjPtr'),
LIST_POBJPTR_ARRAY_TYPE_NUM)
mm.snapshot_range(self._body, ffi.sizeof('PListObject'))
self._body.ob_items = items
self._body.allocated = new_allocated
ffi.cast('PVarObject *', self._body).ob_size = newsize
def _add(self, key):
mm = self._p_mm
khash = fixed_hash(key)
result = ADD_RESULT_RESTART
with mm.transaction():
while result == ADD_RESULT_RESTART:
index, result = self._get_available_entry_slot(key, khash)
if result == ADD_RESULT_FOUND_UNUSED or \
result == ADD_RESULT_FOUND_DUMMY:
table_data = ffi.cast('PSetEntry *',
mm.direct(self._body.table))
mm.snapshot_range(ffi.addressof(table_data, index),
ffi.sizeof('PSetEntry'))
oid = mm.persist(key)
mm.incref(oid)
p_obj = ffi.cast('PObject *', mm.direct(oid))
table_data[index].key = oid
table_data[index].hash = khash
mm.snapshot_range(
ffi.addressof(self._body, 'fill'),
ffi.sizeof('PSetObject') - ffi.sizeof('PObject'))
self._body.used += 1
if result == ADD_RESULT_FOUND_UNUSED:
self._body.fill += 1
if self._body.fill * 3 >= self._body.mask * 2:
self._table_resize(self._body.used)
def __init__(self, filename, flag='w',
pool_size=MIN_POOL_SIZE, mode=0o666, debug=False):
"""Open or create a persistent object pool backed by filename.
If flag is 'w', raise an OSError if the file does not exist and
otherwise open it for reading and writing. If flag is 'x', raise an
OSError if the file *does* exist, otherwise create it and open it for
reading and writing. If flag is 'c', create the file if it does not
exist, otherwise use the existing file.
If the file gets created, use pool_size as the size of the new pool in
bytes and mode as its access mode, otherwise ignore these parameters
and open the existing file.
If debug is True, generate some additional logging, including turning
on some additional sanity-check warnings. This may have an impact
on performance.
See also the open and create functions of nvm.pmemobj, which are
convenience functions for the 'w' and 'x' flags, respectively.
"""
log.debug('PersistentObjectPool.__init__: %r, %r %r, %r',
filename, flag, pool_size, mode)
self.filename = filename
self.debug = debug
exists = os.path.exists(filename)
if flag == 'w' or (flag == 'c' and exists):
self._pool_ptr = _check_null(
lib.pmemobj_open(_coerce_fn(filename),
layout_version))
elif flag == 'x' or (flag == 'c' and not exists):
self._pool_ptr = _check_null(
lib.pmemobj_create(_coerce_fn(filename),
layout_version,
pool_size,
mode))
elif flag == 'r':
raise ValueError("Read-only mode is not supported")
else:
raise ValueError("Invalid flag value {}".format(flag))
mm = self.mm = MemoryManager(self._pool_ptr)
pmem_root = lib.pmemobj_root(self._pool_ptr, ffi.sizeof('PRoot'))
pmem_root = ffi.cast('PRoot *', mm.direct(pmem_root))
type_table_oid = mm.otuple(pmem_root.type_table)
if type_table_oid == mm.OID_NULL:
with mm.transaction():
type_table_oid = mm._create_type_table()
mm.snapshot_range(pmem_root, ffi.sizeof('PObjPtr'))
pmem_root.type_table = type_table_oid
else:
mm._resurrect_type_table(type_table_oid)
self._pmem_root = pmem_root
if exists:
# Make sure any objects orphaned by a crash are cleaned up.
# XXX should fix this to only be called when there is a crash.
self.gc()
def _persist_builtins_str(self, s):
type_code = self._get_type_code(s.__class__)
if sys.version_info[0] > 2:
s = s.encode('utf-8')
with self.transaction():
p_str_oid = self.zalloc(ffi.sizeof('PObject') + len(s) + 1)
p_str = ffi.cast('PObject *', self.direct(p_str_oid))
p_str.ob_type = type_code
body = ffi.cast('char *', p_str) + ffi.sizeof('PObject')
ffi.buffer(body, len(s))[:] = s
return p_str_oid
def _persist_builtins_str(self, s):
type_code = self._get_type_code(s.__class__)
if sys.version_info[0] > 2:
s = s.encode('utf-8')
with self.transaction():
p_str_oid = self.zalloc(ffi.sizeof('PObject') + len(s) + 1)
p_str = ffi.cast('PObject *', self.direct(p_str_oid))
p_str.ob_type = type_code
body = ffi.cast('char *', p_str) + ffi.sizeof('PObject')
ffi.buffer(body, len(s))[:] = s
return p_str_oid
def _persist_nvm_pmemobj_pool_PICKLE_SENTINEL(self, obj):
type_code = self._get_type_code(PICKLE_SENTINEL)
s = dumps(obj)
print(s)
with self.transaction():
p_obj_oid = self.zalloc(ffi.sizeof('PVarObject') + len(s))
p_pickle = ffi.cast('PVarObject *', self.direct(p_obj_oid))
p_pickle.ob_base.ob_type = type_code
p_pickle.ob_size = len(s)
body = ffi.cast('char *', p_pickle) + ffi.sizeof('PVarObject')
ffi.buffer(body, len(s))[:] = s
return p_obj_oid
def _persist_nvm_pmemobj_pool_PICKLE_SENTINEL(self, obj):
type_code = self._get_type_code(PICKLE_SENTINEL)
s = dumps(obj)
print(s)
with self.transaction():
p_obj_oid = self.zalloc(ffi.sizeof('PVarObject') + len(s))
p_pickle = ffi.cast('PVarObject *', self.direct(p_obj_oid))
p_pickle.ob_base.ob_type = type_code
p_pickle.ob_size = len(s)
body = ffi.cast('char *', p_pickle) + ffi.sizeof('PVarObject')
ffi.buffer(body, len(s))[:] = s
return p_obj_oid
def _insertion_resize(self):
# This is modeled on CPython's insertion_resize/dictresize, but
# assuming we always have a combined dict. We copy the keys and values
# into a new dict structure and free the old one. We don't touch the
# refcounts.
mm = self._p_mm
minused = self._growth_rate()
newsize = MIN_SIZE_COMBINED
while newsize <= minused and newsize > 0:
newsize = newsize << 1
oldkeys = self._keys
oldkeys_oid = mm.otuple(self._body.ma_keys)
with mm.transaction():
mm.snapshot_range(ffi.addressof(self._body, 'ma_keys'),
ffi.sizeof('PObjPtr'))
self._body.ma_keys = self._new_keys_object(newsize)
oldsize = oldkeys.dk_size
old_ep0 = ffi.cast('PDictKeyEntry *',
ffi.addressof(oldkeys.dk_entries[0]))
for i in range(oldsize):
old_ep = old_ep0[i]
me_value = mm.otuple(old_ep.me_value)
if me_value != mm.OID_NULL:
me_key = mm.otuple(old_ep.me_key)
assert me_key != DUMMY
me_hash = old_ep.me_hash
new_ep = self._find_empty_slot(me_key, me_hash)
new_ep.me_key = me_key
new_ep.me_hash = me_hash
new_ep.me_value = me_value
self._keys.dk_usable -= self._body.ma_used
mm.free(oldkeys_oid)
def _insert_clean(self, table, mask, key_oid, khash):
mm = self._p_mm
perturb = khash
i = khash & mask
table_data = ffi.cast('PSetEntry *', mm.direct(table))
found_index = -1
while True:
if table_data[i].hash == HASH_UNUSED:
found_index = i
break
for j in range(i + 1, min(i + LINEAR_PROBES, mask) + 1):
if table_data[j].hash == HASH_UNUSED:
found_index = j
break
if found_index != -1:
break
perturb >>= PERTURB_SHIFT
i = (i * 5 + 1 + perturb) & mask
with mm.transaction():
mm.snapshot_range(ffi.addressof(table_data, found_index),
ffi.sizeof('PSetEntry'))
table_data[found_index].hash = khash
table_data[found_index].key = key_oid
def _insert_clean(self, table, mask, key_oid, khash):
mm = self._p_mm
perturb = khash
i = khash & mask
table_data = ffi.cast('PSetEntry *', mm.direct(table))
found_index = -1
while True:
if table_data[i].hash == HASH_UNUSED:
found_index = i
break
for j in range(i + 1, min(i + LINEAR_PROBES, mask) + 1):
if table_data[j].hash == HASH_UNUSED:
found_index = j
break
if found_index != -1:
break
perturb >>= PERTURB_SHIFT
i = (i * 5 + 1 + perturb) & mask
with mm.transaction():
mm.snapshot_range(ffi.addressof(table_data, found_index),
ffi.sizeof('PSetEntry'))
table_data[found_index].hash = khash
table_data[found_index].key = key_oid
def _p_new(self, manager):
mm = self._p_mm = manager
with mm.transaction():
# XXX Will want to implement a freelist here, like CPython
self._p_oid = mm.zalloc(ffi.sizeof('PListObject'))
ob = ffi.cast('PObject *', mm.direct(self._p_oid))
ob.ob_type = mm._get_type_code(PersistentList)
self._body = ffi.cast('PListObject *', mm.direct(self._p_oid))
def _p_new(self, manager):
mm = self._p_mm = manager
with mm.transaction():
# XXX Will want to implement a freelist here, like CPython
self._p_oid = mm.zalloc(ffi.sizeof('PListObject'))
ob = ffi.cast('PObject *', mm.direct(self._p_oid))
ob.ob_type = mm._get_type_code(PersistentList)
self._body = ffi.cast('PListObject *', mm.direct(self._p_oid))
def _persist_builtins_float(self, f):
type_code = self._get_type_code(f.__class__)
with self.transaction():
p_float_oid = self.zalloc(ffi.sizeof('PFloatObject'))
p_float = ffi.cast('PObject *', self.direct(p_float_oid))
p_float.ob_type = type_code
p_float = ffi.cast('PFloatObject *', p_float)
p_float.fval = f
return p_float_oid
def _persist_builtins_float(self, f):
type_code = self._get_type_code(f.__class__)
with self.transaction():
p_float_oid = self.zalloc(ffi.sizeof('PFloatObject'))
p_float = ffi.cast('PObject *', self.direct(p_float_oid))
p_float.ob_type = type_code
p_float = ffi.cast('PFloatObject *', p_float)
p_float.fval = f
return p_float_oid
def root(self, value):
log.debug("setting 'root' to %r (discarding %s)", value, self.root)
with self.mm.transaction(), self.lock:
oid = self.mm.persist(value)
self.mm.snapshot_range(ffi.addressof(self._pmem_root.root_object),
ffi.sizeof('PObjPtr'))
self.mm.xdecref(self._pmem_root.root_object)
self._pmem_root.root_object = oid
self.mm.incref(oid)
def _p_new(self, manager):
mm = self._p_mm = manager
with mm.transaction():
self._p_oid = mm.zalloc(ffi.sizeof('PTupleObject'))
ob = ffi.cast('PObject *', mm.direct(self._p_oid))
ob.ob_type = mm._get_type_code(PersistentTuple)
self._body = ffi.cast('PTupleObject *', mm.direct(self._p_oid))
self._body.ob_items = mm.OID_NULL
def _new_keys_object(self, size):
assert size >= MIN_SIZE_SPLIT
mm = self._p_mm
with mm.transaction():
dk_oid = mm.zalloc(ffi.sizeof('PDictKeysObject')
+ ffi.sizeof('PDictKeyEntry') * (size - 1),
type_num=PDICTKEYSOBJECT_TYPE_NUM)
dk = ffi.cast('PDictKeysObject *', mm.direct(dk_oid))
dk.dk_refcnt = 1
dk.dk_size = size
dk.dk_usable = _usable_fraction(size)
ep = ffi.cast('PDictKeyEntry *', ffi.addressof(dk.dk_entries[0]))
# Hash value of slot 0 is used by popitem, so it must be initizlied
ep[0].me_hash = 0
for i in range(size):
ep[i].me_key = mm.OID_NULL
ep[i].me_value = mm.OID_NULL
# XXX Set dk_lookup to lookdict_unicode_nodummy if we end up using it.
return dk_oid
def root(self, value):
log.debug("setting 'root' to %r (discarding %s)", value, self.root)
with self.mm.transaction(), self.lock:
oid = self.mm.persist(value)
self.mm.snapshot_range(
ffi.addressof(self._pmem_root.root_object),
ffi.sizeof('PObjPtr'))
self.mm.xdecref(self._pmem_root.root_object)
self._pmem_root.root_object = oid
self.mm.incref(oid)
def _table_resize(self, minused):
mm = self._p_mm
if minused > 50000:
minused = (minused << 1)
else:
minused = (minused << 2)
newsize = PERM_SET_MINSIZE
while(newsize <= minused):
newsize = (newsize << 1)
newsize = ffi.cast('size_t', newsize)
if newsize == 0:
raise MemoryError("Out of memory")
newsize = int(newsize)
with mm.transaction():
oldtable = mm.otuple(self._body.table)
oldtable_data = ffi.cast('PSetEntry *', mm.direct(oldtable))
newtable = self._alloc_empty_table(newsize)
newmask = newsize - 1
for i in range(0, self._body.mask + 1):
if oldtable_data[i].hash == HASH_UNUSED or \
oldtable_data[i].hash == HASH_DUMMY:
continue
self._insert_clean(newtable, newmask,
oldtable_data[i].key,
oldtable_data[i].hash)
mm.snapshot_range(ffi.addressof(self._body, 'fill'),
ffi.sizeof('PSetObject') - ffi.sizeof('PObject'))
self._body.mask = newmask
self._body.fill = self._body.used
self._body.table = newtable
mm.free(oldtable)
def _table_resize(self, minused):
mm = self._p_mm
if minused > 50000:
minused = (minused << 1)
else:
minused = (minused << 2)
newsize = PERM_SET_MINSIZE
while (newsize <= minused):
newsize = (newsize << 1)
newsize = ffi.cast('size_t', newsize)
if newsize == 0:
raise MemoryError("Out of memory")
newsize = int(newsize)
with mm.transaction():
oldtable = mm.otuple(self._body.table)
oldtable_data = ffi.cast('PSetEntry *', mm.direct(oldtable))
newtable = self._alloc_empty_table(newsize)
newmask = newsize - 1
for i in range(0, self._body.mask + 1):
if oldtable_data[i].hash == HASH_UNUSED or \
oldtable_data[i].hash == HASH_DUMMY:
continue
self._insert_clean(newtable, newmask, oldtable_data[i].key,
oldtable_data[i].hash)
mm.snapshot_range(ffi.addressof(self._body, 'fill'),
ffi.sizeof('PSetObject') - ffi.sizeof('PObject'))
self._body.mask = newmask
self._body.fill = self._body.used
self._body.table = newtable
mm.free(oldtable)
def _p_new(self, manager):
mm = self._p_mm = manager
with mm.transaction():
self._p_oid = mm.zalloc(ffi.sizeof('PSetObject'))
ob = ffi.cast('PObject *', mm.direct(self._p_oid))
ob.ob_type = mm._get_type_code(self.__class__)
size = PERM_SET_MINSIZE
self._body = ffi.cast('PSetObject *', mm.direct(self._p_oid))
self._body.mask = (size - 1)
self._body.hash = HASH_INVALID
self._body.table = self._alloc_empty_table(PERM_SET_MINSIZE)
def __setitem__(self, index, value):
mm = self._p_mm
index = self._normalize_index(index)
items = self._items
with mm.transaction():
v_oid = mm.persist(value)
mm.snapshot_range(ffi.addressof(items, index),
ffi.sizeof('PObjPtr *'))
mm.xdecref(items[index])
items[index] = v_oid
mm.incref(v_oid)
def _p_new(self, manager):
mm = self._p_mm = manager
with mm.transaction():
self._p_oid = mm.zalloc(ffi.sizeof('PSetObject'))
ob = ffi.cast('PObject *', mm.direct(self._p_oid))
ob.ob_type = mm._get_type_code(self.__class__)
size = PERM_SET_MINSIZE
self._body = ffi.cast('PSetObject *', mm.direct(self._p_oid))
self._body.mask = (size - 1)
self._body.hash = HASH_INVALID
self._body.table = self._alloc_empty_table(PERM_SET_MINSIZE)
def __setitem__(self, index, value):
mm = self._p_mm
index = self._normalize_index(index)
items = self._items
with mm.transaction():
v_oid = mm.persist(value)
mm.snapshot_range(ffi.addressof(items, index),
ffi.sizeof('PObjPtr *'))
mm.xdecref(items[index])
items[index] = v_oid
mm.incref(v_oid)
def _p_new(self, manager):
mm = self._p_mm = manager
with mm.transaction():
# XXX will want to implement a freelist here.
self._p_oid = mm.zalloc(ffi.sizeof('PDictObject'))
ob = ffi.cast('PObject *', mm.direct(self._p_oid))
ob.ob_type = mm._get_type_code(PersistentDict)
d = self._body = ffi.cast('PDictObject *', mm.direct(self._p_oid))
# This code may get moved to a _new_dict method when we implement
# split dicts.
d.ma_keys = self._new_keys_object(MIN_SIZE_COMBINED)
d.ma_values = mm.OID_NULL
def incref(self, oid):
"""Increment the reference count of oid."""
oid = self.otuple(oid)
if oid == OID_NULL:
# Unlike CPython, we don't ref-track our constants.
return
p_obj = ffi.cast('PObject *', self.direct(oid))
log.debug('incref %r %r', oid, p_obj.ob_refcnt + 1)
with self.transaction():
self.snapshot_range(ffi.addressof(p_obj, 'ob_refcnt'),
ffi.sizeof('size_t'))
p_obj.ob_refcnt += 1
def _discard(self, key):
mm = self._p_mm
with mm.transaction():
keyindex = self._lookkey(key, fixed_hash(key))
if keyindex != -1:
table_data = ffi.cast('PSetEntry *',
mm.direct(self._body.table))
mm.snapshot_range(ffi.addressof(table_data, keyindex),
ffi.sizeof('PSetEntry'))
mm.decref(table_data[keyindex].key)
table_data[keyindex].key = mm.OID_NULL
table_data[keyindex].hash = HASH_DUMMY
self._body.used -= 1
def decref(self, oid):
"""Decrement the reference count of oid, and free it if zero."""
oid = self.otuple(oid)
p_obj = ffi.cast('PObject *', self.direct(oid))
log.debug('decref %r %r', oid, p_obj.ob_refcnt - 1)
with self.transaction():
self.snapshot_range(ffi.addressof(p_obj, 'ob_refcnt'),
ffi.sizeof('size_t'))
assert p_obj.ob_refcnt > 0, "{} oid refcount {}".format(
oid, p_obj.ob_refcnt)
p_obj.ob_refcnt -= 1
if p_obj.ob_refcnt < 1:
self._deallocate(oid)
def _discard(self, key):
mm = self._p_mm
with mm.transaction():
keyindex = self._lookkey(key, fixed_hash(key))
if keyindex != -1:
table_data = ffi.cast('PSetEntry *',
mm.direct(self._body.table))
mm.snapshot_range(ffi.addressof(table_data, keyindex),
ffi.sizeof('PSetEntry'))
mm.decref(table_data[keyindex].key)
table_data[keyindex].key = mm.OID_NULL
table_data[keyindex].hash = HASH_DUMMY
self._body.used -= 1
def __init__(self, *args, **kw):
if not args:
return
if len(args) != 1:
raise TypeError("PersistentTuple takes at most 1"
" argument, {} given".format(len(args)))
item_count = len(args[0])
mm = self._p_mm
with mm.transaction():
mm.snapshot_range(ffi.addressof(self._body, 'ob_items'),
ffi.sizeof('PObjPtr'))
self._body.ob_items = mm.zalloc(
item_count * ffi.sizeof('PObjPtr'),
type_num=TUPLE_POBJPTR_ARRAY_TYPE_NUM)
ob = ffi.cast('PVarObject *', self._body)
mm.snapshot_range(ffi.addressof(ob, 'ob_size'),
ffi.sizeof('size_t'))
ob.ob_size = item_count
for index, value in enumerate(args[0]):
super(self.__class__, self).__setitem__(index, value)
def _p_new(self, manager):
self._p_dict = {} # This makes __getattribute__ simpler
mm = self._p_mm = manager
with mm.transaction():
# XXX will want to implement a freelist here.
self._p_oid = mm.zalloc(ffi.sizeof('PObjectObject'))
ob = ffi.cast('PObject *', mm.direct(self._p_oid))
ob.ob_type = mm._get_type_code(self.__class__)
d = self._p_body = ffi.cast('PObjectObject *',
mm.direct(self._p_oid))
self._p_dict = mm.new(PersistentDict)
d.ob_dict = self._p_dict._p_oid
mm.incref(self._p_dict._p_oid)
self._v__init__()
def incref(self, oid):
"""Increment the reference count of oid if it is not a singleton"""
oid = self.otuple(oid)
assert oid != self.OID_NULL
if not oid[0]:
# Unlike CPython, we don't ref-track our constants.
log.debug('not increfing %s', oid)
return
p_obj = ffi.cast('PObject *', self.direct(oid))
log.debug('incref %r %r', oid, p_obj.ob_refcnt + 1)
with self.transaction():
self.snapshot_range(ffi.addressof(p_obj, 'ob_refcnt'),
ffi.sizeof('size_t'))
p_obj.ob_refcnt += 1
def incref(self, oid):
"""Increment the reference count of oid if it is not a singleton"""
oid = self.otuple(oid)
assert oid != self.OID_NULL
if not oid[0]:
# Unlike CPython, we don't ref-track our constants.
log.debug('not increfing %s', oid)
return
p_obj = ffi.cast('PObject *', self.direct(oid))
log.debug('incref %r %r', oid, p_obj.ob_refcnt + 1)
with self.transaction():
self.snapshot_range(ffi.addressof(p_obj, 'ob_refcnt'),
ffi.sizeof('size_t'))
p_obj.ob_refcnt += 1
def decref(self, oid):
"""Decrement the reference count of oid, and free it if zero."""
oid = self.otuple(oid)
if not oid[0]:
# Unlike CPython we do not ref-track our constants.
log.debug('not decrefing %s', oid)
return
p_obj = ffi.cast('PObject *', self.direct(oid))
log.debug('decref %r %r', oid, p_obj.ob_refcnt - 1)
with self.transaction():
self.snapshot_range(ffi.addressof(p_obj, 'ob_refcnt'),
ffi.sizeof('size_t'))
assert p_obj.ob_refcnt > 0, "{} oid refcount {}".format(
oid, p_obj.ob_refcnt)
p_obj.ob_refcnt -= 1
if p_obj.ob_refcnt < 1:
self._deallocate(oid)
def decref(self, oid):
"""Decrement the reference count of oid, and free it if zero."""
oid = self.otuple(oid)
if not oid[0]:
# Unlike CPython we do not ref-track our constants.
log.debug('not decrefing %s', oid)
return
p_obj = ffi.cast('PObject *', self.direct(oid))
log.debug('decref %r %r', oid, p_obj.ob_refcnt - 1)
with self.transaction():
self.snapshot_range(ffi.addressof(p_obj, 'ob_refcnt'),
ffi.sizeof('size_t'))
assert p_obj.ob_refcnt > 0, "{} oid refcount {}".format(
oid, p_obj.ob_refcnt)
p_obj.ob_refcnt -= 1
if p_obj.ob_refcnt < 1:
self._deallocate(oid)
def __init__(self, *args, **kw):
if '__manager__' not in kw:
raise ValueError("__manager__ is required")
mm = self.__manager__ = kw.pop('__manager__')
if '_oid' not in kw:
with mm.transaction():
# XXX Will want to implement a freelist here, like CPython
self._oid = mm.malloc(ffi.sizeof('PListObject'))
ob = ffi.cast('PObject *', mm.direct(self._oid))
ob.ob_type = mm._get_type_code(PersistentList)
else:
self._oid = kw.pop('_oid')
if kw:
raise TypeError("Unrecognized keyword argument(s) {}".format(kw))
self._body = ffi.cast('PListObject *', mm.direct(self._oid))
if args:
if len(args) != 1:
raise TypeError("PersistentList takes at most 1"
" argument, {} given".format(len(args)))
self.extend(args[0])
def _alloc_empty_table(self, tablesize):
return self._p_mm.zalloc(ffi.sizeof('PSetEntry') * tablesize,
type_num=SET_POBJPTR_ARRAY_TYPE_NUM)
def _resurrect_builtins_str(self, obj_ptr):
body = ffi.cast('char *', obj_ptr) + ffi.sizeof('PObject')
s = ffi.string(body)
if sys.version_info[0] > 2:
s = s.decode('utf-8')
return s
def _resurrect_nvm_pmemobj_pool_PICKLE_SENTINEL(self, obj_ptr):
obj_ptr = ffi.cast('PVarObject *', obj_ptr)
body = ffi.cast('char *', obj_ptr) + ffi.sizeof('PVarObject')
s = ffi.buffer(body, obj_ptr.ob_size)[:]
obj = loads(s)
return obj
def _resurrect_builtins_str(self, obj_ptr):
body = ffi.cast('char *', obj_ptr) + ffi.sizeof('PObject')
s = ffi.string(body)
if sys.version_info[0] > 2:
s = s.decode('utf-8')
return s
def __init__(self,
filename,
flag='w',
pool_size=MIN_POOL_SIZE,
mode=0o666,
debug=False):
"""Open or create a persistent object pool backed by filename.
If flag is 'w', raise an OSError if the file does not exist and
otherwise open it for reading and writing. If flag is 'x', raise an
OSError if the file *does* exist, otherwise create it and open it for
reading and writing. If flag is 'c', create the file if it does not
exist, otherwise use the existing file.
If the file gets created, use pool_size as the size of the new pool in
bytes and mode as its access mode, otherwise ignore these parameters
and open the existing file.
If debug is True, generate some additional logging, including turning
on some additional sanity-check warnings. This may have an impact
on performance.
When the pool is opened, if the previous shutdown was not clean the
pool is cleaned up, including running the 'gc' method.
See also the open and create functions of nvm.pmemobj, which are
convenience functions for the 'w' and 'x' flags, respectively.
"""
log.debug('PersistentObjectPool.__init__: %r, %r %r, %r', filename,
flag, pool_size, mode)
self.filename = filename
self.debug = debug
exists = os.path.exists(filename)
if flag == 'w' or (flag == 'c' and exists):
self._pool_ptr = _err_check.check_null(
lib.pmemobj_open(_coerce_fn(filename), layout_version))
elif flag == 'x' or (flag == 'c' and not exists):
self._pool_ptr = _err_check.check_null(
lib.pmemobj_create(_coerce_fn(filename), layout_version,
pool_size, mode))
elif flag == 'r':
raise ValueError("Read-only mode is not supported")
else:
raise ValueError("Invalid flag value {}".format(flag))
mm = self.mm = MemoryManager(self._pool_ptr)
pmem_root = lib.pmemobj_root(self._pool_ptr, ffi.sizeof('PRoot'))
pmem_root = ffi.cast('PRoot *', mm.direct(pmem_root))
type_table_oid = mm.otuple(pmem_root.type_table)
if type_table_oid == mm.OID_NULL:
with mm.transaction():
type_table_oid = mm._create_type_table()
mm.snapshot_range(pmem_root, ffi.sizeof('PObjPtr'))
pmem_root.type_table = type_table_oid
pmem_root.root_object = self.mm.persist(None)
pmem_root.clean_shutdown = self.mm.persist(False)
gc_needed = False
else:
mm._resurrect_type_table(type_table_oid)
gc_needed = not self.mm.resurrect(pmem_root.clean_shutdown)
self._pmem_root = pmem_root
# Make sure any objects orphaned by a crash are cleaned up.
if gc_needed:
self.gc()
def _resurrect_nvm_pmemobj_pool_PICKLE_SENTINEL(self, obj_ptr):
obj_ptr = ffi.cast('PVarObject *', obj_ptr)
body = ffi.cast('char *', obj_ptr) + ffi.sizeof('PVarObject')
s = ffi.buffer(body, obj_ptr.ob_size)[:]
obj = loads(s)
return obj
def _alloc_empty_table(self, tablesize):
return self._p_mm.zalloc(ffi.sizeof('PSetEntry') * tablesize,
type_num=SET_POBJPTR_ARRAY_TYPE_NUM)
