def save_current_state_away(self, shadowstackref, ncontext):
"""Save the current state away into 'shadowstackref'.
This either works, or raise MemoryError and nothing is done.
To do a switch, first call save_current_state_away() or
forget_current_state(), and then call restore_state_from()
or start_fresh_new_state().
"""
fresh_free_fullstack = shadowstackref.prepare_free_slot()
if self.unused_full_stack:
if fresh_free_fullstack:
llmemory.raw_free(fresh_free_fullstack)
elif fresh_free_fullstack:
self.unused_full_stack = fresh_free_fullstack
else:
self._prepare_unused_stack()
#
shadowstackref.base = self.gcdata.root_stack_base
shadowstackref.top = self.gcdata.root_stack_top
shadowstackref.context = ncontext
ll_assert(shadowstackref.base <= shadowstackref.top,
"save_current_state_away: broken shadowstack")
shadowstackref.attach()
#
# cannot use llop.gc_writebarrier() here, because
# we are in a minimally-transformed GC helper :-/
gc = self.gcdata.gc
if hasattr(gc.__class__, 'write_barrier'):
shadowstackadr = llmemory.cast_ptr_to_adr(shadowstackref)
gc.write_barrier(shadowstackadr)
#
self.gcdata.root_stack_top = llmemory.NULL # to detect missing restore
def stacklet_destructor(stacklet):
sscopy = stacklet.s_sscopy
if sscopy:
llmemory.raw_free(sscopy)
h = stacklet.s_handle
if h:
_c.destroy(h)
def f():
addr = raw_malloc(INT_SIZE*100)
ll = AddressStack()
ll.append(addr)
ll.append(addr + INT_SIZE*1)
ll.append(addr + INT_SIZE*2)
a = ll.pop()
res = (a - INT_SIZE*2 == addr)
a = ll.pop()
res = res and (a - INT_SIZE*1 == addr)
res = res and ll.non_empty()
a = ll.pop()
res = res and a == addr
res = res and not ll.non_empty()
ll.append(addr)
for i in range(300):
ll.append(addr + INT_SIZE*i)
for i in range(299, -1, -1):
a = ll.pop()
res = res and (a - INT_SIZE*i == addr)
for i in range(300):
ll.append(addr + INT_SIZE*i)
for i in range(299, -1, -1):
a = ll.pop()
res = res and (a - INT_SIZE*i == addr)
ll.delete()
ll = AddressStack()
ll.append(addr)
ll.append(addr + INT_SIZE*1)
ll.append(addr + INT_SIZE*2)
ll.delete()
raw_free(addr)
return res
def forget_current_state(self):
ll_assert(self.gcdata.root_stack_base == self.gcdata.root_stack_top,
"forget_current_state: shadowstack not empty!")
if self.unused_full_stack:
llmemory.raw_free(self.unused_full_stack)
self.unused_full_stack = self.gcdata.root_stack_base
self.gcdata.root_stack_top = llmemory.NULL # to detect missing restore
def f():
adr = llmemory.raw_malloc(sizeofs)
s = llmemory.cast_adr_to_ptr(adr, STRUCTPTR)
s.y = 5 # does not crash
result = (adr + offsety).signed[0] * 10 + int(offsety < sizeofs)
llmemory.raw_free(adr)
return result
def forget_current_state(self):
ll_assert(self.gcdata.root_stack_base == self.gcdata.root_stack_top,
"forget_current_state: shadowstack not empty!")
if self.unused_full_stack:
llmemory.raw_free(self.unused_full_stack)
self.unused_full_stack = self.gcdata.root_stack_base
self.gcdata.root_stack_top = llmemory.NULL # to detect missing restore
def f():
adr = llmemory.raw_malloc(sizeofs)
s = llmemory.cast_adr_to_ptr(adr, STRUCTPTR)
s.y = 5 # does not crash
result = (adr + offsety).signed[0] * 10 + int(offsety < sizeofs)
llmemory.raw_free(adr)
return result
def stacklet_destructor(stacklet):
sscopy = stacklet.s_sscopy
if sscopy:
llmemory.raw_free(sscopy)
h = stacklet.s_handle
if h:
_c.destroy(h)
def f():
addr = raw_malloc(INT_SIZE * 100)
ll = AddressStack()
ll.append(addr)
ll.append(addr + INT_SIZE * 1)
ll.append(addr + INT_SIZE * 2)
a = ll.pop()
res = (a - INT_SIZE * 2 == addr)
a = ll.pop()
res = res and (a - INT_SIZE * 1 == addr)
res = res and ll.non_empty()
a = ll.pop()
res = res and a == addr
res = res and not ll.non_empty()
ll.append(addr)
for i in range(300):
ll.append(addr + INT_SIZE * i)
for i in range(299, -1, -1):
a = ll.pop()
res = res and (a - INT_SIZE * i == addr)
for i in range(300):
ll.append(addr + INT_SIZE * i)
for i in range(299, -1, -1):
a = ll.pop()
res = res and (a - INT_SIZE * i == addr)
ll.delete()
ll = AddressStack()
ll.append(addr)
ll.append(addr + INT_SIZE * 1)
ll.append(addr + INT_SIZE * 2)
ll.delete()
raw_free(addr)
return res
def sscopy_attach_shadow_stack(sscopy):
base = llop.gc_adr_of_root_stack_base(llmemory.Address).address[0]
ll_assert(llop.gc_adr_of_root_stack_top(llmemory.Address).address[0]==base,
"attach_shadow_stack: ss is not empty?")
length_bytes = sscopy.signed[0]
llmemory.raw_memcopy(sscopy + SIZEADDR, base, length_bytes)
llop.gc_adr_of_root_stack_top(llmemory.Address).address[0] = (
base + length_bytes)
llmemory.raw_free(sscopy)
def call_qsort_rec(r):
if r > 0:
g = GcObject()
g.num += r
call_qsort_rec(r - 1)
assert g.num == 1234 + r
else:
raw = llmemory.raw_malloc(5)
qsort(raw, 5, 1, cb_compare_callback)
llmemory.raw_free(raw)
def sscopy_attach_shadow_stack(sscopy):
base = llop.gc_adr_of_root_stack_base(llmemory.Address).address[0]
ll_assert(
llop.gc_adr_of_root_stack_top(llmemory.Address).address[0] == base,
"attach_shadow_stack: ss is not empty?")
length_bytes = sscopy.signed[0]
llmemory.raw_memcopy(sscopy + SIZEADDR, base, length_bytes)
llop.gc_adr_of_root_stack_top(llmemory.Address).address[0] = (base +
length_bytes)
llmemory.raw_free(sscopy)
def _resize(self, base, used, new_depth):
new_size = sizeofaddr * new_depth
ll_assert(used <= new_size, "shadowstack resize: overflow detected")
addr = llmemory.raw_malloc(new_size)
if addr == llmemory.NULL:
raise MemoryError
# note that we don't know the total memory size of 'base', but we
# know the size of the part that is used right now, and we only need
# to copy that
llmemory.raw_memmove(base, addr, used)
llmemory.raw_free(base)
return addr
def increase_root_stack_depth(self, new_depth):
if new_depth <= self.root_stack_depth:
return # can't easily decrease the size
if self.unused_full_stack:
llmemory.raw_free(self.unused_full_stack)
self.unused_full_stack = llmemory.NULL
used = self.gcdata.root_stack_top - self.gcdata.root_stack_base
addr = self._resize(self.gcdata.root_stack_base, used, new_depth)
self.gcdata.root_stack_base = addr
self.gcdata.root_stack_top = addr + used
# no gc operations above: we just switched shadowstacks
if self.has_threads:
self._resize_thread_shadowstacks(new_depth)
self.root_stack_depth = new_depth
def ll_rebuild(shadowstackref, fullstack_base):
if shadowstackref.fsindex > 0:
shadowstackref.detach()
return fullstack_base
else:
# make an expanded copy of the compact shadowstack stored in
# 'shadowstackref' and free that
compact = shadowstackref.base
size = shadowstackref.top - compact
shadowstackref.base = fullstack_base
shadowstackref.top = fullstack_base + size
llmemory.raw_memcopy(compact, fullstack_base, size)
llmemory.raw_free(compact)
return llmemory.NULL
def shadowstack_destructor(shadowstackref):
if root_walker.stacklet_support:
from rpython.rlib import _rffi_stacklet as _c
h = shadowstackref.context
h = llmemory.cast_adr_to_ptr(h, _c.handle)
shadowstackref.context = llmemory.NULL
#
base = shadowstackref.base
shadowstackref.base = llmemory.NULL
shadowstackref.top = llmemory.NULL
llmemory.raw_free(base)
#
if root_walker.stacklet_support:
if h:
_c.destroy(h)
def shadowstack_destructor(shadowstackref):
if root_walker.stacklet_support:
from rpython.rlib import _rffi_stacklet as _c
h = shadowstackref.context
h = llmemory.cast_adr_to_ptr(h, _c.handle)
shadowstackref.context = llmemory.NULL
#
base = shadowstackref.base
shadowstackref.base = llmemory.NULL
shadowstackref.top = llmemory.NULL
llmemory.raw_free(base)
#
if root_walker.stacklet_support:
if h:
_c.destroy(h)
def test_big_access(self):
AddressStack = get_address_stack()
addrs = [raw_malloc(llmemory.sizeof(lltype.Signed))
for i in range(3000)]
ll = AddressStack()
for i in range(3000):
print i
ll.append(addrs[i])
for i in range(3000)[::-1]:
a = ll.pop()
assert a == addrs[i]
for i in range(3000):
print i
ll.append(addrs[i])
for i in range(3000)[::-1]:
a = ll.pop()
assert a == addrs[i]
ll.delete()
for addr in addrs:
raw_free(addr)
def test_big_access(self):
AddressStack = get_address_stack()
addrs = [
raw_malloc(llmemory.sizeof(lltype.Signed)) for i in range(3000)
]
ll = AddressStack()
for i in range(3000):
print i
ll.append(addrs[i])
for i in range(3000)[::-1]:
a = ll.pop()
assert a == addrs[i]
for i in range(3000):
print i
ll.append(addrs[i])
for i in range(3000)[::-1]:
a = ll.pop()
assert a == addrs[i]
ll.delete()
for addr in addrs:
raw_free(addr)
def test_simple_access(self):
AddressStack = get_address_stack()
addr0 = raw_malloc(llmemory.sizeof(lltype.Signed))
addr1 = raw_malloc(llmemory.sizeof(lltype.Signed))
addr2 = raw_malloc(llmemory.sizeof(lltype.Signed))
ll = AddressStack()
ll.append(addr0)
ll.append(addr1)
ll.append(addr2)
assert ll.non_empty()
a = ll.pop()
assert a == addr2
assert ll.non_empty()
a = ll.pop()
assert a == addr1
assert ll.non_empty()
a = ll.pop()
assert a == addr0
assert not ll.non_empty()
ll.append(addr0)
ll.delete()
ll = AddressStack()
ll.append(addr0)
ll.append(addr1)
ll.append(addr2)
ll.append(NULL)
a = ll.pop()
assert a == NULL
ll.delete()
raw_free(addr2)
raw_free(addr1)
raw_free(addr0)
def test_simple_access(self):
AddressStack = get_address_stack()
addr0 = raw_malloc(llmemory.sizeof(lltype.Signed))
addr1 = raw_malloc(llmemory.sizeof(lltype.Signed))
addr2 = raw_malloc(llmemory.sizeof(lltype.Signed))
ll = AddressStack()
ll.append(addr0)
ll.append(addr1)
ll.append(addr2)
assert ll.non_empty()
a = ll.pop()
assert a == addr2
assert ll.non_empty()
a = ll.pop()
assert a == addr1
assert ll.non_empty()
a = ll.pop()
assert a == addr0
assert not ll.non_empty()
ll.append(addr0)
ll.delete()
ll = AddressStack()
ll.append(addr0)
ll.append(addr1)
ll.append(addr2)
ll.append(NULL)
a = ll.pop()
assert a == NULL
ll.delete()
raw_free(addr2)
raw_free(addr1)
raw_free(addr0)
def op_raw_free(self, addr):
checkadr(addr)
llmemory.raw_free(addr)
def shadowstack_destructor(shadowstackref):
base = shadowstackref.base
shadowstackref.base = llmemory.NULL
shadowstackref.top = llmemory.NULL
llmemory.raw_free(base)
def __del__(self):
llmemory.raw_free(self.addr)
def free1(p):
llmemory.raw_free(p)
def free1(p):
llmemory.raw_free(p)
def __del__(self):
llmemory.raw_free(self.addr)
def shadowstack_destructor(shadowstackref):
base = shadowstackref.base
shadowstackref.base = llmemory.NULL
shadowstackref.top = llmemory.NULL
llmemory.raw_free(base)
def sweep_rawmalloced_objects(self, generation):
# free all the rawmalloced objects of the specified generation
# that have not been marked
if generation == 2:
objects = self.gen2_rawmalloced_objects
# generation 2 sweep: if A points to an object object B that
# moves from gen2 to gen3, it's possible that A no longer points
# to any gen2 object. In this case, A remains a bit too long in
# last_generation_root_objects, but this will be fixed by the
# next collect_last_generation_roots().
elif generation == 3:
objects = self.gen3_rawmalloced_objects
# generation 3 sweep: remove from last_generation_root_objects
# all the objects that we are about to free
gen3roots = self.last_generation_root_objects
newgen3roots = self.AddressStack()
while gen3roots.non_empty():
obj = gen3roots.pop()
if not (self.header(obj).tid & GCFLAG_UNVISITED):
newgen3roots.append(obj)
gen3roots.delete()
self.last_generation_root_objects = newgen3roots
else:
ll_assert(False, "bogus 'generation'")
return 0 # to please the flowspace
surviving_objects = self.AddressStack()
# Help the flow space
alive_count = alive_size = dead_count = dead_size = 0
debug = have_debug_prints()
while objects.non_empty():
obj = objects.pop()
tid = self.header(obj).tid
if tid & GCFLAG_UNVISITED:
if debug:
dead_count+=1
dead_size+=raw_malloc_usage(self.get_size_incl_hash(obj))
addr = obj - self.gcheaderbuilder.size_gc_header
llmemory.raw_free(addr)
else:
if debug:
alive_count+=1
alive_size+=raw_malloc_usage(self.get_size_incl_hash(obj))
if generation == 3:
surviving_objects.append(obj)
elif generation == 2:
ll_assert((tid & GCFLAG_AGE_MASK) < GCFLAG_AGE_MAX,
"wrong age for generation 2 object")
tid += GCFLAG_AGE_ONE
if (tid & GCFLAG_AGE_MASK) == GCFLAG_AGE_MAX:
# the object becomes part of generation 3
self.gen3_rawmalloced_objects.append(obj)
# GCFLAG_NO_HEAP_PTRS not set yet, conservatively
self.last_generation_root_objects.append(obj)
else:
# the object stays in generation 2
tid |= GCFLAG_UNVISITED
surviving_objects.append(obj)
self.header(obj).tid = tid
objects.delete()
if generation == 2:
self.gen2_rawmalloced_objects = surviving_objects
elif generation == 3:
self.gen3_rawmalloced_objects = surviving_objects
debug_print("| [hyb] gen", generation,
"nonmoving now alive: ",
alive_size, "bytes in",
alive_count, "objs")
debug_print("| [hyb] gen", generation,
"nonmoving freed: ",
dead_size, "bytes in",
dead_count, "objs")
return alive_size
def op_raw_free(self, addr):
checkadr(addr)
llmemory.raw_free(addr)
def sweep_rawmalloced_objects(self, generation):
# free all the rawmalloced objects of the specified generation
# that have not been marked
if generation == 2:
objects = self.gen2_rawmalloced_objects
# generation 2 sweep: if A points to an object object B that
# moves from gen2 to gen3, it's possible that A no longer points
# to any gen2 object. In this case, A remains a bit too long in
# last_generation_root_objects, but this will be fixed by the
# next collect_last_generation_roots().
elif generation == 3:
objects = self.gen3_rawmalloced_objects
# generation 3 sweep: remove from last_generation_root_objects
# all the objects that we are about to free
gen3roots = self.last_generation_root_objects
newgen3roots = self.AddressStack()
while gen3roots.non_empty():
obj = gen3roots.pop()
if not (self.header(obj).tid & GCFLAG_UNVISITED):
newgen3roots.append(obj)
gen3roots.delete()
self.last_generation_root_objects = newgen3roots
else:
ll_assert(False, "bogus 'generation'")
return 0 # to please the flowspace
surviving_objects = self.AddressStack()
# Help the flow space
alive_count = alive_size = dead_count = dead_size = 0
debug = have_debug_prints()
while objects.non_empty():
obj = objects.pop()
tid = self.header(obj).tid
if tid & GCFLAG_UNVISITED:
if debug:
dead_count += 1
dead_size += raw_malloc_usage(self.get_size_incl_hash(obj))
addr = obj - self.gcheaderbuilder.size_gc_header
llmemory.raw_free(addr)
else:
if debug:
alive_count += 1
alive_size += raw_malloc_usage(self.get_size_incl_hash(obj))
if generation == 3:
surviving_objects.append(obj)
elif generation == 2:
ll_assert((tid & GCFLAG_AGE_MASK) < GCFLAG_AGE_MAX,
"wrong age for generation 2 object")
tid += GCFLAG_AGE_ONE
if (tid & GCFLAG_AGE_MASK) == GCFLAG_AGE_MAX:
# the object becomes part of generation 3
self.gen3_rawmalloced_objects.append(obj)
# GCFLAG_NO_HEAP_PTRS not set yet, conservatively
self.last_generation_root_objects.append(obj)
else:
# the object stays in generation 2
tid |= GCFLAG_UNVISITED
surviving_objects.append(obj)
self.header(obj).tid = tid
objects.delete()
if generation == 2:
self.gen2_rawmalloced_objects = surviving_objects
elif generation == 3:
self.gen3_rawmalloced_objects = surviving_objects
debug_print("| [hyb] gen", generation, "nonmoving now alive: ",
alive_size, "bytes in", alive_count, "objs")
debug_print("| [hyb] gen", generation, "nonmoving freed: ",
dead_size, "bytes in", dead_count, "objs")
return alive_size
