def collect_var_and_types(self):
#
# collect all variables and constants used in the body,
# and get their types now
#
# NOTE: cannot use dictionaries with Constants as keys, because
# Constants may hash and compare equal but have different lltypes
self.all_cached_consts = None # will be filled after implementation_end
mix = [self.graph.getreturnvar()]
self.more_ll_values = []
for block in self.graph.iterblocks():
mix.extend(block.inputargs)
for op in block.operations:
mix.extend(op.args)
mix.append(op.result)
for link in block.exits:
mix.extend(link.getextravars())
mix.extend(link.args)
if hasattr(link, 'llexitcase'):
self.more_ll_values.append(link.llexitcase)
elif link.exitcase is not None:
mix.append(Constant(link.exitcase))
uniquemix = []
seen = identity_dict()
for v in mix:
if v not in seen:
uniquemix.append(v)
seen[v] = True
self.vars = uniquemix
def track(*ll_objects):
"""Invoke a dot+pygame object reference tracker."""
lst = [MARKER]
size_gc_header = None
seen = identity_dict()
for ll_object in ll_objects:
if isinstance(ll_object, llmemory.GCHeaderOffset):
size_gc_header = ll_object
continue
#if isinstance(lltype.typeOf(ll_object), lltype.Ptr):
# ptr = lltype.normalizeptr(ll_object)
# if ptr is not None:
# ll_object = ptr._obj
# else:
# ll_object = None
if ll_object is not None and ll_object not in seen:
lst.append(ll_object)
seen[ll_object] = ll_object
page = LLRefTrackerPage(lst, size_gc_header)
# auto-expand one level, for now
auto_expand = 1
for i in range(auto_expand):
page = page.content()
for ll_object in lst[1:]:
for name, value in page.enum_content(ll_object):
if not isinstance(value, str) and value not in seen:
lst.append(value)
seen[value] = value
page = page.newpage(lst)
page.display()
def test_dissect_ll_instance():
assert list(dissect_ll_instance(1)) == [(Signed, 1)]
GcS = GcStruct("S", ('x', Signed))
s = malloc(GcS)
s.x = 1
assert list(dissect_ll_instance(s)) == [(Ptr(GcS), s), (GcS, s._obj), (Signed, 1)]
A = GcArray(('x', Signed))
a = malloc(A, 10)
for i in range(10):
a[i].x = i
expected = [(Ptr(A), a), (A, a._obj)]
for t in [((A.OF, a._obj.items[i]), (Signed, i)) for i in range(10)]:
expected.extend(t)
assert list(dissect_ll_instance(a)) == expected
R = GcStruct("R", ('r', Ptr(GcForwardReference())))
R.r.TO.become(R)
r = malloc(R)
r.r = r
r_expected = [(Ptr(R), r), (R, r._obj)]
assert list(dissect_ll_instance(r)) == r_expected
B = GcArray(Ptr(R))
b = malloc(B, 2)
b[0] = b[1] = r
b_expected = [(Ptr(B), b), (B, b._obj)]
assert list(dissect_ll_instance(b)) == b_expected + r_expected
memo = identity_dict()
assert list(dissect_ll_instance(r, None, memo)) == r_expected
assert list(dissect_ll_instance(b, None, memo)) == b_expected
def __init__(self, translator=None, standalone=False,
gcpolicyclass=None,
exctransformer=None,
thread_enabled=False,
sandbox=False):
self.translator = translator
self.standalone = standalone
self.sandbox = sandbox
if gcpolicyclass is None:
gcpolicyclass = gc.RefcountingGcPolicy
self.gcpolicy = gcpolicyclass(self, thread_enabled)
self.exctransformer = exctransformer
self.structdefnodes = {}
self.pendingsetupnodes = []
self.containernodes = {}
self.containerlist = []
self.idelayedfunctionnames = identity_dict()
self.delayedfunctionptrs = []
self.completedcontainers = 0
self.containerstats = {}
self.helpers = OrderedDict()
# late_initializations is for when the value you want to
# assign to a constant object is something C doesn't think is
# constant
self.late_initializations = []
self.namespace = CNameManager()
if translator is not None:
self.gctransformer = self.gcpolicy.gettransformer(translator)
self.completed = False
self.instrument_ncounter = 0
def collect_var_and_types(self):
#
# collect all variables and constants used in the body,
# and get their types now
#
# NOTE: cannot use dictionaries with Constants as keys, because
# Constants may hash and compare equal but have different lltypes
self.all_cached_consts = None # will be filled after implementation_end
mix = [self.graph.getreturnvar()]
self.more_ll_values = []
for block in self.graph.iterblocks():
mix.extend(block.inputargs)
for op in block.operations:
mix.extend(op.args)
mix.append(op.result)
for link in block.exits:
mix.extend(link.getextravars())
mix.extend(link.args)
if hasattr(link, 'llexitcase'):
self.more_ll_values.append(link.llexitcase)
elif link.exitcase is not None:
mix.append(Constant(link.exitcase))
uniquemix = []
seen = identity_dict()
for v in mix:
if v not in seen:
uniquemix.append(v)
seen[v] = True
self.vars = uniquemix
def test_dissect_ll_instance():
assert list(dissect_ll_instance(1)) == [(Signed, 1)]
GcS = GcStruct("S", ('x', Signed))
s = malloc(GcS)
s.x = 1
assert list(dissect_ll_instance(s)) == [(Ptr(GcS), s), (GcS, s._obj),
(Signed, 1)]
A = GcArray(('x', Signed))
a = malloc(A, 10)
for i in range(10):
a[i].x = i
expected = [(Ptr(A), a), (A, a._obj)]
for t in [((A.OF, a._obj.items[i]), (Signed, i)) for i in range(10)]:
expected.extend(t)
assert list(dissect_ll_instance(a)) == expected
R = GcStruct("R", ('r', Ptr(GcForwardReference())))
R.r.TO.become(R)
r = malloc(R)
r.r = r
r_expected = [(Ptr(R), r), (R, r._obj)]
assert list(dissect_ll_instance(r)) == r_expected
B = GcArray(Ptr(R))
b = malloc(B, 2)
b[0] = b[1] = r
b_expected = [(Ptr(B), b), (B, b._obj)]
assert list(dissect_ll_instance(b)) == b_expected + r_expected
memo = identity_dict()
assert list(dissect_ll_instance(r, None, memo)) == r_expected
assert list(dissect_ll_instance(b, None, memo)) == b_expected
def __init__(self, rtyper, classdef, gcflavor='ignored'):
AbstractInstanceRepr.__init__(self, rtyper, classdef)
self.baserepr = None
if self.classdef is None:
self.lowleveltype = OBJECT
else:
b = self.classdef.basedef
if b is not None:
self.baserepr = getinstancerepr(rtyper, b)
b = self.baserepr.lowleveltype
else:
b = OBJECT
if hasattr(classdef.classdesc.pyobj, '_rpython_hints'):
hints = classdef.classdesc.pyobj._rpython_hints
else:
hints = {}
hints = self._check_for_immutable_hints(hints)
self.lowleveltype = ootype.Instance(classdef.name,
b, {}, {},
_hints=hints)
self.iprebuiltinstances = identity_dict()
self.object_type = self.lowleveltype
self.gcflavor = gcflavor
def track(*ll_objects):
"""Invoke a dot+pygame object reference tracker."""
lst = [MARKER]
size_gc_header = None
seen = identity_dict()
for ll_object in ll_objects:
if isinstance(ll_object, llmemory.GCHeaderOffset):
size_gc_header = ll_object
continue
#if isinstance(lltype.typeOf(ll_object), lltype.Ptr):
# ptr = lltype.normalizeptr(ll_object)
# if ptr is not None:
# ll_object = ptr._obj
# else:
# ll_object = None
if ll_object is not None and ll_object not in seen:
lst.append(ll_object)
seen[ll_object] = ll_object
page = LLRefTrackerPage(lst, size_gc_header)
# auto-expand one level, for now
auto_expand = 1
for i in range(auto_expand):
page = page.content()
for ll_object in lst[1:]:
for name, value in page.enum_content(ll_object):
if not isinstance(value, str) and value not in seen:
lst.append(value)
seen[value] = value
page = page.newpage(lst)
page.display()
def break_cycles(vertices, edges):
"""Enumerates a reasonably minimal set of edges that must be removed to
make the graph acyclic."""
import py
py.test.skip("break_cycles() is not used any more")
# the approach is as follows: starting from each root, find some set
# of cycles using a simple depth-first search. Then break the
# edge that is part of the most cycles. Repeat.
remaining_edges = edges.copy()
progress = True
roots_finished = set()
while progress:
roots = list(find_roots(vertices, remaining_edges))
#print '%d inital roots' % (len(roots,))
progress = False
for root in roots:
if root in roots_finished:
continue
cycles = all_cycles(root, vertices, remaining_edges)
if not cycles:
roots_finished.add(root)
continue
#print 'from root %r: %d cycles' % (root, len(cycles))
allcycles = identity_dict()
edge2cycles = {}
for cycle in cycles:
allcycles[cycle] = cycle
for edge in cycle:
edge2cycles.setdefault(edge, []).append(cycle)
edge_weights = {}
for edge, cycle in edge2cycles.iteritems():
edge_weights[edge] = len(cycle)
while allcycles:
max_weight = 0
max_edge = None
for edge, weight in edge_weights.iteritems():
if weight > max_weight:
max_edge = edge
max_weight = weight
if max_edge is None:
break
# kill this edge
yield max_edge
progress = True
# unregister all cycles that have just been broken
for broken_cycle in edge2cycles[max_edge]:
broken_cycle = allcycles.pop(broken_cycle, ())
for edge in broken_cycle:
edge_weights[edge] -= 1
lst = remaining_edges[max_edge.source][:]
lst.remove(max_edge)
remaining_edges[max_edge.source] = lst
assert is_acyclic(vertices, remaining_edges)
def break_cycles(vertices, edges):
"""Enumerates a reasonably minimal set of edges that must be removed to
make the graph acyclic."""
import py; py.test.skip("break_cycles() is not used any more")
# the approach is as follows: starting from each root, find some set
# of cycles using a simple depth-first search. Then break the
# edge that is part of the most cycles. Repeat.
remaining_edges = edges.copy()
progress = True
roots_finished = set()
while progress:
roots = list(find_roots(vertices, remaining_edges))
#print '%d inital roots' % (len(roots,))
progress = False
for root in roots:
if root in roots_finished:
continue
cycles = all_cycles(root, vertices, remaining_edges)
if not cycles:
roots_finished.add(root)
continue
#print 'from root %r: %d cycles' % (root, len(cycles))
allcycles = identity_dict()
edge2cycles = {}
for cycle in cycles:
allcycles[cycle] = cycle
for edge in cycle:
edge2cycles.setdefault(edge, []).append(cycle)
edge_weights = {}
for edge, cycle in edge2cycles.iteritems():
edge_weights[edge] = len(cycle)
while allcycles:
max_weight = 0
max_edge = None
for edge, weight in edge_weights.iteritems():
if weight > max_weight:
max_edge = edge
max_weight = weight
if max_edge is None:
break
# kill this edge
yield max_edge
progress = True
# unregister all cycles that have just been broken
for broken_cycle in edge2cycles[max_edge]:
broken_cycle = allcycles.pop(broken_cycle, ())
for edge in broken_cycle:
edge_weights[edge] -= 1
lst = remaining_edges[max_edge.source][:]
lst.remove(max_edge)
remaining_edges[max_edge.source] = lst
assert is_acyclic(vertices, remaining_edges)
def __init__(self, rtyper, classdef, gcflavor='gc'):
self.rtyper = rtyper
self.classdef = classdef
if classdef is None:
self.object_type = OBJECT_BY_FLAVOR[LLFLAVOR[gcflavor]]
else:
ForwardRef = lltype.FORWARDREF_BY_FLAVOR[LLFLAVOR[gcflavor]]
self.object_type = ForwardRef()
self.iprebuiltinstances = identity_dict()
self.lowleveltype = Ptr(self.object_type)
self.gcflavor = gcflavor
def __init__(self, rtyper, classdef, gcflavor='gc'):
AbstractInstanceRepr.__init__(self, rtyper, classdef)
if classdef is None:
self.object_type = OBJECT_BY_FLAVOR[LLFLAVOR[gcflavor]]
else:
ForwardRef = lltype.FORWARDREF_BY_FLAVOR[LLFLAVOR[gcflavor]]
self.object_type = ForwardRef()
self.iprebuiltinstances = identity_dict()
self.lowleveltype = Ptr(self.object_type)
self.gcflavor = gcflavor
def check_and_print_leaks(self):
rawrefcount._collect()
# check for sane refcnts
import gc
if 1: #ZZZ not self.enable_leak_checking:
leakfinder.stop_tracking_allocations(check=False)
return False
leaking = False
state = self.space.fromcache(RefcountState)
gc.collect()
lost_objects_w = identity_dict()
lost_objects_w.update(
(key, None) for key in self.frozen_refcounts.keys())
for w_obj, obj in state.py_objects_w2r.iteritems():
base_refcnt = self.frozen_refcounts.get(w_obj)
delta = obj.c_ob_refcnt
if base_refcnt is not None:
delta -= base_refcnt
lost_objects_w.pop(w_obj)
if delta != 0:
leaking = True
print >> sys.stderr, "Leaking %r: %i references" % (w_obj,
delta)
try:
weakref.ref(w_obj)
except TypeError:
lifeline = None
else:
lifeline = state.lifeline_dict.get(w_obj)
if lifeline is not None:
refcnt = lifeline.pyo.c_ob_refcnt
if refcnt > 0:
print >> sys.stderr, "\tThe object also held by C code."
else:
referrers_repr = []
for o in gc.get_referrers(w_obj):
try:
repr_str = repr(o)
except TypeError as e:
repr_str = "%s (type of o is %s)" % (str(e),
type(o))
referrers_repr.append(repr_str)
referrers = ", ".join(referrers_repr)
print >>sys.stderr, "\tThe object is referenced by these objects:", \
referrers
for w_obj in lost_objects_w:
print >> sys.stderr, "Lost object %r" % (w_obj, )
leaking = True
# the actual low-level leak checking is done by pypy.tool.leakfinder,
# enabled automatically by pypy.conftest.
return leaking
def check_and_print_leaks(self):
rawrefcount._collect()
# check for sane refcnts
import gc
if 1: #ZZZ not self.enable_leak_checking:
leakfinder.stop_tracking_allocations(check=False)
return False
leaking = False
state = self.space.fromcache(RefcountState)
gc.collect()
lost_objects_w = identity_dict()
lost_objects_w.update((key, None) for key in self.frozen_refcounts.keys())
for w_obj, obj in state.py_objects_w2r.iteritems():
base_refcnt = self.frozen_refcounts.get(w_obj)
delta = obj.c_ob_refcnt
if base_refcnt is not None:
delta -= base_refcnt
lost_objects_w.pop(w_obj)
if delta != 0:
leaking = True
print >>sys.stderr, "Leaking %r: %i references" % (w_obj, delta)
try:
weakref.ref(w_obj)
except TypeError:
lifeline = None
else:
lifeline = state.lifeline_dict.get(w_obj)
if lifeline is not None:
refcnt = lifeline.pyo.c_ob_refcnt
if refcnt > 0:
print >>sys.stderr, "\tThe object also held by C code."
else:
referrers_repr = []
for o in gc.get_referrers(w_obj):
try:
repr_str = repr(o)
except TypeError as e:
repr_str = "%s (type of o is %s)" % (str(e), type(o))
referrers_repr.append(repr_str)
referrers = ", ".join(referrers_repr)
print >>sys.stderr, "\tThe object is referenced by these objects:", \
referrers
for w_obj in lost_objects_w:
print >>sys.stderr, "Lost object %r" % (w_obj, )
leaking = True
# the actual low-level leak checking is done by pypy.tool.leakfinder,
# enabled automatically by pypy.conftest.
return leaking
def implementation_begin(self):
SSI_to_SSA(self.graph)
self.collect_var_and_types()
self.blocknum = {}
for block in self.graph.iterblocks():
self.blocknum[block] = len(self.blocknum)
db = self.db
lltypes = identity_dict()
for v in self.vars:
T = v.concretetype
typename = db.gettype(T)
lltypes[v] = T, typename
self.illtypes = lltypes
self.innerloops = {} # maps the loop's header block to a Loop()
for loop in find_inner_loops(self.graph, Bool):
self.innerloops[loop.headblock] = loop
def implementation_begin(self):
SSI_to_SSA(self.graph)
self.collect_var_and_types()
self.blocknum = {}
for block in self.graph.iterblocks():
self.blocknum[block] = len(self.blocknum)
db = self.db
lltypes = identity_dict()
for v in self.vars:
T = v.concretetype
typename = db.gettype(T)
lltypes[v] = T, typename
self.illtypes = lltypes
self.innerloops = {} # maps the loop's header block to a Loop()
for loop in find_inner_loops(self.graph, Bool):
self.innerloops[loop.headblock] = loop
def __init__(self,
translator=None,
standalone=False,
gcpolicyclass=None,
gchooks=None,
exctransformer=None,
thread_enabled=False,
sandbox=False,
split_gc_address_space=False,
reverse_debugger=False):
self.translator = translator
self.standalone = standalone
self.sandbox = sandbox
self.split_gc_address_space = split_gc_address_space
self.reverse_debugger = reverse_debugger
if gcpolicyclass is None:
gcpolicyclass = gc.RefcountingGcPolicy
self.gcpolicy = gcpolicyclass(self, thread_enabled)
self.exctransformer = exctransformer
self.structdefnodes = {}
self.pendingsetupnodes = []
self.containernodes = {}
self.containerlist = []
self.idelayedfunctionnames = identity_dict()
self.delayedfunctionptrs = []
self.completedcontainers = 0
self.containerstats = {}
# late_initializations is for when the value you want to
# assign to a constant object is something C doesn't think is
# constant
self.late_initializations = []
self.namespace = CNameManager()
if translator is not None:
self.gctransformer = self.gcpolicy.gettransformer(
translator, gchooks)
self.completed = False
self.instrument_ncounter = 0
if translator and translator.config.translation.countfieldaccess:
self.all_field_names = set()
else:
self.all_field_names = None
def __init__(self, GCClass, lltype2vtable=None):
self.GCClass = GCClass
self.lltype2vtable = lltype2vtable
self.make_type_info_group()
self.id_of_type = {} # {LLTYPE: type_id}
self.iseen_roots = identity_dict()
# the following are lists of addresses of gc pointers living inside the
# prebuilt structures. It should list all the locations that could
# possibly point to a GC heap object.
# this lists contains pointers in GcStructs and GcArrays
self.addresses_of_static_ptrs = []
# this lists contains pointers in raw Structs and Arrays
self.addresses_of_static_ptrs_in_nongc = []
# for debugging, the following list collects all the prebuilt
# GcStructs and GcArrays
self.all_prebuilt_gc = []
self._special_funcptrs = {}
self.offsettable_cache = {}
def __init__(self, GCClass, lltype2vtable=None):
self.GCClass = GCClass
self.lltype2vtable = lltype2vtable
self.make_type_info_group()
self.id_of_type = {} # {LLTYPE: type_id}
self.iseen_roots = identity_dict()
# the following are lists of addresses of gc pointers living inside the
# prebuilt structures. It should list all the locations that could
# possibly point to a GC heap object.
# this lists contains pointers in GcStructs and GcArrays
self.addresses_of_static_ptrs = []
# this lists contains pointers in raw Structs and Arrays
self.addresses_of_static_ptrs_in_nongc = []
# for debugging, the following list collects all the prebuilt
# GcStructs and GcArrays
self.all_prebuilt_gc = []
self._special_funcptrs = {}
self.offsettable_cache = {}
def _set_locals(self):
# this code is partly borrowed from
# rpython.translator.c.funcgen.FunctionCodeGenerator
# TODO: refactoring to avoid code duplication
self.blocknum = {}
graph = self.graph
mix = [graph.getreturnvar()]
for block in graph.iterblocks():
self.blocknum[block] = len(self.blocknum)
mix.extend(block.inputargs)
for op in block.operations:
mix.extend(op.args)
mix.append(op.result)
if getattr(op, "cleanup", None) is not None:
cleanup_finally, cleanup_except = op.cleanup
for cleanupop in cleanup_finally + cleanup_except:
mix.extend(cleanupop.args)
mix.append(cleanupop.result)
for link in block.exits:
mix.extend(link.getextravars())
mix.extend(link.args)
# filter only locals variables, i.e.:
# - must be variables
# - must appear only once
# - must not be function parameters
# - must not have 'void' type
args = {}
for ctstype, name in self.args:
args[name] = True
locals = []
seen = identity_dict()
for v in mix:
is_var = isinstance(v, flowmodel.Variable)
if v not in seen and is_var and v.name not in args and v.concretetype is not ootype.Void:
locals.append(self.cts.llvar_to_cts(v))
seen[v] = True
self.locals = locals
def _set_locals(self):
# this code is partly borrowed from
# rpython.translator.c.funcgen.FunctionCodeGenerator
# TODO: refactoring to avoid code duplication
self.blocknum = {}
graph = self.graph
mix = [graph.getreturnvar()]
for block in graph.iterblocks():
self.blocknum[block] = len(self.blocknum)
mix.extend(block.inputargs)
for op in block.operations:
mix.extend(op.args)
mix.append(op.result)
if getattr(op, "cleanup", None) is not None:
cleanup_finally, cleanup_except = op.cleanup
for cleanupop in cleanup_finally + cleanup_except:
mix.extend(cleanupop.args)
mix.append(cleanupop.result)
for link in block.exits:
mix.extend(link.getextravars())
mix.extend(link.args)
# filter only locals variables, i.e.:
# - must be variables
# - must appear only once
# - must not be function parameters
# - must not have 'void' type
args = {}
for ctstype, name in self.args:
args[name] = True
locals = []
seen = identity_dict()
for v in mix:
is_var = isinstance(v, flowmodel.Variable)
if v not in seen and is_var and v.name not in args and v.concretetype is not ootype.Void:
locals.append(self.cts.llvar_to_cts(v))
seen[v] = True
self.locals = locals
def __init__(self, rtyper, classdef, gcflavor='ignored'):
AbstractInstanceRepr.__init__(self, rtyper, classdef)
self.baserepr = None
if self.classdef is None:
self.lowleveltype = OBJECT
else:
b = self.classdef.basedef
if b is not None:
self.baserepr = getinstancerepr(rtyper, b)
b = self.baserepr.lowleveltype
else:
b = OBJECT
if hasattr(classdef.classdesc.pyobj, '_rpython_hints'):
hints = classdef.classdesc.pyobj._rpython_hints
else:
hints = {}
hints = self._check_for_immutable_hints(hints)
self.lowleveltype = ootype.Instance(classdef.name, b, {}, {}, _hints = hints)
self.iprebuiltinstances = identity_dict()
self.object_type = self.lowleveltype
self.gcflavor = gcflavor
def __init__(self):
self.data = identity_dict() # {_key(addr): value}
def __init__(self):
self.data = identity_dict() # {_key(addr): value}
