def _getObjectList(self):
if hasattr(sys, 'getobjects'):
return sys.getobjects(0)
else:
gc.collect()
# grab gc's object list
gc_objects = gc.get_objects()
# use get_referents to find everything else
objects = gc_objects
objects.append(__builtin__.__dict__)
nextObjList = gc_objects
found = set()
found.add(id(objects))
found.add(id(found))
found.add(id(gc_objects))
for obj in objects:
found.add(id(obj))
# repeatedly call get_referents until we can't find any more objects
while len(nextObjList):
curObjList = nextObjList
nextObjList = []
for obj in curObjList:
refs = gc.get_referents(obj)
for ref in refs:
if id(ref) not in found:
found.add(id(ref))
objects.append(ref)
nextObjList.append(ref)
return objects
"""
def _getObjectList(self):
if hasattr(sys, 'getobjects'):
return sys.getobjects(0)
else:
gc.collect()
gc_objects = gc.get_objects()
objects = gc_objects
objects.append(__builtin__.__dict__)
nextObjList = gc_objects
found = set()
found.add(id(objects))
found.add(id(found))
found.add(id(gc_objects))
for obj in objects:
found.add(id(obj))
while len(nextObjList):
curObjList = nextObjList
nextObjList = []
for obj in curObjList:
refs = gc.get_referents(obj)
for ref in refs:
if id(ref) not in found:
found.add(id(ref))
objects.append(ref)
nextObjList.append(ref)
return objects
def update(self, dumpObjects=False):
obs = sys.getobjects(0)
type2count = {}
type2all = {}
type2o = {}
for o in obs:
all = sys.getrefcount(o)
t = type(o)
if t in type2count:
type2count[t] += 1
type2all[t] += all
type2o[t].append(o)
else:
type2count[t] = 1
type2all[t] = all
type2o[t] = [o]
ct = [(type2count[t] - self.type2count.get(t, 0),
type2all[t] - self.type2all.get(t, 0), t)
for t in type2count.iterkeys()]
ct.sort()
ct.reverse()
for delta1, delta2, t in ct:
if delta1 or delta2:
print "%-55s %8d %8d" % (t, delta1, delta2)
if not dumpObjects: continue
oldids = self.type2ids.get(t, [])
for o in type2o[t]:
if id(o) not in oldids:
print 'leak?', o
self.type2count = type2count
self.type2all = type2all
self.type2ids = dict([(k, [id(o) for o in olist])
for k, olist in type2o.iteritems()])
def startTest(self, test):
unittest._TextTestResult.startTest(self, test)
if self.showAll:
N = len(sys.getobjects(0))
self._totnumobj = N
self._totrefcnt = sys.gettotalrefcount()
return
def stopTest(self, test):
if self.showAll:
N = len(sys.getobjects(0))
self.stream.write("objects: %d ===> %d; " % (self._totnumobj, N))
self.stream.write("refcnts: %d ===> %d\n" % (self._totrefcnt,
sys.gettotalrefcount()))
return
def _getObjectList(self):
if hasattr(sys, 'getobjects'):
return sys.getobjects(0)
else:
gc.collect()
gc_objects = gc.get_objects()
objects = gc_objects
objects.append(__builtin__.__dict__)
nextObjList = gc_objects
found = set()
found.add(id(objects))
found.add(id(found))
found.add(id(gc_objects))
for obj in objects:
found.add(id(obj))
while len(nextObjList):
curObjList = nextObjList
nextObjList = []
for obj in curObjList:
refs = gc.get_referents(obj)
for ref in refs:
if id(ref) not in found:
found.add(id(ref))
objects.append(ref)
nextObjList.append(ref)
continue
return objects
def update(self):
obs = sys.getobjects(0)
type2count = {}
type2all = {}
for o in obs:
all = sys.getrefcount(o)
t = type(o)
if t in type2count:
type2count[t] += 1
type2all[t] += all
else:
type2count[t] = 1
type2all[t] = all
ct = [(type2count[t] - self.type2count.get(t, 0),
type2all[t] - self.type2all.get(t, 0), t)
for t in type2count.iterkeys()]
ct.sort()
ct.reverse()
for delta1, delta2, t in ct:
if delta1 or delta2:
print "%-55s %8d %8d" % (t, delta1, delta2)
self.type2count = type2count
self.type2all = type2all
def update(self):
obs = sys.getobjects(0)
type2count = {}
type2all = {}
for o in obs:
all = sys.getrefcount(o)
t = type(o)
if t in type2count:
type2count[t] += 1
type2all[t] += all
else:
type2count[t] = 1
type2all[t] = all
ct = [(type2count[t] - self.type2count.get(t, 0),
type2all[t] - self.type2all.get(t, 0),
t)
for t in type2count.iterkeys()]
ct.sort()
ct.reverse()
for delta1, delta2, t in ct:
if delta1 or delta2:
print ("%-55s %8d %8d" % (t, delta1, delta2))
self.type2count = type2count
self.type2all = type2all
def update(self):
obs = sys.getobjects(0)
type2count = {}
type2all = {}
for o in obs:
all = sys.getrefcount(o)
if type(o) is str and o == '<dummy key>':
# avoid dictionary madness
continue
t = type(o)
if t in type2count:
type2count[t] += 1
type2all[t] += all
else:
type2count[t] = 1
type2all[t] = all
ct = [(type2count[t] - self.type2count.get(t, 0),
type2all[t] - self.type2all.get(t, 0), t)
for t in type2count.iterkeys()]
ct.sort()
ct.reverse()
printed = False
for delta1, delta2, t in ct:
if delta1 or delta2:
if not printed:
print "%-55s %8s %8s" % ('', 'insts', 'refs')
printed = True
print "%-55s %8d %8d" % (t, delta1, delta2)
self.type2count = type2count
self.type2all = type2all
def update(self):
obs = sys.getobjects(0)
type2count = {}
type2all = {}
for o in obs:
all = sys.getrefcount(o)
if type(o) is str and o == '<dummy key>':
# avoid dictionary madness
continue
t = type(o)
if t in type2count:
type2count[t] += 1
type2all[t] += all
else:
type2count[t] = 1
type2all[t] = all
ct = [(type2count[t] - self.type2count.get(t, 0),
type2all[t] - self.type2all.get(t, 0),
t)
for t in type2count.iterkeys()]
ct.sort()
ct.reverse()
printed = False
for delta1, delta2, t in ct:
if delta1 or delta2:
if not printed:
print "%-55s %8s %8s" % ('', 'insts', 'refs')
printed = True
print "%-55s %8d %8d" % (t, delta1, delta2)
self.type2count = type2count
self.type2all = type2all
def _getObjectList(self):
if hasattr(sys, 'getobjects'):
return sys.getobjects(0)
else:
gc.collect()
# grab gc's object list
gc_objects = gc.get_objects()
# use get_referents to find everything else
objects = gc_objects
objects.append(__builtin__.__dict__)
nextObjList = gc_objects
found = set()
found.add(id(objects))
found.add(id(found))
found.add(id(gc_objects))
for obj in objects:
found.add(id(obj))
# repeatedly call get_referents until we can't find any more objects
while len(nextObjList):
curObjList = nextObjList
nextObjList = []
for obj in curObjList:
refs = gc.get_referents(obj)
for ref in refs:
if id(ref) not in found:
found.add(id(ref))
objects.append(ref)
nextObjList.append(ref)
return objects
"""
def _count_object_refs(): # noqa
"""
This function is used for debugging leaking references between business
function calls in the run_daemon function. It relies on a cPython built
with Py_TRACE_REFS. In the absence of such a Python (the standard case)
this function does not called and we don't do this expensive object
counting.
On Ubuntu I was able to get a debug version of python installed by doing:
apt-get install python2.5-dbg
Your mileage may vary on other platforms. I had terrible problems trying
to build Python from source with the Py_TRACE_REFS call and do not
recommend trying that on Ubuntu.
"""
gc.collect()
ref_counts = {}
# Count all of the objects
for obj in sys.getobjects(sys.gettotalrefcount()):
kind = type(obj)
if kind in ref_counts:
ref_counts[kind]['count'] += 1
else:
ref_counts[kind] = dict(kind=kind, count=1, delta=0)
global _ref_counts
if _ref_counts is None:
# first time
_ref_counts = ref_counts
return
# Calculate how many were created since last time
for kind, record in ref_counts.items():
if kind in _ref_counts:
record['delta'] = record['count'] - _ref_counts[kind]['count']
else:
record['delta'] = record['count']
_ref_counts = ref_counts
# Print the top N new objects
N = 20
records = list(ref_counts.values())
records.sort(key=lambda x: (x['delta'], x['count']), reverse=True)
for record in records[:N]:
print "DEBUG: created %d new instances of %s (Total: %d)" % (
record['delta'],
str(record['kind']),
record['count'],
)
if gc.garbage:
print "DEBUG: GARBAGE: %d/%d" % (len(gc.garbage), len(
gc.get_objects()))
def _count_object_refs(): # pragma NO COVERAGE
"""
This function is used for debugging leaking references between business
function calls in the run_daemon function. It relies on a cPython built
with Py_TRACE_REFS. In the absence of such a Python (the standard case)
this function does not called and we don't do this expensive object
counting.
On Ubuntu I was able to get a debug version of python installed by doing:
apt-get install python2.5-dbg
Your mileage may vary on other platforms. I had terrible problems trying
to build Python from source with the Py_TRACE_REFS call and do not
recommend trying that on Ubuntu.
"""
gc.collect()
ref_counts = {}
# Count all of the objects
for obj in sys.getobjects(sys.gettotalrefcount()):
kind = type(obj)
if kind in ref_counts:
ref_counts[kind]["count"] += 1
else:
ref_counts[kind] = dict(kind=kind, count=1, delta=0)
global _ref_counts
if _ref_counts == None:
# first time
_ref_counts = ref_counts
return
# Calculate how many were created since last time
for kind, record in ref_counts.items():
if kind in _ref_counts:
record["delta"] = record["count"] - _ref_counts[kind]["count"]
else:
record["delta"] = record["count"]
_ref_counts = ref_counts
# Print the top N new objects
N = 20
records = list(ref_counts.values())
records.sort(key=lambda x: (x["delta"], x["count"]), reverse=True)
for record in records[:N]:
print "DEBUG: created %d new instances of %s (Total: %d)" % (
record["delta"],
str(record["kind"]),
record["count"],
)
if gc.garbage:
print "DEBUG: GARBAGE: %d/%d" % (len(gc.garbage), len(gc.get_objects()))
def _make_snapshot():
gc.collect()
# get all objects
all = sys.getobjects(0)
# get an array with the refcounts
g = sys.getrefcount
refs = array.array("l", (g(obj) for obj in all))
# the lists have the same endind. Make comparison easier.
all.reverse()
refs.reverse()
return all, refs
def getrefs(i, depth):
"""Get the i'th object in memory, return objects that reference it"""
import sys, gc, types
o = sys.getobjects(i)[-1]
for d in range(depth):
for ref in gc.get_referrers(o):
if type(ref) in (list, dict, types.InstanceType):
if type(ref) is dict and 'copyright' in ref:
continue
o = ref
break
else:
print("Max depth ", d)
return o
return o
def testEnumNew_NoLeak(self):
gc.collect()
total = sys.gettotalrefcount()
for idx in range(1000):
ret = Qt.Key(42)
gc.collect()
delta = sys.gettotalrefcount() - total
print("delta total refcount =", delta)
if abs(delta) >= 10:
all = [(sys.getrefcount(x), x) for x in sys.getobjects(0)]
all.sort(key=lambda x: x[0], reverse=True)
for ob in all[:10]:
print(ob)
self.assertTrue(abs(delta) < 10)
def getrefs(i, depth):
"""Get the i'th object in memory, return objects that reference it"""
import sys, gc, types
o = sys.getobjects(i)[-1]
for d in range(depth):
for ref in gc.get_referrers(o):
if type(ref) in (types.ListType, types.DictType,
types.InstanceType):
if type(ref) is types.DictType and ref.has_key('copyright'):
continue
o = ref
break
else:
print "Max depth ", d
return o
return o
def update(self):
gc.collect()
obs = sys.getobjects(0)
type2count = {}
type2all = {}
n = 0
for o in obs:
if type(o) is str and o == '<dummy key>':
# avoid dictionary madness
continue
all = sys.getrefcount(o) - 3
n += all
t = type(o)
try:
t = o.__class__
except Exception:
pass
if t in type2count:
type2count[t] += 1
type2all[t] += all
else:
type2count[t] = 1
type2all[t] = all
ct = [(
type_or_class_title(t),
type2count[t] - self.type2count.get(t, 0),
type2all[t] - self.type2all.get(t, 0),
)
for t in type2count.keys()]
ct += [(
type_or_class_title(t),
- self.type2count[t],
- self.type2all[t],
)
for t in self.type2count.keys()
if t not in type2count]
ct.sort()
self.delta = ct
self.type2count = type2count
self.type2all = type2all
self.n = n
def update(self):
import sys
import logging
obs = sys.getobjects(0)
type2count = {}
type2all = {}
for o in obs:
all = sys.getrefcount(o)
if type(o) is str and o == '<dummy key>':
# avoid dictionary madness
continue
t = type(o)
if t in type2count:
type2count[t] += 1
type2all[t] += all
else:
type2count[t] = 1
type2all[t] = all
ct = [(type2count[t] - self.type2count.get(t, 0),
type2all[t] - self.type2all.get(t, 0),
t)
for t in type2count.iterkeys()]
ct.sort()
ct.reverse()
printed = False
log = logging.getLogger("cofriend.utils.read_from_pgsql")
log.info("----------------------")
log.info("Memory profiling")
i = 0
for delta1, delta2, t in ct:
if delta1 or delta2:
if not printed:
log.info("%-55s %8s %8s" % ('', 'insts', 'refs'))
printed = True
log.info("%-55s %8d %8d" % (t, delta1, delta2))
i += 1
if i >= self.limit:
break
self.type2count = type2count
self.type2all = type2all
def update(self):
obs = sys.getobjects(0)
type2count = {}
type2all = {}
for o in obs:
all = sys.getrefcount(o)
if type(o) is str and o == '<dummy key>':
# avoid dictionary madness
continue
t = type(o)
if t in type2count:
type2count[t] += 1
type2all[t] += all
else:
type2count[t] = 1
type2all[t] = all
ct = [(type2count[t] - self.type2count.get(t, 0),
type2all[t] - self.type2all.get(t, 0),
t)
for t in type2count.iterkeys()]
ct.sort()
ct.reverse()
printed = False
logger.info("----------------------")
logger.info("Memory profiling")
i = 0
for delta1, delta2, t in ct:
if delta1 or delta2:
if not printed:
logger.info("%-55s %8s %8s" % ('', 'insts', 'refs'))
printed = True
logger.info("%-55s %8d %8d" % (t, delta1, delta2))
i += 1
if i >= self.limit:
break
self.type2count = type2count
self.type2all = type2all
def update(self):
gc.collect()
obs = sys.getobjects(0)
type2count = {}
type2all = {}
n = 0
for o in obs:
if type(o) is str and o == '<dummy key>':
# avoid dictionary madness
continue
all = sys.getrefcount(o) - 3
n += all
t = type(o)
try:
t = o.__class__
except Exception:
pass
if t in type2count:
type2count[t] += 1
type2all[t] += all
else:
type2count[t] = 1
type2all[t] = all
ct = [(
type_or_class_title(t),
type2count[t] - self.type2count.get(t, 0),
type2all[t] - self.type2all.get(t, 0),
) for t in type2count.keys()]
ct += [(
type_or_class_title(t),
-self.type2count[t],
-self.type2all[t],
) for t in self.type2count.keys() if t not in type2count]
ct.sort()
self.delta = ct
self.type2count = type2count
self.type2all = type2all
self.n = n
def update(self, verbose=0):
obs = sys.getobjects(0)
type2count = {}
type2all = {}
for o in obs:
all = sys.getrefcount(o)
t = type(o)
if verbose:
#if t == types.TupleType:
if isinstance(o, Group):
#if isinstance(o, metaIsDescription):
print "-->", o, "refs:", all
refrs = gc.get_referrers(o)
trefrs = []
for refr in refrs:
trefrs.append(type(refr))
print "Referrers -->", refrs
print "Referrers types -->", trefrs
#if t == types.StringType: print "-->",o
if t in type2count:
type2count[t] += 1
type2all[t] += all
else:
type2count[t] = 1
type2all[t] = all
ct = [(type2count[t] - self.type2count.get(t, 0),
type2all[t] - self.type2all.get(t, 0),
t)
for t in type2count.iterkeys()]
ct.sort()
ct.reverse()
for delta1, delta2, t in ct:
if delta1 or delta2:
print "%-55s %8d %8d" % (t, delta1, delta2)
self.type2count = type2count
self.type2all = type2all
def update(self, verbose=0):
obs = sys.getobjects(0)
type2count = {}
type2all = {}
for o in obs:
all = sys.getrefcount(o)
t = type(o)
if verbose:
#if t == types.TupleType:
if isinstance(o, Group):
#if isinstance(o, metaIsDescription):
print "-->", o, "refs:", all
refrs = gc.get_referrers(o)
trefrs = []
for refr in refrs:
trefrs.append(type(refr))
print "Referrers -->", refrs
print "Referrers types -->", trefrs
#if t == types.StringType: print "-->",o
if t in type2count:
type2count[t] += 1
type2all[t] += all
else:
type2count[t] = 1
type2all[t] = all
ct = [(type2count[t] - self.type2count.get(t, 0),
type2all[t] - self.type2all.get(t, 0), t)
for t in type2count.iterkeys()]
ct.sort()
ct.reverse()
for delta1, delta2, t in ct:
if delta1 or delta2:
print "%-55s %8d %8d" % (t, delta1, delta2)
self.type2count = type2count
self.type2all = type2all
def get_objects(self):
return sys.getobjects(0) # pylint:disable=no-member
#before = gc.get_objects() #c.open(apt.progress.OpProgress()) #after = gc.get_objects() # test update() print "refcount before cache.update(): ", sys.gettotalrefcount() c.update() gc.collect() print "refcount after cache.update(): ", sys.gettotalrefcount() c.update() gc.collect() print "refcount after second cache.update(): ", sys.gettotalrefcount() #pprint(sys.getobjects(20)) # test install() c.open(apt.progress.OpProgress()) gc.collect() print "refcount before cache['hello'].markInstall(): ", sys.gettotalrefcount() c["hello"].markInstall() c.commit(apt.progress.FetchProgress(), apt.progress.InstallProgress()) gc.collect() print "refcount after: ", sys.gettotalrefcount() c.open(apt.progress.OpProgress()) c["hello"].markDelete() c.commit(apt.progress.FetchProgress(), apt.progress.InstallProgress()) gc.collect() print "refcount after: ", sys.gettotalrefcount() pprint(sys.getobjects(10))
# FIXME: find a way to get a efficient diff
#before = gc.get_objects()
#c.open(apt.progress.OpProgress())
#after = gc.get_objects()
# test update()
print("refcount before cache.update(): ", sys.gettotalrefcount())
c.update()
gc.collect()
print("refcount after cache.update(): ", sys.gettotalrefcount())
c.update()
gc.collect()
print("refcount after second cache.update(): ", sys.gettotalrefcount())
#pprint(sys.getobjects(20))
# test install()
c.open(apt.progress.OpProgress())
gc.collect()
print("refcount before cache['hello'].mark_install(): ",
sys.gettotalrefcount())
c["hello"].mark_install()
c.commit(apt.progress.FetchProgress(), apt.progress.InstallProgress())
gc.collect()
print("refcount after: ", sys.gettotalrefcount())
c.open(apt.progress.OpProgress())
c["hello"].mark_delete()
c.commit(apt.progress.FetchProgress(), apt.progress.InstallProgress())
gc.collect()
print("refcount after: ", sys.gettotalrefcount())
pprint(sys.getobjects(10))
def _get_objects():
gc.collect()
if hasattr(sys, 'getobjects'):
return sys.getobjects(sys.maxsize)
return gc.get_objects()
def _get_objects():
gc.collect()
if hasattr(sys, 'getobjects'):
return sys.getobjects(sys.maxsize)
return gc.get_objects()
