def _getObjectList(self): if hasattr(sys, 'getobjects'): return sys.getobjects(0) else: gc.collect() # grab gc's object list gc_objects = gc.get_objects() # use get_referents to find everything else objects = gc_objects objects.append(__builtin__.__dict__) nextObjList = gc_objects found = set() found.add(id(objects)) found.add(id(found)) found.add(id(gc_objects)) for obj in objects: found.add(id(obj)) # repeatedly call get_referents until we can't find any more objects while len(nextObjList): curObjList = nextObjList nextObjList = [] for obj in curObjList: refs = gc.get_referents(obj) for ref in refs: if id(ref) not in found: found.add(id(ref)) objects.append(ref) nextObjList.append(ref) return objects """
def _getObjectList(self): if hasattr(sys, 'getobjects'): return sys.getobjects(0) else: gc.collect() gc_objects = gc.get_objects() objects = gc_objects objects.append(__builtin__.__dict__) nextObjList = gc_objects found = set() found.add(id(objects)) found.add(id(found)) found.add(id(gc_objects)) for obj in objects: found.add(id(obj)) while len(nextObjList): curObjList = nextObjList nextObjList = [] for obj in curObjList: refs = gc.get_referents(obj) for ref in refs: if id(ref) not in found: found.add(id(ref)) objects.append(ref) nextObjList.append(ref) return objects
def update(self, dumpObjects=False): obs = sys.getobjects(0) type2count = {} type2all = {} type2o = {} for o in obs: all = sys.getrefcount(o) t = type(o) if t in type2count: type2count[t] += 1 type2all[t] += all type2o[t].append(o) else: type2count[t] = 1 type2all[t] = all type2o[t] = [o] ct = [(type2count[t] - self.type2count.get(t, 0), type2all[t] - self.type2all.get(t, 0), t) for t in type2count.iterkeys()] ct.sort() ct.reverse() for delta1, delta2, t in ct: if delta1 or delta2: print "%-55s %8d %8d" % (t, delta1, delta2) if not dumpObjects: continue oldids = self.type2ids.get(t, []) for o in type2o[t]: if id(o) not in oldids: print 'leak?', o self.type2count = type2count self.type2all = type2all self.type2ids = dict([(k, [id(o) for o in olist]) for k, olist in type2o.iteritems()])
def startTest(self, test): unittest._TextTestResult.startTest(self, test) if self.showAll: N = len(sys.getobjects(0)) self._totnumobj = N self._totrefcnt = sys.gettotalrefcount() return
def stopTest(self, test): if self.showAll: N = len(sys.getobjects(0)) self.stream.write("objects: %d ===> %d; " % (self._totnumobj, N)) self.stream.write("refcnts: %d ===> %d\n" % (self._totrefcnt, sys.gettotalrefcount())) return
def _getObjectList(self): if hasattr(sys, 'getobjects'): return sys.getobjects(0) else: gc.collect() gc_objects = gc.get_objects() objects = gc_objects objects.append(__builtin__.__dict__) nextObjList = gc_objects found = set() found.add(id(objects)) found.add(id(found)) found.add(id(gc_objects)) for obj in objects: found.add(id(obj)) while len(nextObjList): curObjList = nextObjList nextObjList = [] for obj in curObjList: refs = gc.get_referents(obj) for ref in refs: if id(ref) not in found: found.add(id(ref)) objects.append(ref) nextObjList.append(ref) continue return objects
def update(self): obs = sys.getobjects(0) type2count = {} type2all = {} for o in obs: all = sys.getrefcount(o) t = type(o) if t in type2count: type2count[t] += 1 type2all[t] += all else: type2count[t] = 1 type2all[t] = all ct = [(type2count[t] - self.type2count.get(t, 0), type2all[t] - self.type2all.get(t, 0), t) for t in type2count.iterkeys()] ct.sort() ct.reverse() for delta1, delta2, t in ct: if delta1 or delta2: print "%-55s %8d %8d" % (t, delta1, delta2) self.type2count = type2count self.type2all = type2all
def update(self): obs = sys.getobjects(0) type2count = {} type2all = {} for o in obs: all = sys.getrefcount(o) t = type(o) if t in type2count: type2count[t] += 1 type2all[t] += all else: type2count[t] = 1 type2all[t] = all ct = [(type2count[t] - self.type2count.get(t, 0), type2all[t] - self.type2all.get(t, 0), t) for t in type2count.iterkeys()] ct.sort() ct.reverse() for delta1, delta2, t in ct: if delta1 or delta2: print ("%-55s %8d %8d" % (t, delta1, delta2)) self.type2count = type2count self.type2all = type2all
def update(self): obs = sys.getobjects(0) type2count = {} type2all = {} for o in obs: all = sys.getrefcount(o) if type(o) is str and o == '<dummy key>': # avoid dictionary madness continue t = type(o) if t in type2count: type2count[t] += 1 type2all[t] += all else: type2count[t] = 1 type2all[t] = all ct = [(type2count[t] - self.type2count.get(t, 0), type2all[t] - self.type2all.get(t, 0), t) for t in type2count.iterkeys()] ct.sort() ct.reverse() printed = False for delta1, delta2, t in ct: if delta1 or delta2: if not printed: print "%-55s %8s %8s" % ('', 'insts', 'refs') printed = True print "%-55s %8d %8d" % (t, delta1, delta2) self.type2count = type2count self.type2all = type2all
def _count_object_refs(): # noqa """ This function is used for debugging leaking references between business function calls in the run_daemon function. It relies on a cPython built with Py_TRACE_REFS. In the absence of such a Python (the standard case) this function does not called and we don't do this expensive object counting. On Ubuntu I was able to get a debug version of python installed by doing: apt-get install python2.5-dbg Your mileage may vary on other platforms. I had terrible problems trying to build Python from source with the Py_TRACE_REFS call and do not recommend trying that on Ubuntu. """ gc.collect() ref_counts = {} # Count all of the objects for obj in sys.getobjects(sys.gettotalrefcount()): kind = type(obj) if kind in ref_counts: ref_counts[kind]['count'] += 1 else: ref_counts[kind] = dict(kind=kind, count=1, delta=0) global _ref_counts if _ref_counts is None: # first time _ref_counts = ref_counts return # Calculate how many were created since last time for kind, record in ref_counts.items(): if kind in _ref_counts: record['delta'] = record['count'] - _ref_counts[kind]['count'] else: record['delta'] = record['count'] _ref_counts = ref_counts # Print the top N new objects N = 20 records = list(ref_counts.values()) records.sort(key=lambda x: (x['delta'], x['count']), reverse=True) for record in records[:N]: print "DEBUG: created %d new instances of %s (Total: %d)" % ( record['delta'], str(record['kind']), record['count'], ) if gc.garbage: print "DEBUG: GARBAGE: %d/%d" % (len(gc.garbage), len( gc.get_objects()))
def _count_object_refs(): # pragma NO COVERAGE """ This function is used for debugging leaking references between business function calls in the run_daemon function. It relies on a cPython built with Py_TRACE_REFS. In the absence of such a Python (the standard case) this function does not called and we don't do this expensive object counting. On Ubuntu I was able to get a debug version of python installed by doing: apt-get install python2.5-dbg Your mileage may vary on other platforms. I had terrible problems trying to build Python from source with the Py_TRACE_REFS call and do not recommend trying that on Ubuntu. """ gc.collect() ref_counts = {} # Count all of the objects for obj in sys.getobjects(sys.gettotalrefcount()): kind = type(obj) if kind in ref_counts: ref_counts[kind]["count"] += 1 else: ref_counts[kind] = dict(kind=kind, count=1, delta=0) global _ref_counts if _ref_counts == None: # first time _ref_counts = ref_counts return # Calculate how many were created since last time for kind, record in ref_counts.items(): if kind in _ref_counts: record["delta"] = record["count"] - _ref_counts[kind]["count"] else: record["delta"] = record["count"] _ref_counts = ref_counts # Print the top N new objects N = 20 records = list(ref_counts.values()) records.sort(key=lambda x: (x["delta"], x["count"]), reverse=True) for record in records[:N]: print "DEBUG: created %d new instances of %s (Total: %d)" % ( record["delta"], str(record["kind"]), record["count"], ) if gc.garbage: print "DEBUG: GARBAGE: %d/%d" % (len(gc.garbage), len(gc.get_objects()))
def _make_snapshot(): gc.collect() # get all objects all = sys.getobjects(0) # get an array with the refcounts g = sys.getrefcount refs = array.array("l", (g(obj) for obj in all)) # the lists have the same endind. Make comparison easier. all.reverse() refs.reverse() return all, refs
def getrefs(i, depth): """Get the i'th object in memory, return objects that reference it""" import sys, gc, types o = sys.getobjects(i)[-1] for d in range(depth): for ref in gc.get_referrers(o): if type(ref) in (list, dict, types.InstanceType): if type(ref) is dict and 'copyright' in ref: continue o = ref break else: print("Max depth ", d) return o return o
def testEnumNew_NoLeak(self): gc.collect() total = sys.gettotalrefcount() for idx in range(1000): ret = Qt.Key(42) gc.collect() delta = sys.gettotalrefcount() - total print("delta total refcount =", delta) if abs(delta) >= 10: all = [(sys.getrefcount(x), x) for x in sys.getobjects(0)] all.sort(key=lambda x: x[0], reverse=True) for ob in all[:10]: print(ob) self.assertTrue(abs(delta) < 10)
def getrefs(i, depth): """Get the i'th object in memory, return objects that reference it""" import sys, gc, types o = sys.getobjects(i)[-1] for d in range(depth): for ref in gc.get_referrers(o): if type(ref) in (types.ListType, types.DictType, types.InstanceType): if type(ref) is types.DictType and ref.has_key('copyright'): continue o = ref break else: print "Max depth ", d return o return o
def update(self): gc.collect() obs = sys.getobjects(0) type2count = {} type2all = {} n = 0 for o in obs: if type(o) is str and o == '<dummy key>': # avoid dictionary madness continue all = sys.getrefcount(o) - 3 n += all t = type(o) try: t = o.__class__ except Exception: pass if t in type2count: type2count[t] += 1 type2all[t] += all else: type2count[t] = 1 type2all[t] = all ct = [( type_or_class_title(t), type2count[t] - self.type2count.get(t, 0), type2all[t] - self.type2all.get(t, 0), ) for t in type2count.keys()] ct += [( type_or_class_title(t), - self.type2count[t], - self.type2all[t], ) for t in self.type2count.keys() if t not in type2count] ct.sort() self.delta = ct self.type2count = type2count self.type2all = type2all self.n = n
def update(self): import sys import logging obs = sys.getobjects(0) type2count = {} type2all = {} for o in obs: all = sys.getrefcount(o) if type(o) is str and o == '<dummy key>': # avoid dictionary madness continue t = type(o) if t in type2count: type2count[t] += 1 type2all[t] += all else: type2count[t] = 1 type2all[t] = all ct = [(type2count[t] - self.type2count.get(t, 0), type2all[t] - self.type2all.get(t, 0), t) for t in type2count.iterkeys()] ct.sort() ct.reverse() printed = False log = logging.getLogger("cofriend.utils.read_from_pgsql") log.info("----------------------") log.info("Memory profiling") i = 0 for delta1, delta2, t in ct: if delta1 or delta2: if not printed: log.info("%-55s %8s %8s" % ('', 'insts', 'refs')) printed = True log.info("%-55s %8d %8d" % (t, delta1, delta2)) i += 1 if i >= self.limit: break self.type2count = type2count self.type2all = type2all
def update(self): obs = sys.getobjects(0) type2count = {} type2all = {} for o in obs: all = sys.getrefcount(o) if type(o) is str and o == '<dummy key>': # avoid dictionary madness continue t = type(o) if t in type2count: type2count[t] += 1 type2all[t] += all else: type2count[t] = 1 type2all[t] = all ct = [(type2count[t] - self.type2count.get(t, 0), type2all[t] - self.type2all.get(t, 0), t) for t in type2count.iterkeys()] ct.sort() ct.reverse() printed = False logger.info("----------------------") logger.info("Memory profiling") i = 0 for delta1, delta2, t in ct: if delta1 or delta2: if not printed: logger.info("%-55s %8s %8s" % ('', 'insts', 'refs')) printed = True logger.info("%-55s %8d %8d" % (t, delta1, delta2)) i += 1 if i >= self.limit: break self.type2count = type2count self.type2all = type2all
def update(self): gc.collect() obs = sys.getobjects(0) type2count = {} type2all = {} n = 0 for o in obs: if type(o) is str and o == '<dummy key>': # avoid dictionary madness continue all = sys.getrefcount(o) - 3 n += all t = type(o) try: t = o.__class__ except Exception: pass if t in type2count: type2count[t] += 1 type2all[t] += all else: type2count[t] = 1 type2all[t] = all ct = [( type_or_class_title(t), type2count[t] - self.type2count.get(t, 0), type2all[t] - self.type2all.get(t, 0), ) for t in type2count.keys()] ct += [( type_or_class_title(t), -self.type2count[t], -self.type2all[t], ) for t in self.type2count.keys() if t not in type2count] ct.sort() self.delta = ct self.type2count = type2count self.type2all = type2all self.n = n
def update(self, verbose=0): obs = sys.getobjects(0) type2count = {} type2all = {} for o in obs: all = sys.getrefcount(o) t = type(o) if verbose: #if t == types.TupleType: if isinstance(o, Group): #if isinstance(o, metaIsDescription): print "-->", o, "refs:", all refrs = gc.get_referrers(o) trefrs = [] for refr in refrs: trefrs.append(type(refr)) print "Referrers -->", refrs print "Referrers types -->", trefrs #if t == types.StringType: print "-->",o if t in type2count: type2count[t] += 1 type2all[t] += all else: type2count[t] = 1 type2all[t] = all ct = [(type2count[t] - self.type2count.get(t, 0), type2all[t] - self.type2all.get(t, 0), t) for t in type2count.iterkeys()] ct.sort() ct.reverse() for delta1, delta2, t in ct: if delta1 or delta2: print "%-55s %8d %8d" % (t, delta1, delta2) self.type2count = type2count self.type2all = type2all
def get_objects(self): return sys.getobjects(0) # pylint:disable=no-member
#before = gc.get_objects() #c.open(apt.progress.OpProgress()) #after = gc.get_objects() # test update() print "refcount before cache.update(): ", sys.gettotalrefcount() c.update() gc.collect() print "refcount after cache.update(): ", sys.gettotalrefcount() c.update() gc.collect() print "refcount after second cache.update(): ", sys.gettotalrefcount() #pprint(sys.getobjects(20)) # test install() c.open(apt.progress.OpProgress()) gc.collect() print "refcount before cache['hello'].markInstall(): ", sys.gettotalrefcount() c["hello"].markInstall() c.commit(apt.progress.FetchProgress(), apt.progress.InstallProgress()) gc.collect() print "refcount after: ", sys.gettotalrefcount() c.open(apt.progress.OpProgress()) c["hello"].markDelete() c.commit(apt.progress.FetchProgress(), apt.progress.InstallProgress()) gc.collect() print "refcount after: ", sys.gettotalrefcount() pprint(sys.getobjects(10))
# FIXME: find a way to get a efficient diff #before = gc.get_objects() #c.open(apt.progress.OpProgress()) #after = gc.get_objects() # test update() print("refcount before cache.update(): ", sys.gettotalrefcount()) c.update() gc.collect() print("refcount after cache.update(): ", sys.gettotalrefcount()) c.update() gc.collect() print("refcount after second cache.update(): ", sys.gettotalrefcount()) #pprint(sys.getobjects(20)) # test install() c.open(apt.progress.OpProgress()) gc.collect() print("refcount before cache['hello'].mark_install(): ", sys.gettotalrefcount()) c["hello"].mark_install() c.commit(apt.progress.FetchProgress(), apt.progress.InstallProgress()) gc.collect() print("refcount after: ", sys.gettotalrefcount()) c.open(apt.progress.OpProgress()) c["hello"].mark_delete() c.commit(apt.progress.FetchProgress(), apt.progress.InstallProgress()) gc.collect() print("refcount after: ", sys.gettotalrefcount()) pprint(sys.getobjects(10))
def _get_objects(): gc.collect() if hasattr(sys, 'getobjects'): return sys.getobjects(sys.maxsize) return gc.get_objects()