Exemple #1
0
 def _getObjectList(self):
     if hasattr(sys, 'getobjects'):
         return sys.getobjects(0)
     else:
         gc.collect()
         # grab gc's object list
         gc_objects = gc.get_objects()
         # use get_referents to find everything else
         objects = gc_objects
         objects.append(__builtin__.__dict__)
         nextObjList = gc_objects
         found = set()
         found.add(id(objects))
         found.add(id(found))
         found.add(id(gc_objects))
         for obj in objects:
             found.add(id(obj))
         # repeatedly call get_referents until we can't find any more objects
         while len(nextObjList):
             curObjList = nextObjList
             nextObjList = []
             for obj in curObjList:
                 refs = gc.get_referents(obj)
                 for ref in refs:
                     if id(ref) not in found:
                         found.add(id(ref))
                         objects.append(ref)
                         nextObjList.append(ref)
         return objects
     """
    def _getObjectList(self):
        if hasattr(sys, 'getobjects'):
            return sys.getobjects(0)
        else:
            gc.collect()
            gc_objects = gc.get_objects()
            objects = gc_objects
            objects.append(__builtin__.__dict__)
            nextObjList = gc_objects
            found = set()
            found.add(id(objects))
            found.add(id(found))
            found.add(id(gc_objects))
            for obj in objects:
                found.add(id(obj))

            while len(nextObjList):
                curObjList = nextObjList
                nextObjList = []
                for obj in curObjList:
                    refs = gc.get_referents(obj)
                    for ref in refs:
                        if id(ref) not in found:
                            found.add(id(ref))
                            objects.append(ref)
                            nextObjList.append(ref)

            return objects
Exemple #3
0
    def update(self, dumpObjects=False):
        obs = sys.getobjects(0)
        type2count = {}
        type2all = {}
        type2o = {}
        for o in obs:
            all = sys.getrefcount(o)
            t = type(o)
            if t in type2count:
                type2count[t] += 1
                type2all[t] += all
                type2o[t].append(o)
            else:
                type2count[t] = 1
                type2all[t] = all
                type2o[t] = [o]

        ct = [(type2count[t] - self.type2count.get(t, 0),
               type2all[t] - self.type2all.get(t, 0), t)
              for t in type2count.iterkeys()]
        ct.sort()
        ct.reverse()
        for delta1, delta2, t in ct:
            if delta1 or delta2:
                print "%-55s %8d %8d" % (t, delta1, delta2)
                if not dumpObjects: continue
                oldids = self.type2ids.get(t, [])
                for o in type2o[t]:
                    if id(o) not in oldids:
                        print 'leak?', o

        self.type2count = type2count
        self.type2all = type2all
        self.type2ids = dict([(k, [id(o) for o in olist])
                              for k, olist in type2o.iteritems()])
 def startTest(self, test):
     unittest._TextTestResult.startTest(self, test)
     if self.showAll:
         N = len(sys.getobjects(0))
         self._totnumobj = N
         self._totrefcnt = sys.gettotalrefcount()
     return
 def stopTest(self, test):
     if self.showAll:
         N = len(sys.getobjects(0))
         self.stream.write("objects: %d ===> %d;   " % (self._totnumobj, N))
         self.stream.write("refcnts: %d ===> %d\n" % (self._totrefcnt,
                           sys.gettotalrefcount()))
     return
    def _getObjectList(self):
        if hasattr(sys, 'getobjects'):
            return sys.getobjects(0)
        else:
            gc.collect()
            gc_objects = gc.get_objects()
            objects = gc_objects
            objects.append(__builtin__.__dict__)
            nextObjList = gc_objects
            found = set()
            found.add(id(objects))
            found.add(id(found))
            found.add(id(gc_objects))
            for obj in objects:
                found.add(id(obj))

            while len(nextObjList):
                curObjList = nextObjList
                nextObjList = []
                for obj in curObjList:
                    refs = gc.get_referents(obj)
                    for ref in refs:
                        if id(ref) not in found:
                            found.add(id(ref))
                            objects.append(ref)
                            nextObjList.append(ref)
                            continue

            return objects
Exemple #7
0
    def update(self):
        obs = sys.getobjects(0)
        type2count = {}
        type2all = {}
        for o in obs:
            all = sys.getrefcount(o)
            t = type(o)
            if t in type2count:
                type2count[t] += 1
                type2all[t] += all
            else:
                type2count[t] = 1
                type2all[t] = all

        ct = [(type2count[t] - self.type2count.get(t, 0),
               type2all[t] - self.type2all.get(t, 0), t)
              for t in type2count.iterkeys()]
        ct.sort()
        ct.reverse()
        for delta1, delta2, t in ct:
            if delta1 or delta2:
                print "%-55s %8d %8d" % (t, delta1, delta2)

        self.type2count = type2count
        self.type2all = type2all
Exemple #8
0
    def update(self):
        obs = sys.getobjects(0)
        type2count = {}
        type2all = {}
        for o in obs:
            all = sys.getrefcount(o)
            t = type(o)
            if t in type2count:
                type2count[t] += 1
                type2all[t] += all
            else:
                type2count[t] = 1
                type2all[t] = all

        ct = [(type2count[t] - self.type2count.get(t, 0),
               type2all[t] - self.type2all.get(t, 0),
               t)
              for t in type2count.iterkeys()]
        ct.sort()
        ct.reverse()
        for delta1, delta2, t in ct:
            if delta1 or delta2:
                print ("%-55s %8d %8d" % (t, delta1, delta2))

        self.type2count = type2count
        self.type2all = type2all
Exemple #9
0
    def update(self):
        obs = sys.getobjects(0)
        type2count = {}
        type2all = {}
        for o in obs:
            all = sys.getrefcount(o)

            if type(o) is str and o == '<dummy key>':
                # avoid dictionary madness
                continue
            t = type(o)
            if t in type2count:
                type2count[t] += 1
                type2all[t] += all
            else:
                type2count[t] = 1
                type2all[t] = all

        ct = [(type2count[t] - self.type2count.get(t, 0),
               type2all[t] - self.type2all.get(t, 0), t)
              for t in type2count.iterkeys()]
        ct.sort()
        ct.reverse()
        printed = False
        for delta1, delta2, t in ct:
            if delta1 or delta2:
                if not printed:
                    print "%-55s %8s %8s" % ('', 'insts', 'refs')
                    printed = True
                print "%-55s %8d %8d" % (t, delta1, delta2)

        self.type2count = type2count
        self.type2all = type2all
Exemple #10
0
    def update(self):
        obs = sys.getobjects(0)
        type2count = {}
        type2all = {}
        for o in obs:
            all = sys.getrefcount(o)

            if type(o) is str and o == '<dummy key>':
                # avoid dictionary madness
                continue
            t = type(o)
            if t in type2count:
                type2count[t] += 1
                type2all[t] += all
            else:
                type2count[t] = 1
                type2all[t] = all

        ct = [(type2count[t] - self.type2count.get(t, 0),
               type2all[t] - self.type2all.get(t, 0),
               t)
              for t in type2count.iterkeys()]
        ct.sort()
        ct.reverse()
        printed = False
        for delta1, delta2, t in ct:
            if delta1 or delta2:
                if not printed:
                    print "%-55s %8s %8s" % ('', 'insts', 'refs')
                    printed = True
                print "%-55s %8d %8d" % (t, delta1, delta2)

        self.type2count = type2count
        self.type2all = type2all
Exemple #11
0
 def _getObjectList(self):
     if hasattr(sys, 'getobjects'):
         return sys.getobjects(0)
     else:
         gc.collect()
         # grab gc's object list
         gc_objects = gc.get_objects()
         # use get_referents to find everything else
         objects = gc_objects
         objects.append(__builtin__.__dict__)
         nextObjList = gc_objects
         found = set()
         found.add(id(objects))
         found.add(id(found))
         found.add(id(gc_objects))
         for obj in objects:
             found.add(id(obj))
         # repeatedly call get_referents until we can't find any more objects
         while len(nextObjList):
             curObjList = nextObjList
             nextObjList = []
             for obj in curObjList:
                 refs = gc.get_referents(obj)
                 for ref in refs:
                     if id(ref) not in found:
                         found.add(id(ref))
                         objects.append(ref)
                         nextObjList.append(ref)
         return objects
     """
Exemple #12
0
def _count_object_refs():  # noqa
    """
    This function is used for debugging leaking references between business
    function calls in the run_daemon function.  It relies on a cPython built
    with Py_TRACE_REFS.  In the absence of such a Python (the standard case)
    this function does not called and we don't do this expensive object
    counting.

    On Ubuntu I was able to get a debug version of python installed by doing:

        apt-get install python2.5-dbg

    Your mileage may vary on other platforms.  I had terrible problems trying
    to build Python from source with the Py_TRACE_REFS call and do not
    recommend trying that on Ubuntu.
    """
    gc.collect()
    ref_counts = {}

    # Count all of the objects
    for obj in sys.getobjects(sys.gettotalrefcount()):
        kind = type(obj)
        if kind in ref_counts:
            ref_counts[kind]['count'] += 1
        else:
            ref_counts[kind] = dict(kind=kind, count=1, delta=0)

    global _ref_counts
    if _ref_counts is None:
        # first time
        _ref_counts = ref_counts
        return

    # Calculate how many were created since last time
    for kind, record in ref_counts.items():
        if kind in _ref_counts:
            record['delta'] = record['count'] - _ref_counts[kind]['count']
        else:
            record['delta'] = record['count']
    _ref_counts = ref_counts

    # Print the top N new objects
    N = 20
    records = list(ref_counts.values())
    records.sort(key=lambda x: (x['delta'], x['count']), reverse=True)
    for record in records[:N]:
        print "DEBUG: created %d new instances of %s (Total: %d)" % (
            record['delta'],
            str(record['kind']),
            record['count'],
        )

    if gc.garbage:
        print "DEBUG: GARBAGE: %d/%d" % (len(gc.garbage), len(
            gc.get_objects()))
Exemple #13
0
def _count_object_refs():  # pragma NO COVERAGE
    """
    This function is used for debugging leaking references between business
    function calls in the run_daemon function.  It relies on a cPython built
    with Py_TRACE_REFS.  In the absence of such a Python (the standard case)
    this function does not called and we don't do this expensive object
    counting.

    On Ubuntu I was able to get a debug version of python installed by doing:

        apt-get install python2.5-dbg

    Your mileage may vary on other platforms.  I had terrible problems trying
    to build Python from source with the Py_TRACE_REFS call and do not
    recommend trying that on Ubuntu.
    """
    gc.collect()
    ref_counts = {}

    # Count all of the objects
    for obj in sys.getobjects(sys.gettotalrefcount()):
        kind = type(obj)
        if kind in ref_counts:
            ref_counts[kind]["count"] += 1
        else:
            ref_counts[kind] = dict(kind=kind, count=1, delta=0)

    global _ref_counts
    if _ref_counts == None:
        # first time
        _ref_counts = ref_counts
        return

    # Calculate how many were created since last time
    for kind, record in ref_counts.items():
        if kind in _ref_counts:
            record["delta"] = record["count"] - _ref_counts[kind]["count"]
        else:
            record["delta"] = record["count"]
    _ref_counts = ref_counts

    # Print the top N new objects
    N = 20
    records = list(ref_counts.values())
    records.sort(key=lambda x: (x["delta"], x["count"]), reverse=True)
    for record in records[:N]:
        print "DEBUG: created %d new instances of %s (Total: %d)" % (
            record["delta"],
            str(record["kind"]),
            record["count"],
        )

    if gc.garbage:
        print "DEBUG: GARBAGE: %d/%d" % (len(gc.garbage), len(gc.get_objects()))
 def _make_snapshot():
     gc.collect()
     # get all objects
     all = sys.getobjects(0)
     # get an array with the refcounts
     g = sys.getrefcount
     refs = array.array("l", (g(obj) for obj in all))
     # the lists have the same endind. Make comparison easier.
     all.reverse()
     refs.reverse()
     return all, refs
Exemple #15
0
def getrefs(i, depth):
    """Get the i'th object in memory, return objects that reference it"""
    import sys, gc, types
    o = sys.getobjects(i)[-1]
    for d in range(depth):
        for ref in gc.get_referrers(o):
            if type(ref) in (list, dict, types.InstanceType):
                if type(ref) is dict and 'copyright' in ref:
                    continue
                o = ref
                break
        else:
            print("Max depth ", d)
            return o
    return o
Exemple #16
0
    def testEnumNew_NoLeak(self):
        gc.collect()
        total = sys.gettotalrefcount()
        for idx in range(1000):
            ret = Qt.Key(42)

        gc.collect()
        delta = sys.gettotalrefcount() - total
        print("delta total refcount =", delta)
        if abs(delta) >= 10:
            all = [(sys.getrefcount(x), x) for x in sys.getobjects(0)]
            all.sort(key=lambda x: x[0], reverse=True)
            for ob in all[:10]:
                print(ob)
        self.assertTrue(abs(delta) < 10)
Exemple #17
0
def getrefs(i, depth):
	"""Get the i'th object in memory, return objects that reference it"""
	import sys, gc, types
	o = sys.getobjects(i)[-1]
	for d in range(depth):
		for ref in gc.get_referrers(o):
			if type(ref) in (types.ListType, types.DictType,
								types.InstanceType):
				if type(ref) is types.DictType and ref.has_key('copyright'):
					continue
				o = ref
				break
		else:
			print "Max depth ", d
			return o
	return o
Exemple #18
0
    def update(self):
        gc.collect()
        obs = sys.getobjects(0)
        type2count = {}
        type2all = {}
        n = 0
        for o in obs:
            if type(o) is str and o == '<dummy key>':
                # avoid dictionary madness
                continue

            all = sys.getrefcount(o) - 3
            n += all

            t = type(o)
            try:
                t = o.__class__
            except Exception:
                pass

            if t in type2count:
                type2count[t] += 1
                type2all[t] += all
            else:
                type2count[t] = 1
                type2all[t] = all


        ct = [(
               type_or_class_title(t),
               type2count[t] - self.type2count.get(t, 0),
               type2all[t] - self.type2all.get(t, 0),
               )
              for t in type2count.keys()]
        ct += [(
                type_or_class_title(t),
                - self.type2count[t],
                - self.type2all[t],
                )
               for t in self.type2count.keys()
               if t not in type2count]
        ct.sort()
        self.delta = ct
        self.type2count = type2count
        self.type2all = type2all
        self.n = n
Exemple #19
0
    def update(self):
        import sys
        import logging
        obs = sys.getobjects(0)
        type2count = {}
        type2all = {}
        for o in obs:
            all = sys.getrefcount(o)

            if type(o) is str and o == '<dummy key>':
                # avoid dictionary madness
                continue
            t = type(o)
            if t in type2count:
                type2count[t] += 1
                type2all[t] += all
            else:
                type2count[t] = 1
                type2all[t] = all

        ct = [(type2count[t] - self.type2count.get(t, 0),
               type2all[t] - self.type2all.get(t, 0),
               t)
              for t in type2count.iterkeys()]
        ct.sort()
        ct.reverse()
        printed = False
        log = logging.getLogger("cofriend.utils.read_from_pgsql")
        log.info("----------------------")
        log.info("Memory profiling")
        i = 0
        for delta1, delta2, t in ct:
            if delta1 or delta2:
                if not printed:
                    log.info("%-55s %8s %8s" % ('', 'insts', 'refs'))
                    printed = True

                log.info("%-55s %8d %8d" % (t, delta1, delta2))

                i += 1
                if i >= self.limit:
                    break 

        self.type2count = type2count
        self.type2all = type2all
Exemple #20
0
    def update(self):
        obs = sys.getobjects(0)
        type2count = {}
        type2all = {}
        for o in obs:
            all = sys.getrefcount(o)

            if type(o) is str and o == '<dummy key>':
                # avoid dictionary madness
                continue
            t = type(o)
            if t in type2count:
                type2count[t] += 1
                type2all[t] += all
            else:
                type2count[t] = 1
                type2all[t] = all

        ct = [(type2count[t] - self.type2count.get(t, 0),
               type2all[t] - self.type2all.get(t, 0),
               t)
              for t in type2count.iterkeys()]
        ct.sort()
        ct.reverse()
        printed = False

        logger.info("----------------------")
        logger.info("Memory profiling")
        i = 0
        for delta1, delta2, t in ct:
            if delta1 or delta2:
                if not printed:
                    logger.info("%-55s %8s %8s" % ('', 'insts', 'refs'))
                    printed = True

                logger.info("%-55s %8d %8d" % (t, delta1, delta2))

                i += 1
                if i >= self.limit:
                    break

        self.type2count = type2count
        self.type2all = type2all
Exemple #21
0
    def update(self):
        gc.collect()
        obs = sys.getobjects(0)
        type2count = {}
        type2all = {}
        n = 0
        for o in obs:
            if type(o) is str and o == '<dummy key>':
                # avoid dictionary madness
                continue

            all = sys.getrefcount(o) - 3
            n += all

            t = type(o)
            try:
                t = o.__class__
            except Exception:
                pass

            if t in type2count:
                type2count[t] += 1
                type2all[t] += all
            else:
                type2count[t] = 1
                type2all[t] = all

        ct = [(
            type_or_class_title(t),
            type2count[t] - self.type2count.get(t, 0),
            type2all[t] - self.type2all.get(t, 0),
        ) for t in type2count.keys()]
        ct += [(
            type_or_class_title(t),
            -self.type2count[t],
            -self.type2all[t],
        ) for t in self.type2count.keys() if t not in type2count]
        ct.sort()
        self.delta = ct
        self.type2count = type2count
        self.type2all = type2all
        self.n = n
Exemple #22
0
    def update(self, verbose=0):
        obs = sys.getobjects(0)
        type2count = {}
        type2all = {}
        for o in obs:
            all = sys.getrefcount(o)
            t = type(o)
            if verbose:
                #if t == types.TupleType:
                if isinstance(o, Group):
                #if isinstance(o, metaIsDescription):
                    print "-->", o, "refs:", all
                    refrs = gc.get_referrers(o)
                    trefrs = []
                    for refr in refrs:
                        trefrs.append(type(refr))
                    print "Referrers -->", refrs
                    print "Referrers types -->", trefrs
            #if t == types.StringType: print "-->",o
            if t in type2count:
                type2count[t] += 1
                type2all[t] += all
            else:
                type2count[t] = 1
                type2all[t] = all

        ct = [(type2count[t] - self.type2count.get(t, 0),
               type2all[t] - self.type2all.get(t, 0),
               t)
              for t in type2count.iterkeys()]
        ct.sort()
        ct.reverse()
        for delta1, delta2, t in ct:
            if delta1 or delta2:
                print "%-55s %8d %8d" % (t, delta1, delta2)

        self.type2count = type2count
        self.type2all = type2all
Exemple #23
0
    def update(self, verbose=0):
        obs = sys.getobjects(0)
        type2count = {}
        type2all = {}
        for o in obs:
            all = sys.getrefcount(o)
            t = type(o)
            if verbose:
                #if t == types.TupleType:
                if isinstance(o, Group):
                    #if isinstance(o, metaIsDescription):
                    print "-->", o, "refs:", all
                    refrs = gc.get_referrers(o)
                    trefrs = []
                    for refr in refrs:
                        trefrs.append(type(refr))
                    print "Referrers -->", refrs
                    print "Referrers types -->", trefrs
            #if t == types.StringType: print "-->",o
            if t in type2count:
                type2count[t] += 1
                type2all[t] += all
            else:
                type2count[t] = 1
                type2all[t] = all

        ct = [(type2count[t] - self.type2count.get(t, 0),
               type2all[t] - self.type2all.get(t, 0), t)
              for t in type2count.iterkeys()]
        ct.sort()
        ct.reverse()
        for delta1, delta2, t in ct:
            if delta1 or delta2:
                print "%-55s %8d %8d" % (t, delta1, delta2)

        self.type2count = type2count
        self.type2all = type2all
Exemple #24
0
 def get_objects(self):
     return sys.getobjects(0)  # pylint:disable=no-member
Exemple #25
0
#before = gc.get_objects()
#c.open(apt.progress.OpProgress())
#after = gc.get_objects()


# test update()
print "refcount before cache.update(): ", sys.gettotalrefcount()
c.update()
gc.collect()
print "refcount after cache.update(): ", sys.gettotalrefcount()
c.update()
gc.collect()
print "refcount after second cache.update(): ", sys.gettotalrefcount()
#pprint(sys.getobjects(20))


# test install()
c.open(apt.progress.OpProgress())
gc.collect()
print "refcount before cache['hello'].markInstall(): ", sys.gettotalrefcount()
c["hello"].markInstall()
c.commit(apt.progress.FetchProgress(), apt.progress.InstallProgress())
gc.collect()
print "refcount after: ", sys.gettotalrefcount()
c.open(apt.progress.OpProgress())
c["hello"].markDelete()
c.commit(apt.progress.FetchProgress(), apt.progress.InstallProgress())
gc.collect()
print "refcount after: ", sys.gettotalrefcount()
pprint(sys.getobjects(10))
Exemple #26
0
# FIXME: find a way to get a efficient diff
#before = gc.get_objects()
#c.open(apt.progress.OpProgress())
#after = gc.get_objects()

# test update()
print("refcount before cache.update(): ", sys.gettotalrefcount())
c.update()
gc.collect()
print("refcount after cache.update(): ", sys.gettotalrefcount())
c.update()
gc.collect()
print("refcount after second cache.update(): ", sys.gettotalrefcount())
#pprint(sys.getobjects(20))

# test install()
c.open(apt.progress.OpProgress())
gc.collect()
print("refcount before cache['hello'].mark_install(): ",
      sys.gettotalrefcount())
c["hello"].mark_install()
c.commit(apt.progress.FetchProgress(), apt.progress.InstallProgress())
gc.collect()
print("refcount after: ", sys.gettotalrefcount())
c.open(apt.progress.OpProgress())
c["hello"].mark_delete()
c.commit(apt.progress.FetchProgress(), apt.progress.InstallProgress())
gc.collect()
print("refcount after: ", sys.gettotalrefcount())
pprint(sys.getobjects(10))
Exemple #27
0
def _get_objects():
    gc.collect()
    if hasattr(sys, 'getobjects'):
        return sys.getobjects(sys.maxsize)
    return gc.get_objects()
def _get_objects():
    gc.collect()
    if hasattr(sys, 'getobjects'):
        return sys.getobjects(sys.maxsize)
    return gc.get_objects()