Пример #1
0
    def test_get_referents(self):
        if test_support.due_to_ironpython_incompatibility(
            "http://tkbgitvstfat01:8080/WorkItemTracking/WorkItem.aspx?artifactMoniker=314470"
        ):
            return
        alist = [1, 3, 5]
        got = gc.get_referents(alist)
        got.sort()
        self.assertEqual(got, alist)

        atuple = tuple(alist)
        got = gc.get_referents(atuple)
        got.sort()
        self.assertEqual(got, alist)

        adict = {1: 3, 5: 7}
        expected = [1, 3, 5, 7]
        got = gc.get_referents(adict)
        got.sort()
        self.assertEqual(got, expected)

        got = gc.get_referents([1, 2], {3: 4}, (0, 0, 0))
        got.sort()
        self.assertEqual(got, [0, 0] + range(5))

        self.assertEqual(gc.get_referents(1, "a", 4j), [])
Пример #2
0
def test_gc():
    a = flowlet(lambda: None)

    assert len(get_referents(a.greenlet)) == 3
    b = ref(a.greenlet)
    del a
    collect()
    assert len(get_referents(b)) == 0
Пример #3
0
def test_get_referents():
    if is_cli or is_silverlight:
        AssertError(NotImplementedError, gc.get_referents,1,"hello",True)
        AssertError(NotImplementedError, gc.get_referents)
    else:
        gc.get_referents(1,"hello",True)
        gc.get_referents()
        
        class TempClass: pass
        AreEqual(gc.get_referents(TempClass).count('TempClass'), 1)
Пример #4
0
 def test_get_referents(self):
     if is_cli:
         self.assertRaises(NotImplementedError, gc.get_referents,1,"hello",True)
         self.assertRaises(NotImplementedError, gc.get_referents)
     else:
         gc.get_referents(1,"hello",True)
         gc.get_referents()
         
         class TempClass: pass
         self.assertEqual(gc.get_referents(TempClass).count('TempClass'), 1)
Пример #5
0
	def show_edge(source, target):
		if collapse_dicts and id(source) in dicts:
			for key, value in source.iteritems():
				if value is target:
					membership_edges.append((id(dicts[id(source)]), mapping.get(id(target), id(target)), quote(key)))
					return

		if mapping.get(id(target), None)==id(source):
			return

		if isinstance(target, dict):
			source_dict=getattr(source, "__dict__", None)
			if isinstance(source_dict, types.DictProxyType):
				source_dict=gc.get_referents(gc.get_referents(source_dict))[0]
			if target is source_dict:
				elementary_edges.append((id(source), mapping.get(id(target), id(target)), "__dict__"))
				return
		if isinstance(source, type):
			if target in source.__bases__:
				inheritance_edges.append((id(source), mapping.get(id(target), id(target)), ""))
				return

		if getattr(source, "__class__", None) is target:
			instance_edges.append((id(source), mapping.get(id(target), id(target)), ""))
			return
		elif getattr(source, "__self__", None) is target:
			ownership_edges.append((id(source), mapping.get(id(target), id(target)), ""))
			return
		elif isinstance(source, dict):
			for key, value in source.iteritems():
				if value is target:
					elementary_edges.append((id(source), mapping.get(id(target), id(target)), quote(repr(key))))
			return
		elif isinstance(source, types.FrameType) and source.f_back is target:
			traceback_edges.append((id(source), mapping.get(id(target), id(target)), ""))
			return

		try:
			if target in source:
				elementary_edges.append((id(source), mapping.get(id(target), id(target)), ""))
				return
		except:
			pass

		for name in dir(source):
			try:
				if target is getattr(source, name):
					elementary_edges.append((id(source), mapping.get(id(target), id(target)), name))
					return
			except:
				pass

		reference_edges.append((id(source), mapping.get(id(target), id(target)), ""))
Пример #6
0
def test_gc_traversal(w):
    sentinel = mock.Mock()
    w.set_serializer(sentinel)
    assert sentinel in gc.get_referents(w)

    sentinel = mock.Mock()
    w.set_log_fn(sentinel, 0)
    assert sentinel in gc.get_referents(w)

    w.add_function("test_method", 60, echo_function)
    print gc.get_referents(w)
    assert {'test_method': echo_function} in gc.get_referents(w)
Пример #7
0
 def hasPcapObj():
     count = 0
     for obj in gc.get_objects():
         #t = type(obj)
         if isinstance(obj, pcap.pcapObject) or \
                 isinstance(obj, event.EventCallbackHandle) or \
                 isinstance(obj, Capture._CaptureDescriptor) or \
                 isinstance(obj, Capture):
             if isinstance(obj, Capture):
                 print obj, gc.get_referents(gc.get_referents(obj))
             count += 1
     print "objects found:", count
Пример #8
0
    def check_owners(target):
        if id(target) in owners:
            return

        ignore.add(id(sys._getframe()))
        sources = gc.get_referrers(target)
        ignore.add(id(sources))
        try:
            master = None

            if isinstance(target, dict):
                for source in sources:
                    if id(source) in ignore:
                        continue

                    if skip_functions and \
                            isinstance(source, types.FunctionType) and \
                            source.__globals__ is target:
                        continue

                    reference = getattr(source, "__dict__", None)
                    if isinstance(reference, types.DictProxyType):
                        reference = gc.get_referents(gc.get_referents(reference))[0]

                    if target is reference:
                        if master is None:
                            master = source
                        else:
                            owners[id(target)] = None
                            return
            elif isinstance(target, tuple):
                for source in sources:
                    if id(source) in ignore:
                        continue

                    if getattr(source, "__bases__", None) is target or getattr(source, "__mro__", None) is target:
                        if master is None:
                            master = source
                        else:
                            owners[id(target)] = None
                            return

            if master is None:
                return
            else:
                owners[id(target)] = master
                return
        finally:
            ignore.remove(id(sources))
            ignore.remove(id(sys._getframe()))
            del sources
 def debug_info(self):
     result = []
     for oid, klass in self.persistent_classes.items():
         result.append((oid,
                         len(gc.get_referents(klass)),
                         type(klass).__name__,
                         klass._p_state,
                         ))
     for oid, value in self.data.items():
         result.append((oid,
                         len(gc.get_referents(value)),
                         type(value).__name__,
                         value._p_state,
                         ))
     return result
Пример #10
0
 def test_get_referents(self):
     import gc
     y = 12345
     z = 23456
     x = [y, z]
     lst = gc.get_referents(x)
     assert y in lst and z in lst
Пример #11
0
  def _inner(tar, gar = None, index = 0):
    if gar == tar:
      return [('end', 'find')]
    if index > TIMES:
      return
    if gar:
      res = [gar]
    else:
      res = [tar]

    for obj in res:
      subres = []
      if hasattr(obj, '__dict__'):
        subres.extend(obj.__dict__.items())
      for attr in gc.get_referents(obj):
        item = ('mem', attr)
        subres.append(item)

      for key, y in subres:
        if inspect.isbuiltin(y):
          continue
        tmp = _inner(tar, y, index + 1)
        if tmp:
          tmp.append((key, y))
          return tmp
    return []
Пример #12
0
def dump_description_of_object_refs(o, f):
    # This holds the ids of all referents that we've already dumped.
    dumped = set()

    # First, any __dict__ items
    try:
        itemsiter = o.__dict__.iteritems()
    except:
        pass
    else:
        for k, v in itemsiter:
            try:
                idr = id(v)
                if idr not in dumped:
                    dumped.add(idr)
                    f.write("%d:"%len(k))
                    f.write(k)
                    f.write(",")
                    f.write("%0x,"%idr)
            except:
                pass

    # Then anything else that gc.get_referents() returns.
    for r in gc.get_referents(o):
        idr = id(r)
        if idr not in dumped:
            dumped.add(idr)
            f.write("0:,%0x,"%idr)
Пример #13
0
def mem_dump(path):
    dump = open(os.path.join(root, path), 'w')
    with dump as wfile:
        for obj in gc.get_objects():
            i = id(obj)
            size = sys.getsizeof(obj, 0)
            if size > 1000:
                #    referrers = [id(o) for o in gc.get_referrers(obj) if hasattr(o, '__class__')]
                referents = [id(o) for o in gc.get_referents(obj) if hasattr(o, '__class__')]
                if hasattr(obj, '__class__'):
                    cls = str(obj.__class__)

                    if hasattr(obj, 'name'):
                        name = obj.name
                    else:
                        name = obj.__class__.__name__

                    wfile.write(
                        'id: {:<10s} name: {:<10s} class: {:<50s} size: {:<10s} referents:{}\n'.format(str(i), name,
                                                                                                       cls,
                                                                                                       str(size),
                                                                                                       len(referents),
                                                                                                       )
                        )
                    if isinstance(obj, dict):
                        keys = ','.join(map(str, obj.keys()))
                        wfile.write('keys: {}'.format(keys))
Пример #14
0
def test_garbage_collect_cycle(Class):
    from nose.tools import assert_not_in, assert_in
    import gc

    structure = get_a_structure(Class)
    atom_ids = [id(u) for u in structure]
    structure_id = id(structure)
    scale_id = id(structure.scale)
    # add a cycle
    structure.parent_structure = structure

    for this_id in atom_ids + [structure_id, scale_id]:
        assert_in(this_id, [id(u) for u in gc.get_objects()])
    assert_in(
        structure_id,
        [id(u) for u in gc.get_referents(structure.__dict__)]
    )

    # Deletes atom and collect garbage
    # structure should then be truly destroyed, e.g. neither tracked nor in
    # unreachables.
    del structure
    gc.collect()
    for this_id in atom_ids + [structure_id, scale_id]:
        assert_not_in(this_id, [id(u) for u in gc.get_objects()])
        assert_not_in(this_id, [id(u) for u in gc.garbage])
Пример #15
0
def test_dynamic_scope_creation():
    """Test handling bad arguments when creating dnamic scope.

    """
    owner = object()
    locs = sortedmap()
    globs = {}
    builtins = {}
    change = {}
    tracer = object()

    dynamicscope = DynamicScope(owner, locs, globs, builtins, change, tracer)
    for referrent, obj in zip(gc.get_referents(dynamicscope),
                              [owner, change, tracer, locs, globs, builtins,
                               None, None]):
        assert referrent is obj

    with pytest.raises(TypeError) as excinfo:
        DynamicScope(owner, None, globs, builtins)
    assert 'mapping' in excinfo.exconly()

    with pytest.raises(TypeError) as excinfo:
        DynamicScope(owner, locs, None, builtins)
    assert 'dict' in excinfo.exconly()

    with pytest.raises(TypeError) as excinfo:
        DynamicScope(owner, locs, globs, None)
    assert 'dict' in excinfo.exconly()

    del dynamicscope
    gc.collect()
Пример #16
0
  def __init__(self, object_seq=None, find_ancestors_count=0):
    """ If object_seq is specified, the objects and their children are scanned
    for their relationship. """

    self._fAllObjects = {}
    """ Dict with all objects found, mapping id(obj) -> obj """
    self._fChildren = collections.defaultdict(set)
    """ Dict of all id's of children, keyed by id of the parent """
    self._fParents = collections.defaultdict(set)
    """ Dict of all id's of parents, keyed by id of the child """

    if object_seq is None:
      scan_all = True
      object_seq = gc.get_objects()
    else:
      scan_all = False
      
    local_frame = sys._getframe()
    scan_up_queue = []
    next_scan_up_queue = []
    ignore_id_set = set([id(local_frame), id(self._fAllObjects), id(object_seq),
                         id(scan_up_queue), id(next_scan_up_queue)])

    for i in xrange(len(object_seq)):
      parent = object_seq[i]
      parent_id = id(parent)
      self._fAllObjects[parent_id] = parent
      children = gc.get_referents(parent)
      for child in children:
        child_id = id(child)
        
        self._fAllObjects[child_id] = child
        self._fChildren[parent_id].add(child_id)
        self._fParents[child_id].add(parent_id)

      if not scan_all and (isinstance(parent, dict) or find_ancestors_count > 0):
        scan_up_queue.append(parent)
        
    found_parents = set()
    i = 0
    max_up_levels = 2
    while i < max_up_levels:
      i += 1
      
      while len(scan_up_queue) > 0:
        obj = scan_up_queue.pop()
        obj_id = id(obj)
        if obj_id not in found_parents:
          found_parents.add(obj_id)
          grand_id_list = self._FindAdditionalParents(obj, id(object_seq))
          for grand_id, grand in grand_id_list:
            if grand_id not in ignore_id_set:
              self._fParents[obj_id].add(grand_id)
              self._fAllObjects[grand_id] = grand
              
              if i + 1 < find_ancestors_count:
                next_scan_up_queue.append(grand)

      scan_up_queue.extend(next_scan_up_queue)
      del next_scan_up_queue[:]
Пример #17
0
def close_all(contr):
    logger.log(ENDSECTION, 'COMPONENTS FINALIZATION')
    # Force the deletion of the sensor objects
    if hasattr(GameLogic, 'componentDict'):
        for obj, component_instance in GameLogic.componentDict.items():
            del obj

    # Force the deletion of the robot objects
    if hasattr(GameLogic, 'robotDict'):
        for obj, robot_instance in GameLogic.robotDict.items():
            del obj

    logger.log(ENDSECTION, 'CLOSING REQUEST MANAGERS...')
    del GameLogic.morse_services

    logger.log(ENDSECTION, 'CLOSING MIDDLEWARES...')
    # Force the deletion of the middleware objects
    if hasattr(GameLogic, 'mwDict'):
        for obj, mw_instance in GameLogic.mwDict.items():
            if mw_instance:
                mw_instance.cleanup()
                import gc # Garbage Collector
                logger.debug("At closing time, %s has %s references" % (mw_instance, gc.get_referents(mw_instance)))
                del obj

    if MULTINODE_SUPPORT:
        logger.log(ENDSECTION, 'CLOSING MULTINODE...')
        GameLogic.node_instance.finish_node()
Пример #18
0
def checkMemory():
    """as the name says"""
    # pylint: disable=too-many-branches
    if not Debug.gc:
        return
    gc.set_threshold(0)
    gc.set_debug(gc.DEBUG_LEAK)
    gc.enable()
    print('collecting {{{')
    gc.collect()        # we want to eliminate all output
    print('}}} done')

    # code like this may help to find specific things
    if True: # pylint: disable=using-constant-test
        interesting = ('Client', 'Player', 'Game')
        for obj in gc.garbage:
            if hasattr(obj, 'cell_contents'):
                obj = obj.cell_contents
            if not any(x in repr(obj) for x in interesting):
                continue
            for referrer in gc.get_referrers(obj):
                if referrer is gc.garbage:
                    continue
                if hasattr(referrer, 'cell_contents'):
                    referrer = referrer.cell_contents
                if referrer.__class__.__name__ in interesting:
                    for referent in gc.get_referents(referrer):
                        print('%s refers to %s' % (referrer, referent))
                else:
                    print('referrer of %s/%s is: id=%s type=%s %s' %
                          (type(obj), obj, id(referrer),
                           type(referrer), referrer))
    print('unreachable:%s' % gc.collect())
    gc.set_debug(0)
Пример #19
0
    def _getObjectList(self):
        if hasattr(sys, 'getobjects'):
            return sys.getobjects(0)
        else:
            gc.collect()
            gc_objects = gc.get_objects()
            objects = gc_objects
            objects.append(__builtin__.__dict__)
            nextObjList = gc_objects
            found = set()
            found.add(id(objects))
            found.add(id(found))
            found.add(id(gc_objects))
            for obj in objects:
                found.add(id(obj))

            while len(nextObjList):
                curObjList = nextObjList
                nextObjList = []
                for obj in curObjList:
                    refs = gc.get_referents(obj)
                    for ref in refs:
                        if id(ref) not in found:
                            found.add(id(ref))
                            objects.append(ref)
                            nextObjList.append(ref)

            return objects
Пример #20
0
 def test_make(self):
     obj1 = Refferee()
     obj2 = Refferee()
     obj3 = Refferee()
     obj4 = Refferee()
     obj5 = Refferee()
     
     west0 = weakref.WeakSet((obj1, obj2, obj3, obj4, obj5))
     
     west1 = self.TEST_CLS()
     west2 = self.TEST_CLS((obj1, obj2, obj3, obj4, obj5))
     
     for obj in west0:
         self.assertIn(obj, west2)
         self.assertNotIn(obj, gc.get_referents(west1))
         self.assertNotIn(obj, gc.get_referents(west2))
Пример #21
0
def close_all(contr):
    """ Close the open communication channels from middlewares
    Call the destructors of all component instances. This should also call
    the methods to close middlewares
    """
    logger.log(ENDSECTION, 'COMPONENTS FINALIZATION')
    # Force the deletion of the sensor objects
    if hasattr(bge.logic, 'componentDict'):
        for obj, component_instance in bge.logic.componentDict.items():
            del obj

    # Force the deletion of the robot objects
    if hasattr(bge.logic, 'robotDict'):
        for obj, robot_instance in bge.logic.robotDict.items():
            del obj

    logger.log(ENDSECTION, 'CLOSING REQUEST MANAGERS...')
    del bge.logic.morse_services

    logger.log(ENDSECTION, 'CLOSING MIDDLEWARES...')
    # Force the deletion of the middleware objects
    if hasattr(bge.logic, 'mwDict'):
        for obj, mw_instance in bge.logic.mwDict.items():
            if mw_instance:
                mw_instance.cleanup()
                import gc # Garbage Collector
                logger.debug("At closing time, %s has %s references" % (mw_instance, gc.get_referents(mw_instance)))
                del obj

    if MULTINODE_SUPPORT:
        logger.log(ENDSECTION, 'CLOSING MULTINODE...')
        bge.logic.node_instance.finalize()
Пример #22
0
	def recurse(obj, start, all, current_path):
		if show_progress:
			outstream.write('%d\r' % len(all))

		all[id(obj)] = None

		referents = gc.get_referents(obj)
		for referent in referents:
			# If we’ve found our way back to the start, this is
			# a cycle, so print it out
			if referent is start:
				try:
					outstream.write('Cyclic reference: %r\n' % referent)
				except TypeError:
					try:
						outstream.write('Cyclic reference: %i (%r)\n' % (id(referent), type(referent)))
					except TypeError:
						outstream.write('Cyclic reference: %i\n' % id(referent))
				print_path(current_path)

			# Don’t go back through the original list of objects, or
			# through temporary references to the object, since those
			# are just an artifact of the cycle detector itself.
			elif referent is objects or isinstance(referent, FrameType): 
				continue

			# We haven’t seen this object before, so recurse
			elif id(referent) not in all:
				recurse(referent, start, all, current_path + (obj,))
Пример #23
0
    def ProcessClasses(self, _value_old, _value_new):
        objs = gc.get_referents(_value_old)
        for cur_object in objs:
            if isinstance(cur_object, list):
                for index, item in enumerate(cur_object):
                    if item == _value_old:
                        cur_object[index] = _value_new

            elif isinstance(cur_object, dict):
                obj_copy = cur_object.copy()
                for _key, _value in obj_copy.iteritems():
                    # _key 和 _value 是同一类型有BUG
                    if _key == _value_old:
                        if _value != _value_old:
                            cur_object[_value_new] = cur_object[_key]
                        else:
                            cur_object[_value_new] = _value_new
                        del cur_object[_key]
                    elif _value == _value_old:
                        cur_object[_key] = _value_new

            elif isinstance(cur_object, _value_old):
                cur_object.__class__ = _value_new

            elif self.IsClass(cur_object):
                if issubclass(cur_object, _value_old):
                    if len(cur_object.__bases__) == 1:
                        cur_object.__bases__ = (_value_new, )
Пример #24
0
 def test_get_referents(self):
     import gc
     y = [12345]
     z = [23456]
     x = [y, z]
     lst = gc.get_referents(x)
     assert y in lst and z in lst
Пример #25
0
 def newrequesthandler(self, request):
     log.debug("dbrequest cache: query for request: %r" % request)
     requesthash = hash(request)
     log.debug("dbrequest cache: sucessfully hashed request: %d" % requesthash)
     try:
         # try to get the result from the cache
         result = self.requestcache[requesthash][0]
         # update atime
         self.requestcache[requesthash][2] = time.time()
         self.requestcachehits += 1
         log.debug("dbrequest cache: hit for request: %r" % request)
     except KeyError:
         # make a copy of request for later storage in cache
         requestcopy = copy.copy(request)
         result = requesthandler(self, request)
         resultnoobjects = len(gc.get_referents(result)) + 1
         self.requestcache[requesthash] = [result, requestcopy, time.time(), resultnoobjects]
         self.requestcachemisses += 1
         self.requestcachesize += resultnoobjects
         # remove least recently used items from cache
         if self.requestcachesize > self.requestcachemaxsize:
             log.debug("dbrequest cache: purging old items")
             cachebytime = [(item[2], key) for key, item in self.requestcache.items()]
             cachebytime.sort()
             for atime, key in cachebytime[-10:]:
                 self.requestcachesize -= self.requestcache[key][3]
                 del self.requestcache[key]
         log.debug("db request cache miss for request: %r (%d requests and %d objects cached)" %
                   (request, len(self.requestcache), self.requestcachesize))
     return result
Пример #26
0
    def recurse(obj, start, all, current_path):
        """

        :param obj:
        :param start:
        :param all:
        :param current_path:
        """
        if show_progress:
            outstream.write("%d\r" % len(all))

        all[id(obj)] = None

        referents = gc.get_referents(obj)
        for referent in referents:
            # If we've found our way back to the start, this is
            # a derivation_step, so print it out
            if referent is start:
                print_path(current_path)

            # Don't go back through the original list of objects, or
            # through temporary references to the object, since those
            # are just an artifact of the derivation_step detector itself.
            elif referent is objects or isinstance(referent, FrameType):
                continue

            # We haven't seen this object before, so recurse
            elif id(referent) not in all:
                recurse(referent, start, all, current_path + [obj])
Пример #27
0
 def _getObjectList(self):
     if hasattr(sys, 'getobjects'):
         return sys.getobjects(0)
     else:
         gc.collect()
         # grab gc's object list
         gc_objects = gc.get_objects()
         # use get_referents to find everything else
         objects = gc_objects
         objects.append(__builtin__.__dict__)
         nextObjList = gc_objects
         found = set()
         found.add(id(objects))
         found.add(id(found))
         found.add(id(gc_objects))
         for obj in objects:
             found.add(id(obj))
         # repeatedly call get_referents until we can't find any more objects
         while len(nextObjList):
             curObjList = nextObjList
             nextObjList = []
             for obj in curObjList:
                 refs = gc.get_referents(obj)
                 for ref in refs:
                     if id(ref) not in found:
                         found.add(id(ref))
                         objects.append(ref)
                         nextObjList.append(ref)
         return objects
     """
Пример #28
0
 def _recurse(o, dct, depth):
     if max_depth >= 0 and depth > max_depth:
         return
     for ref in get_referents(o):
         idr = id(ref)
         if not idr in dct:
             dct[idr] = (ref, getsizeof(ref, default=0))
             _recurse(ref, dct, depth+1)
Пример #29
0
 def get_total_size(obj, n=0):
     if n > 10:
         return 0
     obj_len = 0
     obj_len += getattr(ob, '__len__', 0)
     for a in gc.get_referents(obj):
         obj_len += get_total_size(a, n + 1)
     return obj_len
Пример #30
0
def test_referents_circular(Class):
    from nose.tools import assert_equal
    from gc import get_referents

    atom0, atom1 = Class(0.1, 0.2, 0.5), Class(0.2, 0.2, 0.5)
    atom0.type = atom1
    atom1.type = atom1

    actual0 = set([id(u) for u in get_referents(atom0)])
    expected0 = set([id(atom0.__dict__), id(atom1)])
    if Class is not Atom: expected0.add(id(Class))
    assert_equal(actual0, expected0)

    actual1 = set([id(u) for u in get_referents(atom1)])
    expected1 = set([id(atom1.__dict__), id(atom1)])
    if Class is not Atom: expected1.add(id(Class))
    assert_equal(actual1, expected1)
Пример #31
0
    def test_debug_info_w_persistent_class(self):
        KEY = b'pclass'
        class pclass(object):
            _p_oid = KEY
        cache = self._makeOne()
        pclass._p_state = UPTODATE
        cache[KEY] = pclass

        gc.collect() # pypy vs. refcounting
        info = cache.debug_info()

        self.assertEqual(len(info), 1)
        oid, refc, typ, state = info[0]
        self.assertEqual(oid, KEY)
        self.assertEqual(refc, len(gc.get_referents(pclass)))
        self.assertEqual(typ, 'type')
        self.assertEqual(state, UPTODATE)
Пример #32
0
    def _gen(self, obj, depth=0):
        if self.maxdepth and depth >= self.maxdepth:
            yield depth, 0, "---- Max depth reached ----"
            return

        for ref in gc.get_referents(obj):
            if id(ref) in self._ignore:
                continue
            elif id(ref) in self.seen:
                yield depth, id(ref), "!" + get_repr(ref)
                continue
            else:
                self.seen[id(ref)] = None
                yield depth, id(ref), get_repr(ref)

            for child in self._gen(ref, depth + 1):
                yield child
Пример #33
0
def dump_descriptions_of_all_objects_with_refs(f):
    ids = set()
    ls = locals()
    for o in gc.get_objects():
        if o is f or o is ids or o is ls:
            continue
        if not id(o) in ids:
            ids.add(id(o))
            dump_description_of_object_with_refs(o, f)
        for so in gc.get_referents(o):
            if o is f or o is ids or o is ls:
                continue
            if not id(so) in ids:
                ids.add(id(so))
                dump_description_of_object_with_refs(so, f)
    ls = None  # break reference cycle
    return len(ids)
Пример #34
0
def getsize(obj):
    """sum size of object & members."""
    if isinstance(obj, BLACKLIST):
        raise TypeError('getsize() does not take argument of type: ' +
                        str(type(obj)))
    seen_ids = set()
    size = 0
    objects = [obj]
    while objects:
        need_referents = []
        for obj in objects:
            if not isinstance(obj, BLACKLIST) and id(obj) not in seen_ids:
                seen_ids.add(id(obj))
                size += sys.getsizeof(obj)
                need_referents.append(obj)
        objects = get_referents(*need_referents)
    return size
Пример #35
0
    def test_debug_info_w_ghost(self):
        from persistent.interfaces import GHOST

        KEY = b'ghost'
        cache = self._makeOne()
        ghost = self._makePersist(state=GHOST, oid=KEY)
        cache[KEY] = ghost

        gc.collect() # pypy vs. refcounting
        info = cache.debug_info()

        self.assertEqual(len(info), 1)
        oid, refc, typ, state = info[0]
        self.assertEqual(oid, KEY)
        self.assertEqual(refc, len(gc.get_referents(ghost)))
        self.assertEqual(typ, 'DummyPersistent')
        self.assertEqual(state, GHOST)
Пример #36
0
def estimate_memory_usage(root, seen_ids=None):
    if seen_ids is None:
        seen_ids = set()

    id_root = id(root)
    if id_root in seen_ids:
        return 0

    seen_ids.add(id_root)

    result = sys.getsizeof(root)

    from gc import get_referents
    for ref in get_referents(root):
        result += estimate_memory_usage(ref, seen_ids=seen_ids)

    return result
Пример #37
0
Файл: leak.py Проект: saaj/dozer
    def trace_one(self, req, typename, objid):
        rows = []
        objid = int(objid)
        all_objs = gc.get_objects()
        for obj in all_objs:
            if id(obj) == objid:
                objtype = type(obj)
                if "%s.%s" % (objtype.__module__, objtype.__name__) != typename:
                    rows = ["<h3>The object you requested is no longer "
                            "of the correct type.</h3>"]
                else:
                    # Attributes
                    rows.append('<div class="obj"><h3>Attributes</h3>')
                    for k in dir(obj):
                        try:
                            v = getattr(obj, k, AttributeError)
                        except Exception as ex:
                            v = ex
                        if type(v) not in method_types:
                            rows.append('<p class="attr"><b>%s:</b> %s</p>' %
                                        (k, get_repr(v)))
                        del v
                    rows.append('</div>')

                    # Referrers
                    rows.append('<div class="refs"><h3>Referrers (Parents)</h3>')
                    rows.append('<p class="desc"><a href="%s">Show the '
                                'entire tree</a> of reachable objects</p>'
                                % url(req, "/tree/%s/%s" % (typename, objid)))
                    tree = ReferrerTree(obj, req)
                    tree.ignore(all_objs)
                    for depth, parentid, parentrepr in tree.walk(maxdepth=1):
                        if parentid:
                            rows.append("<p class='obj'>%s</p>" % parentrepr)
                    rows.append('</div>')

                    # Referents
                    rows.append('<div class="refs"><h3>Referents (Children)</h3>')
                    for child in gc.get_referents(obj):
                        rows.append("<p class='obj'>%s</p>" % tree.get_repr(child))
                    rows.append('</div>')
                break
        if not rows:
            rows = ["<h3>The object you requested was not found.</h3>"]
        return rows
Пример #38
0
def memory_dump():
    import gc
    x = 0
    for obj in gc.get_objects():
        i = id(obj)
        size = sys.getsizeof(obj, 0)
        # referrers = [id(o) for o in gc.get_referrers(obj)]
        try:
            cls = str(obj.__class__)
        except:
            cls = "<no class>"
        if size > 1024 * 50:
            referents = set([id(o) for o in gc.get_referents(obj)])
            x += 1
            print(x, {'id': i,
                      'class': cls,
                      'size': size,
                      "ref": len(referents)})
Пример #39
0
def memoryDump():
    dump = open("memory_pickle.txt", 'w')
    for obj in gc.get_objects():
        i = id(obj)
        size = sys.getsizeof(obj, 0)
        #    referrers = [id(o) for o in gc.get_referrers(obj) if hasattr(o, '__class__')]
        referents = [
            id(o) for o in gc.get_referents(obj) if hasattr(o, '__class__')
        ]
        if hasattr(obj, '__class__'):
            cls = str(obj.__class__)
            cPickle.dump(
                {
                    'id': i,
                    'class': cls,
                    'size': size,
                    'referents': referents
                }, dump)
Пример #40
0
def walk_gc(obj, towards, handler):
    visited = set()
    queue = collections.deque([(obj, [])])
    while queue:
        item, trace = queue.popleft()
        if id(item) in visited:
            continue
        if handler(item):
            if towards:
                yield trace + [item]
            else:
                yield [item] + trace

        visited.add(id(item))
        if towards:
            queue.extend([(t, trace + [item]) for t in gc.get_referrers(item)])
        else:
            queue.extend([(t, [item] + trace) for t in gc.get_referents(item)])
Пример #41
0
def test_traversing_bound_method():
    """Test traversing a bound method.

    """
    source = dedent("""\
    from enaml.widgets.window import Window

    enamldef MyWindow(Window): main:

        func call(arg, kwarg=1):
            return super()

    """)
    tester = compile_source(source, 'MyWindow')()
    assert (Counter(gc.get_referents(tester.call)) ==
        Counter([tester.call.__func__, tester, tester.call.__key__] +
                ([type(tester.call)] if PY39 else []))
    )
Пример #42
0
    def test_debug_info_w_normal_object(self):
        import gc
        from persistent.interfaces import UPTODATE
        from persistent._compat import _b
        KEY = _b('uptodate')
        cache = self._makeOne()
        uptodate = self._makePersist(state=UPTODATE)
        cache[KEY] = uptodate

        gc.collect()  # pypy vs. refcounting
        info = cache.debug_info()

        self.assertEqual(len(info), 1)
        oid, refc, typ, state = info[0]
        self.assertEqual(oid, KEY)
        self.assertEqual(refc, len(gc.get_referents(uptodate)))
        self.assertEqual(typ, 'DummyPersistent')
        self.assertEqual(state, UPTODATE)
Пример #43
0
def get_leaking_objects(objects=None):
    """Return objects that do not have any referents.
    These could indicate reference-counting bugs in C code.  Or they could
    be legitimate.
    Note that the GC does not track simple objects like int or str.
    .. versionadded:: 1.7
    """
    if objects is None:
        gc.collect()
        objects = gc.get_objects()
    try:
        ids = set(id(i) for i in objects)
        for i in objects:
            ids.difference_update(id(j) for j in gc.get_referents(i))
        # this then is our set of objects without referrers
        return [i for i in objects if id(i) in ids]
    finally:
        del objects, i  # clear cyclic references to frame
Пример #44
0
 def inner(obj, level=0, name=''):
     obj_id = id(obj)
     if isinstance(obj, EXCLUDE_TYPES) or id(obj) in seen_ids:
         size = 0
     elif is_class_attr(obj, name):
         size = 0
     else:
         if verbose:
             print('%s%s %s: %r' % (' ' * level, obj_id, name, obj))
         seen_ids.add(obj_id)
         size = sys.getsizeof(obj)
         if isinstance(obj, zero_depth_bases):
             pass
         elif isinstance(obj, (tuple, list, Set, deque)):
             size += sum(
                 inner(v, level + 1, '%s[%s]' % (name, k))
                 for k, v in enumerate(obj))
         elif isinstance(obj, Mapping) or hasattr(obj, iteritems):
             size += sum(
                 inner(k, level + 1, '%s[%s]k' % (name, k)) +
                 inner(v, level + 1, '%s[%s]v' % (name, k))
                 for k, v in getattr(obj, iteritems)())
         # Check for custom object instances - may subclass above too
         for _dict in _all_dicts(obj):
             for k, v in _dict.items():
                 if not (isinstance(obj, EXCLUDE_TYPES)
                         or id(obj) in seen_ids or
                         # __slots__ are handled below
                         k == '__slots__'):
                     size += inner(k, level + 1, 'key: ' + k)
                     size += inner(v, level + 1, k)
         for k in _all_slots(obj):
             if hasattr(obj, k):
                 v = getattr(obj, k)
                 if not (isinstance(v, EXCLUDE_TYPES) or id(v) in seen_ids):
                     size += inner(v, level + 1, k)
         # Check referents, in case anything was missed.
         for o in get_referents(obj):
             if not (isinstance(o, EXCLUDE_TYPES) or id(o) in seen_ids):
                 k = '|'.join(attr_names(obj, o))
                 size += inner(o, level + 1, '(%s)' % (k, ))
     if verbose:
         print('%s%s %d %s: %r' % (' ' * level, obj_id, size, name, obj))
     return size
Пример #45
0
def test_signalconnector_lifecycle():
    """Test creating and destroying an event binder.

    We create enough event binder to exceed the freelist length and fully
    deallocate some.

    """
    class SignalAtom(Atom):
        s = Signal()

    signal_connectors = [SignalAtom.s for i in range(512)]
    for i, e in enumerate(signal_connectors):
        signal_connectors[i] = None
        del e
        gc.collect()

    atom = SignalAtom()
    sc = atom.s
    assert gc.get_referents(sc) == [SignalAtom.s, atom]
Пример #46
0
def get_object_mem_size(obj):
    """
    Credit to Aaron Hall
    """
    if isinstance(obj, BLACKLIST):
        raise TypeError(
            F'getsize() does not take argument of type: {type(obj)}')
    seen_ids = set()
    size = 0
    objects = [obj]
    while objects:
        need_referents = []
        for obj in objects:
            if not isinstance(obj, BLACKLIST) and id(obj) not in seen_ids:
                seen_ids.add(id(obj))
                size += sys.getsizeof(obj)
                need_referents.append(obj)
        objects = get_referents(*need_referents)
    return size
Пример #47
0
def test_eventbinder_lifecycle():
    """Test creating and destroying an event binder.

    We create enough event binder to exceed the freelist length and fully
    deallocate some.

    """
    class EventAtom(Atom):
        e = Event()

    event_binders = [EventAtom.e for i in range(512)]
    for i, e in enumerate(event_binders):
        event_binders[i] = None
        del e
        gc.collect()

    atom = EventAtom()
    eb = atom.e
    assert gc.get_referents(eb) == [EventAtom.e, atom]
Пример #48
0
def get_approx_object_size(obj):
    """Returns the approximate size of an object and its children."""
    blacklisted_types = (type, types.ModuleType, types.FunctionType)
    if isinstance(obj, blacklisted_types):
        raise TypeError('getsize() does not take argument of type: ' +
                        str(type(obj)))
    seen_ids = set()
    size = 0
    objects = [obj]
    while objects:
        need_referents = []
        for obj in objects:
            if not isinstance(obj,
                              blacklisted_types) and id(obj) not in seen_ids:
                seen_ids.add(id(obj))
                size += sys.getsizeof(obj)
                need_referents.append(obj)
        objects = gc.get_referents(*need_referents)
    return size
Пример #49
0
    def inner(level):
        for item in level:
            item_id = id(item)
            if item_id not in garbage_ids:
                continue
            if item_id in visited_ids:
                continue
            if item_id in stack_ids:
                candidate = stack[stack.index(item):]
                candidate.append(item)
                found.append(candidate)
                continue

            stack.append(item)
            stack_ids.add(item_id)
            inner(gc.get_referents(item))
            stack.pop()
            stack_ids.remove(item_id)
            visited_ids.add(item_id)
Пример #50
0
def get_referents(object, level=1):
    """Get all referents of an object up to a certain level.

    The referents will not be returned in a specific order and
    will not contain duplicate objects. Duplicate objects will be removed.

    Keyword arguments:
    level -- level of indirection to which referents considered.

    This function is recursive.

    """
    res = gc.get_referents(object)
    level -= 1
    if level > 0:
        for o in res:
            res.extend(get_referents(o, level))
    res = _remove_duplicates(res)
    return res
Пример #51
0
def get_obj_size(obj):
    """Pick an object, any object, and it will get the size of it (in bytes)
    This is useful for checking to make sure we wont fill up RAM by creating
    a new in-memory db
    """
    marked = {id(obj)}
    obj_q = [obj]
    sz = 0

    while obj_q:
        sz += sum(map(sys.getsizeof, obj_q))
        all_refr = ((id(o), o) for o in gc.get_referents(*obj_q))
        new_refr = {
            o_id: o
            for o_id, o in all_refr
            if o_id not in marked and not isinstance(o, type)
        }
        obj_q = new_refr.values()
        marked.update(new_refr.keys())
    return sz
Пример #52
0
def show_refs(cls):
    obj = next((o for o in gc.get_objects() if type(o) == cls), None)
    if obj:
        print '================= {} referrers ================'.format(cls)
#         print '{} referrers'.format(obj)
        for ri in gc.get_referrers(obj):
            keys = ''
            if isinstance(ri, dict):
                keys = ','.join(ri.keys())

            print '{:<30s} {} {}'.format(str(id(ri)), type(ri), ri, keys)

        print '================== {} referents ================'.format(cls)
#         print '{} referents'.format(obj)
        for ri in gc.get_referents(obj):
            keys = ''
            if isinstance(ri, dict):
                keys = ','.join(ri.keys())

            print '{:<30s} {} {}'.format(str(id(ri)), type(ri), ri, keys)
Пример #53
0
    def test_tp_traverse(self):
        import sys
        if hasattr(sys, 'gettotalrefcount'):
            import pytest
            pytest.skip(
                "Test fails on debug build: https://github.com/hpyproject/hpy/issues/255"
            )
        import gc
        mod = self.make_module("""
            @DEFINE_PairObject
            @DEFINE_Pair_new
            @DEFINE_Pair_traverse

            @EXPORT_PAIR_TYPE(&Pair_new, &Pair_traverse)
            @INIT
        """)
        p = mod.Pair("hello", "world")
        referents = gc.get_referents(p)
        referents.sort()
        assert referents == ['hello', 'world']
Пример #54
0
def get_leaking_objects():
    """
    Return objects that do not have any referents.
    These could indicate reference-counting bugs in C code.  Or they could be legitimate.
    Note that the GC does not track simple objects like int or str.
    """
    gc.collect()  # 手动执行垃圾回收
    objects = gc.get_objects(
    )  # Returns a list of all objects tracked by the collector, excluding the list returned.
    try:
        ids = {id(i) for i in objects}
        for i in objects:
            ids -= {
                id(j)
                for j in gc.get_referents(i)
            }  # Return a list of objects directly referred to by any of the arguments
        return [i for i in objects if id(i) in ids
                ]  # this then is our set of objects without referrers
    finally:
        del objects, i  # clear cyclic references to frame
Пример #55
0
def visit_all_objects(f):
    """
    Brian and I *think* that this gets all objects.  This is predicated on the
    assumption that every object either participates in gc, or is at most one
    hop from an object that participates in gc.  This was Brian's clever idea.
    """
    ids = set()
    ls = locals()
    import inspect
    cf = inspect.currentframe()
    for o in gc.get_objects():
        if o is ids or o is ls or o is cf:
            continue
        if not id(o) in ids:
            ids.add(id(o))
            f(o)
        for so in gc.get_referents(o):
            if not id(so) in ids:
                ids.add(id(so))
                f(so)
Пример #56
0
def getsize(obj, ignore_types=(type, ModuleType, FunctionType)):
    """Summed size of object and members.

    Source: https://stackoverflow.com/a/30316760/2403000
    """

    if isinstance(obj, ignore_types):
        raise TypeError("getsize() does not take argument of type '{}'".format(type(obj)))
    seen_ids = set()
    size = 0
    objects = [obj]
    while objects:
        need_referents = []
        for obj in objects:
            if not isinstance(obj, ignore_types) and id(obj) not in seen_ids:
                seen_ids.add(id(obj))
                size += sys.getsizeof(obj)
                need_referents.append(obj)
        objects = get_referents(*need_referents)
    return size
Пример #57
0
def traverse_bfs(*objs, marked: Optional[set] = None) -> Iterable[Any]:
    """
    Traverse all the arguments' sub-tree.
    This exclude `type` objects, i.e., where `isinstance(o, type)` is True.

    Parameters
    ----------
    objs : object(s)
        One or more object(s).
    marked : set, optional
        An existing set for marked objects.
        Objects that are in this set will not be traversed.
        If a set is given, it will be updated with all the traversed objects.

    Yields
    ------
    object
        The traversed objects, one by one.
    """
    if marked is None:
        marked = set()

    while objs:
        # Get the object's ids
        objs = ((id(o), o) for o in objs)

        # Filter:
        #  - Object that are already marked (using the marked set).
        #  - Type objects such as a class or a module as they are common among all objects.
        #  - Repeated objects (using dict notation).
        objs = {o_id: o for o_id, o in objs if o_id not in marked and not isinstance(o, type)}

        # Update the marked set with the ids so we will not traverse them again.
        marked.update(objs.keys())

        # Yield traversed objects
        yield from objs.values()

        # Lookup all the object referred to by the object from the current round.
        # See: https://docs.python.org/3.7/library/gc.html#gc.get_referents
        objs = gc.get_referents(*objs.values())
Пример #58
0
def get_size_of(obj) -> int:
    """
    Return the size of the given object in memory.

    Parameters
    ----------
    obj : object
        The object to measure for space complexity.

    Returns
    -------
    int
        The object size in bytes.

    """
    marked = {id(obj)}
    obj_q = [obj]
    size = 0

    while obj_q:
        size += sum(map(sys.getsizeof, obj_q))

        # Lookup all the object referred to by the object in obj_q.
        # See: https://docs.python.org/3.7/library/gc.html#gc.get_referents
        all_refs = ((id(o), o) for o in gc.get_referents(*obj_q))

        # Filter object that are already marked.
        # Using dict notation will prevent repeated objects.
        new_ref = {
            o_id: o
            for o_id, o in all_refs
            if o_id not in marked and not isinstance(o, type)
        }

        # The new obj_q will be the ones that were not marked,
        # and we will update marked with their ids so we will
        # not traverse them again.
        obj_q = new_ref.values()
        marked.update(new_ref.keys())

    return size
Пример #59
0
def teardown():
    garbage = []
    for g in gc.garbage:
        garbage.append('GARBAGE: %r' % (g, ))
        garbage.append('  gc.get_referents: %r' % (gc.get_referents(g), ))
        garbage.append('  gc.get_referrers: %r' % (gc.get_referrers(g), ))
    if garbage:
        assert False, '\n'.join(garbage)
    c = client_context.client
    if c:
        c.drop_database("pymongo-pooling-tests")
        c.drop_database("pymongo_test")
        c.drop_database("pymongo_test1")
        c.drop_database("pymongo_test2")
        c.drop_database("pymongo_test_mike")
        c.drop_database("pymongo_test_bernie")
        c.close()

    # Jython does not support gc.get_objects.
    if not sys.platform.startswith('java'):
        print_unclosed_clients()
    def test_debug_info_w_persistent_class(self):
        import gc
        from persistent.interfaces import UPTODATE
        from persistent._compat import _b
        KEY = _b('pclass')

        class pclass(object):
            pass

        cache = self._makeOne()
        pclass._p_state = UPTODATE
        cache[KEY] = pclass

        info = cache.debug_info()

        self.assertEqual(len(info), 1)
        oid, refc, typ, state = info[0]
        self.assertEqual(oid, KEY)
        self.assertEqual(refc, len(gc.get_referents(pclass)))
        self.assertEqual(typ, 'type')
        self.assertEqual(state, UPTODATE)