def test_clear_traces(self):
     obj, obj_traceback = allocate_bytes(123)
     traceback = tracemalloc.get_object_traceback(obj)
     self.assertIsNotNone(traceback)
     tracemalloc.clear_traces()
     traceback2 = tracemalloc.get_object_traceback(obj)
     self.assertIsNone(traceback2)
Пример #2
0
    def test_clear_traces(self):
        obj, obj_traceback = allocate_bytes(123)
        traceback = tracemalloc.get_object_traceback(obj)
        self.assertIsNotNone(traceback)

        tracemalloc.clear_traces()
        traceback2 = tracemalloc.get_object_traceback(obj)
        self.assertIsNone(traceback2)
Пример #3
0
def big_objs(traced_only=False):

    big_objs = (
        (sys.getsizeof(obj), obj) for obj in gc.get_objects()
        if sys.getsizeof(obj) > 20000 and (
            tracemalloc.get_object_traceback(obj) if traced_only else True))
    for size, obj in big_objs:
        print(type(obj), size, tracemalloc.get_object_traceback(obj))
Пример #4
0
def _perform_leak_detection() -> None:
    # Log potentially useful memory usage metrics
    logger.info(f"GC count before collect {gc.get_count()}")
    traced_memory_size, traced_memory_peak = tracemalloc.get_traced_memory()
    logger.info(
        f"Traced Memory: size={traced_memory_size}, peak={traced_memory_peak}")
    num_unreacheable_objects = gc.collect()
    logger.info(f"Number of unreachable objects = {num_unreacheable_objects}")
    logger.info(f"GC count after collect {gc.get_count()}")

    # Collect unique traces of all live objects in the garbage - these have potential leaks.
    unique_traces_to_objects: Dict[Union[tracemalloc.Traceback, int],
                                   List[object]] = defaultdict(list)
    for obj in gc.garbage:
        obj_trace = tracemalloc.get_object_traceback(obj)
        if obj_trace is not None:
            if _trace_has_file(obj_trace, "*datahub/*.py"):
                # Leaking object
                unique_traces_to_objects[obj_trace].append(obj)
            else:
                unique_traces_to_objects[id(obj)].append(obj)
    logger.info("Potentially leaking objects start")
    for key, obj_list in sorted(
            unique_traces_to_objects.items(),
            key=lambda item: sum([sys.getsizeof(o) for o in item[1]]),
            reverse=True,
    ):
        if isinstance(key, tracemalloc.Traceback):
            obj_traceback: tracemalloc.Traceback = cast(
                tracemalloc.Traceback, key)
            logger.info(
                f"#Objects:{len(obj_list)}; Total memory:{sum([sys.getsizeof(obj) for obj in obj_list])};"
                + " Allocation Trace:\n\t" +
                "\n\t".join(obj_traceback.format(limit=25)))
        else:
            logger.info(
                f"#Objects:{len(obj_list)}; Total memory:{sum([sys.getsizeof(obj) for obj in obj_list])};"
                + " No Allocation Trace available!")
        # Print details about the live referrers of each object in the obj_list (same trace).
        for obj in obj_list:
            referrers = [r for r in gc.get_referrers(obj) if r in gc.garbage]
            logger.info(
                f"Referrers[{len(referrers)}] for object(addr={hex(id(obj))}):'{obj}':"
            )
            for ref_index, referrer in enumerate(referrers):
                ref_trace = tracemalloc.get_object_traceback(referrer)
                logger.info(
                    f"Referrer[{ref_index}] referrer_obj(addr={hex(id(referrer))}):{referrer}, RefTrace:\n\t\t"
                    + "\n\t\t".join(
                        ref_trace.format(limit=5) if ref_trace else []) + "\n")
    logger.info("Potentially leaking objects end")

    tracemalloc.stop()
Пример #5
0
def _formatwarnmsg_impl(msg):
    import linecache
    s =  ("%s:%s: %s: %s\n"
          % (msg.filename, msg.lineno, msg.category.__name__,
             msg.message))
    if msg.line is None:
        line = linecache.getline(msg.filename, msg.lineno)
    else:
        line = msg.line
    if line:
        line = line.strip()
        s += "  %s\n" % line
    if msg.source is not None:
        import tracemalloc
        tb = tracemalloc.get_object_traceback(msg.source)
        if tb is not None:
            s += 'Object allocated at (most recent call first):\n'
            for frame in tb:
                s += ('  File "%s", lineno %s\n'
                      % (frame.filename, frame.lineno))
                line = linecache.getline(frame.filename, frame.lineno)
                if line:
                    line = line.strip()
                    s += '    %s\n' % line
    return s
Пример #6
0
def warning_record_to_str(warning_message: warnings.WarningMessage) -> str:
    """Convert a warnings.WarningMessage to a string."""
    warn_msg = warning_message.message
    msg = warnings.formatwarning(
        str(warn_msg),
        warning_message.category,
        warning_message.filename,
        warning_message.lineno,
        warning_message.line,
    )
    if warning_message.source is not None:
        try:
            import tracemalloc
        except ImportError:
            pass
        else:
            tb = tracemalloc.get_object_traceback(warning_message.source)
            if tb is not None:
                formatted_tb = "\n".join(tb.format())
                # Use a leading new line to better separate the (large) output
                # from the traceback to the previous warning text.
                msg += f"\nObject allocated at:\n{formatted_tb}"
            else:
                # No need for a leading new line.
                url = "https://docs.pytest.org/en/stable/how-to/capture-warnings.html#resource-warnings"
                msg += "Enable tracemalloc to get traceback where the object was allocated.\n"
                msg += f"See {url} for more info."
    return msg
 def test_set_traceback_limit(self):
     obj_size = 10
     tracemalloc.stop()
     self.assertRaises(ValueError, tracemalloc.start, -1)
     tracemalloc.stop()
     tracemalloc.start(10)
     obj2, obj2_traceback = allocate_bytes(obj_size)
     traceback = tracemalloc.get_object_traceback(obj2)
     self.assertEqual(len(traceback), 10)
     self.assertEqual(traceback, obj2_traceback)
     tracemalloc.stop()
     tracemalloc.start(1)
     obj, obj_traceback = allocate_bytes(obj_size)
     traceback = tracemalloc.get_object_traceback(obj)
     self.assertEqual(len(traceback), 1)
     self.assertEqual(traceback, obj_traceback)
Пример #8
0
def memory_obj(args: tuple, packet: ircp.Packet, ___: dict):
    """ Print the biggest memory hogs """
    if not _IS_TRACING:
        return packet.notice(
            "Sorry, but tracing is currently disabled. "
            'Please restart probot with the "PYTHONTRACEMALLOC=NFRAME" '
            "environment variable."
        )

    snapshot = tracemalloc.take_snapshot()
    top_stats = snapshot.statistics("filename")

    num = 0
    if len(args) >= 2:
        try:
            num = int(args[1])
        except ValueError:
            return packet.notice("Your argument must be an integer")
    else:
        return packet.notice("You must specify an object to inspect!")

    if len(top_stats) >= num:
        output = [packet.notice("Memory hog #{}".format(num))]
        obj = top_stats[num]
        trace = tracemalloc.get_object_traceback(obj)
        for line in trace:
            output.append(packet.notice(line))
        return output
    else:
        return packet.notice("Sorry, but that object does not exist")
Пример #9
0
def memory_obj(args: tuple, packet: ircp.Packet, ___: dict):
    """ Print the biggest memory hogs """
    if not _IS_TRACING:
        return packet.notice(
            'Sorry, but tracing is currently disabled. '
            'Please restart probot with the "PYTHONTRACEMALLOC=NFRAME" '
            'environment variable.')

    snapshot = tracemalloc.take_snapshot()
    top_stats = snapshot.statistics('filename')

    num = 0
    if len(args) >= 2:
        try:
            num = int(args[1])
        except ValueError:
            return packet.notice('Your argument must be an integer')
    else:
        return packet.notice('You must specify an object to inspect!')

    if len(top_stats) >= num:
        output = [packet.notice('Memory hog #{}'.format(num))]
        obj = top_stats[num]
        trace = tracemalloc.get_object_traceback(obj)
        for line in trace:
            output.append(packet.notice(line))
        return output
    else:
        return packet.notice('Sorry, but that object does not exist')
Пример #10
0
def show(filename=None):
    global before_objects
    gc.collect()
    after_objects = gc.get_objects()
    frame = sys._getframe()
    globals_ = globals()
    num_leaks = 0
    before_id_set = set(map(id, before_objects))
    if filename is not None:
        f = open(filename, 'w')
    else:
        f = sys.stderr

    for obj in after_objects:
        if id(obj) not in before_id_set:
            if obj in (before_objects, frame):
                continue
            num_leaks += 1
            print("Leak: id=0x{:x} type={} {}".format(id(obj), type(obj),
                                                      _get_detail(obj)),
                  file=f)
            tb = tracemalloc.get_object_traceback(obj)
            if tb is None:
                print("Traceback: None", file=f)
            else:
                print("Traceback:", file=f)
                print("\n".join(tb.format(MAX_FRAMES)), file=f)
            print(file=f)
            print("Referrers:", file=f)
            for ref in gc.get_referrers(obj):
                if ref in (after_objects, before_objects, frame, globals_):
                    continue
                print("     id=0x{:x} type={} {}".format(
                    id(ref), type(ref), _get_detail(ref)),
                      file=f)
                print("     traceback: {}".format(
                    tracemalloc.get_object_traceback(ref)),
                      file=f)
            print(file=f)
            print(file=f)
    print("Total leaks: {}".format(num_leaks), file=f)
    if filename is not None:
        f.close()
    tracemalloc.stop()
    gc.enable()
Пример #11
0
def _formatwarnmsg_impl(msg):
    category = msg.category.__name__
    s =  f"{msg.filename}:{msg.lineno}: {category}: {msg.message}\n"

    if msg.line is None:
        try:
            import linecache
            line = linecache.getline(msg.filename, msg.lineno)
        except Exception:
            # When a warning is logged during Python shutdown, linecache
            # and the import machinery don't work anymore
            line = None
            linecache = None
    else:
        line = msg.line
    if line:
        line = line.strip()
        s += "  %s\n" % line

    if msg.source is not None:
        try:
            import tracemalloc
        # Logging a warning should not raise a new exception:
        # catch Exception, not only ImportError and RecursionError.
        except Exception:
            # don't suggest to enable tracemalloc if it's not available
            tracing = True
            tb = None
        else:
            tracing = tracemalloc.is_tracing()
            try:
                tb = tracemalloc.get_object_traceback(msg.source)
            except Exception:
                # When a warning is logged during Python shutdown, tracemalloc
                # and the import machinery don't work anymore
                tb = None

        if tb is not None:
            s += 'Object allocated at (most recent call last):\n'
            for frame in tb:
                s += ('  File "%s", lineno %s\n'
                      % (frame.filename, frame.lineno))

                try:
                    if linecache is not None:
                        line = linecache.getline(frame.filename, frame.lineno)
                    else:
                        line = None
                except Exception:
                    line = None
                if line:
                    line = line.strip()
                    s += '    %s\n' % line
        elif not tracing:
            s += (f'{category}: Enable tracemalloc to get the object '
                  f'allocation traceback\n')
    return s
Пример #12
0
def _formatwarnmsg_impl(msg):
    category = msg.category.__name__
    s = f"{msg.filename}:{msg.lineno}: {category}: {msg.message}\n"

    if msg.line is None:
        try:
            import linecache
            line = linecache.getline(msg.filename, msg.lineno)
        except Exception:
            # When a warning is logged during Python shutdown, linecache
            # and the import machinery don't work anymore
            line = None
            linecache = None
    else:
        line = msg.line
    if line:
        line = line.strip()
        s += "  %s\n" % line

    if msg.source is not None:
        try:
            import tracemalloc
        # Logging a warning should not raise a new exception:
        # catch Exception, not only ImportError and RecursionError.
        except Exception:
            # don't suggest to enable tracemalloc if it's not available
            tracing = True
            tb = None
        else:
            tracing = tracemalloc.is_tracing()
            try:
                tb = tracemalloc.get_object_traceback(msg.source)
            except Exception:
                # When a warning is logged during Python shutdown, tracemalloc
                # and the import machinery don't work anymore
                tb = None

        if tb is not None:
            s += 'Object allocated at (most recent call last):\n'
            for frame in tb:
                s += ('  File "%s", lineno %s\n' %
                      (frame.filename, frame.lineno))

                try:
                    if linecache is not None:
                        line = linecache.getline(frame.filename, frame.lineno)
                    else:
                        line = None
                except Exception:
                    line = None
                if line:
                    line = line.strip()
                    s += '    %s\n' % line
        elif not tracing:
            s += (f'{category}: Enable tracemalloc to get the object '
                  f'allocation traceback\n')
    return s
Пример #13
0
 def fork_child(self):
     if not tracemalloc.is_tracing():
         return 2
     obj_size = 12345
     obj, obj_traceback = allocate_bytes(obj_size)
     traceback = tracemalloc.get_object_traceback(obj)
     if traceback is None:
         return 3
     return 0
Пример #14
0
    def test_set_traceback_limit(self):
        obj_size = 10

        tracemalloc.stop()
        self.assertRaises(ValueError, tracemalloc.start, -1)

        tracemalloc.stop()
        tracemalloc.start(10)
        obj2, obj2_traceback = allocate_bytes(obj_size)
        traceback = tracemalloc.get_object_traceback(obj2)
        self.assertEqual(len(traceback), 10)
        self.assertEqual(traceback, obj2_traceback)

        tracemalloc.stop()
        tracemalloc.start(1)
        obj, obj_traceback = allocate_bytes(obj_size)
        traceback = tracemalloc.get_object_traceback(obj)
        self.assertEqual(len(traceback), 1)
        self.assertEqual(traceback, obj_traceback)
Пример #15
0
        def __del__(self):
            if self._watcher:
                tb = get_object_traceback(self)
                tb_msg = ''
                if tb is not None:
                    tb_msg = '\n'.join(tb.format())
                    tb_msg = '\nTraceback:\n' + tb_msg
                warnings.warn("Failed to close watcher %r%s" % (self, tb_msg),
                              ResourceWarning)

                # may fail if __init__ did; will be harmlessly printed
                self.close()
Пример #16
0
    def fork_child(self):
        if not tracemalloc.is_tracing():
            return 2

        obj_size = 12345
        obj, obj_traceback = allocate_bytes(obj_size)
        traceback = tracemalloc.get_object_traceback(obj)
        if traceback is None:
            return 3

        # everything is fine
        return 0
Пример #17
0
        def __del__(self):
            if self._watcher:
                tb = get_object_traceback(self)
                tb_msg = ''
                if tb is not None:
                    tb_msg = '\n'.join(tb.format())
                    tb_msg = '\nTraceback:\n' + tb_msg
                warnings.warn("Failed to close watcher %r%s" % (self, tb_msg),
                              ResourceWarning)

                # may fail if __init__ did; will be harmlessly printed
                self.close()
Пример #18
0
def show(filename=None):
    global before_objects
    gc.collect()
    after_objects = gc.get_objects()
    frame = sys._getframe()
    globals_ = globals()
    num_leaks = 0
    before_id_set = set(map(id, before_objects))
    if filename is not None:
        f = open(filename, 'w')
    else:
        f = sys.stderr

    for obj in after_objects:
        if id(obj) not in before_id_set:
            if obj in (before_objects, frame):
                continue
            num_leaks += 1
            print("Leak: id=0x{:x} type={} {}".format(id(obj), type(obj), _get_detail(obj)), file=f)
            tb = tracemalloc.get_object_traceback(obj)
            if tb is None:
                print("Traceback: None", file=f)
            else:
                print("Traceback:", file=f)
                print("\n".join(tb.format(MAX_FRAMES)), file=f)
            print(file=f)
            print("Referrers:", file=f)
            for ref in gc.get_referrers(obj):
                if ref in (after_objects, before_objects, frame, globals_):
                    continue
                print("     id=0x{:x} type={} {}".format(id(ref), type(ref), _get_detail(ref)), file=f)
                print("     traceback: {}".format(tracemalloc.get_object_traceback(ref)), file=f)
            print(file=f)
            print(file=f)
    print("Total leaks: {}".format(num_leaks), file=f)
    if filename is not None:
        f.close()
    tracemalloc.stop()
    gc.enable()
Пример #19
0
def _formatwarnmsg_impl(msg):
    s = "%s:%s: %s: %s\n" % (
        msg.filename,
        msg.lineno,
        msg.category.__name__,
        msg.message,
    )

    if msg.line is None:
        try:
            import linecache

            line = linecache.getline(msg.filename, msg.lineno)
        except Exception:
            # When a warning is logged during Python shutdown, linecache
            # and the import machinery don't work anymore
            line = None
            linecache = None
    else:
        line = msg.line
    if line:
        line = line.strip()
        s += "  %s\n" % line

    if msg.source is not None:
        try:
            import tracemalloc

            tb = tracemalloc.get_object_traceback(msg.source)
        except Exception:
            # When a warning is logged during Python shutdown, tracemalloc
            # and the import machinery don't work anymore
            tb = None

        if tb is not None:
            s += "Object allocated at (most recent call first):\n"
            for frame in tb:
                s += '  File "%s", lineno %s\n' % (frame.filename, frame.lineno)

                try:
                    if linecache is not None:
                        line = linecache.getline(frame.filename, frame.lineno)
                    else:
                        line = None
                except Exception:
                    line = None
                if line:
                    line = line.strip()
                    s += "    %s\n" % line
    return s
Пример #20
0
def _formatwarnmsg_impl(msg):
    s =  ("%s:%s: %s: %s\n"
          % (msg.filename, msg.lineno, msg.category.__name__,
             msg.message))

    if msg.line is None:
        try:
            import linecache
            line = linecache.getline(msg.filename, msg.lineno)
        except Exception:
            # When a warning is logged during Python shutdown, linecache
            # and the import machinery don't work anymore
            line = None
            linecache = None
    else:
        line = msg.line
    if line:
        line = line.strip()
        s += "  %s\n" % line

    if msg.source is not None:
        try:
            import tracemalloc
            tb = tracemalloc.get_object_traceback(msg.source)
        except Exception:
            # When a warning is logged during Python shutdown, tracemalloc
            # and the import machinery don't work anymore
            tb = None

        if tb is not None:
            s += 'Object allocated at (most recent call first):\n'
            for frame in tb:
                s += ('  File "%s", lineno %s\n'
                      % (frame.filename, frame.lineno))

                try:
                    if linecache is not None:
                        line = linecache.getline(frame.filename, frame.lineno)
                    else:
                        line = None
                except Exception:
                    line = None
                if line:
                    line = line.strip()
                    s += '    %s\n' % line
    return s
Пример #21
0
    def test_new_reference(self):
        tracemalloc.clear_traces()
        # gc.collect() indirectly calls PyList_ClearFreeList()
        support.gc_collect()

        # Create a list and "destroy it": put it in the PyListObject free list
        obj = []
        obj = None

        # Create a list which should reuse the previously created empty list
        obj = []

        nframe = tracemalloc.get_traceback_limit()
        frames = get_frames(nframe, -3)
        obj_traceback = tracemalloc.Traceback(frames, min(len(frames), nframe))

        traceback = tracemalloc.get_object_traceback(obj)
        self.assertIsNotNone(traceback)
        self.assertEqual(traceback, obj_traceback)
Пример #22
0
def _formatwarnmsg_impl(msg):
    s = '%s:%s: %s: %s\n' % (msg.filename, msg.lineno, msg.category.__name__,
                             msg.message)
    if msg.line is None:
        try:
            import linecache
            line = linecache.getline(msg.filename, msg.lineno)
        except Exception:
            line = None
            linecache = None
    else:
        line = msg.line
    if line:
        line = line.strip()
        s += '  %s\n' % line
    if msg.source is not None:
        try:
            import tracemalloc
            tb = tracemalloc.get_object_traceback(msg.source)
        except Exception:
            tb = None
        if tb is not None:
            s += 'Object allocated at (most recent call first):\n'
            for frame in tb:
                s += '  File "%s", lineno %s\n' % (frame.filename,
                                                   frame.lineno)
                try:
                    if linecache is not None:
                        line = linecache.getline(frame.filename, frame.lineno)
                    else:
                        line = None
                except Exception:
                    line = None
                if line:
                    line = line.strip()
                    s += '    %s\n' % line
    return s
Пример #23
0
 def test_get_object_traceback(self):
     tracemalloc.clear_traces()
     obj_size = 12345
     obj, obj_traceback = allocate_bytes(obj_size)
     traceback = tracemalloc.get_object_traceback(obj)
     self.assertEqual(traceback, obj_traceback)
Пример #24
0
 def update_event(self, inp=-1):
     self.set_output_val(0, tracemalloc.get_object_traceback(self.input(0)))
Пример #25
0
 def test_get_object_traceback(self):
     tracemalloc.clear_traces()
     obj_size = 12345
     obj, obj_traceback = allocate_bytes(obj_size)
     traceback = tracemalloc.get_object_traceback(obj)
     self.assertEqual(traceback, obj_traceback)