def test_clear_traces(self): obj, obj_traceback = allocate_bytes(123) traceback = tracemalloc.get_object_traceback(obj) self.assertIsNotNone(traceback) tracemalloc.clear_traces() traceback2 = tracemalloc.get_object_traceback(obj) self.assertIsNone(traceback2)
def big_objs(traced_only=False): big_objs = ( (sys.getsizeof(obj), obj) for obj in gc.get_objects() if sys.getsizeof(obj) > 20000 and ( tracemalloc.get_object_traceback(obj) if traced_only else True)) for size, obj in big_objs: print(type(obj), size, tracemalloc.get_object_traceback(obj))
def _perform_leak_detection() -> None: # Log potentially useful memory usage metrics logger.info(f"GC count before collect {gc.get_count()}") traced_memory_size, traced_memory_peak = tracemalloc.get_traced_memory() logger.info( f"Traced Memory: size={traced_memory_size}, peak={traced_memory_peak}") num_unreacheable_objects = gc.collect() logger.info(f"Number of unreachable objects = {num_unreacheable_objects}") logger.info(f"GC count after collect {gc.get_count()}") # Collect unique traces of all live objects in the garbage - these have potential leaks. unique_traces_to_objects: Dict[Union[tracemalloc.Traceback, int], List[object]] = defaultdict(list) for obj in gc.garbage: obj_trace = tracemalloc.get_object_traceback(obj) if obj_trace is not None: if _trace_has_file(obj_trace, "*datahub/*.py"): # Leaking object unique_traces_to_objects[obj_trace].append(obj) else: unique_traces_to_objects[id(obj)].append(obj) logger.info("Potentially leaking objects start") for key, obj_list in sorted( unique_traces_to_objects.items(), key=lambda item: sum([sys.getsizeof(o) for o in item[1]]), reverse=True, ): if isinstance(key, tracemalloc.Traceback): obj_traceback: tracemalloc.Traceback = cast( tracemalloc.Traceback, key) logger.info( f"#Objects:{len(obj_list)}; Total memory:{sum([sys.getsizeof(obj) for obj in obj_list])};" + " Allocation Trace:\n\t" + "\n\t".join(obj_traceback.format(limit=25))) else: logger.info( f"#Objects:{len(obj_list)}; Total memory:{sum([sys.getsizeof(obj) for obj in obj_list])};" + " No Allocation Trace available!") # Print details about the live referrers of each object in the obj_list (same trace). for obj in obj_list: referrers = [r for r in gc.get_referrers(obj) if r in gc.garbage] logger.info( f"Referrers[{len(referrers)}] for object(addr={hex(id(obj))}):'{obj}':" ) for ref_index, referrer in enumerate(referrers): ref_trace = tracemalloc.get_object_traceback(referrer) logger.info( f"Referrer[{ref_index}] referrer_obj(addr={hex(id(referrer))}):{referrer}, RefTrace:\n\t\t" + "\n\t\t".join( ref_trace.format(limit=5) if ref_trace else []) + "\n") logger.info("Potentially leaking objects end") tracemalloc.stop()
def _formatwarnmsg_impl(msg): import linecache s = ("%s:%s: %s: %s\n" % (msg.filename, msg.lineno, msg.category.__name__, msg.message)) if msg.line is None: line = linecache.getline(msg.filename, msg.lineno) else: line = msg.line if line: line = line.strip() s += " %s\n" % line if msg.source is not None: import tracemalloc tb = tracemalloc.get_object_traceback(msg.source) if tb is not None: s += 'Object allocated at (most recent call first):\n' for frame in tb: s += (' File "%s", lineno %s\n' % (frame.filename, frame.lineno)) line = linecache.getline(frame.filename, frame.lineno) if line: line = line.strip() s += ' %s\n' % line return s
def warning_record_to_str(warning_message: warnings.WarningMessage) -> str: """Convert a warnings.WarningMessage to a string.""" warn_msg = warning_message.message msg = warnings.formatwarning( str(warn_msg), warning_message.category, warning_message.filename, warning_message.lineno, warning_message.line, ) if warning_message.source is not None: try: import tracemalloc except ImportError: pass else: tb = tracemalloc.get_object_traceback(warning_message.source) if tb is not None: formatted_tb = "\n".join(tb.format()) # Use a leading new line to better separate the (large) output # from the traceback to the previous warning text. msg += f"\nObject allocated at:\n{formatted_tb}" else: # No need for a leading new line. url = "https://docs.pytest.org/en/stable/how-to/capture-warnings.html#resource-warnings" msg += "Enable tracemalloc to get traceback where the object was allocated.\n" msg += f"See {url} for more info." return msg
def test_set_traceback_limit(self): obj_size = 10 tracemalloc.stop() self.assertRaises(ValueError, tracemalloc.start, -1) tracemalloc.stop() tracemalloc.start(10) obj2, obj2_traceback = allocate_bytes(obj_size) traceback = tracemalloc.get_object_traceback(obj2) self.assertEqual(len(traceback), 10) self.assertEqual(traceback, obj2_traceback) tracemalloc.stop() tracemalloc.start(1) obj, obj_traceback = allocate_bytes(obj_size) traceback = tracemalloc.get_object_traceback(obj) self.assertEqual(len(traceback), 1) self.assertEqual(traceback, obj_traceback)
def memory_obj(args: tuple, packet: ircp.Packet, ___: dict): """ Print the biggest memory hogs """ if not _IS_TRACING: return packet.notice( "Sorry, but tracing is currently disabled. " 'Please restart probot with the "PYTHONTRACEMALLOC=NFRAME" ' "environment variable." ) snapshot = tracemalloc.take_snapshot() top_stats = snapshot.statistics("filename") num = 0 if len(args) >= 2: try: num = int(args[1]) except ValueError: return packet.notice("Your argument must be an integer") else: return packet.notice("You must specify an object to inspect!") if len(top_stats) >= num: output = [packet.notice("Memory hog #{}".format(num))] obj = top_stats[num] trace = tracemalloc.get_object_traceback(obj) for line in trace: output.append(packet.notice(line)) return output else: return packet.notice("Sorry, but that object does not exist")
def memory_obj(args: tuple, packet: ircp.Packet, ___: dict): """ Print the biggest memory hogs """ if not _IS_TRACING: return packet.notice( 'Sorry, but tracing is currently disabled. ' 'Please restart probot with the "PYTHONTRACEMALLOC=NFRAME" ' 'environment variable.') snapshot = tracemalloc.take_snapshot() top_stats = snapshot.statistics('filename') num = 0 if len(args) >= 2: try: num = int(args[1]) except ValueError: return packet.notice('Your argument must be an integer') else: return packet.notice('You must specify an object to inspect!') if len(top_stats) >= num: output = [packet.notice('Memory hog #{}'.format(num))] obj = top_stats[num] trace = tracemalloc.get_object_traceback(obj) for line in trace: output.append(packet.notice(line)) return output else: return packet.notice('Sorry, but that object does not exist')
def show(filename=None): global before_objects gc.collect() after_objects = gc.get_objects() frame = sys._getframe() globals_ = globals() num_leaks = 0 before_id_set = set(map(id, before_objects)) if filename is not None: f = open(filename, 'w') else: f = sys.stderr for obj in after_objects: if id(obj) not in before_id_set: if obj in (before_objects, frame): continue num_leaks += 1 print("Leak: id=0x{:x} type={} {}".format(id(obj), type(obj), _get_detail(obj)), file=f) tb = tracemalloc.get_object_traceback(obj) if tb is None: print("Traceback: None", file=f) else: print("Traceback:", file=f) print("\n".join(tb.format(MAX_FRAMES)), file=f) print(file=f) print("Referrers:", file=f) for ref in gc.get_referrers(obj): if ref in (after_objects, before_objects, frame, globals_): continue print(" id=0x{:x} type={} {}".format( id(ref), type(ref), _get_detail(ref)), file=f) print(" traceback: {}".format( tracemalloc.get_object_traceback(ref)), file=f) print(file=f) print(file=f) print("Total leaks: {}".format(num_leaks), file=f) if filename is not None: f.close() tracemalloc.stop() gc.enable()
def _formatwarnmsg_impl(msg): category = msg.category.__name__ s = f"{msg.filename}:{msg.lineno}: {category}: {msg.message}\n" if msg.line is None: try: import linecache line = linecache.getline(msg.filename, msg.lineno) except Exception: # When a warning is logged during Python shutdown, linecache # and the import machinery don't work anymore line = None linecache = None else: line = msg.line if line: line = line.strip() s += " %s\n" % line if msg.source is not None: try: import tracemalloc # Logging a warning should not raise a new exception: # catch Exception, not only ImportError and RecursionError. except Exception: # don't suggest to enable tracemalloc if it's not available tracing = True tb = None else: tracing = tracemalloc.is_tracing() try: tb = tracemalloc.get_object_traceback(msg.source) except Exception: # When a warning is logged during Python shutdown, tracemalloc # and the import machinery don't work anymore tb = None if tb is not None: s += 'Object allocated at (most recent call last):\n' for frame in tb: s += (' File "%s", lineno %s\n' % (frame.filename, frame.lineno)) try: if linecache is not None: line = linecache.getline(frame.filename, frame.lineno) else: line = None except Exception: line = None if line: line = line.strip() s += ' %s\n' % line elif not tracing: s += (f'{category}: Enable tracemalloc to get the object ' f'allocation traceback\n') return s
def fork_child(self): if not tracemalloc.is_tracing(): return 2 obj_size = 12345 obj, obj_traceback = allocate_bytes(obj_size) traceback = tracemalloc.get_object_traceback(obj) if traceback is None: return 3 return 0
def __del__(self): if self._watcher: tb = get_object_traceback(self) tb_msg = '' if tb is not None: tb_msg = '\n'.join(tb.format()) tb_msg = '\nTraceback:\n' + tb_msg warnings.warn("Failed to close watcher %r%s" % (self, tb_msg), ResourceWarning) # may fail if __init__ did; will be harmlessly printed self.close()
def fork_child(self): if not tracemalloc.is_tracing(): return 2 obj_size = 12345 obj, obj_traceback = allocate_bytes(obj_size) traceback = tracemalloc.get_object_traceback(obj) if traceback is None: return 3 # everything is fine return 0
def show(filename=None): global before_objects gc.collect() after_objects = gc.get_objects() frame = sys._getframe() globals_ = globals() num_leaks = 0 before_id_set = set(map(id, before_objects)) if filename is not None: f = open(filename, 'w') else: f = sys.stderr for obj in after_objects: if id(obj) not in before_id_set: if obj in (before_objects, frame): continue num_leaks += 1 print("Leak: id=0x{:x} type={} {}".format(id(obj), type(obj), _get_detail(obj)), file=f) tb = tracemalloc.get_object_traceback(obj) if tb is None: print("Traceback: None", file=f) else: print("Traceback:", file=f) print("\n".join(tb.format(MAX_FRAMES)), file=f) print(file=f) print("Referrers:", file=f) for ref in gc.get_referrers(obj): if ref in (after_objects, before_objects, frame, globals_): continue print(" id=0x{:x} type={} {}".format(id(ref), type(ref), _get_detail(ref)), file=f) print(" traceback: {}".format(tracemalloc.get_object_traceback(ref)), file=f) print(file=f) print(file=f) print("Total leaks: {}".format(num_leaks), file=f) if filename is not None: f.close() tracemalloc.stop() gc.enable()
def _formatwarnmsg_impl(msg): s = "%s:%s: %s: %s\n" % ( msg.filename, msg.lineno, msg.category.__name__, msg.message, ) if msg.line is None: try: import linecache line = linecache.getline(msg.filename, msg.lineno) except Exception: # When a warning is logged during Python shutdown, linecache # and the import machinery don't work anymore line = None linecache = None else: line = msg.line if line: line = line.strip() s += " %s\n" % line if msg.source is not None: try: import tracemalloc tb = tracemalloc.get_object_traceback(msg.source) except Exception: # When a warning is logged during Python shutdown, tracemalloc # and the import machinery don't work anymore tb = None if tb is not None: s += "Object allocated at (most recent call first):\n" for frame in tb: s += ' File "%s", lineno %s\n' % (frame.filename, frame.lineno) try: if linecache is not None: line = linecache.getline(frame.filename, frame.lineno) else: line = None except Exception: line = None if line: line = line.strip() s += " %s\n" % line return s
def _formatwarnmsg_impl(msg): s = ("%s:%s: %s: %s\n" % (msg.filename, msg.lineno, msg.category.__name__, msg.message)) if msg.line is None: try: import linecache line = linecache.getline(msg.filename, msg.lineno) except Exception: # When a warning is logged during Python shutdown, linecache # and the import machinery don't work anymore line = None linecache = None else: line = msg.line if line: line = line.strip() s += " %s\n" % line if msg.source is not None: try: import tracemalloc tb = tracemalloc.get_object_traceback(msg.source) except Exception: # When a warning is logged during Python shutdown, tracemalloc # and the import machinery don't work anymore tb = None if tb is not None: s += 'Object allocated at (most recent call first):\n' for frame in tb: s += (' File "%s", lineno %s\n' % (frame.filename, frame.lineno)) try: if linecache is not None: line = linecache.getline(frame.filename, frame.lineno) else: line = None except Exception: line = None if line: line = line.strip() s += ' %s\n' % line return s
def test_new_reference(self): tracemalloc.clear_traces() # gc.collect() indirectly calls PyList_ClearFreeList() support.gc_collect() # Create a list and "destroy it": put it in the PyListObject free list obj = [] obj = None # Create a list which should reuse the previously created empty list obj = [] nframe = tracemalloc.get_traceback_limit() frames = get_frames(nframe, -3) obj_traceback = tracemalloc.Traceback(frames, min(len(frames), nframe)) traceback = tracemalloc.get_object_traceback(obj) self.assertIsNotNone(traceback) self.assertEqual(traceback, obj_traceback)
def _formatwarnmsg_impl(msg): s = '%s:%s: %s: %s\n' % (msg.filename, msg.lineno, msg.category.__name__, msg.message) if msg.line is None: try: import linecache line = linecache.getline(msg.filename, msg.lineno) except Exception: line = None linecache = None else: line = msg.line if line: line = line.strip() s += ' %s\n' % line if msg.source is not None: try: import tracemalloc tb = tracemalloc.get_object_traceback(msg.source) except Exception: tb = None if tb is not None: s += 'Object allocated at (most recent call first):\n' for frame in tb: s += ' File "%s", lineno %s\n' % (frame.filename, frame.lineno) try: if linecache is not None: line = linecache.getline(frame.filename, frame.lineno) else: line = None except Exception: line = None if line: line = line.strip() s += ' %s\n' % line return s
def test_get_object_traceback(self): tracemalloc.clear_traces() obj_size = 12345 obj, obj_traceback = allocate_bytes(obj_size) traceback = tracemalloc.get_object_traceback(obj) self.assertEqual(traceback, obj_traceback)
def update_event(self, inp=-1): self.set_output_val(0, tracemalloc.get_object_traceback(self.input(0)))