def _get_snapshot(): snapshot = tracemalloc.take_snapshot() return snapshot.filter_traces( (tracemalloc.Filter(False, '<frozen importlib._bootstrap>'), tracemalloc.Filter(False, '<frozen importlib._bootstrap_external>'), tracemalloc.Filter(False, '<unknown>'), tracemalloc.Filter(False, tracemalloc.__file__)))
def display_top(snapshot, key_type="lineno", limit=3): snapshot = snapshot.filter_traces( ( tracemalloc.Filter(False, "<frozen importlib._bootstrap>"), tracemalloc.Filter(False, "<unknown>"), ) ) top_stats = snapshot.statistics(key_type) print("Top %s lines" % limit) for index, stat in enumerate(top_stats[:limit], 1): frame = stat.traceback[0] # replace "/path/to/module/file.py" with "module/file.py" filename = os.sep.join(frame.filename.split(os.sep)[-2:]) print( "#%s: %s:%s: %.1f KiB" % (index, filename, frame.lineno, stat.size / 1024) ) line = linecache.getline(frame.filename, frame.lineno).strip() if line: print(" %s" % line) other = top_stats[limit:] if other: size = sum(stat.size for stat in other) print("%s other: %.1f KiB" % (len(other), size / 1024)) total = sum(stat.size for stat in top_stats) print("Total allocated size: %.1f KiB" % (total / 1024))
def display_top(self, snapshot, key_type="lineno", limit=10): self.set_status(200, "Ok") self.set_header("Content-type", "text/plain") snapshot = snapshot.filter_traces(( tracemalloc.Filter(False, "<frozen importlib._bootstrap>"), tracemalloc.Filter(False, "<unknown>"), )) top_stats = snapshot.statistics(key_type) self.write(bytes(f"Top {limit} lines\n".encode("utf-8"))) for index, stat in enumerate(top_stats[:limit], 1): frame = stat.traceback[0] # replace "/path/to/module/file.py" with "module/file.py" filename = os.sep.join(frame.filename.split(os.sep)[-2:]) self.write( bytes(("#%s: %s:%s: %.1f KiB\n" % (index, filename, frame.lineno, stat.size / 1024)).encode("utf-8"))) line = linecache.getline(frame.filename, frame.lineno).strip() if line: self.write(f" {line}\n") other = top_stats[limit:] if other: size = sum(stat.size for stat in other) self.write( bytes(("%s other: %.1f KiB\n" % (len(other), size / 1024)).encode("utf-8"))) total = sum(stat.size for stat in top_stats) self.write( bytes(("Total allocated size: %.1f KiB\n" % (total / 1024)).encode("utf-8")))
def test_filter_traces(self): snapshot, snapshot2 = create_snapshots() filter1 = tracemalloc.Filter(False, "b.py") filter2 = tracemalloc.Filter(True, "a.py", 2) filter3 = tracemalloc.Filter(True, "a.py", 5) original_traces = list(snapshot.traces._traces) # exclude b.py snapshot3 = snapshot.filter_traces((filter1, )) self.assertEqual(snapshot3.traces._traces, [ (10, (('a.py', 2), ('b.py', 4))), (10, (('a.py', 2), ('b.py', 4))), (10, (('a.py', 2), ('b.py', 4))), (2, (('a.py', 5), ('b.py', 4))), (7, (('<unknown>', 0), )), ]) # filter_traces() must not touch the original snapshot self.assertEqual(snapshot.traces._traces, original_traces) # only include two lines of a.py snapshot4 = snapshot3.filter_traces((filter2, filter3)) self.assertEqual(snapshot4.traces._traces, [ (10, (('a.py', 2), ('b.py', 4))), (10, (('a.py', 2), ('b.py', 4))), (10, (('a.py', 2), ('b.py', 4))), (2, (('a.py', 5), ('b.py', 4))), ]) # No filter: just duplicate the snapshot snapshot5 = snapshot.filter_traces(()) self.assertIsNot(snapshot5, snapshot) self.assertIsNot(snapshot5.traces, snapshot.traces) self.assertEqual(snapshot5.traces, snapshot.traces)
def memory_tracer(): tracemalloc.start() tracemalloc.clear_traces() filters = ( tracemalloc.Filter(True, aiormq.__file__), tracemalloc.Filter(True, pamqp.__file__), tracemalloc.Filter(True, aio_pika.__file__), ) snapshot_before = tracemalloc.take_snapshot().filter_traces(filters) try: yield gc.collect() snapshot_after = tracemalloc.take_snapshot().filter_traces(filters) top_stats = snapshot_after.compare_to(snapshot_before, 'lineno', cumulative=True) assert not top_stats finally: tracemalloc.stop()
def display_top(snapshot, key_type='lineno', limit=10): """https://docs.python.org/3/library/tracemalloc.html""" # TODO write this as a context manager, then import as library snapshot = snapshot.filter_traces(( tracemalloc.Filter(False, "<frozen importlib._bootstrap>"), tracemalloc.Filter(False, "<unknown>"), )) top_stats = snapshot.statistics(key_type) print(f'Top {limit} lines') for index, stat in enumerate(top_stats[:limit], 1): frame = stat.traceback[0] # replace "/path/to/module/file.py" with "module/file.py" filename = os.sep.join(frame.filename.split(os.sep)[-2:]) print(f'#{index}: {filename}:{frame.lineno}: ' f'{stat.size / 1048576:.1f} MB') line = linecache.getline(frame.filename, frame.lineno).strip() if line: print(f' {line}') other = top_stats[limit:] if other: size = sum(stat.size for stat in other) / 1048576 print(f"{len(other)} other: {size:.1f} MB") total = sum(stat.size for stat in top_stats) / 1048576 print(f'Total allocated size: {total:.1f} MB')
def display_top_memory_users(key_type='lineno', limit=3, censored=True): snapshot = tracemalloc.take_snapshot() if censored: snapshot = snapshot.filter_traces(( tracemalloc.Filter(False, "<frozen importlib._bootstrap>"), tracemalloc.Filter(False, "<unknown>"), )) print("Top %s memory usage lines" % limit) else: limit = 0 top_stats = snapshot.statistics(key_type) print(("Unc" if not censored else "C") + "ensored memory usage:") for index, stat in enumerate(top_stats[:limit], 1): frame = stat.traceback[0] # replace "/path/to/module/file.py" with "module/file.py" filename = os.sep.join(frame.filename.split(os.sep)[-2:]) print("#%s: %s:%s: %.1f KiB" % (index, filename, frame.lineno, stat.size / 1024)) line = linecache.getline(frame.filename, frame.lineno).strip() if line: print(' %s' % line) other = top_stats[limit:] if other and censored: size = sum(stat.size for stat in other) print("%s other: %.1f KiB" % (len(other), size / 1024)) total = sum(stat.size for stat in top_stats) print("Total allocated size: %.1f KiB" % (total / 1024)) print()
def get_top_memory_usage(snapshot, key_type='lineno', limit=3, logger = None): snapshot = snapshot.filter_traces(( tracemalloc.Filter(False, "<frozen importlib._bootstrap>"), tracemalloc.Filter(False, "<unknown>"), )) top_stats = snapshot.statistics(key_type) out = [] out.append("Top %s lines" % limit) for index, stat in enumerate(top_stats[:limit], 1): frame = stat.traceback[0] # replace "/path/to/module/file.py" with "module/file.py" filename = os.sep.join(frame.filename.split(os.sep)[-2:]) out.append("#%s: %s:%s: %.1f KiB" % (index, filename, frame.lineno, stat.size / 1024)) line = linecache.getline(frame.filename, frame.lineno).strip() if line: out.append(' %s' % line) other = top_stats[limit:] if other: size = sum(stat.size for stat in other) out.append("%s other: %.1f KiB" % (len(other), size / 1024)) total = sum(stat.size for stat in top_stats) out.append("Total allocated size: %.1f KiB" % (total / 1024)) if logger is not None: for line in out: logger.debug("{}".format(line)) return out
def trace_memory_leaks(key_type='lineno', limit=50): snapshot = tracemalloc.take_snapshot() snapshot = snapshot.filter_traces(( # tracemalloc.Filter(True, "fkhttp.py"), tracemalloc.Filter(False, "<frozen importlib._bootstrap>"), tracemalloc.Filter(False, "<frozen importlib._bootstrap_external>"), tracemalloc.Filter(False, threading.__file__), tracemalloc.Filter(False, "<unknown>"), )) top_stats = snapshot.statistics(key_type) total_leak_size = sum(stat.size for stat in top_stats) print("Total Leak size: %.1f KiB" % (total_leak_size / 1024)) print("Top %s lines" % limit) for index, stat in enumerate(top_stats[:limit], 1): frame = stat.traceback[0] print(frame.filename) # replace "/path/to/module/file.py" with "module/file.py" filename = os.sep.join(frame.filename.split(os.sep)[-2:]) print("#%s: %s:%s: %.1f KiB" % (index, filename, frame.lineno, stat.size / 1024)) line = linecache.getline(frame.filename, frame.lineno).strip() if line: print(' %s' % line) other = top_stats[limit:] if other: size = sum(stat.size for stat in other) print("%s other: %.1f KiB" % (len(other), size / 1024)) total = sum(stat.size for stat in top_stats) print("Total allocated size: %.1f KiB" % (total / 1024))
def profile_memory(stats_mode: str = "lineno", top_limit: int = 10): """ Provides a context manager which will activates memory profiling of our source code. """ tracemalloc.start() yield snapshot = tracemalloc.take_snapshot() snapshot = snapshot.filter_traces(( tracemalloc.Filter(False, "<frozen importlib._bootstrap>"), tracemalloc.Filter(False, "<unknown>"), tracemalloc.Filter(False, "<frozen importlib._bootstrap_external>"), )) top_stats = snapshot.statistics(stats_mode) print("Top %s lines" % top_limit) for index, stat in enumerate(top_stats[:top_limit], 1): frame = stat.traceback[0] print("#%s: %s:%s: %.1f KiB" % (index, frame.filename, frame.lineno, stat.size / 1024)) line = linecache.getline(frame.filename, frame.lineno).strip() if line: print(" %s" % line) other = top_stats[top_limit:] if other: size = sum(stat.size for stat in other) print("%s other: %.1f KiB" % (len(other), size / 1024)) total = sum(stat.size for stat in top_stats) print("Total allocated size: %.1f KiB" % (total / 1024))
def _log_memory_top(snapshot, group_by='lineno', limit=30): try: import tracemalloc import linecache except ImportError: return snapshot = snapshot.filter_traces(( tracemalloc.Filter(False, "<frozen importlib._bootstrap>"), tracemalloc.Filter(False, "<unknown>"), )) top_stats = snapshot.statistics(group_by) log_info("Top %s lines", limit) for index, stat in enumerate(top_stats[:limit], 1): frame = stat.traceback[0] # replace "/path/to/module/file.py" with "module/file.py" filename = os.sep.join(frame.filename.split(os.sep)[-2:]) log_info("#%s: %s:%s: %.1f KiB", index, filename, frame.lineno, stat.size / 1024) line = linecache.getline(frame.filename, frame.lineno).strip() if line: log_info(' %s', line) other = top_stats[limit:] if other: size = sum(stat.size for stat in other) log_info("%s other: %.1f KiB", len(other), size / 1024) total = sum(stat.size for stat in top_stats) log_info("Total allocated size: %.1f KiB", total / 1024)
def test_filter_traces(self): snapshot, snapshot2 = create_snapshots() filter1 = tracemalloc.Filter(False, 'b.py') filter2 = tracemalloc.Filter(True, 'a.py', 2) filter3 = tracemalloc.Filter(True, 'a.py', 5) original_traces = list(snapshot.traces._traces) snapshot3 = snapshot.filter_traces((filter1, )) self.assertEqual(snapshot3.traces._traces, [(0, 10, (('a.py', 2), ('b.py', 4))), (0, 10, (('a.py', 2), ('b.py', 4))), (0, 10, (('a.py', 2), ('b.py', 4))), (1, 2, (('a.py', 5), ('b.py', 4))), (3, 7, (('<unknown>', 0), ))]) self.assertEqual(snapshot.traces._traces, original_traces) snapshot4 = snapshot3.filter_traces((filter2, filter3)) self.assertEqual(snapshot4.traces._traces, [(0, 10, (('a.py', 2), ('b.py', 4))), (0, 10, (('a.py', 2), ('b.py', 4))), (0, 10, (('a.py', 2), ('b.py', 4))), (1, 2, (('a.py', 5), ('b.py', 4)))]) snapshot5 = snapshot.filter_traces(()) self.assertIsNot(snapshot5, snapshot) self.assertIsNot(snapshot5.traces, snapshot.traces) self.assertEqual(snapshot5.traces, snapshot.traces) self.assertRaises(TypeError, snapshot.filter_traces, filter1)
def log_top(snapshot, key_type='traceback', limit=10): snapshot = snapshot.filter_traces(( tracemalloc.Filter(False, "<frozen importlib._bootstrap>"), tracemalloc.Filter(False, "<unknown>"), )) top_stats = snapshot.statistics(key_type) top = f"Top {limit} memory usage lines\n" for index, stat in enumerate(top_stats[:limit], 1): frame = stat.traceback[0] # replace "/path/to/module/file.py" with "module/file.py" filename = os.sep.join(frame.filename.split(os.sep)[-2:]) top += f"#{index}. {filename}:{frame.lineno} {stat.size / 1024:.1f} KiB\n" line = linecache.getline(frame.filename, frame.lineno).strip() if line: top += f" {line}\n" other = top_stats[limit:] if other: size = sum(stat.size for stat in other) top += f"{len(other)} other: {size / 1024:.1f} KiB\n" total = sum(stat.size for stat in top_stats) top += f"Total allocated size: {total / 1024:.1f} KiB\n" logging.debug(top)
async def show_memory(): key_type='lineno' limit=10 snapshot = tracemalloc.take_snapshot() snapshot = snapshot.filter_traces(( tracemalloc.Filter(False, "<frozen importlib._bootstrap>"), tracemalloc.Filter(False, "<unknown>"), )) top_stats = snapshot.statistics(key_type) logger.info("------- Trace --------") logger.info("Top %s lines" % limit) for index, stat in enumerate(top_stats[:limit], 1): frame = stat.traceback[0] # replace "/path/to/module/file.py" with "module/file.py" filename = os.sep.join(frame.filename.split(os.sep)[-2:]) logger.info("#%s: %s:%s: %.1f KiB" % (index, filename, frame.lineno, stat.size / 1024)) line = linecache.getline(frame.filename, frame.lineno).strip() if line: logger.info(' %s' % line) other = top_stats[limit:] if other: size = sum(stat.size for stat in other) logger.info("%s other: %.1f KiB" % (len(other), size / 1024)) total = sum(stat.size for stat in top_stats) logger.info("Total allocated size: %.1f KiB" % (total / 1024)) logger.info("-------- End ---------") await asyncio.sleep(30) asyncio.Task(show_memory())
def display_top(snapshot, key_type='lineno', limit=3): """ Usage ----- from nkrpy.logger import memory_trace as mt; import tracemalloc as tm; from nkrpy.math import pairwise; import numpy as np; tm.start(); list(pairwise(np.arange(1,200000).tolist(), 4)); sn = tm.take_snapshot(); mt.display_top(sn, limit=0)""" snapshot = snapshot.filter_traces(( tracemalloc.Filter(False, "<frozen importlib._bootstrap>"), tracemalloc.Filter(False, "<unknown>"), )) top_stats = snapshot.statistics(key_type) print("Top %s lines" % limit) for index, stat in enumerate(top_stats[:limit], 1): frame = stat.traceback[0] # replace "/path/to/module/file.py" with "module/file.py" filename = os.sep.join(frame.filename.split(os.sep)[-2:]) print("#%s: %s:%s: %.1f KiB" % (index, filename, frame.lineno, stat.size / 1024)) line = linecache.getline(frame.filename, frame.lineno).strip() if line: print(' %s' % line) other = top_stats[limit:] if other: size = sum(stat.size for stat in other) print("%s other: %.1f KiB" % (len(other), size / 1024)) total = sum(stat.size for stat in top_stats) print("Total allocated size: %.1f KiB" % (total / 1024))
def on_get(self, req, resp, limit=5): # Mostly copied from https://stackoverflow.com/a/45679009 output = [] snapshot = tracemalloc.take_snapshot() snapshot = snapshot.filter_traces(( tracemalloc.Filter(False, "<frozen importlib._bootstrap>"), tracemalloc.Filter(False, "<unknown>"), )) top_stats = snapshot.statistics('lineno') output.append("Top %s lines" % limit) for index, stat in enumerate(top_stats[:limit], 1): frame = stat.traceback[0] # replace "/path/to/module/file.py" with "module/file.py" filename = os.sep.join(frame.filename.split(os.sep)[-2:]) output.append("#%s: %s:%s: %.1f KiB" % (index, filename, frame.lineno, stat.size / 1024)) line = linecache.getline(frame.filename, frame.lineno).strip() if line: output.append(' %s' % line) other = top_stats[limit:] if other: size = sum(stat.size for stat in other) output.append("%s other: %.1f KiB" % (len(other), size / 1024)) total = sum(stat.size for stat in top_stats) output.append("Total allocated size: %.1f KiB" % (total / 1024)) resp.body = "\n".join(output)
def display_top(snapshot, key_type='lineno', limit=10): snapshot = snapshot.filter_traces(( tracemalloc.Filter(False, "<frozen importlib._bootstrap>"), tracemalloc.Filter(False, "<unknown>"), )) top_stats = snapshot.statistics(key_type) print("Top %s lines" % limit) for index, stat in enumerate(top_stats[:limit], 1): frame = stat.traceback[0] print("#%s: %s:%s: %.1f KiB" % (index, frame.filename, frame.lineno, stat.size / 1024), "({:d})".format(stat.size)) line = linecache.getline(frame.filename, frame.lineno).strip() if line: print(' %s' % line) other = top_stats[limit:] if other: size = sum(stat.size for stat in other) print("%s other: %.1f KiB" % (len(other), size / 1024), "({:d})".format(size)) total = sum(stat.size for stat in top_stats) print("Total allocated size: %.1f KiB" % (total / 1024), "({:d})".format(total))
def display_top(snapshot, key_type="lineno", limit=3, unit="kb"): if unit == "kb": size = 1000 elif unit == "mb": size = 1000000 elif unit == "gb": size = 1000000000 else: raise ValueError("Unit '{}' not supported.") snapshot = snapshot.filter_traces(( tracemalloc.Filter(False, "<frozen importlib._bootstrap>"), tracemalloc.Filter(False, "<unknown>"), )) top_stats = snapshot.statistics(key_type) print("Top {} lines".format(limit)) for index, stat in enumerate(top_stats[:limit], 1): frame = stat.traceback[0] # replace "/path/to/module/file.py" with "module/file.py" filename = os.sep.join(frame.filename.split(os.sep)[-2:]) print("#{0}: {1}:{2}: {3:.1f} {4}".format(index, filename, frame.lineno, stat.size / size, unit)) line = linecache.getline(frame.filename, frame.lineno).strip() if line: print(' {}'.format(line)) other = top_stats[limit:] if other: size = sum([stat.size for stat in other]) print("{0} other: {1:.1f} {2}".format(len(other), size / size, unit)) total = sum([stat.size for stat in top_stats]) print("Total allocated size: {0:.1f} {1}".format(total / size, unit))
def take_snapshot(sig, frame=None): global __old_snapshot gc.collect() if not __old_snapshot: print(f"Start SnapShot Tracking") tracemalloc.start() snapshot = tracemalloc.take_snapshot() snapshot = snapshot.filter_traces(( tracemalloc.Filter(False, "<frozen importlib._bootstrap>"), tracemalloc.Filter(False, "<unknown>"), tracemalloc.Filter(False, "*linecache.py"), )) key = __get_configs(MallocEnvsKey.statics_key) rank_num = __get_configs(MallocEnvsKey.statics_rank_number) top_stats = snapshot.statistics(key) __old_snapshot = snapshot if not top_stats: return print(f"Top {rank_num}:") for index, stat in enumerate(top_stats[:int(rank_num)], 1): frame = stat.traceback[0] print("#%s: %s:%s: %.1f KiB" % (index, frame.filename, frame.lineno, stat.size / 1024)) line = linecache.getline(frame.filename, frame.lineno).strip() if line: print(' %s' % line)
def main(): args = parse_args() files = os.listdir(args.snapshot_dir) if args.cv_only is True: files = [f for f in files if "cv" in f] paths = [os.path.join(args.snapshot_dir, f) for f in files] snapshots = [] tm_filters = (tracemalloc.Filter(False, "<frozen importlib._bootstrap>"), tracemalloc.Filter(False, "<unknown>")) for fn in paths: print(os.path.basename(fn)) snapshot = tracemalloc.Snapshot.load(fn) snapshots.append(snapshot) display_top(snapshot, unit=args.unit) print("-" * 10) print() if len(snapshots) > 1: stats = snapshot.filter_traces(tm_filters).compare_to( snapshots[-2], 'filename') for stat in stats[:5]: print( "{} new kb, {} total kb, {} new, {} total memory blocks: ". format(stat.size_diff / 1000, stat.size / 1000, stat.count_diff, stat.count)) for line in stat.traceback.format(): print(line) print()
def display_top(snapshot, key_type='lineno', limit=20): """Function to display the top traces. Note: Start by ignoring <frozen importlib._bootstrap> and <unknown> files and using statistics group by line no. Then we enumerate the top_stats and get the frame, filename, line number, line name and the RSS bytes for each of the top 20 traces. Based on: https://pytracemalloc.readthedocs.io/examples.html. :param snapshot : A snapshot of traces. In this case the Max RSS. :param key_type : Group by the line number, defaults to 'lineno'. :param limit (int) : Number of profiles to monitor, defaults to 20. """ snapshot = snapshot.filter_traces(( tracemalloc.Filter(False, "<frozen importlib._bootstrap>"), tracemalloc.Filter(False, "<unknown>"), )) top_stats = snapshot.statistics(key_type) print("Top {} lines".format(limit)) for index, stat in enumerate(top_stats[:limit], 1): frame = stat.traceback[0] filename = os.sep.join(frame.filename.split(os.sep)[-2:]) line = linecache.getline(frame.filename, frame.lineno).strip() print("#{:3d}: {:23s} | LineNo: {:>4} | RSS: {:>8} | LINE: {:>8}".format(index, filename, frame.lineno, format_bytes(stat.size), line)) other = top_stats[limit:] if other: size = sum(stat.size for stat in other) print("{} other calls: {}".format(len(other), format_bytes(size))) total = sum(stat.size for stat in top_stats) print("Total allocated size: {}".format(format_bytes(total)))
def trace_memory_alloc_pretty_top(snapshot, key_type='lineno', limit=10): """From https://docs.python.org/3/library/tracemalloc.html""" print('-----------------------------------------------------------------') snapshot = snapshot.filter_traces(( tracemalloc.Filter(False, "<frozen importlib._bootstrap>"), tracemalloc.Filter(False, "<unknown>"), )) top_stats = snapshot.statistics(key_type) print("Top %s lines" % limit) for index, stat in enumerate(top_stats[:limit], 1): frame = stat.traceback[0] # replace "/path/to/module/file.py" with "module/file.py" filename = sep.join(frame.filename.split(sep)[-2:]) print("#%s: %s:%s: %.1f KiB" % (index, filename, frame.lineno, stat.size / 1024)) line = linecache.getline(frame.filename, frame.lineno).strip() if line: print(' %s' % line) other = top_stats[limit:] if other: size = sum(stat.size for stat in other) print("%s other: %.1f KiB" % (len(other), size / 1024)) total = sum(stat.size for stat in top_stats) print("Total allocated size: %.1f KiB" % (total / 1024))
def test_filter_attributes(self): # test default values f = tracemalloc.Filter(True, "abc") self.assertEqual(f.inclusive, True) self.assertEqual(f.filename_pattern, "abc") self.assertIsNone(f.lineno) self.assertEqual(f.all_frames, False) # test custom values f = tracemalloc.Filter(False, "test.py", 123, True) self.assertEqual(f.inclusive, False) self.assertEqual(f.filename_pattern, "test.py") self.assertEqual(f.lineno, 123) self.assertEqual(f.all_frames, True) # parameters passed by keyword f = tracemalloc.Filter(inclusive=False, filename_pattern="test.py", lineno=123, all_frames=True) self.assertEqual(f.inclusive, False) self.assertEqual(f.filename_pattern, "test.py") self.assertEqual(f.lineno, 123) self.assertEqual(f.all_frames, True) # read-only attribute self.assertRaises(AttributeError, setattr, f, "filename_pattern", "abc")
def test_object_hook_and_default(): tracemalloc.start() data = [] for i in range(1, 100): data.append({ "name": "a%d" % i, "timestamp": datetime.timedelta(seconds=i) }) snapshot1 = tracemalloc.take_snapshot().filter_traces( (tracemalloc.Filter(True, __file__), )) for _ in range(1000): a = rj.dumps(data, default=default) rj.loads(a, object_hook=object_hook) gc.collect() snapshot2 = tracemalloc.take_snapshot().filter_traces( (tracemalloc.Filter(True, __file__), )) top_stats = snapshot2.compare_to(snapshot1, 'lineno') tracemalloc.stop() for stat in top_stats[:10]: assert stat.count_diff < 3
def test_filter_traces_domain(self): snapshot, snapshot2 = create_snapshots() filter1 = tracemalloc.Filter(False, "a.py", domain=1) filter2 = tracemalloc.Filter(True, "a.py", domain=1) original_traces = list(snapshot.traces._traces) # exclude a.py of domain 1 snapshot3 = snapshot.filter_traces((filter1,)) self.assertEqual(snapshot3.traces._traces, [ (0, 10, (('a.py', 2), ('b.py', 4))), (0, 10, (('a.py', 2), ('b.py', 4))), (0, 10, (('a.py', 2), ('b.py', 4))), (2, 66, (('b.py', 1),)), (3, 7, (('<unknown>', 0),)), ]) # include domain 1 snapshot3 = snapshot.filter_traces((filter1,)) self.assertEqual(snapshot3.traces._traces, [ (0, 10, (('a.py', 2), ('b.py', 4))), (0, 10, (('a.py', 2), ('b.py', 4))), (0, 10, (('a.py', 2), ('b.py', 4))), (2, 66, (('b.py', 1),)), (3, 7, (('<unknown>', 0),)), ])
def display_tracemalloc(snapshot, key_type='lineno', limit=10): snapshot = snapshot.filter_traces(( tracemalloc.Filter(False, "<frozen importlib._bootstrap*"), tracemalloc.Filter(False, "<unknown>"), tracemalloc.Filter(False, "*/idna/*.py"), )) top_stats = snapshot.statistics(key_type) print("Top {} lines".format(limit)) for index, stat in enumerate(top_stats[:limit], 1): frame = stat.traceback[0] # replace "/path/to/module/file.py" with "module/file.py" filename = os.sep.join(frame.filename.split(os.sep)[-2:]) print("#{}: {}:{}: {:.1f} KiB".format(index, filename, frame.lineno, stat.size // 1024)) line = linecache.getline(frame.filename, frame.lineno).strip() if line: print(' {}'.format(line)) other = top_stats[limit:] if other: size = sum(stat.size for stat in other) print("{} other: {:.1f} KiB".format(len(other), size // 1024)) total = sum(stat.size for stat in top_stats) current, peak = tracemalloc.get_traced_memory() print("Total allocated size: {:8.1f} KiB".format(total // 1024)) print("Current size: {:8.1f} KiB".format(current // 1024)) print("Peak size: {:8.1f} KiB".format(peak // 1024))
def _display_top(snapshot: tracemalloc.Snapshot, *, key_type: str = "lineno", limit: int = 10) -> None: """Print top consumers. Inspired by Python docs: https://docs.python.org/3/library/tracemalloc.html#get-the-traceback-of-a-memory-block """ snapshot = snapshot.filter_traces(( tracemalloc.Filter(False, "<frozen importlib._bootstrap>"), tracemalloc.Filter(False, "<unknown>"), )) top_stats = snapshot.statistics(key_type) print("Top %s lines" % limit) for index, stat in enumerate(top_stats[:limit], 1): frame = stat.traceback[0] print("#%s: %s:%s: %.1f KiB" % (index, frame.filename, frame.lineno, stat.size / 1024)) line = linecache.getline(frame.filename, frame.lineno).strip() if line: print(" %s" % line) other = top_stats[limit:] if other: size = sum(stat.size for stat in other) print("%s other: %.1f KiB" % (len(other), size / 1024)) total = sum(stat.size for stat in top_stats) print("Total allocated size: %.1f KiB" % (total / 1024))
def get_detailed_mem_use(snapshot, key_type='lineno', limit=30): import tracemalloc import linecache wr_line = "" snapshot = snapshot.filter_traces(( tracemalloc.Filter(False, "<frozen importlib._bootstrap>"), tracemalloc.Filter(False, "<unknown>"), )) top_stats = snapshot.statistics(key_type) wr_line += ("Top %s lines" % limit) for index, stat in enumerate(top_stats[:limit], 1): frame = stat.traceback[0] # replace "/path/to/module/file.py" with "module/file.py" filename = os.sep.join(frame.filename.split(os.sep)[-2:]) wr_line += ("\n #%s: %s:%s: %.1f KiB" % (index, filename, frame.lineno, stat.size / 1024)) line = linecache.getline(frame.filename, frame.lineno).strip() if line: wr_line += ('\n %s' % line) other = top_stats[limit:] if other: size = sum(stat.size for stat in other) wr_line += ("\n %s other: %.1f KiB" % (len(other), size / 1024)) total = sum(stat.size for stat in top_stats) wr_line += ("\n Total allocated size: %.1f KiB" % (total / 1024)) return wr_line
def take_snapshot(): snapshot = tracemalloc.take_snapshot() return snapshot.filter_traces(( tracemalloc.Filter(False, "<frozen importlib._bootstrap>"), tracemalloc.Filter(False, "tracemalloc"), tracemalloc.Filter(False, "<unknown>"), ))
def trace_print(): global snapshot global oldmem snapshot2 = tracemalloc.take_snapshot() snapshot2 = snapshot2.filter_traces( ( tracemalloc.Filter(False, "<frozen importlib._bootstrap>"), tracemalloc.Filter(False, "<unknown>"), tracemalloc.Filter(False, tracemalloc.__file__), ) ) if snapshot is not None: thismem = PROCESS.memory_info().rss / 1024 ** 2 diff = thismem - oldmem print( "===================== Begin Trace (TOTAL MEM={:1.4e} MB... [{:+1.4e} MB]):".format( thismem, diff ) ) top_stats = snapshot2.compare_to(snapshot, "lineno", cumulative=True) for stat in top_stats[:4]: print(stat) print("End Trace ===========================================") print() oldmem = thismem snapshot = snapshot2