def _get_snapshot():
snapshot = tracemalloc.take_snapshot()
return snapshot.filter_traces(
(tracemalloc.Filter(False, '<frozen importlib._bootstrap>'),
tracemalloc.Filter(False, '<frozen importlib._bootstrap_external>'),
tracemalloc.Filter(False, '<unknown>'),
tracemalloc.Filter(False, tracemalloc.__file__)))
def display_top(snapshot, key_type="lineno", limit=3):
snapshot = snapshot.filter_traces(
(
tracemalloc.Filter(False, "<frozen importlib._bootstrap>"),
tracemalloc.Filter(False, "<unknown>"),
)
)
top_stats = snapshot.statistics(key_type)
print("Top %s lines" % limit)
for index, stat in enumerate(top_stats[:limit], 1):
frame = stat.traceback[0]
# replace "/path/to/module/file.py" with "module/file.py"
filename = os.sep.join(frame.filename.split(os.sep)[-2:])
print(
"#%s: %s:%s: %.1f KiB" % (index, filename, frame.lineno, stat.size / 1024)
)
line = linecache.getline(frame.filename, frame.lineno).strip()
if line:
print(" %s" % line)
other = top_stats[limit:]
if other:
size = sum(stat.size for stat in other)
print("%s other: %.1f KiB" % (len(other), size / 1024))
total = sum(stat.size for stat in top_stats)
print("Total allocated size: %.1f KiB" % (total / 1024))
def display_top(self, snapshot, key_type="lineno", limit=10):
self.set_status(200, "Ok")
self.set_header("Content-type", "text/plain")
snapshot = snapshot.filter_traces((
tracemalloc.Filter(False, "<frozen importlib._bootstrap>"),
tracemalloc.Filter(False, "<unknown>"),
))
top_stats = snapshot.statistics(key_type)
self.write(bytes(f"Top {limit} lines\n".encode("utf-8")))
for index, stat in enumerate(top_stats[:limit], 1):
frame = stat.traceback[0]
# replace "/path/to/module/file.py" with "module/file.py"
filename = os.sep.join(frame.filename.split(os.sep)[-2:])
self.write(
bytes(("#%s: %s:%s: %.1f KiB\n" %
(index, filename, frame.lineno,
stat.size / 1024)).encode("utf-8")))
line = linecache.getline(frame.filename, frame.lineno).strip()
if line:
self.write(f" {line}\n")
other = top_stats[limit:]
if other:
size = sum(stat.size for stat in other)
self.write(
bytes(("%s other: %.1f KiB\n" %
(len(other), size / 1024)).encode("utf-8")))
total = sum(stat.size for stat in top_stats)
self.write(
bytes(("Total allocated size: %.1f KiB\n" %
(total / 1024)).encode("utf-8")))
def test_filter_traces(self):
snapshot, snapshot2 = create_snapshots()
filter1 = tracemalloc.Filter(False, "b.py")
filter2 = tracemalloc.Filter(True, "a.py", 2)
filter3 = tracemalloc.Filter(True, "a.py", 5)
original_traces = list(snapshot.traces._traces)
# exclude b.py
snapshot3 = snapshot.filter_traces((filter1, ))
self.assertEqual(snapshot3.traces._traces, [
(10, (('a.py', 2), ('b.py', 4))),
(10, (('a.py', 2), ('b.py', 4))),
(10, (('a.py', 2), ('b.py', 4))),
(2, (('a.py', 5), ('b.py', 4))),
(7, (('<unknown>', 0), )),
])
# filter_traces() must not touch the original snapshot
self.assertEqual(snapshot.traces._traces, original_traces)
# only include two lines of a.py
snapshot4 = snapshot3.filter_traces((filter2, filter3))
self.assertEqual(snapshot4.traces._traces, [
(10, (('a.py', 2), ('b.py', 4))),
(10, (('a.py', 2), ('b.py', 4))),
(10, (('a.py', 2), ('b.py', 4))),
(2, (('a.py', 5), ('b.py', 4))),
])
# No filter: just duplicate the snapshot
snapshot5 = snapshot.filter_traces(())
self.assertIsNot(snapshot5, snapshot)
self.assertIsNot(snapshot5.traces, snapshot.traces)
self.assertEqual(snapshot5.traces, snapshot.traces)
def memory_tracer():
tracemalloc.start()
tracemalloc.clear_traces()
filters = (
tracemalloc.Filter(True, aiormq.__file__),
tracemalloc.Filter(True, pamqp.__file__),
tracemalloc.Filter(True, aio_pika.__file__),
)
snapshot_before = tracemalloc.take_snapshot().filter_traces(filters)
try:
yield
gc.collect()
snapshot_after = tracemalloc.take_snapshot().filter_traces(filters)
top_stats = snapshot_after.compare_to(snapshot_before,
'lineno',
cumulative=True)
assert not top_stats
finally:
tracemalloc.stop()
def display_top(snapshot, key_type='lineno', limit=10):
"""https://docs.python.org/3/library/tracemalloc.html"""
# TODO write this as a context manager, then import as library
snapshot = snapshot.filter_traces((
tracemalloc.Filter(False, "<frozen importlib._bootstrap>"),
tracemalloc.Filter(False, "<unknown>"),
))
top_stats = snapshot.statistics(key_type)
print(f'Top {limit} lines')
for index, stat in enumerate(top_stats[:limit], 1):
frame = stat.traceback[0]
# replace "/path/to/module/file.py" with "module/file.py"
filename = os.sep.join(frame.filename.split(os.sep)[-2:])
print(f'#{index}: {filename}:{frame.lineno}: '
f'{stat.size / 1048576:.1f} MB')
line = linecache.getline(frame.filename, frame.lineno).strip()
if line:
print(f' {line}')
other = top_stats[limit:]
if other:
size = sum(stat.size for stat in other) / 1048576
print(f"{len(other)} other: {size:.1f} MB")
total = sum(stat.size for stat in top_stats) / 1048576
print(f'Total allocated size: {total:.1f} MB')
def display_top_memory_users(key_type='lineno', limit=3, censored=True):
snapshot = tracemalloc.take_snapshot()
if censored:
snapshot = snapshot.filter_traces((
tracemalloc.Filter(False, "<frozen importlib._bootstrap>"),
tracemalloc.Filter(False, "<unknown>"),
))
print("Top %s memory usage lines" % limit)
else:
limit = 0
top_stats = snapshot.statistics(key_type)
print(("Unc" if not censored else "C") + "ensored memory usage:")
for index, stat in enumerate(top_stats[:limit], 1):
frame = stat.traceback[0]
# replace "/path/to/module/file.py" with "module/file.py"
filename = os.sep.join(frame.filename.split(os.sep)[-2:])
print("#%s: %s:%s: %.1f KiB" %
(index, filename, frame.lineno, stat.size / 1024))
line = linecache.getline(frame.filename, frame.lineno).strip()
if line:
print(' %s' % line)
other = top_stats[limit:]
if other and censored:
size = sum(stat.size for stat in other)
print("%s other: %.1f KiB" % (len(other), size / 1024))
total = sum(stat.size for stat in top_stats)
print("Total allocated size: %.1f KiB" % (total / 1024))
print()
def get_top_memory_usage(snapshot, key_type='lineno', limit=3, logger = None):
snapshot = snapshot.filter_traces((
tracemalloc.Filter(False, "<frozen importlib._bootstrap>"),
tracemalloc.Filter(False, "<unknown>"),
))
top_stats = snapshot.statistics(key_type)
out = []
out.append("Top %s lines" % limit)
for index, stat in enumerate(top_stats[:limit], 1):
frame = stat.traceback[0]
# replace "/path/to/module/file.py" with "module/file.py"
filename = os.sep.join(frame.filename.split(os.sep)[-2:])
out.append("#%s: %s:%s: %.1f KiB"
% (index, filename, frame.lineno, stat.size / 1024))
line = linecache.getline(frame.filename, frame.lineno).strip()
if line:
out.append(' %s' % line)
other = top_stats[limit:]
if other:
size = sum(stat.size for stat in other)
out.append("%s other: %.1f KiB" % (len(other), size / 1024))
total = sum(stat.size for stat in top_stats)
out.append("Total allocated size: %.1f KiB" % (total / 1024))
if logger is not None:
for line in out:
logger.debug("{}".format(line))
return out
def trace_memory_leaks(key_type='lineno', limit=50):
snapshot = tracemalloc.take_snapshot()
snapshot = snapshot.filter_traces((
# tracemalloc.Filter(True, "fkhttp.py"),
tracemalloc.Filter(False, "<frozen importlib._bootstrap>"),
tracemalloc.Filter(False, "<frozen importlib._bootstrap_external>"),
tracemalloc.Filter(False, threading.__file__),
tracemalloc.Filter(False, "<unknown>"),
))
top_stats = snapshot.statistics(key_type)
total_leak_size = sum(stat.size for stat in top_stats)
print("Total Leak size: %.1f KiB" % (total_leak_size / 1024))
print("Top %s lines" % limit)
for index, stat in enumerate(top_stats[:limit], 1):
frame = stat.traceback[0]
print(frame.filename)
# replace "/path/to/module/file.py" with "module/file.py"
filename = os.sep.join(frame.filename.split(os.sep)[-2:])
print("#%s: %s:%s: %.1f KiB" %
(index, filename, frame.lineno, stat.size / 1024))
line = linecache.getline(frame.filename, frame.lineno).strip()
if line:
print(' %s' % line)
other = top_stats[limit:]
if other:
size = sum(stat.size for stat in other)
print("%s other: %.1f KiB" % (len(other), size / 1024))
total = sum(stat.size for stat in top_stats)
print("Total allocated size: %.1f KiB" % (total / 1024))
def profile_memory(stats_mode: str = "lineno", top_limit: int = 10):
"""
Provides a context manager which will activates memory profiling of our
source code.
"""
tracemalloc.start()
yield
snapshot = tracemalloc.take_snapshot()
snapshot = snapshot.filter_traces((
tracemalloc.Filter(False, "<frozen importlib._bootstrap>"),
tracemalloc.Filter(False, "<unknown>"),
tracemalloc.Filter(False, "<frozen importlib._bootstrap_external>"),
))
top_stats = snapshot.statistics(stats_mode)
print("Top %s lines" % top_limit)
for index, stat in enumerate(top_stats[:top_limit], 1):
frame = stat.traceback[0]
print("#%s: %s:%s: %.1f KiB" %
(index, frame.filename, frame.lineno, stat.size / 1024))
line = linecache.getline(frame.filename, frame.lineno).strip()
if line:
print(" %s" % line)
other = top_stats[top_limit:]
if other:
size = sum(stat.size for stat in other)
print("%s other: %.1f KiB" % (len(other), size / 1024))
total = sum(stat.size for stat in top_stats)
print("Total allocated size: %.1f KiB" % (total / 1024))
def _log_memory_top(snapshot, group_by='lineno', limit=30):
try:
import tracemalloc
import linecache
except ImportError:
return
snapshot = snapshot.filter_traces((
tracemalloc.Filter(False, "<frozen importlib._bootstrap>"),
tracemalloc.Filter(False, "<unknown>"),
))
top_stats = snapshot.statistics(group_by)
log_info("Top %s lines", limit)
for index, stat in enumerate(top_stats[:limit], 1):
frame = stat.traceback[0]
# replace "/path/to/module/file.py" with "module/file.py"
filename = os.sep.join(frame.filename.split(os.sep)[-2:])
log_info("#%s: %s:%s: %.1f KiB", index, filename, frame.lineno,
stat.size / 1024)
line = linecache.getline(frame.filename, frame.lineno).strip()
if line:
log_info(' %s', line)
other = top_stats[limit:]
if other:
size = sum(stat.size for stat in other)
log_info("%s other: %.1f KiB", len(other), size / 1024)
total = sum(stat.size for stat in top_stats)
log_info("Total allocated size: %.1f KiB", total / 1024)
def test_filter_traces(self):
snapshot, snapshot2 = create_snapshots()
filter1 = tracemalloc.Filter(False, 'b.py')
filter2 = tracemalloc.Filter(True, 'a.py', 2)
filter3 = tracemalloc.Filter(True, 'a.py', 5)
original_traces = list(snapshot.traces._traces)
snapshot3 = snapshot.filter_traces((filter1, ))
self.assertEqual(snapshot3.traces._traces,
[(0, 10, (('a.py', 2), ('b.py', 4))),
(0, 10, (('a.py', 2), ('b.py', 4))),
(0, 10, (('a.py', 2), ('b.py', 4))),
(1, 2, (('a.py', 5), ('b.py', 4))),
(3, 7, (('<unknown>', 0), ))])
self.assertEqual(snapshot.traces._traces, original_traces)
snapshot4 = snapshot3.filter_traces((filter2, filter3))
self.assertEqual(snapshot4.traces._traces,
[(0, 10, (('a.py', 2), ('b.py', 4))),
(0, 10, (('a.py', 2), ('b.py', 4))),
(0, 10, (('a.py', 2), ('b.py', 4))),
(1, 2, (('a.py', 5), ('b.py', 4)))])
snapshot5 = snapshot.filter_traces(())
self.assertIsNot(snapshot5, snapshot)
self.assertIsNot(snapshot5.traces, snapshot.traces)
self.assertEqual(snapshot5.traces, snapshot.traces)
self.assertRaises(TypeError, snapshot.filter_traces, filter1)
def log_top(snapshot, key_type='traceback', limit=10):
snapshot = snapshot.filter_traces((
tracemalloc.Filter(False, "<frozen importlib._bootstrap>"),
tracemalloc.Filter(False, "<unknown>"),
))
top_stats = snapshot.statistics(key_type)
top = f"Top {limit} memory usage lines\n"
for index, stat in enumerate(top_stats[:limit], 1):
frame = stat.traceback[0]
# replace "/path/to/module/file.py" with "module/file.py"
filename = os.sep.join(frame.filename.split(os.sep)[-2:])
top += f"#{index}. {filename}:{frame.lineno} {stat.size / 1024:.1f} KiB\n"
line = linecache.getline(frame.filename, frame.lineno).strip()
if line:
top += f" {line}\n"
other = top_stats[limit:]
if other:
size = sum(stat.size for stat in other)
top += f"{len(other)} other: {size / 1024:.1f} KiB\n"
total = sum(stat.size for stat in top_stats)
top += f"Total allocated size: {total / 1024:.1f} KiB\n"
logging.debug(top)
async def show_memory():
key_type='lineno'
limit=10
snapshot = tracemalloc.take_snapshot()
snapshot = snapshot.filter_traces((
tracemalloc.Filter(False, "<frozen importlib._bootstrap>"),
tracemalloc.Filter(False, "<unknown>"),
))
top_stats = snapshot.statistics(key_type)
logger.info("------- Trace --------")
logger.info("Top %s lines" % limit)
for index, stat in enumerate(top_stats[:limit], 1):
frame = stat.traceback[0]
# replace "/path/to/module/file.py" with "module/file.py"
filename = os.sep.join(frame.filename.split(os.sep)[-2:])
logger.info("#%s: %s:%s: %.1f KiB"
% (index, filename, frame.lineno, stat.size / 1024))
line = linecache.getline(frame.filename, frame.lineno).strip()
if line:
logger.info(' %s' % line)
other = top_stats[limit:]
if other:
size = sum(stat.size for stat in other)
logger.info("%s other: %.1f KiB" % (len(other), size / 1024))
total = sum(stat.size for stat in top_stats)
logger.info("Total allocated size: %.1f KiB" % (total / 1024))
logger.info("-------- End ---------")
await asyncio.sleep(30)
asyncio.Task(show_memory())
def display_top(snapshot, key_type='lineno', limit=3):
"""
Usage
-----
from nkrpy.logger import memory_trace as mt;
import tracemalloc as tm; from nkrpy.math import pairwise;
import numpy as np;
tm.start();
list(pairwise(np.arange(1,200000).tolist(), 4));
sn = tm.take_snapshot();
mt.display_top(sn, limit=0)"""
snapshot = snapshot.filter_traces((
tracemalloc.Filter(False, "<frozen importlib._bootstrap>"),
tracemalloc.Filter(False, "<unknown>"),
))
top_stats = snapshot.statistics(key_type)
print("Top %s lines" % limit)
for index, stat in enumerate(top_stats[:limit], 1):
frame = stat.traceback[0]
# replace "/path/to/module/file.py" with "module/file.py"
filename = os.sep.join(frame.filename.split(os.sep)[-2:])
print("#%s: %s:%s: %.1f KiB" %
(index, filename, frame.lineno, stat.size / 1024))
line = linecache.getline(frame.filename, frame.lineno).strip()
if line:
print(' %s' % line)
other = top_stats[limit:]
if other:
size = sum(stat.size for stat in other)
print("%s other: %.1f KiB" % (len(other), size / 1024))
total = sum(stat.size for stat in top_stats)
print("Total allocated size: %.1f KiB" % (total / 1024))
def on_get(self, req, resp, limit=5):
# Mostly copied from https://stackoverflow.com/a/45679009
output = []
snapshot = tracemalloc.take_snapshot()
snapshot = snapshot.filter_traces((
tracemalloc.Filter(False, "<frozen importlib._bootstrap>"),
tracemalloc.Filter(False, "<unknown>"),
))
top_stats = snapshot.statistics('lineno')
output.append("Top %s lines" % limit)
for index, stat in enumerate(top_stats[:limit], 1):
frame = stat.traceback[0]
# replace "/path/to/module/file.py" with "module/file.py"
filename = os.sep.join(frame.filename.split(os.sep)[-2:])
output.append("#%s: %s:%s: %.1f KiB" %
(index, filename, frame.lineno, stat.size / 1024))
line = linecache.getline(frame.filename, frame.lineno).strip()
if line:
output.append(' %s' % line)
other = top_stats[limit:]
if other:
size = sum(stat.size for stat in other)
output.append("%s other: %.1f KiB" % (len(other), size / 1024))
total = sum(stat.size for stat in top_stats)
output.append("Total allocated size: %.1f KiB" % (total / 1024))
resp.body = "\n".join(output)
def display_top(snapshot, key_type='lineno', limit=10):
snapshot = snapshot.filter_traces((
tracemalloc.Filter(False, "<frozen importlib._bootstrap>"),
tracemalloc.Filter(False, "<unknown>"),
))
top_stats = snapshot.statistics(key_type)
print("Top %s lines" % limit)
for index, stat in enumerate(top_stats[:limit], 1):
frame = stat.traceback[0]
print("#%s: %s:%s: %.1f KiB"
% (index, frame.filename, frame.lineno, stat.size / 1024),
"({:d})".format(stat.size))
line = linecache.getline(frame.filename, frame.lineno).strip()
if line:
print(' %s' % line)
other = top_stats[limit:]
if other:
size = sum(stat.size for stat in other)
print("%s other: %.1f KiB" % (len(other), size / 1024),
"({:d})".format(size))
total = sum(stat.size for stat in top_stats)
print("Total allocated size: %.1f KiB" % (total / 1024), "({:d})".format(total))
def display_top(snapshot, key_type="lineno", limit=3, unit="kb"):
if unit == "kb":
size = 1000
elif unit == "mb":
size = 1000000
elif unit == "gb":
size = 1000000000
else:
raise ValueError("Unit '{}' not supported.")
snapshot = snapshot.filter_traces((
tracemalloc.Filter(False, "<frozen importlib._bootstrap>"),
tracemalloc.Filter(False, "<unknown>"),
))
top_stats = snapshot.statistics(key_type)
print("Top {} lines".format(limit))
for index, stat in enumerate(top_stats[:limit], 1):
frame = stat.traceback[0]
# replace "/path/to/module/file.py" with "module/file.py"
filename = os.sep.join(frame.filename.split(os.sep)[-2:])
print("#{0}: {1}:{2}: {3:.1f} {4}".format(index, filename,
frame.lineno,
stat.size / size, unit))
line = linecache.getline(frame.filename, frame.lineno).strip()
if line:
print(' {}'.format(line))
other = top_stats[limit:]
if other:
size = sum([stat.size for stat in other])
print("{0} other: {1:.1f} {2}".format(len(other), size / size, unit))
total = sum([stat.size for stat in top_stats])
print("Total allocated size: {0:.1f} {1}".format(total / size, unit))
def take_snapshot(sig, frame=None):
global __old_snapshot
gc.collect()
if not __old_snapshot:
print(f"Start SnapShot Tracking")
tracemalloc.start()
snapshot = tracemalloc.take_snapshot()
snapshot = snapshot.filter_traces((
tracemalloc.Filter(False, "<frozen importlib._bootstrap>"),
tracemalloc.Filter(False, "<unknown>"),
tracemalloc.Filter(False, "*linecache.py"),
))
key = __get_configs(MallocEnvsKey.statics_key)
rank_num = __get_configs(MallocEnvsKey.statics_rank_number)
top_stats = snapshot.statistics(key)
__old_snapshot = snapshot
if not top_stats:
return
print(f"Top {rank_num}:")
for index, stat in enumerate(top_stats[:int(rank_num)], 1):
frame = stat.traceback[0]
print("#%s: %s:%s: %.1f KiB"
% (index, frame.filename, frame.lineno, stat.size / 1024))
line = linecache.getline(frame.filename, frame.lineno).strip()
if line:
print(' %s' % line)
def main():
args = parse_args()
files = os.listdir(args.snapshot_dir)
if args.cv_only is True:
files = [f for f in files if "cv" in f]
paths = [os.path.join(args.snapshot_dir, f) for f in files]
snapshots = []
tm_filters = (tracemalloc.Filter(False, "<frozen importlib._bootstrap>"),
tracemalloc.Filter(False, "<unknown>"))
for fn in paths:
print(os.path.basename(fn))
snapshot = tracemalloc.Snapshot.load(fn)
snapshots.append(snapshot)
display_top(snapshot, unit=args.unit)
print("-" * 10)
print()
if len(snapshots) > 1:
stats = snapshot.filter_traces(tm_filters).compare_to(
snapshots[-2], 'filename')
for stat in stats[:5]:
print(
"{} new kb, {} total kb, {} new, {} total memory blocks: ".
format(stat.size_diff / 1000, stat.size / 1000,
stat.count_diff, stat.count))
for line in stat.traceback.format():
print(line)
print()
def display_top(snapshot, key_type='lineno', limit=20):
"""Function to display the top traces.
Note:
Start by ignoring <frozen importlib._bootstrap> and <unknown> files and using statistics group by line no.
Then we enumerate the top_stats and get the frame, filename, line number, line name and the RSS bytes
for each of the top 20 traces.
Based on: https://pytracemalloc.readthedocs.io/examples.html.
:param snapshot : A snapshot of traces. In this case the Max RSS.
:param key_type : Group by the line number, defaults to 'lineno'.
:param limit (int) : Number of profiles to monitor, defaults to 20.
"""
snapshot = snapshot.filter_traces((
tracemalloc.Filter(False, "<frozen importlib._bootstrap>"),
tracemalloc.Filter(False, "<unknown>"),
))
top_stats = snapshot.statistics(key_type)
print("Top {} lines".format(limit))
for index, stat in enumerate(top_stats[:limit], 1):
frame = stat.traceback[0]
filename = os.sep.join(frame.filename.split(os.sep)[-2:])
line = linecache.getline(frame.filename, frame.lineno).strip()
print("#{:3d}: {:23s} | LineNo: {:>4} | RSS: {:>8} | LINE: {:>8}".format(index, filename, frame.lineno,
format_bytes(stat.size), line))
other = top_stats[limit:]
if other:
size = sum(stat.size for stat in other)
print("{} other calls: {}".format(len(other), format_bytes(size)))
total = sum(stat.size for stat in top_stats)
print("Total allocated size: {}".format(format_bytes(total)))
def trace_memory_alloc_pretty_top(snapshot, key_type='lineno', limit=10):
"""From https://docs.python.org/3/library/tracemalloc.html"""
print('-----------------------------------------------------------------')
snapshot = snapshot.filter_traces((
tracemalloc.Filter(False, "<frozen importlib._bootstrap>"),
tracemalloc.Filter(False, "<unknown>"),
))
top_stats = snapshot.statistics(key_type)
print("Top %s lines" % limit)
for index, stat in enumerate(top_stats[:limit], 1):
frame = stat.traceback[0]
# replace "/path/to/module/file.py" with "module/file.py"
filename = sep.join(frame.filename.split(sep)[-2:])
print("#%s: %s:%s: %.1f KiB" %
(index, filename, frame.lineno, stat.size / 1024))
line = linecache.getline(frame.filename, frame.lineno).strip()
if line:
print(' %s' % line)
other = top_stats[limit:]
if other:
size = sum(stat.size for stat in other)
print("%s other: %.1f KiB" % (len(other), size / 1024))
total = sum(stat.size for stat in top_stats)
print("Total allocated size: %.1f KiB" % (total / 1024))
def test_filter_attributes(self):
# test default values
f = tracemalloc.Filter(True, "abc")
self.assertEqual(f.inclusive, True)
self.assertEqual(f.filename_pattern, "abc")
self.assertIsNone(f.lineno)
self.assertEqual(f.all_frames, False)
# test custom values
f = tracemalloc.Filter(False, "test.py", 123, True)
self.assertEqual(f.inclusive, False)
self.assertEqual(f.filename_pattern, "test.py")
self.assertEqual(f.lineno, 123)
self.assertEqual(f.all_frames, True)
# parameters passed by keyword
f = tracemalloc.Filter(inclusive=False,
filename_pattern="test.py",
lineno=123,
all_frames=True)
self.assertEqual(f.inclusive, False)
self.assertEqual(f.filename_pattern, "test.py")
self.assertEqual(f.lineno, 123)
self.assertEqual(f.all_frames, True)
# read-only attribute
self.assertRaises(AttributeError, setattr, f, "filename_pattern",
"abc")
def test_object_hook_and_default():
tracemalloc.start()
data = []
for i in range(1, 100):
data.append({
"name": "a%d" % i,
"timestamp": datetime.timedelta(seconds=i)
})
snapshot1 = tracemalloc.take_snapshot().filter_traces(
(tracemalloc.Filter(True, __file__), ))
for _ in range(1000):
a = rj.dumps(data, default=default)
rj.loads(a, object_hook=object_hook)
gc.collect()
snapshot2 = tracemalloc.take_snapshot().filter_traces(
(tracemalloc.Filter(True, __file__), ))
top_stats = snapshot2.compare_to(snapshot1, 'lineno')
tracemalloc.stop()
for stat in top_stats[:10]:
assert stat.count_diff < 3
def test_filter_traces_domain(self):
snapshot, snapshot2 = create_snapshots()
filter1 = tracemalloc.Filter(False, "a.py", domain=1)
filter2 = tracemalloc.Filter(True, "a.py", domain=1)
original_traces = list(snapshot.traces._traces)
# exclude a.py of domain 1
snapshot3 = snapshot.filter_traces((filter1,))
self.assertEqual(snapshot3.traces._traces, [
(0, 10, (('a.py', 2), ('b.py', 4))),
(0, 10, (('a.py', 2), ('b.py', 4))),
(0, 10, (('a.py', 2), ('b.py', 4))),
(2, 66, (('b.py', 1),)),
(3, 7, (('<unknown>', 0),)),
])
# include domain 1
snapshot3 = snapshot.filter_traces((filter1,))
self.assertEqual(snapshot3.traces._traces, [
(0, 10, (('a.py', 2), ('b.py', 4))),
(0, 10, (('a.py', 2), ('b.py', 4))),
(0, 10, (('a.py', 2), ('b.py', 4))),
(2, 66, (('b.py', 1),)),
(3, 7, (('<unknown>', 0),)),
])
def display_tracemalloc(snapshot, key_type='lineno', limit=10):
snapshot = snapshot.filter_traces((
tracemalloc.Filter(False, "<frozen importlib._bootstrap*"),
tracemalloc.Filter(False, "<unknown>"),
tracemalloc.Filter(False, "*/idna/*.py"),
))
top_stats = snapshot.statistics(key_type)
print("Top {} lines".format(limit))
for index, stat in enumerate(top_stats[:limit], 1):
frame = stat.traceback[0]
# replace "/path/to/module/file.py" with "module/file.py"
filename = os.sep.join(frame.filename.split(os.sep)[-2:])
print("#{}: {}:{}: {:.1f} KiB".format(index, filename, frame.lineno,
stat.size // 1024))
line = linecache.getline(frame.filename, frame.lineno).strip()
if line:
print(' {}'.format(line))
other = top_stats[limit:]
if other:
size = sum(stat.size for stat in other)
print("{} other: {:.1f} KiB".format(len(other), size // 1024))
total = sum(stat.size for stat in top_stats)
current, peak = tracemalloc.get_traced_memory()
print("Total allocated size: {:8.1f} KiB".format(total // 1024))
print("Current size: {:8.1f} KiB".format(current // 1024))
print("Peak size: {:8.1f} KiB".format(peak // 1024))
def _display_top(snapshot: tracemalloc.Snapshot,
*,
key_type: str = "lineno",
limit: int = 10) -> None:
"""Print top consumers.
Inspired by Python docs:
https://docs.python.org/3/library/tracemalloc.html#get-the-traceback-of-a-memory-block
"""
snapshot = snapshot.filter_traces((
tracemalloc.Filter(False, "<frozen importlib._bootstrap>"),
tracemalloc.Filter(False, "<unknown>"),
))
top_stats = snapshot.statistics(key_type)
print("Top %s lines" % limit)
for index, stat in enumerate(top_stats[:limit], 1):
frame = stat.traceback[0]
print("#%s: %s:%s: %.1f KiB" %
(index, frame.filename, frame.lineno, stat.size / 1024))
line = linecache.getline(frame.filename, frame.lineno).strip()
if line:
print(" %s" % line)
other = top_stats[limit:]
if other:
size = sum(stat.size for stat in other)
print("%s other: %.1f KiB" % (len(other), size / 1024))
total = sum(stat.size for stat in top_stats)
print("Total allocated size: %.1f KiB" % (total / 1024))
def get_detailed_mem_use(snapshot, key_type='lineno', limit=30):
import tracemalloc
import linecache
wr_line = ""
snapshot = snapshot.filter_traces((
tracemalloc.Filter(False, "<frozen importlib._bootstrap>"),
tracemalloc.Filter(False, "<unknown>"),
))
top_stats = snapshot.statistics(key_type)
wr_line += ("Top %s lines" % limit)
for index, stat in enumerate(top_stats[:limit], 1):
frame = stat.traceback[0]
# replace "/path/to/module/file.py" with "module/file.py"
filename = os.sep.join(frame.filename.split(os.sep)[-2:])
wr_line += ("\n #%s: %s:%s: %.1f KiB" %
(index, filename, frame.lineno, stat.size / 1024))
line = linecache.getline(frame.filename, frame.lineno).strip()
if line:
wr_line += ('\n %s' % line)
other = top_stats[limit:]
if other:
size = sum(stat.size for stat in other)
wr_line += ("\n %s other: %.1f KiB" % (len(other), size / 1024))
total = sum(stat.size for stat in top_stats)
wr_line += ("\n Total allocated size: %.1f KiB" % (total / 1024))
return wr_line
def take_snapshot():
snapshot = tracemalloc.take_snapshot()
return snapshot.filter_traces((
tracemalloc.Filter(False, "<frozen importlib._bootstrap>"),
tracemalloc.Filter(False, "tracemalloc"),
tracemalloc.Filter(False, "<unknown>"),
))
def trace_print():
global snapshot
global oldmem
snapshot2 = tracemalloc.take_snapshot()
snapshot2 = snapshot2.filter_traces(
(
tracemalloc.Filter(False, "<frozen importlib._bootstrap>"),
tracemalloc.Filter(False, "<unknown>"),
tracemalloc.Filter(False, tracemalloc.__file__),
)
)
if snapshot is not None:
thismem = PROCESS.memory_info().rss / 1024 ** 2
diff = thismem - oldmem
print(
"===================== Begin Trace (TOTAL MEM={:1.4e} MB... [{:+1.4e} MB]):".format(
thismem, diff
)
)
top_stats = snapshot2.compare_to(snapshot, "lineno", cumulative=True)
for stat in top_stats[:4]:
print(stat)
print("End Trace ===========================================")
print()
oldmem = thismem
snapshot = snapshot2
