def test_get_overload(self):
"""
Tests the _get_overload function.
"""
with self.subTest("test _get_overload with empty statistics"):
snapshot = Snapshot((), 0)
expected = []
result = _get_overload(snapshot, threshold=10)
self.assertListEqual(expected, result)
with self.subTest(
"test _get_overload with filled statistics and low threshold"):
tracemalloc.start()
list(range(1000000))
snapshot = tracemalloc.take_snapshot()
tracemalloc.stop()
expected = snapshot.statistics("lineno")
result = _get_overload(snapshot, threshold=0)
self.assertListEqual(expected, result)
with self.subTest(
"test _get_overload with filled statistics and high threshold"
):
tracemalloc.start()
list(range(0))
snapshot = tracemalloc.take_snapshot()
tracemalloc.stop()
expected = []
result = _get_overload(snapshot, threshold=100)
self.assertListEqual(expected, result)
def _display_top(snapshot: tracemalloc.Snapshot,
*,
key_type: str = "lineno",
limit: int = 10) -> None:
"""Print top consumers.
Inspired by Python docs:
https://docs.python.org/3/library/tracemalloc.html#get-the-traceback-of-a-memory-block
"""
snapshot = snapshot.filter_traces((
tracemalloc.Filter(False, "<frozen importlib._bootstrap>"),
tracemalloc.Filter(False, "<unknown>"),
))
top_stats = snapshot.statistics(key_type)
print("Top %s lines" % limit)
for index, stat in enumerate(top_stats[:limit], 1):
frame = stat.traceback[0]
print("#%s: %s:%s: %.1f KiB" %
(index, frame.filename, frame.lineno, stat.size / 1024))
line = linecache.getline(frame.filename, frame.lineno).strip()
if line:
print(" %s" % line)
other = top_stats[limit:]
if other:
size = sum(stat.size for stat in other)
print("%s other: %.1f KiB" % (len(other), size / 1024))
total = sum(stat.size for stat in top_stats)
print("Total allocated size: %.1f KiB" % (total / 1024))
def display_top(
snapshot: tracemalloc.Snapshot,
key_type: str = "lineno",
limit: int = 10,
short_filename: bool = False,
) -> None:
"""
Display e.g. lines of code allocating the most memory.
Args:
snapshot: a :class:`tracemalloc.Snapshot` object
key_type: thing to group by
limit: show the top *n*
short_filename: make source code filenames shorter?
"""
# Modified from https://docs.python.org/3/library/tracemalloc.html
print("Calculating memory allocation...")
snapshot = snapshot.filter_traces(
(
tracemalloc.Filter(False, "<frozen importlib._bootstrap>"),
tracemalloc.Filter(
False, "<frozen importlib._bootstrap_external>"
),
tracemalloc.Filter(False, "<unknown>"),
)
)
top_stats = snapshot.statistics(key_type)
print(f"Top {limit} lines")
for index, stat in enumerate(top_stats[:limit], 1):
frame = stat.traceback[0]
if short_filename:
# replace "/path/to/module/file.py" with "module/file.py"
filename = os.sep.join(frame.filename.split(os.sep)[-2:])
else:
filename = frame.filename
print(
f"#{index}: {filename}:{frame.lineno}: "
f"{stat.size / 1024:.1f} KiB"
)
line = linecache.getline(frame.filename, frame.lineno).strip()
if line:
print(f" {line}")
other = top_stats[limit:]
if other:
size = sum(stat.size for stat in other)
print(f"{len(other)} other: {size / 1024:.1f} KiB")
total = sum(stat.size for stat in top_stats)
print(f"Total allocated size: {total / 1024:.1f} KiB")
def get_top_allocation_info(
snapshot: tracemalloc.Snapshot,
*,
key_type: str = 'lineno',
cumulative: bool = False,
limit: int = 20,
filters: Optional[List[tracemalloc.Filter]] = None,
) -> List[str]:
"""
Get the top allocations from a given tracemalloc snapshot in a list.
Parameters
----------
snapshot : tracemalloc.Snapshot
Snapshot to get information from.
key_type : str, optional
Key for the snapshot statistics.
cumulative : bool, optional
Cumulative statistics.
limit : int, optional
Limit the number of results.
filters : list of tracemalloc.Filter
Filters to apply to the snapshot.
Returns
-------
text_lines : list of str
"""
if filters is None:
filters = TRACEMALLOC_FILTERS
snapshot = snapshot.filter_traces(filters)
top_stats = snapshot.statistics(key_type, cumulative=cumulative)
result = []
for index, stat in enumerate(top_stats[:limit], 1):
frame = stat.traceback[0]
kbytes = stat.size / 1024
result.append(
f"#{index}: {frame.filename}:{frame.lineno}: {kbytes:.1f} KiB")
line = linecache.getline(frame.filename, frame.lineno).strip()
if line:
result.append(f' {line}')
return result, top_stats
def get_trace(client, force_start):
(started, path) = client.tracemalloc_dump()
if force_start and not started:
client.tracemalloc_toggle()
(started, path) = client.tracemalloc_dump()
if not started:
raise TraceCantStart
elif not started:
raise TraceNotStarted
return Snapshot.load(path)
def get_trace(client, force_start):
(started, path) = client.tracemalloc_dump()
if force_start and not started:
client.tracemalloc_toggle()
(started, path) = client.tracemalloc_dump()
if not started:
raise TraceCantStart
elif not started:
raise TraceNotStarted
return Snapshot.load(path)
def _get_overload(snapshot: Snapshot, threshold: float, key_type: str = "lineno") -> \
List[Statistic]:
"""
Returns a list of statistics that exceed the given threshold in KiB.
:param snapshot: snapshot to analyze
:param threshold: threshold not to exceed (in KiB)
:param key_type: key used to order the snapshot's statistics
:return: a list of statistics that exceed the given threshold
"""
stats = snapshot.statistics(key_type)
return [stat for stat in stats if stat.size / 1024 > threshold]
def _print_memory_statistics(snapshot: tracemalloc.Snapshot,
key: str,
topx: int = 10):
"""
Print memory statistics from a tracemalloc.Snapshot in certain formats.
:param snapshot:
:param key:
:param topx:
:return:
"""
if key not in ["filename", "lineno", "traceback"]:
raise KeyError
stats = snapshot.statistics(key)
if key == "lineno":
for idx, stat in enumerate(stats[:topx]):
frame = stat.traceback[0]
print(
"\n{idx:02d}: {filename}:{lineno} {size:.1f} KiB, count {count}"
.format(idx=idx + 1,
filename=frame.filename,
lineno=frame.lineno,
size=stat.size / 1024,
count=stat.count))
lines = []
lines_whitespaces = []
for lineshift in range(-3, 2):
stat = linecache.getline(frame.filename,
frame.lineno + lineshift)
lines_whitespaces.append(
len(stat) - len(stat.lstrip(" "))) # count
lines.append(stat.strip())
lines_whitespaces = [
x - min([y for y in lines_whitespaces if y > 0])
for x in lines_whitespaces
]
for lidx, stat in enumerate(lines):
print(" {}{}".format(
"> " if lidx == 3 else "| ",
" " * lines_whitespaces.pop(0) + stat))
elif key == "filename":
for idx, stat in enumerate(stats[:topx]):
frame = stat.traceback[0]
print(
"{idx:02d}: {filename:80s} {size:6.1f} KiB, count {count:5<d}"
.format(idx=idx + 1,
filename=frame.filename,
size=stat.size / 1024,
count=stat.count))
def summarize(snapshot: tracemalloc.Snapshot, key_type: str, limit: int):
"""Summarize snapshot in console."""
snapshot = snapshot.filter_traces((
tracemalloc.Filter(False, "<frozen importlib._bootstrap>"),
tracemalloc.Filter(False, "<unknown>"),
))
top_statistics = snapshot.statistics(key_type)
for i, statistic in enumerate(top_statistics[:limit], 1):
frame = statistic.traceback[0]
print("#%s: %s:%s: %.1f KiB" %
(i, frame.filename, frame.lineno, statistic.size / 1024))
line = linecache.getline(frame.filename, frame.lineno).strip()
if line:
print(" %s" % line)
other = top_statistics[limit:]
if other:
size = sum(statistic.size for statistic in other)
print("%s other: %.1f KiB" % (len(other), size / 1024))
total = sum(statistic.size for statistic in top_statistics)
print("Total allocated size: %.1f KiB" % (total / 1024))
def _get_leaks(snapshot1: Snapshot,
snapshot2: Snapshot,
threshold: float,
key_type: str = "lineno") -> List[StatisticDiff]:
"""
Returns a list of statistics that contains detected memory leaks above
the given threshold in bytes.
:param snapshot1: first snapshot (before the tested function execution)
:param snapshot2: second snapshot (after the tested function execution)
:param threshold: threshold to be exceeded to retain the leaks (in B)
:param key_type: key used to order the snapshot's statistics
:return: a list of statistics that contains detected memory leaks
"""
return [
s for s in snapshot2.compare_to(snapshot1, key_type)
if s.size_diff / 1024 >= threshold
]
def _filter_snapshot(snapshot: Snapshot,
exclude: ExcludeType = None) -> tracemalloc.Snapshot:
"""
Filters the given snapshot using the exclude value and the DEFAULT_FILTERS.
:param snapshot: snapshot to be filtered
:param exclude: element(s) to be excluded from the snapshot
:return: the filtered snapshot
"""
filters = deepcopy(DEFAULT_FILTERS)
if isinstance(exclude, str):
filters.add(tracemalloc.Filter(False, exclude))
elif isinstance(exclude, abc.Iterable):
filters.update({
tracemalloc.Filter(False, e)
for e in exclude if isinstance(e, str)
})
return snapshot.filter_traces(tuple(filters))
def analyze(self):
output = []
for i, tracedump in enumerate(self.get_input_files('*.tracemalloc')):
logging.debug('Analyzing "%s" tracemalloc dump' % tracedump)
try:
snapshot = Snapshot.load(tracedump)
snapshot = filter_traces(snapshot)
except Exception, e:
logging.error('Exception in PyTraceMallocSummary: "%s"' % e)
continue
top = format_top(snapshot)
output.append(('Measurement #%s' % i, top))
trace = format_tracebacks(snapshot)
output.append(('Traceback #%s' % i, trace))
def compare_snapshot(
self,
snapshot_from: tracemalloc.Snapshot,
snapshot_to: tracemalloc.Snapshot,
) -> List[tracemalloc.StatisticDiff]:
return snapshot_to.compare_to(snapshot_from, self.group_by.value)
def compare_snapshot(
self, snapshot_from: tracemalloc.Snapshot,
snapshot_to: tracemalloc.Snapshot,
):
return snapshot_to.compare_to(snapshot_from, self.group_by.value)
print("Top %s lines" % limit)
for index, stat in enumerate(top_stats[:limit], 1):
frame = stat.traceback[0]
print("#%s: %s:%s: %.1f KiB" %
(index, frame.filename, frame.lineno, stat.size / 1024))
line = linecache.getline(frame.filename, frame.lineno).strip()
if line:
print(' %s' % line)
other = top_stats[limit:]
if other:
size = sum(stat.size for stat in other)
print("%s other: %.1f KiB" % (len(other), size / 1024))
total = sum(stat.size for stat in top_stats)
print("Total allocated size: %.1f KiB" % (total / 1024))
tracemalloc.start(nframe)
s = Snapshot.load('/Users/jx/Downloads/tracemalloc_2.log')
top_stats = s.statistics('traceback')
# pick the biggest memory block
stat = top_stats[0]
print("%s memory blocks: %.1f KiB" % (stat.count, stat.size / 1024))
for line in stat.traceback.format():
print(line)
display_top(s, limit=1000)