def test_clear_traces(self):
obj, obj_traceback = allocate_bytes(123)
traceback = tracemalloc.get_object_traceback(obj)
self.assertIsNotNone(traceback)
tracemalloc.clear_traces()
traceback2 = tracemalloc.get_object_traceback(obj)
self.assertIsNone(traceback2)
def test_clear_traces(self):
obj, obj_traceback = allocate_bytes(123)
traceback = tracemalloc.get_object_traceback(obj)
self.assertIsNotNone(traceback)
tracemalloc.clear_traces()
traceback2 = tracemalloc.get_object_traceback(obj)
self.assertIsNone(traceback2)
def big_objs(traced_only=False):
big_objs = (
(sys.getsizeof(obj), obj) for obj in gc.get_objects()
if sys.getsizeof(obj) > 20000 and (
tracemalloc.get_object_traceback(obj) if traced_only else True))
for size, obj in big_objs:
print(type(obj), size, tracemalloc.get_object_traceback(obj))
def _perform_leak_detection() -> None:
# Log potentially useful memory usage metrics
logger.info(f"GC count before collect {gc.get_count()}")
traced_memory_size, traced_memory_peak = tracemalloc.get_traced_memory()
logger.info(
f"Traced Memory: size={traced_memory_size}, peak={traced_memory_peak}")
num_unreacheable_objects = gc.collect()
logger.info(f"Number of unreachable objects = {num_unreacheable_objects}")
logger.info(f"GC count after collect {gc.get_count()}")
# Collect unique traces of all live objects in the garbage - these have potential leaks.
unique_traces_to_objects: Dict[Union[tracemalloc.Traceback, int],
List[object]] = defaultdict(list)
for obj in gc.garbage:
obj_trace = tracemalloc.get_object_traceback(obj)
if obj_trace is not None:
if _trace_has_file(obj_trace, "*datahub/*.py"):
# Leaking object
unique_traces_to_objects[obj_trace].append(obj)
else:
unique_traces_to_objects[id(obj)].append(obj)
logger.info("Potentially leaking objects start")
for key, obj_list in sorted(
unique_traces_to_objects.items(),
key=lambda item: sum([sys.getsizeof(o) for o in item[1]]),
reverse=True,
):
if isinstance(key, tracemalloc.Traceback):
obj_traceback: tracemalloc.Traceback = cast(
tracemalloc.Traceback, key)
logger.info(
f"#Objects:{len(obj_list)}; Total memory:{sum([sys.getsizeof(obj) for obj in obj_list])};"
+ " Allocation Trace:\n\t" +
"\n\t".join(obj_traceback.format(limit=25)))
else:
logger.info(
f"#Objects:{len(obj_list)}; Total memory:{sum([sys.getsizeof(obj) for obj in obj_list])};"
+ " No Allocation Trace available!")
# Print details about the live referrers of each object in the obj_list (same trace).
for obj in obj_list:
referrers = [r for r in gc.get_referrers(obj) if r in gc.garbage]
logger.info(
f"Referrers[{len(referrers)}] for object(addr={hex(id(obj))}):'{obj}':"
)
for ref_index, referrer in enumerate(referrers):
ref_trace = tracemalloc.get_object_traceback(referrer)
logger.info(
f"Referrer[{ref_index}] referrer_obj(addr={hex(id(referrer))}):{referrer}, RefTrace:\n\t\t"
+ "\n\t\t".join(
ref_trace.format(limit=5) if ref_trace else []) + "\n")
logger.info("Potentially leaking objects end")
tracemalloc.stop()
def _formatwarnmsg_impl(msg):
import linecache
s = ("%s:%s: %s: %s\n"
% (msg.filename, msg.lineno, msg.category.__name__,
msg.message))
if msg.line is None:
line = linecache.getline(msg.filename, msg.lineno)
else:
line = msg.line
if line:
line = line.strip()
s += " %s\n" % line
if msg.source is not None:
import tracemalloc
tb = tracemalloc.get_object_traceback(msg.source)
if tb is not None:
s += 'Object allocated at (most recent call first):\n'
for frame in tb:
s += (' File "%s", lineno %s\n'
% (frame.filename, frame.lineno))
line = linecache.getline(frame.filename, frame.lineno)
if line:
line = line.strip()
s += ' %s\n' % line
return s
def warning_record_to_str(warning_message: warnings.WarningMessage) -> str:
"""Convert a warnings.WarningMessage to a string."""
warn_msg = warning_message.message
msg = warnings.formatwarning(
str(warn_msg),
warning_message.category,
warning_message.filename,
warning_message.lineno,
warning_message.line,
)
if warning_message.source is not None:
try:
import tracemalloc
except ImportError:
pass
else:
tb = tracemalloc.get_object_traceback(warning_message.source)
if tb is not None:
formatted_tb = "\n".join(tb.format())
# Use a leading new line to better separate the (large) output
# from the traceback to the previous warning text.
msg += f"\nObject allocated at:\n{formatted_tb}"
else:
# No need for a leading new line.
url = "https://docs.pytest.org/en/stable/how-to/capture-warnings.html#resource-warnings"
msg += "Enable tracemalloc to get traceback where the object was allocated.\n"
msg += f"See {url} for more info."
return msg
def test_set_traceback_limit(self):
obj_size = 10
tracemalloc.stop()
self.assertRaises(ValueError, tracemalloc.start, -1)
tracemalloc.stop()
tracemalloc.start(10)
obj2, obj2_traceback = allocate_bytes(obj_size)
traceback = tracemalloc.get_object_traceback(obj2)
self.assertEqual(len(traceback), 10)
self.assertEqual(traceback, obj2_traceback)
tracemalloc.stop()
tracemalloc.start(1)
obj, obj_traceback = allocate_bytes(obj_size)
traceback = tracemalloc.get_object_traceback(obj)
self.assertEqual(len(traceback), 1)
self.assertEqual(traceback, obj_traceback)
def memory_obj(args: tuple, packet: ircp.Packet, ___: dict):
""" Print the biggest memory hogs """
if not _IS_TRACING:
return packet.notice(
"Sorry, but tracing is currently disabled. "
'Please restart probot with the "PYTHONTRACEMALLOC=NFRAME" '
"environment variable."
)
snapshot = tracemalloc.take_snapshot()
top_stats = snapshot.statistics("filename")
num = 0
if len(args) >= 2:
try:
num = int(args[1])
except ValueError:
return packet.notice("Your argument must be an integer")
else:
return packet.notice("You must specify an object to inspect!")
if len(top_stats) >= num:
output = [packet.notice("Memory hog #{}".format(num))]
obj = top_stats[num]
trace = tracemalloc.get_object_traceback(obj)
for line in trace:
output.append(packet.notice(line))
return output
else:
return packet.notice("Sorry, but that object does not exist")
def memory_obj(args: tuple, packet: ircp.Packet, ___: dict):
""" Print the biggest memory hogs """
if not _IS_TRACING:
return packet.notice(
'Sorry, but tracing is currently disabled. '
'Please restart probot with the "PYTHONTRACEMALLOC=NFRAME" '
'environment variable.')
snapshot = tracemalloc.take_snapshot()
top_stats = snapshot.statistics('filename')
num = 0
if len(args) >= 2:
try:
num = int(args[1])
except ValueError:
return packet.notice('Your argument must be an integer')
else:
return packet.notice('You must specify an object to inspect!')
if len(top_stats) >= num:
output = [packet.notice('Memory hog #{}'.format(num))]
obj = top_stats[num]
trace = tracemalloc.get_object_traceback(obj)
for line in trace:
output.append(packet.notice(line))
return output
else:
return packet.notice('Sorry, but that object does not exist')
def show(filename=None):
global before_objects
gc.collect()
after_objects = gc.get_objects()
frame = sys._getframe()
globals_ = globals()
num_leaks = 0
before_id_set = set(map(id, before_objects))
if filename is not None:
f = open(filename, 'w')
else:
f = sys.stderr
for obj in after_objects:
if id(obj) not in before_id_set:
if obj in (before_objects, frame):
continue
num_leaks += 1
print("Leak: id=0x{:x} type={} {}".format(id(obj), type(obj),
_get_detail(obj)),
file=f)
tb = tracemalloc.get_object_traceback(obj)
if tb is None:
print("Traceback: None", file=f)
else:
print("Traceback:", file=f)
print("\n".join(tb.format(MAX_FRAMES)), file=f)
print(file=f)
print("Referrers:", file=f)
for ref in gc.get_referrers(obj):
if ref in (after_objects, before_objects, frame, globals_):
continue
print(" id=0x{:x} type={} {}".format(
id(ref), type(ref), _get_detail(ref)),
file=f)
print(" traceback: {}".format(
tracemalloc.get_object_traceback(ref)),
file=f)
print(file=f)
print(file=f)
print("Total leaks: {}".format(num_leaks), file=f)
if filename is not None:
f.close()
tracemalloc.stop()
gc.enable()
def _formatwarnmsg_impl(msg):
category = msg.category.__name__
s = f"{msg.filename}:{msg.lineno}: {category}: {msg.message}\n"
if msg.line is None:
try:
import linecache
line = linecache.getline(msg.filename, msg.lineno)
except Exception:
# When a warning is logged during Python shutdown, linecache
# and the import machinery don't work anymore
line = None
linecache = None
else:
line = msg.line
if line:
line = line.strip()
s += " %s\n" % line
if msg.source is not None:
try:
import tracemalloc
# Logging a warning should not raise a new exception:
# catch Exception, not only ImportError and RecursionError.
except Exception:
# don't suggest to enable tracemalloc if it's not available
tracing = True
tb = None
else:
tracing = tracemalloc.is_tracing()
try:
tb = tracemalloc.get_object_traceback(msg.source)
except Exception:
# When a warning is logged during Python shutdown, tracemalloc
# and the import machinery don't work anymore
tb = None
if tb is not None:
s += 'Object allocated at (most recent call last):\n'
for frame in tb:
s += (' File "%s", lineno %s\n'
% (frame.filename, frame.lineno))
try:
if linecache is not None:
line = linecache.getline(frame.filename, frame.lineno)
else:
line = None
except Exception:
line = None
if line:
line = line.strip()
s += ' %s\n' % line
elif not tracing:
s += (f'{category}: Enable tracemalloc to get the object '
f'allocation traceback\n')
return s
def _formatwarnmsg_impl(msg):
category = msg.category.__name__
s = f"{msg.filename}:{msg.lineno}: {category}: {msg.message}\n"
if msg.line is None:
try:
import linecache
line = linecache.getline(msg.filename, msg.lineno)
except Exception:
# When a warning is logged during Python shutdown, linecache
# and the import machinery don't work anymore
line = None
linecache = None
else:
line = msg.line
if line:
line = line.strip()
s += " %s\n" % line
if msg.source is not None:
try:
import tracemalloc
# Logging a warning should not raise a new exception:
# catch Exception, not only ImportError and RecursionError.
except Exception:
# don't suggest to enable tracemalloc if it's not available
tracing = True
tb = None
else:
tracing = tracemalloc.is_tracing()
try:
tb = tracemalloc.get_object_traceback(msg.source)
except Exception:
# When a warning is logged during Python shutdown, tracemalloc
# and the import machinery don't work anymore
tb = None
if tb is not None:
s += 'Object allocated at (most recent call last):\n'
for frame in tb:
s += (' File "%s", lineno %s\n' %
(frame.filename, frame.lineno))
try:
if linecache is not None:
line = linecache.getline(frame.filename, frame.lineno)
else:
line = None
except Exception:
line = None
if line:
line = line.strip()
s += ' %s\n' % line
elif not tracing:
s += (f'{category}: Enable tracemalloc to get the object '
f'allocation traceback\n')
return s
def fork_child(self):
if not tracemalloc.is_tracing():
return 2
obj_size = 12345
obj, obj_traceback = allocate_bytes(obj_size)
traceback = tracemalloc.get_object_traceback(obj)
if traceback is None:
return 3
return 0
def test_set_traceback_limit(self):
obj_size = 10
tracemalloc.stop()
self.assertRaises(ValueError, tracemalloc.start, -1)
tracemalloc.stop()
tracemalloc.start(10)
obj2, obj2_traceback = allocate_bytes(obj_size)
traceback = tracemalloc.get_object_traceback(obj2)
self.assertEqual(len(traceback), 10)
self.assertEqual(traceback, obj2_traceback)
tracemalloc.stop()
tracemalloc.start(1)
obj, obj_traceback = allocate_bytes(obj_size)
traceback = tracemalloc.get_object_traceback(obj)
self.assertEqual(len(traceback), 1)
self.assertEqual(traceback, obj_traceback)
def __del__(self):
if self._watcher:
tb = get_object_traceback(self)
tb_msg = ''
if tb is not None:
tb_msg = '\n'.join(tb.format())
tb_msg = '\nTraceback:\n' + tb_msg
warnings.warn("Failed to close watcher %r%s" % (self, tb_msg),
ResourceWarning)
# may fail if __init__ did; will be harmlessly printed
self.close()
def fork_child(self):
if not tracemalloc.is_tracing():
return 2
obj_size = 12345
obj, obj_traceback = allocate_bytes(obj_size)
traceback = tracemalloc.get_object_traceback(obj)
if traceback is None:
return 3
# everything is fine
return 0
def __del__(self):
if self._watcher:
tb = get_object_traceback(self)
tb_msg = ''
if tb is not None:
tb_msg = '\n'.join(tb.format())
tb_msg = '\nTraceback:\n' + tb_msg
warnings.warn("Failed to close watcher %r%s" % (self, tb_msg),
ResourceWarning)
# may fail if __init__ did; will be harmlessly printed
self.close()
def show(filename=None):
global before_objects
gc.collect()
after_objects = gc.get_objects()
frame = sys._getframe()
globals_ = globals()
num_leaks = 0
before_id_set = set(map(id, before_objects))
if filename is not None:
f = open(filename, 'w')
else:
f = sys.stderr
for obj in after_objects:
if id(obj) not in before_id_set:
if obj in (before_objects, frame):
continue
num_leaks += 1
print("Leak: id=0x{:x} type={} {}".format(id(obj), type(obj), _get_detail(obj)), file=f)
tb = tracemalloc.get_object_traceback(obj)
if tb is None:
print("Traceback: None", file=f)
else:
print("Traceback:", file=f)
print("\n".join(tb.format(MAX_FRAMES)), file=f)
print(file=f)
print("Referrers:", file=f)
for ref in gc.get_referrers(obj):
if ref in (after_objects, before_objects, frame, globals_):
continue
print(" id=0x{:x} type={} {}".format(id(ref), type(ref), _get_detail(ref)), file=f)
print(" traceback: {}".format(tracemalloc.get_object_traceback(ref)), file=f)
print(file=f)
print(file=f)
print("Total leaks: {}".format(num_leaks), file=f)
if filename is not None:
f.close()
tracemalloc.stop()
gc.enable()
def _formatwarnmsg_impl(msg):
s = "%s:%s: %s: %s\n" % (
msg.filename,
msg.lineno,
msg.category.__name__,
msg.message,
)
if msg.line is None:
try:
import linecache
line = linecache.getline(msg.filename, msg.lineno)
except Exception:
# When a warning is logged during Python shutdown, linecache
# and the import machinery don't work anymore
line = None
linecache = None
else:
line = msg.line
if line:
line = line.strip()
s += " %s\n" % line
if msg.source is not None:
try:
import tracemalloc
tb = tracemalloc.get_object_traceback(msg.source)
except Exception:
# When a warning is logged during Python shutdown, tracemalloc
# and the import machinery don't work anymore
tb = None
if tb is not None:
s += "Object allocated at (most recent call first):\n"
for frame in tb:
s += ' File "%s", lineno %s\n' % (frame.filename, frame.lineno)
try:
if linecache is not None:
line = linecache.getline(frame.filename, frame.lineno)
else:
line = None
except Exception:
line = None
if line:
line = line.strip()
s += " %s\n" % line
return s
def _formatwarnmsg_impl(msg):
s = ("%s:%s: %s: %s\n"
% (msg.filename, msg.lineno, msg.category.__name__,
msg.message))
if msg.line is None:
try:
import linecache
line = linecache.getline(msg.filename, msg.lineno)
except Exception:
# When a warning is logged during Python shutdown, linecache
# and the import machinery don't work anymore
line = None
linecache = None
else:
line = msg.line
if line:
line = line.strip()
s += " %s\n" % line
if msg.source is not None:
try:
import tracemalloc
tb = tracemalloc.get_object_traceback(msg.source)
except Exception:
# When a warning is logged during Python shutdown, tracemalloc
# and the import machinery don't work anymore
tb = None
if tb is not None:
s += 'Object allocated at (most recent call first):\n'
for frame in tb:
s += (' File "%s", lineno %s\n'
% (frame.filename, frame.lineno))
try:
if linecache is not None:
line = linecache.getline(frame.filename, frame.lineno)
else:
line = None
except Exception:
line = None
if line:
line = line.strip()
s += ' %s\n' % line
return s
def test_new_reference(self):
tracemalloc.clear_traces()
# gc.collect() indirectly calls PyList_ClearFreeList()
support.gc_collect()
# Create a list and "destroy it": put it in the PyListObject free list
obj = []
obj = None
# Create a list which should reuse the previously created empty list
obj = []
nframe = tracemalloc.get_traceback_limit()
frames = get_frames(nframe, -3)
obj_traceback = tracemalloc.Traceback(frames, min(len(frames), nframe))
traceback = tracemalloc.get_object_traceback(obj)
self.assertIsNotNone(traceback)
self.assertEqual(traceback, obj_traceback)
def _formatwarnmsg_impl(msg):
s = '%s:%s: %s: %s\n' % (msg.filename, msg.lineno, msg.category.__name__,
msg.message)
if msg.line is None:
try:
import linecache
line = linecache.getline(msg.filename, msg.lineno)
except Exception:
line = None
linecache = None
else:
line = msg.line
if line:
line = line.strip()
s += ' %s\n' % line
if msg.source is not None:
try:
import tracemalloc
tb = tracemalloc.get_object_traceback(msg.source)
except Exception:
tb = None
if tb is not None:
s += 'Object allocated at (most recent call first):\n'
for frame in tb:
s += ' File "%s", lineno %s\n' % (frame.filename,
frame.lineno)
try:
if linecache is not None:
line = linecache.getline(frame.filename, frame.lineno)
else:
line = None
except Exception:
line = None
if line:
line = line.strip()
s += ' %s\n' % line
return s
def test_get_object_traceback(self):
tracemalloc.clear_traces()
obj_size = 12345
obj, obj_traceback = allocate_bytes(obj_size)
traceback = tracemalloc.get_object_traceback(obj)
self.assertEqual(traceback, obj_traceback)
def update_event(self, inp=-1):
self.set_output_val(0, tracemalloc.get_object_traceback(self.input(0)))
def test_get_object_traceback(self):
tracemalloc.clear_traces()
obj_size = 12345
obj, obj_traceback = allocate_bytes(obj_size)
traceback = tracemalloc.get_object_traceback(obj)
self.assertEqual(traceback, obj_traceback)