def _formatwarnmsg_impl(msg):
category = msg.category.__name__
s = f"{msg.filename}:{msg.lineno}: {category}: {msg.message}\n"
if msg.line is None:
try:
import linecache
line = linecache.getline(msg.filename, msg.lineno)
except Exception:
# When a warning is logged during Python shutdown, linecache
# and the import machinery don't work anymore
line = None
linecache = None
else:
line = msg.line
if line:
line = line.strip()
s += " %s\n" % line
if msg.source is not None:
try:
import tracemalloc
# Logging a warning should not raise a new exception:
# catch Exception, not only ImportError and RecursionError.
except Exception:
# don't suggest to enable tracemalloc if it's not available
tracing = True
tb = None
else:
tracing = tracemalloc.is_tracing()
try:
tb = tracemalloc.get_object_traceback(msg.source)
except Exception:
# When a warning is logged during Python shutdown, tracemalloc
# and the import machinery don't work anymore
tb = None
if tb is not None:
s += 'Object allocated at (most recent call last):\n'
for frame in tb:
s += (' File "%s", lineno %s\n'
% (frame.filename, frame.lineno))
try:
if linecache is not None:
line = linecache.getline(frame.filename, frame.lineno)
else:
line = None
except Exception:
line = None
if line:
line = line.strip()
s += ' %s\n' % line
elif not tracing:
s += (f'{category}: Enable tracemalloc to get the object '
f'allocation traceback\n')
return s
def setUp(self):
if tracemalloc.is_tracing():
self.skipTest("tracemalloc must be stopped before the test")
self.domain = 5
self.size = 123
self.obj = allocate_bytes(self.size)[0]
# for the type "object", id(obj) is the address of its memory block.
# This type is not tracked by the garbage collector
self.ptr = id(self.obj)
def fork_child(self):
if not tracemalloc.is_tracing():
return 2
obj_size = 12345
obj, obj_traceback = allocate_bytes(obj_size)
traceback = tracemalloc.get_object_traceback(obj)
if traceback is None:
return 3
# everything is fine
return 0
def stop_profiler(self):
self.agent.log('Deactivating memory allocation profiler.')
with self.profile_lock:
if self.overhead_monitor:
self.overhead_monitor.cancel()
self.overhead_monitor = None
if tracemalloc.is_tracing():
snapshot = tracemalloc.take_snapshot()
self.agent.log('Allocation profiler memory overhead {0} bytes'.format(tracemalloc.get_tracemalloc_memory()))
tracemalloc.stop()
self.process_snapshot(snapshot, time.time() - self.start_ts)
def exec_with_profiler(filename, profiler, backend):
choose_backend(backend)
if _backend == 'tracemalloc' and has_tracemalloc:
tracemalloc.start()
builtins.__dict__['profile'] = profiler
# shadow the profile decorator defined above
ns = dict(_CLEAN_GLOBALS, profile=profiler)
try:
with open(filename) as f:
exec(compile(f.read(), filename, 'exec'), ns, ns)
finally:
if has_tracemalloc and tracemalloc.is_tracing():
tracemalloc.stop()
def exec_with_profiler(filename, profiler, backend, passed_args=[]):
from runpy import run_module
builtins.__dict__['profile'] = profiler
ns = dict(_CLEAN_GLOBALS, profile=profiler)
_backend = choose_backend(backend)
sys.argv = [filename] + passed_args
try:
if _backend == 'tracemalloc' and has_tracemalloc:
tracemalloc.start()
with open(filename) as f:
exec(compile(f.read(), filename, 'exec'), ns, ns)
finally:
if has_tracemalloc and tracemalloc.is_tracing():
tracemalloc.stop()
def run_module_with_profiler(module, profiler, backend, passed_args=[]):
from runpy import run_module
builtins.__dict__['profile'] = profiler
ns = dict(_CLEAN_GLOBALS, profile=profiler)
_backend = choose_backend(backend)
sys.argv = [module] + passed_args
if PY2:
run_module(module, run_name="__main__", init_globals=ns)
else:
if _backend == 'tracemalloc' and has_tracemalloc:
tracemalloc.start()
try:
run_module(module, run_name="__main__", init_globals=ns)
finally:
if has_tracemalloc and tracemalloc.is_tracing():
tracemalloc.stop()
def profile(func=None, stream=None, precision=1, backend='psutil'):
"""
Decorator that will run the function and print a line-by-line profile
"""
global _backend
_backend = backend
if not _backend_chosen:
choose_backend()
if _backend == 'tracemalloc' and not tracemalloc.is_tracing():
tracemalloc.start()
if func is not None:
def wrapper(*args, **kwargs):
prof = LineProfiler()
val = prof(func)(*args, **kwargs)
show_results(prof, stream=stream, precision=precision)
return val
return wrapper
else:
def inner_wrapper(f):
return profile(f, stream=stream, precision=precision, backend=backend)
return inner_wrapper
def profile(func=None, stream=None, precision=1, backend='psutil'):
"""
Decorator that will run the function and print a line-by-line profile
"""
backend = choose_backend(backend)
if backend == 'tracemalloc' and has_tracemalloc:
if not tracemalloc.is_tracing():
tracemalloc.start()
if func is not None:
if not asyncio.iscoroutinefunction(func):
def wrapper(*args, **kwargs):
prof = LineProfiler(backend=backend)
val = prof(func)(*args, **kwargs)
show_results(prof, stream=stream, precision=precision)
return val
else:
@asyncio.coroutine
def wrapper(*args, **kwargs):
prof = LineProfiler(backend=backend)
val = yield from prof(func)(*args, **kwargs)
show_results(prof, stream=stream, precision=precision)
return val
else:
def wrapper(f):
return profile(f,
stream=stream,
precision=precision,
backend=backend)
functools.update_wrapper(wrapper, func)
return wrapper
def _dump_memory_usage(self):
"""Dumps memory to files in /tmp/
Inspiration: https://github.com/mozilla-services/mozservices/blob/master/mozsvc/gunicorn_worker.py
Only will run if MEMORY_DEBUG is enabled.
"""
# pylint: disable=import-outside-toplevel
self.log.error('received signal, dumping memory')
from datetime import datetime
import tracemalloc
if not tracemalloc.is_tracing():
self.log.error('tracemalloc not enabled, will not dump memory')
return
filename = "/tmp/memdump.tracemalloc.%d.%s" % (
os.getpid(), datetime.now().strftime("%d-%m-%Y_%I-%M-%S_%p"))
try:
snapshot = tracemalloc.take_snapshot()
snapshot.dump(filename)
self.log.error("tracemalloc dump to %s", filename)
except Exception as e:
self.log.error("tracemalloc failed: %s", str(e))
def tracemalloc_dump() -> None:
if not tracemalloc.is_tracing():
logger.warning("pid {}: tracemalloc off, nothing to dump"
.format(os.getpid()))
return
# Despite our name for it, `timezone_now` always deals in UTC.
basename = "snap.{}.{}".format(os.getpid(),
timezone_now().strftime("%F-%T"))
path = os.path.join(settings.TRACEMALLOC_DUMP_DIR, basename)
os.makedirs(settings.TRACEMALLOC_DUMP_DIR, exist_ok=True)
gc.collect()
tracemalloc.take_snapshot().dump(path)
procstat = open('/proc/{}/stat'.format(os.getpid()), 'rb').read().split()
rss_pages = int(procstat[23])
logger.info("tracemalloc dump: tracing {} MiB ({} MiB peak), using {} MiB; rss {} MiB; dumped {}"
.format(tracemalloc.get_traced_memory()[0] // 1048576,
tracemalloc.get_traced_memory()[1] // 1048576,
tracemalloc.get_tracemalloc_memory() // 1048576,
rss_pages // 256,
basename))
def profile(func=None, stream=None, precision=1, backend='psutil'):
"""
Decorator that will run the function and print a line-by-line profile
"""
backend = choose_backend(backend)
if backend == 'tracemalloc' and has_tracemalloc:
if not tracemalloc.is_tracing():
tracemalloc.start()
if func is not None:
get_prof = partial(LineProfiler, backend=backend)
show_results_bound = partial(show_results,
stream=stream,
precision=precision)
if iscoroutinefunction(func):
@coroutine
def wrapper(*args, **kwargs):
prof = get_prof()
val = yield from prof(func)(*args, **kwargs)
show_results_bound(prof)
return val
else:
def wrapper(*args, **kwargs):
prof = get_prof()
val = prof(func)(*args, **kwargs)
show_results_bound(prof)
return val
return wrapper
else:
def inner_wrapper(f):
return profile(f,
stream=stream,
precision=precision,
backend=backend)
return inner_wrapper
def tracemalloc_dump() -> None:
if not tracemalloc.is_tracing():
logger.warning("pid {}: tracemalloc off, nothing to dump".format(
os.getpid()))
return
# Despite our name for it, `timezone_now` always deals in UTC.
basename = "snap.{}.{}".format(os.getpid(),
timezone_now().strftime("%F-%T"))
path = os.path.join(settings.TRACEMALLOC_DUMP_DIR, basename)
os.makedirs(settings.TRACEMALLOC_DUMP_DIR, exist_ok=True)
gc.collect()
tracemalloc.take_snapshot().dump(path)
procstat = open('/proc/{}/stat'.format(os.getpid()), 'rb').read().split()
rss_pages = int(procstat[23])
logger.info(
"tracemalloc dump: tracing {} MiB ({} MiB peak), using {} MiB; rss {} MiB; dumped {}"
.format(tracemalloc.get_traced_memory()[0] // 1048576,
tracemalloc.get_traced_memory()[1] // 1048576,
tracemalloc.get_tracemalloc_memory() // 1048576,
rss_pages // 256, basename))
def exec_with_profiler(filename, profiler, backend, passed_args=[]):
from runpy import run_module
builtins.__dict__['profile'] = profiler
ns = dict(_CLEAN_GLOBALS,
profile=profiler,
# Make sure the __file__ variable is usable
# by the script we're profiling
__file__=filename)
# Make sure the script's directory in on sys.path
# credit to line_profiler
sys.path.insert(0, os.path.dirname(script_filename))
_backend = choose_backend(backend)
sys.argv = [filename] + passed_args
try:
if _backend == 'tracemalloc' and has_tracemalloc:
tracemalloc.start()
with open(filename) as f:
exec(compile(f.read(), filename, 'exec'), ns, ns)
finally:
if has_tracemalloc and tracemalloc.is_tracing():
tracemalloc.stop()
def profile(func=None, stream=None, precision=1, backend='psutil'):
"""
Decorator that will run the function and print a line-by-line profile
"""
backend = choose_backend(backend)
if backend == 'tracemalloc' and has_tracemalloc:
if not tracemalloc.is_tracing():
tracemalloc.start()
if func is not None:
def wrapper(*args, **kwargs):
prof = LineProfiler(backend=backend)
val = prof(func)(*args, **kwargs)
show_results(prof, stream=stream, precision=precision)
return val
return wrapper
else:
def inner_wrapper(f):
return profile(f, stream=stream, precision=precision,
backend=backend)
return inner_wrapper
def get(self, uri=None):
if tracemalloc.is_tracing():
try:
uri = 10 if uri is None else int(uri)
except ValueError:
uri = 10
snapshot = tracemalloc.take_snapshot()
snapshot = snapshot.filter_traces(
(tracemalloc.Filter(False, '<frozen importlib._bootstrap>'),
tracemalloc.Filter(False, '<unknown>')))
top_stats = snapshot.statistics('lineno')
output = []
for index, stat in enumerate(top_stats[:uri], 1):
frame = stat.traceback[0]
filename = sep.join(frame.filename.split(sep)[-2:])
output.append(
'#%s: %s:%s: %1.1f KiB<br />' %
(index, filename, frame.lineno, stat.size / 1024))
line = linecache.getline(frame.filename, frame.lineno).strip()
if line:
output.append(' %s<br />' % line)
self.write('<html><body>' + ''.join(output) + '</body></html>')
else:
self.set_status(404)
def monitor_overhead():
if tracemalloc.is_tracing() and tracemalloc.get_tracemalloc_memory() > self.MAX_MEMORY_OVERHEAD:
self.agent.log('Allocation profiler memory overhead limit exceeded: {0} bytes'.format(tracemalloc.get_tracemalloc_memory()))
self.stop_profiler()
def test_is_tracing(self):
tracemalloc.stop()
self.assertFalse(tracemalloc.is_tracing())
tracemalloc.start()
self.assertTrue(tracemalloc.is_tracing())
)
elif incomplete:
print('\n'.join(tracemalloc.Traceback(incomplete[0]).format()))
print(' ...')
else:
print('no traceback found')
# abbreviations
do_q = do_quit
do_l = do_list
do_n = do_next
do_p = do_print
do_tb = do_traceback
if __name__ == '__main__':
if not tracemalloc.is_tracing():
sys.exit('specify tracing depth using -X tracemalloc=N')
del sys.argv[0]
path = sys.argv[0]
with open(path, 'r') as f:
src = f.read()
code = compile(src, path, 'exec')
gc.set_debug(gc.DEBUG_SAVEALL)
print('executing {} tracing at depth {}'.format(
path, tracemalloc.get_traceback_limit()))
try:
exec(code)
finally:
explore_cycles()
async def debug(self, ctx):
if tracemalloc.is_tracing():
snapshot = tracemalloc.take_snapshot()
top_stats = snapshot.statistics("lineno")
await ctx.send("```" + "\n".join([str(x) for x in top_stats[:10]]) + "```")
def run_task(self):
"""
@postcondition: Check Python VM's memory pressure and clean if necessary. Cleaning means flushing objects, setting
all fields to none (to allow GC to work better) and remove from retained references. If volatile or pending to register,
we remove it once registered.
"""
if self.is_flushing_all or self.is_processing_gc:
self.logger.debug(
"[==GC==] Not running since is being processed or flush all is being done"
)
return
# No race condition possible here since there is a time interval for run_gc that MUST be > than time spend to check
# flag and set to True. Should we do it atomically?
self.is_processing_gc = True
try:
self.logger.trace("[==GC==] Running GC")
self.exec_env.prepareThread()
is_pressure = self.__check_memory_pressure()
if is_pressure:
self.logger.verbose(
"System memory is under pressure, proceeding to clean up objects"
)
if self.logger.isEnabledFor(
logging.DEBUG
) and tracemalloc is not None and tracemalloc.is_tracing():
self.logger.debug("Doing a snapshot...")
snapshot = tracemalloc.take_snapshot()
top_stats = snapshot.statistics('lineno')
print("[ Top 10 ]")
for stat in top_stats[:10]:
print(stat)
"""
TODO: CORRECT THESE LOGS
cur_frame = sys._getframe(0)
self.logger.debug("[==GC==] Is enabled? %s", str(gc.isenabled()))
self.logger.debug("[==GC==] Is retained objects tracked? %s", str(gc.is_tracked(self.retained_objects)))
self.logger.debug("[==GC==] Inmemory map %s", str(id(HeapManager.get_heap(self))))
self.logger.debug("[==GC==] Retained map %s", str(id(self.retained_objects)))
self.logger.debug("[==GC==] Threshold: %s", str(gc.get_threshold()))
self.logger.debug("[==GC==] Count: %s", str(gc.get_count()))
self.logger.debug("[==GC==] Current frame ID: %s", str(id(cur_frame)))
"""
# Copy the references in order to process it in a plain fashion
retained_objects_copy = self.retained_objects[:]
self.logger.debug(
"Starting iteration with #%d retained objects",
len(self.retained_objects))
while retained_objects_copy:
# We iterate through while-pop to ensure that the reference is freed
dc_obj = retained_objects_copy.pop()
if dc_obj.get_memory_pinned():
self.logger.trace(
"Object %s is memory pinned, ignoring it",
dc_obj.get_object_id())
continue
if dc_obj.get_object_id(
) in self.runtime.volatiles_under_deserialitzation:
self.logger.trace(
"[==GC==] Not collecting since it is under deserialization."
)
continue
"""
self.logger.debug("[==GC==] Object address in memory: %s", str(id(dc_obj)))
self.logger.debug("[==GC==] Is tracked? %s", str(gc.is_tracked(dc_obj)))
for r in gc.get_referents(dc_obj):
if r == self.retained_objects:
self.logger.debug("[==GC==] REFERENT BEFORE CLEAN FOR %s is retained map. ", (dc_obj.get_object_id()))
elif r == HeapManager.get_heap(self):
self.logger.debug("[==GC==] REFERENT BEFORE CLEAN FOR %s is inmemory map. ", (dc_obj.get_object_id()))
else:
self.logger.debug("[==GC==] REFERENT BEFORE CLEAN FOR %s is: %s ", (dc_obj.get_object_id(), str(id(r))))
# self.logger.debug("[==GC==] REFERENT BEFORE CLEAN FOR %s is: %s ", (dc_obj.get_object_id(), pprint.pformat(r)))
for r in gc.get_referrers(dc_obj):
if r == self.retained_objects:
self.logger.debug("[==GC==] REFERRER BEFORE CLEAN FOR %s is retained map. ", (dc_obj.get_object_id()))
elif r == HeapManager.get_heap(self):
self.logger.debug("[==GC==] REFERRER BEFORE CLEAN FOR %s is inmemory map. ", (dc_obj.get_object_id()))
else:
self.logger.debug("[==GC==] REFERRER BEFORE CLEAN FOR %s is: %s ", (dc_obj.get_object_id(), str(id(r))))
# self.logger.debug("[==GC==] REFERRER BEFORE CLEAN FOR %s is: %s ", (dc_obj.get_object_id(), pprint.pformat(r)))
"""
self.__clean_object(dc_obj)
"""
for r in gc.get_referents(dc_obj):
if r == self.retained_objects:
self.logger.debug("[==GC==] REFERENT FOR %s is retained map. ", (dc_obj.get_object_id()))
elif r == HeapManager.get_heap(self):
self.logger.debug("[==GC==] REFERENT FOR %s is inmemory map. ", (dc_obj.get_object_id()))
else:
self.logger.debug("[==GC==] REFERENT FOR %s is: %s ", (dc_obj.get_object_id(), str(id(r))))
# self.logger.debug("[==GC==] REFERENTS FOR %s is: %s ", (dc_obj.get_object_id(), pprint.pformat(r)))
"""
"""
for r in gc.get_referrers(dc_obj):
if r == self.retained_objects:
self.logger.debug("[==GC==] REFERRER FOR %s is retained map. ", (dc_obj.get_object_id()))
elif r == HeapManager.get_heap(self):
self.logger.debug("[==GC==] REFERRER FOR %s is inmemory map. ", (dc_obj.get_object_id()))
else:
self.logger.debug("[==GC==] ID REFERRER FOR %s is: %s ", (dc_obj.get_object_id(), str(id(r))))
self.logger.debug("[==GC==] REFERRER FOR %s are: %s ", (dc_obj.get_object_id(), pprint.pformat(r)))
"""
del dc_obj # Remove reference from Frame
# Big heaps
n = gc.collect()
if self.logger.isEnabledFor(
logging.DEBUG) or self.logger.isEnabledFor(
logging.TRACE):
if n > 0:
self.logger.debug("[==GC==] Collected %d", n)
else:
self.logger.trace("[==GC==] No objects collected")
if gc.garbage:
self.logger.debug("[==GC==] Uncollectable: %s",
gc.garbage)
else:
self.logger.trace(
"[==GC==] No uncollectable objects")
# Check memory
at_ease = self.__check_memory_ease()
if at_ease:
self.logger.trace(
"[==GC==] Not collecting since memory is 'at ease' now"
)
break
if self.is_flushing_all:
self.logger.debug(
"[==GC==] Interrupted due to flush all.")
break
else:
# This block is only entered when the while has finished (with no break statement)
# which means that all the objects have been iterated
self.logger.warning(
"I did my best and ended up cleaning all retained_objects. This typically means"
" that there is a huge global memory pressure. Problematic."
)
self.logger.debug(
"Finishing iteration with #%d retained objects",
len(self.retained_objects))
self.cleanReferencesAndLockers()
# For cyclic references
n = gc.collect()
if self.logger.isEnabledFor(
logging.DEBUG) or self.logger.isEnabledFor(
logging.TRACE):
if retained_objects_copy:
self.logger.debug(
"There are #%d remaining objects after Garbage Collection",
len(retained_objects_copy))
if n > 0:
self.logger.debug("[==GC==] Finally Collected %d", n)
else:
self.logger.trace("[==GC==] No objects collected")
if gc.garbage:
self.logger.debug("[==GC==] Uncollectable: %s",
gc.garbage)
else:
self.logger.trace("[==GC==] No uncollectable objects")
del retained_objects_copy
if self.logger.isEnabledFor(
logging.DEBUG
) and tracemalloc is not None and tracemalloc.is_tracing():
self.logger.debug("Doing a snapshot...")
snapshot2 = tracemalloc.take_snapshot()
top_stats = snapshot2.compare_to(snapshot, 'lineno')
print("[ Top 10 differences ]")
for stat in top_stats[:10]:
print(stat)
except:
# TODO: Interrupted thread for some reason (sigkill?). Make sure to flag is_processing_gc to false.
pass
self.is_processing_gc = False
return
def setUp(self):
if tracemalloc.is_tracing():
self.skipTest("tracemalloc must be stopped before the test")
tracemalloc.start(1)
def test_is_tracing(self):
tracemalloc.stop()
self.assertFalse(tracemalloc.is_tracing())
tracemalloc.start()
self.assertTrue(tracemalloc.is_tracing())
def is_running():
return tracemalloc and tracemalloc.is_tracing()
Statistical information
Give statistical information about this bot (such as the number
of messages parsed)
"""
import datetime
from irctools import require_auth, CLR_HGLT, CLR_RESET
import ircpacket as ircp
_IS_TRACING = False
try:
import tracemalloc # pylint: disable=wrong-import-order,wrong-import-position
if tracemalloc.is_tracing():
_IS_TRACING = True
except ImportError:
pass
__plugin_description__ = "Stats about this bot"
__plugin_version__ = "v0.1"
__plugin_author__ = "Cameron Conn"
__plugin_type__ = "command"
__plugin_enabled__ = True
def _uptime(start_time: int) -> str:
""" Get the current uptime as a string
start_time - the Unix time this bot was started
def update_event(self, inp=-1):
self.set_output_val(0, tracemalloc.is_tracing())
# along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
Statistical information
Give statistical information about this bot (such as the number
of messages parsed)
"""
import datetime
from irctools import require_auth, CLR_HGLT, CLR_RESET
import ircpacket as ircp
_IS_TRACING = False
try:
import tracemalloc # pylint: disable=wrong-import-order,wrong-import-position
if tracemalloc.is_tracing():
_IS_TRACING = True
except ImportError:
pass
__plugin_description__ = 'Stats about this bot'
__plugin_version__ = 'v0.1'
__plugin_author__ = 'Cameron Conn'
__plugin_type__ = 'command'
__plugin_enabled__ = True
def _uptime(start_time: int) -> str:
''' Get the current uptime as a string
start_time - the Unix time this bot was started
def setUp(self):
if tracemalloc.is_tracing():
self.skipTest("tracemalloc must be stopped before the test")
tracemalloc.start(1)
def is_running():
return tracemalloc and tracemalloc.is_tracing()
