def run(self, profiler):
profile = functools.partial(self._profile, profiler)
sys.setprofile(profile)
threading.setprofile(profile)
yield
threading.setprofile(None)
sys.setprofile(None)
def start_tracing(output_path: str):
"""Start tracing allocations."""
path = os.path.join(output_path, timestamp_now()).encode("utf-8")
preload.fil_reset(path)
preload.fil_start_tracking()
threading.setprofile(_start_thread_trace)
preload.register_fil_tracer()
def start(builtins=False, profile_threads=True):
"""
Start profiler.
"""
if profile_threads:
threading.setprofile(_profile_thread_callback)
_yappi.start(builtins, profile_threads)
def install(self):
"""Install the trace function and open the JSON output stream."""
self._open_collection() # Open the JSON output.
self.writer.start() # Start the writer thread.
sys.setprofile(self.tracer) # Set the trace/profile function.
threading.setprofile(
self.tracer) # Set the trace/profile function for threads.
def profile(f, *args, **kwds):
"""XXX docstring"""
global _g_threadmap
p = Profiler()
p.enable(subcalls=True)
threading.setprofile(_thread_profile)
# Note: The except clause is needed below so that profiling data still
# gets dumped even when exceptions are encountered. The except clause code
# is taken straight from run_bzr_catch_errrors() in commands.py and ought
# to be kept in sync with it.
try:
try:
ret = f(*args, **kwds)
except (KeyboardInterrupt, Exception), e:
import bzrlib.trace
bzrlib.trace.report_exception(sys.exc_info(), sys.stderr)
ret = 3
finally:
p.disable()
for pp in _g_threadmap.values():
pp.disable()
threading.setprofile(None)
threads = {}
for tid, pp in _g_threadmap.items():
threads[tid] = Stats(pp.getstats(), {})
_g_threadmap = {}
return ret, Stats(p.getstats(), threads)
def run(self, profiler):
profile = functools.partial(self._profile, profiler)
sys.setprofile(profile)
threading.setprofile(profile)
yield
threading.setprofile(None)
sys.setprofile(None)
def shutdown(self):
sys.setprofile(None) # Clear the trace/profile function.
threading.setprofile(
None) # Clear the trace/profile function for threads.
self._close_collection() # Close the JSON output.
self.terminator.set() # Stop the writer thread.
self.writer.join() # Join the writer thread.
def start(builtins=False, profile_threads=True):
"""
Start profiler.
"""
if profile_threads:
threading.setprofile(_callback)
_yappi.start(builtins, profile_threads)
def profile_on(*filenames):
global p_stats, p_start_time, files
p_stats = []
p_start_time = time.time()
files = filenames
threading.setprofile(profiler)
sys.setprofile(profiler)
def trace(self, predicate):
"""
Starts tracing with the given callable.
Args:
predicate (callable that accepts a single :obj:`~hunter.event.Event` argument):
Return:
self
"""
self._handler = predicate
if self.profiling_mode:
if self.threading_support is None or self.threading_support:
self._threading_previous = getattr(threading, '_profile_hook',
None)
threading.setprofile(self)
self._previous = sys.getprofile()
sys.setprofile(self)
else:
if self.threading_support is None or self.threading_support:
self._threading_previous = getattr(threading, '_trace_hook',
None)
threading.settrace(self)
self._previous = sys.gettrace()
sys.settrace(self)
return self
def stop_types_collection():
# type: () -> None
"""
Remove profiler hooks.
"""
sys.setprofile(None)
threading.setprofile(None) # type: ignore
def shutdown(self):
"""Unset the profile function and stop the writer thread."""
sys.setprofile(None) # Clear the trace/profile function.
threading.setprofile(
None) # Clear the trace/profile function for threads.
self.terminator.set() # Stop the writer thread.
self.writer.join() # Join the writer thread.
def init_types_collection():
# type: () -> None
"""
Setup profiler hooks to enable type collection.
Call this one time from the main thread.
"""
sys.setprofile(_trace_dispatch)
threading.setprofile(_trace_dispatch)
def run(self, profiler):
profile = functools.partial(self._profile, profiler)
with deferral() as defer:
sys.setprofile(profile)
defer(sys.setprofile, None)
threading.setprofile(profile)
defer(threading.setprofile, None)
yield
def stop_profiler():
# we keep the _state around for the user until the next session
# Unregister the profiler in this order, otherwise we will have extra
# measurements in the end
sys.setprofile(None)
threading.setprofile(None)
greenlet.settrace(None) # pylint: disable=no-member
def stop_profiler():
# we keep the _state around for the user until the next session
# Unregister the profiler in this order, otherwise we will have extra
# measurements in the end
sys.setprofile(None)
threading.setprofile(None)
greenlet.settrace(None) # pylint: disable=no-member
def run(self, profiler):
profile = functools.partial(self._profile, profiler)
with deferral() as defer:
sys.setprofile(profile)
defer(sys.setprofile, None)
threading.setprofile(profile)
defer(threading.setprofile, None)
yield
def start(self):
if sys.getprofile() is not None:
raise RuntimeError('Another profiler already registered.')
self._running = True
sys.setprofile(self._profile)
threading.setprofile(self._profile)
self.timer.start()
self.stats.record_starting(time.clock())
def activate_hook():
""" Activates the thread monitor hook. Note that this interferes
with any kind of profiler and some debugging extensions. """
global hook_enabled
sys.setprofile(dump_hook)
threading.setprofile(dump_hook)
hook_enabled = True
def start(self):
if sys.getprofile() is not None:
raise RuntimeError('Another profiler already registered.')
self._running = True
sys.setprofile(self._profile)
threading.setprofile(self._profile)
self.timer.start()
self.stats.record_starting(time.clock())
def trace_execution(fn, args, save_to=None):
import inspect
if save_to:
os.environ["INTEL_SEA_SAVE_TO"] = save_to
itt = ITT("python")
if itt.lib:
file_id = itt.get_string_id("__FILE__")
line_id = itt.get_string_id("__LINE__")
module_id = itt.get_string_id("__MODULE__")
trace_execution.frames = {}
trace_execution.recurrent = False
high_part = 2 ** 32
def profiler(frame, event, arg): # https://pymotw.com/2/sys/tracing.html
if trace_execution.recurrent:
return
trace_execution.recurrent = True
task_id = id(frame.f_code)
if "call" in event:
if task_id in trace_execution.frames:
trace_execution.frames[task_id] += 1
else:
trace_execution.frames[task_id] = 1
task_id += trace_execution.frames[task_id] * high_part
name = frame.f_code.co_name + ((" (%s)" % arg.__name__) if arg else "")
if "self" in frame.f_locals:
cls = frame.f_locals["self"].__class__.__name__
name = cls + "." + name
# print event, name, task_id, arg
mdl = inspect.getmodule(frame)
itt.lib.itt_task_begin_overlapped(itt.domain, task_id, 0, itt.get_string_id(name), 0)
itt.lib.itt_metadata_add_str(itt.domain, task_id, file_id, frame.f_code.co_filename)
itt.lib.itt_metadata_add(itt.domain, task_id, line_id, frame.f_code.co_firstlineno)
if mdl:
itt.lib.itt_metadata_add_str(itt.domain, task_id, module_id, mdl.__name__)
elif "return" in event:
# print event, frame.f_code.co_name, task_id + trace_execution.frames[task_id] * high_part
if task_id in trace_execution.frames:
itt.lib.itt_task_end_overlapped(
itt.domain, 0, task_id + trace_execution.frames[task_id] * high_part
)
if trace_execution.frames[task_id] > 1:
trace_execution.frames[task_id] -= 1
else:
del trace_execution.frames[task_id]
trace_execution.recurrent = False
print trace_execution.frames
old_profiler = sys.getprofile()
sys.setprofile(profiler)
old_threading_profiler = threading.setprofile(profiler)
fn(*args)
sys.setprofile(old_profiler)
threading.setprofile(old_threading_profiler)
else:
fn(*args)
def runcall(self, func, *args, **kw):
self.set_cmd(repr(func))
threading.setprofile(self.dispatcher)
sys.setprofile(self.dispatcher)
try:
return func(*args, **kw)
finally:
sys.setprofile(None)
threading.setprofile(None)
def profile_off(self):
sys.setprofile(None)
threading.setprofile(None)
self._do_poll = False
self._tPoll.join()
self.outfile.flush()
self.outfile.close()
def stop_tracing(output_path: str) -> str:
"""Finish tracing allocations, and dump to disk.
Returns path to the index HTML page of the report.
"""
sys.setprofile(None)
threading.setprofile(None)
preload.fil_shutting_down()
return create_report(output_path)
def profile_on(self, regex='.*PYME.*', outfile='profile.txt'):
self.regex = re.compile(regex)
self.outfile = open(outfile, 'w')
sys.setprofile(self.prof_callback)
threading.setprofile(self.prof_callback)
self._tPoll = threading.Thread(target=self._poll)
self._tPoll.start()
def stop(self) -> None:
# Unregister the profiler in this order, otherwise we will have extra
# measurements in the end
sys.setprofile(None)
threading.setprofile(None)
greenlet.settrace(self.previous_callback) # pylint: disable=c-extension-no-member
self.collector.stop()
self.collector = None
def trace_execution(fn, args, save_to=None):
import inspect
if save_to:
os.environ['INTEL_SEA_SAVE_TO'] = save_to
itt = ITT("python")
if itt.lib:
file_id = itt.get_string_id('__FILE__')
line_id = itt.get_string_id('__LINE__')
module_id = itt.get_string_id('__MODULE__')
trace_execution.frames = {}
trace_execution.recurrent = False
high_part = 2**32
def profiler(frame, event, arg): # https://pymotw.com/2/sys/tracing.html
if trace_execution.recurrent:
return
trace_execution.recurrent = True
task_id = id(frame.f_code)
if 'call' in event:
if task_id in trace_execution.frames:
trace_execution.frames[task_id] += 1
else:
trace_execution.frames[task_id] = 1
task_id += trace_execution.frames[task_id] * high_part
name = frame.f_code.co_name + ((' (%s)' % arg.__name__) if arg else '')
if 'self' in frame.f_locals:
cls = frame.f_locals['self'].__class__.__name__
name = cls + "." + name
# print event, name, task_id, arg
mdl = inspect.getmodule(frame)
itt.lib.itt_task_begin_overlapped(itt.domain, task_id, 0, itt.get_string_id(name), 0)
itt.lib.itt_metadata_add_str(itt.domain, task_id, file_id, frame.f_code.co_filename)
itt.lib.itt_metadata_add(itt.domain, task_id, line_id, frame.f_code.co_firstlineno)
if mdl:
itt.lib.itt_metadata_add_str(itt.domain, task_id, module_id, mdl.__name__)
elif 'return' in event:
# print event, frame.f_code.co_name, task_id + trace_execution.frames[task_id] * high_part
if task_id in trace_execution.frames:
itt.lib.itt_task_end_overlapped(itt.domain, 0, task_id + trace_execution.frames[task_id] * high_part)
if trace_execution.frames[task_id] > 1:
trace_execution.frames[task_id] -= 1
else:
del trace_execution.frames[task_id]
itt.counter('MEMORY_USAGE', get_memory_usage())
trace_execution.recurrent = False
old_profiler = sys.getprofile()
sys.setprofile(profiler)
old_threading_profiler = threading.setprofile(profiler)
if fn:
fn(*args)
sys.setprofile(old_profiler)
threading.setprofile(old_threading_profiler)
elif fn:
fn(*args)
def stop(self):
"""
Removes the module profiler globally and from future threads.
"""
if sys.getprofile() != self.profiler:
logger.warning(
"ModuleProfiler: The currently enabled profile function was not ours - unbinding anyways"
)
threading.setprofile(None)
sys.setprofile(None)
def start(builtins=False):
'''
Args:
builtins: If set true, then builtin functions are profiled too.
timing_sample: will cause the profiler to do timing measuresements
according to the value. Will increase profiler speed but
decrease accuracy.
'''
threading.setprofile(__callback)
_yappi.start(builtins)
def _set_threading_profile(on, _):
def _profile_thread_callback(frame, event, arg):
"""_profile_thread_callback will only be called once per-thread.
"""
_bfext._profile_event(frame, event, arg)
if on:
threading.setprofile(_profile_thread_callback)
else:
threading.setprofile(None)
def runctx(self, cmd, globals, locals):
self.set_cmd(cmd)
threading.setprofile(self.dispatcher)
sys.setprofile(self.dispatcher)
try:
exec cmd in globals, locals
finally:
sys.setprofile(None)
threading.setprofile(None)
return self
def start(builtins=False):
'''
Args:
builtins: If set true, then builtin functions are profiled too.
timing_sample: will cause the profiler to do timing measuresements
according to the value. Will increase profiler speed but
decrease accuracy.
'''
threading.setprofile(__callback)
_yappi.start(builtins)
def enforce(self, enable=True):
"""Raises an exception when Qt method calls are made from a
non-main thread without the mainloop blocked. Only takes effect
on threads created after enforce() is called."""
def enforce(frame, event, func):
if event == 'c_call':
if isinstance(func.__self__, QObject):
if not self.held():
message = 'qtlock was not acquired for this Qt call, and we are not in the main thread.'
raise threading.ThreadError(message)
threading.setprofile(enforce if enable else None)
def stop_tracing(output_path: str) -> str:
"""Finish tracing allocations, and dump to disk.
Returns path to the index HTML page of the report.
"""
sys.setprofile(None)
threading.setprofile(None)
preload.fil_stop_tracking()
result = create_report(output_path)
# Clear allocations; we don't need them anymore, and they're just wasting
# memory:
preload.fil_reset("/tmp")
return result
def interpret(source, file_name=''):
stdout = sys.stdout
sys.stdout = None
objs = {}
rtinfo = RuntimeInfo()
builder = RuntimeInfoBuilder(rtinfo)
# We have to save the environment to avoid any import side-effect by the interpreter
saved_sys_path = sys.path
saved_sys_modules = sys.modules
sys.path = sys.path.copy()
sys.modules = sys.modules.copy()
if file_name:
dir_name = os.path.dirname(file_name)
sys.path.append(dir_name)
threading.setprofile(builder._profile_func)
thread = threading.Thread(target=_do_interpret,
args=(source, file_name, objs))
thread.start()
# TODO: busy loop?
while thread.is_alive():
thread.join(1)
threading.setprofile(None)
sys.stdout = stdout
if thread.exc_info:
_, exc, tb = thread.exc_info
if isinstance(exc, InterpretError):
raise exc
else:
if env.verbose_level:
traceback.print_tb(tb)
warn(None, Warnings.EXCEPTION_RAISED)
builder.build()
rtinfo.pyfuncs = _make_pyfuncs(objs)
module_classes = _find_module_classes(rtinfo.ctor_nodes)
for cls in module_classes:
pyfuncs = _make_pyfuncs(cls.__dict__)
rtinfo.pyfuncs.update(pyfuncs)
instances = _find_module_instances(objs, module_classes)
rtinfo.module_classes = module_classes
rtinfo.module_instances = instances
namespace_names = [
scp.name for scp in env.scopes.values()
if scp.is_namespace() and not scp.is_global()
]
_vars, _namespaces = _find_vars('__main__', objs, set(), namespace_names)
_namespaces['__main__'] = _vars['__main__']
rtinfo.global_vars = _namespaces
env.runtime_info = rtinfo
sys.path = saved_sys_path
sys.modules = saved_sys_modules
def recover(self):
""" Unset the current function in the sys.setprofile.
If available the previous method is recovered in setprofile. A
RuntimeError is raised if the `previous` attribute does not exist.
"""
if hasattr(self, 'previous'):
sys.setprofile(self.previous)
if has_threading:
threading.setprofile(self.previous)
del self.previous
else:
raise RuntimeError('A profile function has not been set')
def recover(self):
""" Unset the current function in the sys.setprofile.
If available the previous method is recovered in setprofile. A
RuntimeError is raised if the `previous` attribute does not exist.
"""
if hasattr(self, 'previous'):
sys.setprofile(self.previous)
if has_threading:
threading.setprofile(self.previous)
del self.previous
else:
raise RuntimeError('A profile function has not been set')
def stop(self):
if self._active:
if sys.getprofile() is self:
sys.setprofile(self._previous_profiler)
else: # pragma: no cover
warn('the system profiling hook has changed unexpectedly')
if self.all_threads:
if threading._profile_hook is self:
threading.setprofile(self._previous_thread_profiler)
else: # pragma: no cover
warn('the threading profiling hook has changed unexpectedly')
self._active = False
def init_types_collection(filter_filename=default_filter_filename):
# type: (Callable[[Optional[str]], Optional[str]]) -> None
"""
Setup profiler hooks to enable type collection.
Call this one time from the main thread.
The optional argument is a filter that maps a filename (from
code.co_filename) to either a normalized filename or None.
For the default filter see default_filter_filename().
"""
global _filter_filename
_filter_filename = filter_filename
sys.setprofile(_trace_dispatch)
threading.setprofile(_trace_dispatch)
def start_profiler():
global _state
_state = GlobalState()
frame = sys._getframe(0)
current_greenlet = greenlet.getcurrent() # pylint: disable=no-member
thread_state = ensure_thread_state(current_greenlet, frame)
_state.last = thread_state
# this needs to be instantiate before the handler is installed
greenlet.settrace(greenlet_profiler) # pylint: disable=no-member
sys.setprofile(thread_profiler)
threading.setprofile(thread_profiler)
def run(self):
if sys.getprofile() is not None:
# NOTE: There's no threading.getprofile().
# The profiling function will be stored at threading._profile_hook
# but it's not documented.
raise RuntimeError('Another profiler already registered')
with deferral() as defer:
sys.setprofile(self._profile)
defer(sys.setprofile, None)
threading.setprofile(self._profile)
defer(threading.setprofile, None)
self.timer.start(self)
defer(self.timer.stop)
yield
self._times_entered.clear()
def start(self):
if self._active:
raise RuntimeError('type checker already running')
self._active = True
# Install this instance as the current profiler
self._previous_profiler = sys.getprofile()
sys.setprofile(self)
# If requested, set this instance as the default profiler for all future threads
# (does not affect existing threads)
if self.all_threads:
self._previous_thread_profiler = threading._profile_hook
threading.setprofile(self)
def start_profiler():
global _state
_state = GlobalState()
frame = sys._getframe(0)
current_greenlet = greenlet.getcurrent() # pylint: disable=no-member
thread_state = ensure_thread_state(current_greenlet, frame)
_state.last = thread_state
# this needs to be instantiate before the handler is installed
greenlet.settrace(greenlet_profiler) # pylint: disable=no-member
sys.setprofile(thread_profiler)
threading.setprofile(thread_profiler)
def __init__(self):
super(Hooks, self).__init__()
sort = os.getenv('PEAS_PYTHON_PROFILE', default='time')
self.__stat_sort = sort.split(';')
self.__stats = None
self.__stats_lock = threading.Lock()
self.__thread_refs = []
self.__thread_local = threading.local()
threading.setprofile(self.__init_thread)
self.__profile = cProfile.Profile()
self.__profile.enable()
def replace(self, function):
""" Set a new function in sys.setprofile.
If the function has been already set and it is not the same as before
then RuntimeError is raised.
"""
if hasattr(self, 'previous'):
if function != sys.getprofile():
raise RuntimeError(
'Cannot replace profile function more than once')
return
else:
self.previous = sys.getprofile()
if has_threading:
threading.setprofile(function)
sys.setprofile(function)
def start(self):
"""Start profiling.
This hooks into threading and will record all calls made until
stop() is called.
"""
self._g_threadmap = {}
self.p = Profiler()
permitted = self.__class__.profiler_lock.acquire(
self.__class__.profiler_block)
if not permitted:
raise errors.InternalBzrError(msg="Already profiling something")
try:
self.p.enable(subcalls=True)
threading.setprofile(self._thread_profile)
except:
self.__class__.profiler_lock.release()
raise
def stop(self):
"""Stop profiling.
This unhooks from threading and cleans up the profiler, returning
the gathered Stats object.
:return: A bzrlib.lsprof.Stats object.
"""
try:
self.p.disable()
for pp in self._g_threadmap.values():
pp.disable()
threading.setprofile(None)
p = self.p
self.p = None
threads = {}
for tid, pp in self._g_threadmap.items():
threads[tid] = Stats(pp.getstats(), {})
self._g_threadmap = None
return Stats(p.getstats(), threads)
finally:
self.__class__.profiler_lock.release()
def trace_execution(fn, *args):
import inspect
itt = ITT("python")
if itt.lib:
trace_execution.depth = 0
file_id = itt.get_string_id('__FILE__')
line_id = itt.get_string_id('__LINE__')
module_id = itt.get_string_id('__MODULE__')
def profiler(frame, event, arg):
if 'call' in event:
trace_execution.depth += 1
name = frame.f_code.co_name
if 'self' in frame.f_locals:
cls = frame.f_locals['self'].__class__.__name__
name = cls + "." + name
mdl = inspect.getmodule(frame)
itt.lib.itt_task_begin(itt.domain, trace_execution.depth, 0, itt.get_string_id(name), 0)
itt.lib.itt_metadata_add_str(itt.domain, trace_execution.depth, file_id, frame.f_code.co_filename)
itt.lib.itt_metadata_add(itt.domain, trace_execution.depth, line_id, frame.f_code.co_firstlineno)
if mdl:
itt.lib.itt_metadata_add_str(itt.domain, trace_execution.depth, module_id, mdl.__name__)
if 'return' in event:
itt.lib.itt_task_end(itt.domain, 0)
trace_execution.depth -= 1
old_profiler = sys.getprofile()
sys.setprofile(profiler)
old_threading_profiler = threading.setprofile(profiler)
fn(*args)
sys.setprofile(old_profiler)
threading.setprofile(old_threading_profiler)
else:
fn(*args)
def profile_off():
threading.setprofile(None)
sys.setprofile(None)
def profile_on():
global p_stats, p_start_time
p_stats = {}
p_start_time = time.time()
threading.setprofile(profiler)
sys.setprofile(profiler)
def shutdown(self): sys.setprofile(None) # Clear the trace/profile function. threading.setprofile(None) # Clear the trace/profile function for threads. self._close_collection() # Close the JSON output. self.terminator.set() # Stop the writer thread. self.writer.join() # Join the writer thread.
index = 4
if event == 'c_return':
index += 1
stats[lockobj][name][index] = time()
stats[lockobj][name][6] = index == 1
doCheck = event == 'c_return' or (event == 'c_call' and callname == 'release')
if doCheck:
took = stats[lockobj][name][index] - stats[lockobj][name][index-1]
if took > 2:
print >> sys.stderr, "%s waited more than %.2f to %s lock %s:%d"%(name, took, callname, filename, lineno)
if hasattr(threadlocal, "lines"):
for line in threadlocal.lines:
print >> sys.stderr, "\t", line
if index == 0:
for otherthread in stats[lockobj]:
if otherthread != name:
if stats[lockobj][otherthread][6]:
print >> sys.stderr, "%s waiting for lock acquired by %s"%(name, otherthread)
if False and hasattr(threadlocal, "lines"):
for line in threadlocal.lines:
print >> sys.stderr, "\t", line
if __name__ == '__main__':
sys.setprofile(lock_profile)
setprofile(lock_profile)
run()
def start(builtins=False, timing_sample=1):
threading.setprofile(__callback)
_yappi.start(builtins, timing_sample)
def stop(self):
threading.setprofile(None)
sys.setprofile(None)
self.active = False
def start(self):
sys.setprofile(self.event)
threading.setprofile(self.event)
self.active = True
def stop():
"""
Stop profiler.
"""
_yappi.stop()
threading.setprofile(None)
def stop():
threading.setprofile(None)
_yappi.stop()
def activate_hook(cls):
sys.setprofile(cls.dump_hook)
threading.setprofile(cls.dump_hook)
cls.hook_enabled = True
def stop(self):
self.stats.record_stopping(time.clock())
self.timer.stop()
threading.setprofile(None)
sys.setprofile(None)
self._running = False
