def test_enable_disable(self):
lp = LineProfiler()
self.assertEqual(lp.enable_count, 0)
lp.enable_by_count()
self.assertEqual(lp.enable_count, 1)
lp.enable_by_count()
self.assertEqual(lp.enable_count, 2)
lp.disable_by_count()
self.assertEqual(lp.enable_count, 1)
lp.disable_by_count()
self.assertEqual(lp.enable_count, 0)
self.assertEqual(lp.last_time, {})
lp.disable_by_count()
self.assertEqual(lp.enable_count, 0)
with lp:
self.assertEqual(lp.enable_count, 1)
with lp:
self.assertEqual(lp.enable_count, 2)
self.assertEqual(lp.enable_count, 1)
self.assertEqual(lp.enable_count, 0)
self.assertEqual(lp.last_time, {})
with self.assertRaises(RuntimeError):
self.assertEqual(lp.enable_count, 0)
with lp:
self.assertEqual(lp.enable_count, 1)
raise RuntimeError()
self.assertEqual(lp.enable_count, 0)
self.assertEqual(lp.last_time, {})
class ProfilingPanel(Panel):
"""
Panel that displays profiling information.
"""
title = _('Profiling')
template = 'debug_toolbar_line_profiler/panels/profiling.html'
def _unwrap_closure_and_profile(self, func):
if not hasattr(func, '__code__'):
return
self.line_profiler.add_function(func)
for subfunc in getattr(func, 'profile_additional', []):
self._unwrap_closure_and_profile(subfunc)
if func.__closure__:
for cell in func.__closure__:
target = cell.cell_contents
if inspect.isclass(target) and View in inspect.getmro(target):
for name, value in inspect.getmembers(target):
if name[0] != '_' and inspect.ismethod(value):
self._unwrap_closure_and_profile(value)
else:
self._unwrap_closure_and_profile(target)
def process_view(self, request, view_func, view_args, view_kwargs):
self.profiler = cProfile.Profile()
args = (request,) + view_args
self.line_profiler = LineProfiler()
self._unwrap_closure_and_profile(view_func)
self.line_profiler.enable_by_count()
out = self.profiler.runcall(view_func, *args, **view_kwargs)
self.line_profiler.disable_by_count()
return out
def add_node(self, func_list, func, max_depth, cum_time=0.1):
func_list.append(func)
func.has_subfuncs = False
if func.depth < max_depth:
for subfunc in func.subfuncs():
if (subfunc.stats[3] >= cum_time or
(hasattr(self.stats, 'line_stats') and
(subfunc.func in self.stats.line_stats.timings))):
func.has_subfuncs = True
self.add_node(func_list, subfunc, max_depth, cum_time=cum_time)
def process_response(self, request, response):
if not hasattr(self, 'profiler'):
return None
# Could be delayed until the panel content is requested (perf. optim.)
self.profiler.create_stats()
self.stats = DjangoDebugToolbarStats(self.profiler)
self.stats.line_stats = self.line_profiler.get_stats()
self.stats.calc_callees()
root = FunctionCall(self.stats, self.stats.get_root_func(), depth=0)
func_list = []
self.add_node(func_list, root, 10, root.stats[3] / 8)
self.record_stats({'func_list': func_list})
def jungleprofiler_wrapped_f(*args, **kwargs):
''' Wrapper that collects time and system usage data on wrapped function f'''
# Set up LineProfiler
lp = LineProfiler()
lp.add_function(f)
# Set up MemoryProfiler
pass # todo add Memory Profiler
# Start Counters
if self.t_prof: lp.enable_by_count()
if self.m_prof: pass
try:
t0 = time.time()
sio = io.StringIO() # Collects redirected stdout
with redirect_stdout(sio):
preturn = f(*args, **kwargs)
self.stdout = sio.getvalue()
t1 = time.time()
finally:
# Stop Counters
if self.m_prof: lp.disable_by_count()
if self.m_prof: pass # todo add Memory Profiler
# Collect Stats
# print('Get Stats: %s' % lp.print_stats())
self.walltime = t1 - t0
return preturn, copy.deepcopy(self)
def profiled_func(*args, **kwargs):
profiler = LineProfiler()
profiler.add_function(func)
for f in follow:
profiler.add_function(f)
profiler.enable_by_count()
return func(*args, **kwargs)
def profiled_func(*args, **kwargs):
try:
lp = LineProfiler()
lp.add_function(f)
lp.enable_by_count()
return f(*args, **kwargs)
finally:
lp.print_stats()
def wrap(*args, **kwargs):
profile = LineProfiler()
profile.add_function(f)
profile.enable_by_count()
result = f(*args, **kwargs)
profile.disable_by_count()
profile.print_stats(sys.stdout)
return result
def profiled_func(*args, **kwargs):
try:
profiler = LineProfiler()
profiler.add_function(func)
profiler.enable_by_count()
return func(*args, **kwargs)
finally:
profiler.print_stats()
def profiled_func(*args, **kwargs):
try:
profiler = LineProfiler()
profiler.add_function(func)
profiler.enable_by_count()
return func(*args, **kwargs)
finally:
profiler.print_stats()
def profiled_func(*args, **kwargs):
line_profiler = LineProfiler()
line_profiler.add_function(func)
map(lambda x: line_profiler.add_function(x), self.follow)
line_profiler.enable_by_count()
result = func(*args, **kwargs)
line_profiler.disable_by_count()
line_profiler.print_stats(stripzeros=True)
return result
def decorator(*args, **kwargs):
# line-profiler==3.0.2
profiler = LineProfiler()
try:
profiler.add_function(func)
profiler.enable_by_count()
return func(*args, **kwargs)
finally:
profiler.dump_stats('/var/log/{}.lprof'.format(time.time()))
def wrapper(*args, **kwargs):
try:
profiler = LineProfiler()
profiler.add_function(func)
for f in follow:
profiler.add_function(f)
profiler.enable_by_count()
return func(*args, **kwargs)
finally:
profiler.print_stats()
def profiled_func(*args, **kwargs):
try:
profiler = LineProfiler()
profiler.add_function(func)
for f in follow:
profiler.add_function(getattr(args[0], f))
profiler.enable_by_count()
return func(*args, **kwargs)
finally:
profiler.print_stats()
def profiled_func(*args, **kwargs):
try:
pf = LineProfiler()
pf.add_function(func)
for f in follow:
pf.add_function(f)
pf.enable_by_count()
return func(*args, **kwargs)
finally:
pf.print_stats()
def wrapped_fn(*args, **kwargs):
try:
profiler = LineProfiler()
profiler.add_function(fn)
for f in follow:
profiler.add_function(f)
profiler.enable_by_count()
return fn(*args, **kwargs)
finally:
profiler.print_stats()
def wrapper(*args, **kwargs):
profiler = LineProfiler()
profiler.add_function(fn)
try:
for f in follow:
profiler.add_function(f)
profiler.enable_by_count()
return fn(*args, **kwargs)
finally:
profiler.print_stats(output_unit=1)
profiler.dump_stats(report)
def timetest(func, *para):
p = LineProfiler()
p.add_function(func)
p.enable_by_count()
p_wrapper = p(func)
p_wrapper(*para)
# Printing
print(func(*para))
p.print_stats()
def profiled_func(*args, **kwargs):
try:
profiler = LineProfiler()
profiler.add_function(func)
for f in follow:
if isinstance(f, six.string_types):
f = to_function(f)
profiler.add_function(f)
profiler.enable_by_count()
return func(*args, **kwargs)
finally:
profiler.print_stats()
def profiled_func(*args, **kwargs):
try:
profiler = LineProfiler()
profiler.add_function(func)
for f in follow:
if isinstance(f, basestring):
f = to_function(f)
profiler.add_function(f)
profiler.enable_by_count()
return func(*args, **kwargs)
finally:
profiler.print_stats()
def create_line_profile(*args):
line_profiler = LineProfiler()
line_profiler.enable_by_count()
for klass in args:
for attr in dir(klass):
func = getattr(klass, attr)
condition = (not inspect.isfunction(func),
not inspect.ismethod(func))
if all(condition):
continue
line_profiler.add_function(func)
return line_profiler
def __call__(self, *args, **kwargs):
result = None
if logger.getEffectiveLevel() == logging.MPROFILE:
import sys
result = self.function(*args, **kwargs)
logger.debug(">>Tracing memory size : return object is %s bytes",
sys.getsizeof(result))
elif logger.getEffectiveLevel() == logging.CPROFILE:
import cProfile
profile = cProfile.Profile()
try:
profile.enable()
result = self.function(*args, **kwargs)
profile.disable()
finally:
profile.print_stats()
elif logger.getEffectiveLevel() == logging.LPROFILE:
try:
from line_profiler import LineProfiler
try:
profiler = LineProfiler()
profiler.add_function(self.function)
profiler.enable_by_count()
result = self.function(*args, **kwargs)
finally:
profiler.print_stats()
except ImportError:
logger.debug("Error importing line_profiler.")
return self.function(*args, **kwargs)
elif logger.getEffectiveLevel() == logging.MEMPROFILE:
try:
from memory_profiler import LineProfiler, show_results
try:
profiler = LineProfiler()
profiler.add_function(self.function)
profiler.enable_by_count()
result = self.function(*args, **kwargs)
finally:
show_results(profiler)
except ImportError:
logger.debug("Error importing memory_profiler.")
result = self.function(*args, **kwargs)
else:
result = self.function(*args, **kwargs)
return result
def profiled_func(*args, **kwargs):
try:
profiler = LineProfiler()
profiler.add_function(func)
if follow_all_methods:
cls = args[0] # class instance
_add_all_class_methods(profiler, cls,
except_=func.__name__)
for f in follow:
_add_function_or_classmethod(profiler, f, args)
profiler.enable_by_count()
return func(*args, **kwargs)
finally:
profiler.print_stats()
def speedtest_validate_transaction():
# create a transaction
b = bigchaindb.Bigchain()
tx = b.create_transaction(b.me, b.me, None, 'CREATE')
tx_signed = b.sign_transaction(tx, b.me_private)
# setup the profiler
profiler = LineProfiler()
profiler.enable_by_count()
profiler.add_function(bigchaindb.Bigchain.validate_transaction)
# validate_transaction 1000 times
for i in range(1000):
b.validate_transaction(tx_signed)
profiler.print_stats()
def run_and_profile(self):
if len(self._funcs_to_profile) > 0:
profiler = LineProfiler()
for func in self._funcs_to_profile:
profiler.add_function(func)
profiler.enable_by_count()
try:
yield
finally:
with io.StringIO() as str_stream:
profiler.print_stats(str_stream)
string = str_stream.getvalue()
print(f'Writing profile data to "{self._output_filenmae}"')
with open(self._output_filenmae, 'w') as fid:
fid.write(string)
else:
yield
def profile():
import termcolor
import time
from line_profiler import LineProfiler
epoch = int(time.time())
outfile = f"compare50_profile_{epoch}.txt"
profiler = LineProfiler()
for f in PROFILE:
profiler.add_function(f)
profiler.enable_by_count()
try:
yield
finally:
with open(outfile, "w") as f:
profiler.print_stats(stream=f)
termcolor.cprint(f"Profiling data written to {outfile}", "yellow")
class Profiler(object):
def __init__(self, *args):
self.profile = LineProfiler()
if len(args) > 0:
for func in args:
if callable(func):
self.add_function(func)
def add_function(self, func):
self.profile.add_function(func)
def __enter__(self):
self.profile.enable_by_count()
def __exit__(self, type, value, traceback):
self.profile.disable_by_count()
self.profile.print_stats()
class LineProfilerDebug(DebugDecorator):
def __init__(self, logger=None):
super().__init__(logger)
if not has_line_profiler:
raise ImportError("'line_profiler' not found.")
def setup(self, *args, **kwargs):
self.lprof = LineProfiler()
def cleanup(self, *args, **kwargs):
pass
def debug_func(self, *args, **kwargs):
self.lprof.add_function(self.func)
self.lprof.enable_by_count()
ret = self.func(*args, **kwargs)
self.lprof.print_stats()
return ret
def set_profiler(func_list=default_profile_funcs,
rank=0,
outfile_prefix='time_profile.out',
dump_raw=False):
"""
Applies a line profiler to the listed functions, wherever they appear in pyuvsim.
Places a LineProfiler object in the module namespace, and registers its dumping/printing
functions to run at the end.
Parameters
----------
func_list: list
List of function names (strings) to profile.
Defaults to ``profiling.default_profile_funcs``.
rank: int, optional
Which rank process should write out to file? (only one rank at a time will).
outfile_prefix: str
Filename prefix for printing profiling results.
Human-readable line by line profiling goes to <outfile_prefix>.out
LineStats data goes to <outfile_prefix>.lprof (if dump_raw)
Axis sizes go to <outfile_prefix>_axes.npz
dump_raw: bool
Write out a pickled LineStats object to <outfile_name>.lprof (Default False)
Sets
----
prof : LineProfiler
An instance of LineProfiler in the module namespace.
exit functions:
When the Python environment closes, the profiler functions
print_stats (and dump_stats, if dump_raw is True) will execute,
saving profiler data to file.
"""
global prof
if outfile_prefix.endswith(".out"):
outfile_prefix = outfile_prefix[:-4] # Strip extension
outfile_name = outfile_prefix + '.out'
# Can only set up profiling once per Python session.
if prof is not None: # pragma: nocover
warnings.warn("Profiler already set. Returning now.")
return
prof = LineProfiler()
if mpi is None or prof is None: # pragma: no cover
raise ImportError("You need mpi4py and line_profiler to use the "
"profiling module. Install them both by running pip "
"install pyuvsim[all].")
mpi.start_mpi()
# Add module functions to profiler.
mod_iter = chain(_pyuvsim.__dict__.values(), _pyradiosky.__dict__.values())
for mod_it in mod_iter:
if isfunction(mod_it):
if mod_it.__name__ in func_list:
prof.add_function(mod_it)
if isclass(mod_it):
for item in mod_it.__dict__.values():
if isfunction(item):
if item.__name__ in func_list:
prof.add_function(item)
# Write out profiling report to file.
if mpi.get_rank() == rank:
ofile = open(outfile_name, 'w')
atexit.register(ofile.close)
atexit.register(prof.print_stats, stream=ofile)
if dump_raw:
outfile_raw_name = outfile_prefix + ".lprof"
atexit.register(prof.dump_stats, outfile_raw_name)
setattr(prof, 'rank',
rank) # Add "rank" as an attribute to the profiler.
setattr(prof, 'meta_file', outfile_prefix + '_meta.out')
prof.enable_by_count()
class SpecialTestRunner(SpecialTest):
"""
Test runner, calls the specified test under specified profiler
Mode = None - no profiler, "c" - cProfile, "l" - LineProfiler, "h" - hotshot
"""
def __init__(self, test, mode=None):
super(SpecialTestRunner, self).__init__()
self.mode = mode
self.test = test
self.profiler = None
def setup(self):
if self.mode == 'c':
import cProfile
self.profiler = cProfile.Profile()
elif self.mode == 'l':
from line_profiler import LineProfiler
self.profiler = LineProfiler()
elif self.mode == 'h':
import hotshot
self.info['name'] = 'special.prof'
self.profiler = hotshot.Profile(self.info['name'])
self.test.setup()
def run(self):
if self.mode == 'c':
self.profiler.enable()
elif self.mode == 'l':
self.profiler.enable_by_count()
self.profiler.add_function(Handler.handle)
self.profiler.add_function(Condition.check_string_match)
self.profiler.add_function(Condition.check_function)
self.profiler.add_function(Condition.check_list)
t = Timer()
# Run itself
if self.mode == 'h':
self.profiler.runcall(self.test.run)
else:
self.test.run()
print('Test time: %s' % t.delta())
if self.mode == 'c':
import pstats
import StringIO
self.profiler.disable()
sio = StringIO.StringIO()
ps = pstats.Stats(self.profiler, stream=sio).sort_stats('time')
ps.print_stats()
print(sio.getvalue())
elif self.mode == 'h':
import hotshot.stats
print('Processing results...')
self.profiler.close()
name = self.info['name']
stats = hotshot.stats.load(name)
stats.strip_dirs()
stats.sort_stats('time', 'calls')
stats.print_stats(50)
print('Run "hotshot2calltree -o %s.out %s" to generate the cachegrind file' % (name, name))
elif self.mode == 'l':
self.profiler.disable()
self.profiler.print_stats()
return data
def for_range():
z = {
"data": {
"extra": {
"event": {
"outComeData": "[2,3,45]",
"key": "value"
}
}
}
}
for _ in range(100000):
k = remove_data_in_spin(z)
remove_data_in_spin(z)
# pj = simdjson.ParsedJson(z)
# print(pj.items('.data'))
if __name__ == '__main__':
prof = LineProfiler()
prof.add_function(remove_data_in_spin)
prof.enable_by_count()
for_range()
prof.print_stats()
# python -m cProfile -s cumtime copy_test.py
#
class ProfilingDebugPanel(DebugPanel):
"""
Panel that displays the Django version.
"""
name = 'Profiling'
template = 'debug_toolbar/panels/profiling.html'
has_content = True
def nav_title(self):
return _('Profiling')
def url(self):
return ''
def title(self):
return _('Profiling')
def _unwrap_closure_and_profile(self, func):
if not hasattr(func, 'func_code'):
return
self.line_profiler.add_function(func)
if func.func_closure:
for cell in func.func_closure:
if hasattr(cell.cell_contents, 'func_code'):
self._unwrap_closure_and_profile(cell.cell_contents)
def process_view(self, request, view_func, view_args, view_kwargs):
__traceback_hide__ = True
self.profiler = cProfile.Profile()
args = (request,) + view_args
if DJ_PROFILE_USE_LINE_PROFILER:
self.line_profiler = LineProfiler()
self._unwrap_closure_and_profile(view_func)
self.line_profiler.enable_by_count()
out = self.profiler.runcall(view_func, *args, **view_kwargs)
self.line_profiler.disable_by_count()
else:
self.line_profiler = None
out = self.profiler.runcall(view_func, *args, **view_kwargs)
return out
def add_node(self, func_list, func, max_depth, cum_time=0.1):
func_list.append(func)
func.has_subfuncs = False
if func.depth < max_depth:
for subfunc in func.subfuncs():
if (subfunc.stats[3] >= cum_time or
(hasattr(self.stats, 'line_stats') and
(subfunc.func in self.stats.line_stats.timings))):
func.has_subfuncs = True
self.add_node(func_list, subfunc, max_depth, cum_time=cum_time)
def process_response(self, request, response):
self.profiler.create_stats()
self.stats = DjangoDebugToolbarStats(self.profiler)
if DJ_PROFILE_USE_LINE_PROFILER:
self.stats.line_stats = self.line_profiler.get_stats()
self.stats.calc_callees()
root = FunctionCall(self.stats, self.stats.get_root_func(), depth=0)
func_list = []
self.add_node(func_list, root, 10, root.stats[3]/8)
self.stats_record({'func_list': func_list})
class ProfilingPanel(Panel):
"""
Panel that displays profiling information.
"""
title = _('Profiling')
template = 'debug_toolbar_line_profiler/panels/profiling.html'
def _unwrap_closure_and_profile(self, func):
if not hasattr(func, '__code__'):
return
self.line_profiler.add_function(func)
for subfunc in getattr(func, 'profile_additional', []):
self._unwrap_closure_and_profile(subfunc)
if PY2:
func_closure = func.func_closure
else:
func_closure = func.__closure__
if func_closure:
for cell in func_closure:
target = cell.cell_contents
if hasattr(target, '__code__'):
self._unwrap_closure_and_profile(cell.cell_contents)
if inspect.isclass(target) and View in inspect.getmro(target):
for name, value in inspect.getmembers(target):
if name[0] != '_' and inspect.ismethod(value):
self._unwrap_closure_and_profile(value)
def process_view(self, request, view_func, view_args, view_kwargs):
self.view_func = view_func
self.profiler = cProfile.Profile()
args = (request,) + view_args
self.line_profiler = LineProfiler()
self._unwrap_closure_and_profile(view_func)
signals.profiler_setup.send(sender=self,
profiler=self.line_profiler,
view_func=view_func,
view_args=view_args,
view_kwargs=view_kwargs)
self.line_profiler.enable_by_count()
out = self.profiler.runcall(view_func, *args, **view_kwargs)
self.line_profiler.disable_by_count()
return out
def add_node(self, func_list, func, max_depth, cum_time=0.1):
"""
add_node does a depth first traversal of the call graph, appending a
FunctionCall object to func_list, so that the Django template only
has to do a single for loop over func_list that can render a tree
structure
Parameters:
func_list is an array that will have a FunctionCall for each call
added to it
func is a FunctionCall object that will have all its callees added
max_depth is the maximum depth we should recurse
cum_time is the minimum cum_time a function should have to be
included in the output
"""
func_list.append(func)
func.has_subfuncs = False
# this function somewhat dangerously relies on FunctionCall to set its
# subfuncs' depth argument correctly
if func.depth >= max_depth:
return
# func.subfuncs returns FunctionCall objects
subs = sorted(func.subfuncs(), key=FunctionCall.cumtime, reverse=True)
for subfunc in subs:
# a sub function is important if it takes a long time or it has
# line_stats
if (subfunc.cumtime() >= cum_time or
(hasattr(self.stats, 'line_stats') and
subfunc.func in self.stats.line_stats.timings)):
func.has_subfuncs = True
self.add_node(
func_list=func_list,
func=subfunc,
max_depth=max_depth,
cum_time=subfunc.cumtime()/16)
def process_response(self, request, response):
if not hasattr(self, 'profiler'):
return None
# Could be delayed until the panel content is requested (perf. optim.)
self.profiler.create_stats()
self.stats = DjangoDebugToolbarStats(self.profiler)
self.stats.line_stats = self.line_profiler.get_stats()
self.stats.calc_callees()
func_list = []
root_func = self.stats.get_root_func(self.view_func)
if root_func is not None:
root_node = FunctionCall(statobj=self.stats,
func=root_func,
depth=0)
self.add_node(
func_list=func_list,
func=root_node,
max_depth=10,
cum_time=root_node.cumtime() / 8
)
# else:
# what should we do if we didn't detect a root function? It's not
# clear what causes this, but there are real world examples of it (see
# https://github.com/dmclain/django-debug-toolbar-line-profiler/issues/11)
self.record_stats({'func_list': func_list})
class ProfilingDebugPanel(DebugPanel):
"""
Panel that displays the Django version.
"""
name = 'Profiling'
template = 'debug_toolbar/panels/profiling.html'
has_content = True
def nav_title(self):
return _('Profiling')
def url(self):
return ''
def title(self):
return _('Profiling')
def _unwrap_closure_and_profile(self, func):
if not hasattr(func, 'func_code'):
return
self.line_profiler.add_function(func)
if func.func_closure:
for cell in func.func_closure:
if hasattr(cell.cell_contents, 'func_code'):
self._unwrap_closure_and_profile(cell.cell_contents)
def process_view(self, request, view_func, view_args, view_kwargs):
__traceback_hide__ = True
self.profiler = cProfile.Profile()
args = (request, ) + view_args
if DJ_PROFILE_USE_LINE_PROFILER:
self.line_profiler = LineProfiler()
self._unwrap_closure_and_profile(view_func)
self.line_profiler.enable_by_count()
out = self.profiler.runcall(view_func, *args, **view_kwargs)
self.line_profiler.disable_by_count()
else:
self.line_profiler = None
out = self.profiler.runcall(view_func, *args, **view_kwargs)
return out
def add_node(self, func_list, func, max_depth, cum_time=0.1):
func_list.append(func)
func.has_subfuncs = False
if func.depth < max_depth:
for subfunc in func.subfuncs():
if (subfunc.stats[3] >= cum_time
or (hasattr(self.stats, 'line_stats') and
(subfunc.func in self.stats.line_stats.timings))):
func.has_subfuncs = True
self.add_node(func_list,
subfunc,
max_depth,
cum_time=cum_time)
def process_response(self, request, response):
__traceback_hide__ = True
if not hasattr(self, 'profiler'):
return None
self.profiler.create_stats()
self.stats = DjangoDebugToolbarStats(self.profiler)
if DJ_PROFILE_USE_LINE_PROFILER:
self.stats.line_stats = self.line_profiler.get_stats()
self.stats.calc_callees()
root = FunctionCall(self.stats, self.stats.get_root_func(), depth=0)
func_list = []
self.add_node(func_list, root, 10, root.stats[3] / 8)
self.record_stats({'func_list': func_list})
m = DFMessage(fmt, elements, False)
except ValueError:
return self._parse_next()
self._add_msg(m)
return m
if __name__ == "__main__":
use_profiler = False
if use_profiler:
from line_profiler import LineProfiler
profiler = LineProfiler()
profiler.add_function(DFReader_binary._parse_next)
profiler.add_function(DFReader_binary._add_msg)
profiler.add_function(DFReader._set_time)
profiler.enable_by_count()
filename = sys.argv[1]
if filename.endswith('.log'):
log = DFReader_text(filename)
else:
log = DFReader_binary(filename)
while True:
m = log.recv_msg()
if m is None:
break
if use_profiler:
profiler.print_stats()
self._add_msg(m)
return m
if __name__ == "__main__":
import sys
use_profiler = False
if use_profiler:
from line_profiler import LineProfiler
profiler = LineProfiler()
profiler.add_function(DFReader_binary._parse_next)
profiler.add_function(DFReader_binary._add_msg)
profiler.add_function(DFReader._set_time)
profiler.enable_by_count()
filename = sys.argv[1]
if filename.endswith('.log'):
log = DFReader_text(filename)
else:
log = DFReader_binary(filename)
while True:
m = log.recv_msg()
if m is None:
break
#print(m)
if use_profiler:
profiler.print_stats()
class ProfilingPanel(Panel):
"""
Panel that displays profiling information.
"""
title = _('Profiling')
template = 'debug_toolbar_line_profiler/panels/profiling.html'
def _unwrap_closure_and_profile(self, func):
if not hasattr(func, '__code__'):
return
self.line_profiler.add_function(func)
for subfunc in getattr(func, 'profile_additional', []):
self._unwrap_closure_and_profile(subfunc)
if PY2:
func_closure = func.func_closure
else:
func_closure = func.__closure__
if func_closure:
for cell in func_closure:
target = cell.cell_contents
if hasattr(target, '__code__'):
self._unwrap_closure_and_profile(cell.cell_contents)
if inspect.isclass(target) and View in inspect.getmro(target):
for name, value in inspect.getmembers(target):
if name[0] != '_' and inspect.ismethod(value):
self._unwrap_closure_and_profile(value)
def process_view(self, request, view_func, view_args, view_kwargs):
self.view_func = view_func
self.profiler = cProfile.Profile()
args = (request,) + view_args
self.line_profiler = LineProfiler()
self._unwrap_closure_and_profile(view_func)
signals.profiler_setup.send(sender=self,
profiler=self.line_profiler,
view_func=view_func,
view_args=view_args,
view_kwargs=view_kwargs)
self.line_profiler.enable_by_count()
out = self.profiler.runcall(view_func, *args, **view_kwargs)
self.line_profiler.disable_by_count()
return out
def add_node(self, func_list, func, max_depth, cum_time=0.1):
"""
add_node does a depth first traversal of the call graph, appending a
FunctionCall object to func_list, so that the Django template only
has to do a single for loop over func_list that can render a tree
structure
Parameters:
func_list is an array that will have a FunctionCall for each call
added to it
func is a FunctionCall object that will have all its callees added
max_depth is the maximum depth we should recurse
cum_time is the minimum cum_time a function should have to be
included in the output
"""
func_list.append(func)
func.has_subfuncs = False
# this function somewhat dangerously relies on FunctionCall to set its
# subfuncs' depth argument correctly
if func.depth >= max_depth:
return
# func.subfuncs returns FunctionCall objects
subs = sorted(func.subfuncs(), key=FunctionCall.cumtime, reverse=True)
for subfunc in subs:
# a sub function is important if it takes a long time or it has
# line_stats
if (subfunc.cumtime() >= cum_time or
(hasattr(self.stats, 'line_stats') and
subfunc.func in self.stats.line_stats.timings)):
func.has_subfuncs = True
self.add_node(
func_list=func_list,
func=subfunc,
max_depth=max_depth,
cum_time=subfunc.cumtime()/16)
def process_response(self, request, response):
if not hasattr(self, 'profiler'):
return None
# Could be delayed until the panel content is requested (perf. optim.)
self.profiler.create_stats()
self.stats = DjangoDebugToolbarStats(self.profiler)
self.stats.line_stats = self.line_profiler.get_stats()
self.stats.calc_callees()
func_list = []
root_func = self.stats.get_root_func(self.view_func)
if root_func is not None:
root_node = FunctionCall(statobj=self.stats,
func=root_func,
depth=0)
self.add_node(
func_list=func_list,
func=root_node,
max_depth=10,
cum_time=root_node.cumtime() / 8
)
# else:
# what should we do if we didn't detect a root function? It's not
# clear what causes this, but there are real world examples of it (see
# https://github.com/dmclain/django-debug-toolbar-line-profiler/issues/11)
self.record_stats({'func_list': func_list})
class SpecialTestRunner(SpecialTest):
"""
Test runner, calls the specified test under specified profiler
Mode = None - no profiler, "c" - cProfile, "l" - LineProfiler, "h" - hotshot
"""
def __init__(self, test, mode=None):
super(SpecialTestRunner, self).__init__()
self.mode = mode
self.test = test
self.profiler = None
def setup(self):
if self.mode == 'c':
import cProfile
self.profiler = cProfile.Profile()
elif self.mode == 'l':
from line_profiler import LineProfiler
self.profiler = LineProfiler()
elif self.mode == 'h':
import hotshot
self.info['name'] = 'special.prof'
self.profiler = hotshot.Profile(self.info['name'])
self.test.setup()
def run(self):
if self.mode == 'c':
self.profiler.enable()
elif self.mode == 'l':
self.profiler.enable_by_count()
self.profiler.add_function(Handler.handle)
self.profiler.add_function(Condition.check_string_match)
self.profiler.add_function(Condition.check_function)
self.profiler.add_function(Condition.check_list)
t = Timer()
# Run itself
if self.mode == 'h':
self.profiler.runcall(self.test.run)
else:
self.test.run()
print('Test time: %s' % t.delta())
if self.mode == 'c':
import pstats
import StringIO
self.profiler.disable()
sio = StringIO.StringIO()
ps = pstats.Stats(self.profiler, stream=sio).sort_stats('time')
ps.print_stats()
print(sio.getvalue())
elif self.mode == 'h':
import hotshot.stats
print('Processing results...')
self.profiler.close()
name = self.info['name']
stats = hotshot.stats.load(name)
stats.strip_dirs()
stats.sort_stats('time', 'calls')
stats.print_stats(50)
print(
'Run "hotshot2calltree -o %s.out %s" to generate the cachegrind file'
% (name, name))
elif self.mode == 'l':
self.profiler.disable()
self.profiler.print_stats()
