def measure_memory(*args, **kwargs):
gc.collect()
result = {}
profiler = LineProfiler(backend='psutil')
func = profiler(_measure_memory_inner)
func(*args, **kwargs)
for (filename, lines) in profiler.code_map.items():
all_lines = linecache.getlines(filename)
last_mem = None
for (lineno, mem) in lines:
if mem is None:
result[all_lines[lineno - 1].strip()[2:]] = last_mem[1]
last_mem = mem
return result
def test_loop_count(self):
def some_loop():
for i in range(12): # line -2
a = 1 # line -1
profiler = LineProfiler()
wrapped = profiler(some_loop)
wrapped()
show_results(profiler)
for_line = list(list(profiler.code_map.values())[0].values())[-2]
looped_instruction = list(
list(profiler.code_map.values())[0].values())[-1]
self.assertEqual(for_line[2], 13)
self.assertEqual(looped_instruction[2], 12)
def enable_mem_profiler(fun):
"""
Enable memory profiler if specified in arguments.
:param fun: function to wrap
:return: LineProfiler instance
"""
global parsed_args
mem = None
if parsed_args.memory:
mem = LineProfiler()
mem.add_function(fun)
mem.enable()
return mem
def test_decr(self):
def del_stuff():
b = [2] * (2 * 10**7)
del b
profiler = LineProfiler()
wrapped = profiler(del_stuff)
wrapped()
show_results(profiler)
b_line = list(list(profiler.code_map.values())[0].values())[-2]
del_line = list(list(profiler.code_map.values())[0].values())[-1]
self.assertGreater(0, del_line[0])
self.assertGreater(del_line[1], 0)
self.assertAlmostEqual(-del_line[0], b_line[0], delta=1)
def test_loop_incr(self):
def loop_incr():
a = []
b = [2] * (2 * 10**7) # line -4
for i in range(3):
c = [2] * (2 * 10**7) # line -2
a.append(c)
profiler = LineProfiler()
wrapped = profiler(loop_incr)
wrapped()
show_results(profiler)
b_line = list(list(profiler.code_map.values())[0].values())[-4]
c_line = list(list(profiler.code_map.values())[0].values())[-2]
self.assertAlmostEqual(b_line[2] * 3, c_line[2], delta=1)
self.assertEqual(c_line[2], 3)
def wrapper(*args, **kwargs):
prof = LineProfiler(backend=backend)
val = prof(func)(*args, **kwargs)
show_results(prof, stream=stream, precision=precision)
return val
def mprun(self, parameter_s='', cell=None):
""" Execute a statement under the line-by-line memory profiler from the
memory_profiler module.
Usage, in line mode:
%mprun -f func1 -f func2 <statement>
Usage, in cell mode:
%%mprun -f func1 -f func2 [statement]
code...
code...
In cell mode, the additional code lines are appended to the (possibly
empty) statement in the first line. Cell mode allows you to easily
profile multiline blocks without having to put them in a separate
function.
The given statement (which doesn't require quote marks) is run via the
LineProfiler. Profiling is enabled for the functions specified by the -f
options. The statistics will be shown side-by-side with the code through
the pager once the statement has completed.
Options:
-f <function>: LineProfiler only profiles functions and methods it is told
to profile. This option tells the profiler about these functions. Multiple
-f options may be used. The argument may be any expression that gives
a Python function or method object. However, one must be careful to avoid
spaces that may confuse the option parser. Additionally, functions defined
in the interpreter at the In[] prompt or via %run currently cannot be
displayed. Write these functions out to a separate file and import them.
One or more -f options are required to get any useful results.
-T <filename>: dump the text-formatted statistics with the code
side-by-side out to a text file.
-r: return the LineProfiler object after it has completed profiling.
-c: If present, add the memory usage of any children process to the report.
"""
from io import StringIO
from memory_profiler import show_results, LineProfiler
# Local imports to avoid hard dependency.
from distutils.version import LooseVersion
import IPython
ipython_version = LooseVersion(IPython.__version__)
if ipython_version < '0.11':
from IPython.genutils import page
from IPython.ipstruct import Struct
from IPython.ipapi import UsageError
else:
from IPython.core.page import page
from IPython.utils.ipstruct import Struct
from IPython.core.error import UsageError
# Escape quote markers.
opts_def = Struct(T=[''], f=[])
parameter_s = parameter_s.replace('"', r'\"').replace("'", r"\'")
opts, arg_str = self.parse_options(parameter_s, 'rf:T:c',
list_all=True)
opts.merge(opts_def)
global_ns = self.shell.user_global_ns
local_ns = self.shell.user_ns
if cell is not None:
arg_str += '\n' + cell
# Get the requested functions.
funcs = []
for name in opts.f:
try:
funcs.append(eval(name, global_ns, local_ns))
except Exception as e:
raise UsageError('Could not find function %r.\n%s: %s' % (name,
e.__class__.__name__,
e))
include_children = 'c' in opts
profile = LineProfiler(include_children=include_children)
for func in funcs:
profile(func)
# Add the profiler to the builtins for @profile.
if 'profile' in builtins.__dict__:
had_profile = True
old_profile = builtins.__dict__['profile']
else:
had_profile = False
old_profile = None
builtins.__dict__['profile'] = profile
try:
profile.runctx(arg_str, global_ns, local_ns)
message = ''
except SystemExit:
message = "*** SystemExit exception caught in code being profiled."
except KeyboardInterrupt:
message = ("*** KeyboardInterrupt exception caught in code being "
"profiled.")
finally:
if had_profile:
builtins.__dict__['profile'] = old_profile
# Trap text output.
stdout_trap = StringIO()
show_results(profile, stdout_trap)
output = stdout_trap.getvalue()
output = output.rstrip()
if ipython_version < '0.11':
page(output, screen_lines=self.shell.rc.screen_length)
else:
page(output)
print(message, )
text_file = opts.T[0]
if text_file:
with open(text_file, 'w') as pfile:
pfile.write(output)
print('\n*** Profile printout saved to text file %s. %s' % (
text_file,
message))
return_value = None
if 'r' in opts:
return_value = profile
return return_value
default='psutil',
help='backend using for getting memory info '
'(one of the {tracemalloc, psutil, posix})')
if not sys.argv[1:]:
parser.print_help()
sys.exit(2)
(options, args) = parser.parse_args()
sys.argv[:] = args # Remove every memory_profiler arguments
script_filename = _find_script(args[0])
_backend = choose_backend(options.backend)
if options.timestamp:
prof = TimeStamper(_backend)
else:
prof = LineProfiler(max_mem=options.max_mem, backend=_backend)
try:
exec_with_profiler(script_filename, prof, options.backend)
finally:
if options.out_filename is not None:
out_file = open(options.out_filename, "a")
else:
out_file = sys.stdout
if options.timestamp:
prof.show_results(stream=out_file)
else:
show_results(prof, precision=options.precision, stream=out_file)
nargs=REMAINDER,
help=
'python script or module followed by command line arguements to run')
args = parser.parse_args()
if len(args.program) == 0:
print("A program to run must be provided. Use -h for help")
sys.exit(1)
target = args.program[0]
script_args = args.program[1:]
_backend = choose_backend(args.backend)
if args.timestamp:
prof = TimeStamper(_backend)
else:
prof = LineProfiler(max_mem=args.max_mem, backend=_backend)
try:
if args.program[0].endswith('.py'):
script_filename = _find_script(args.program[0])
exec_with_profiler(script_filename, prof, args.backend,
script_args)
else:
run_module_with_profiler(target, prof, args.backend, script_args)
finally:
if args.out_filename is not None:
out_file = open(args.out_filename, "a")
else:
out_file = sys.stdout
if args.timestamp:
dest="timestamp",
default=False,
action="store_true",
help="""print timestamp instead of memory measurement for
decorated functions""")
if not sys.argv[1:]:
parser.print_help()
sys.exit(2)
(options, args) = parser.parse_args()
sys.argv[:] = args # Remove every memory_profiler arguments
if options.timestamp:
prof = TimeStamper()
else:
prof = LineProfiler(max_mem=options.max_mem)
script_filename = _find_script(args[0])
try:
exec_with_profiler(script_filename, prof)
finally:
if options.out_filename is not None:
out_file = open(options.out_filename, "a")
else:
out_file = sys.stdout
if options.timestamp:
prof.show_results(stream=out_file)
else:
show_results(prof, precision=options.precision, stream=out_file)
def _profile(*args, **kwargs):
prof = LineProfiler() # backend=backend)
val = prof(f)(*args, **kwargs)
show_results(prof) # , stream=stream, precision=precision)
return val
