def record_postexecute(self, execution_time=None):
"""
Function used to record runtime information after the task we want to track is comleted, e.g.
the `execute_analysis(...)` function of a standard analysis.
The function may be overwritten in child classes to add recording of
additional runtime information.
When overwriting the function we should call super(...,self).runinfo_record_postexecute(execution_time)
in the custom version to ensure that the execution and end_time are properly
recorded.
:param execution_time: The total time it took to execute the analysis. May be None, in which
case the function will attempt to compute the execution time based on the start_time
(if available) and the the current time.
:param comm: Used for logging only. The MPI communicator to be used. Default value is None,
in which case MPI.COMM_WORLD is used.
"""
log_helper.debug(__name__, 'Recording post-execution runtime data', root=self.mpi_root, comm=self.mpi_comm)
# Finalize recording of post execution provenance
self['end_time'] = unicode(datetime.datetime.now())
if execution_time is not None:
self['execution_time'] = unicode(execution_time)
elif 'start_time' in self:
start_time = run_info_dict.string_to_time(self['start_time'])
stop_time = run_info_dict.string_to_time(self['end_time'])
self['execution_time'] = unicode(stop_time - start_time) # TODO: This only gives execution time in full seconds right now
else:
self['execution_time'] = None
# Attempt to record psutil data
try:
import psutil
process = psutil.Process()
self['memory_info_after'] = unicode(process.memory_info())
except ImportError:
log_helper.warning(__name__, 'psutil not installed. Recording of part of runtime information not possible',
root=self.mpi_root, comm=self.mpi_comm)
except:
warnings.warn("Recording of psutil-based runtime information failed: "+str(sys.exc_info()))
# Record the time and use profiling data if possible
if self.__time_and_use_profiler is not None:
self.__time_and_use_profiler.disable()
self.__time_and_use_profiler.create_stats()
self['profile'] = unicode(self.__time_and_use_profiler.stats)
# Save the summary statistics for the profiling data
stats_io = StringIO.StringIO()
profiler_stats = pstats.Stats(self.__time_and_use_profiler, stream=stats_io).sort_stats('cumulative')
profiler_stats.print_stats()
self['profile_stats'] = stats_io.getvalue()
# Record the memory profiling data if possible
if self.__memory_profiler is not None and self.get_profile_memory():
log_helper.debug(__name__, 'Recording memory profiling data', root=self.mpi_root, comm=self.mpi_comm)
mem_stats_io = StringIO.StringIO()
memory_profiler.show_results(self.__memory_profiler, stream=mem_stats_io)
self['profile_mem'] = unicode(self.__memory_profiler.code_map)
self['profile_mem_stats'] = mem_stats_io.getvalue()
def show_mem_profiler_results(mem):
"""
Show results of memory profiling if enabled.
:param mem: profiler instance
"""
if mem:
show_results(mem)
mem.disable()
def wrapper(*args, **kwargs):
if PROFILE_MEMORY:
prof = memory_profiler.LineProfiler()
val = prof(func)(*args, **kwargs)
memory_profiler.show_results(prof)
else:
val = func(*args, **kwargs)
return val
def _profile_memory_per_line(self, get_maximum=False):
profiler = self._get_memory_line_profiler()
decorators.ACTIVE_PROFILER = self._get_memory_line_wrapper(
profiler, get_maximum
)
# "a"ppend if output per element; "w"rite (once) for maximum.
# append will append a file with potentially already-existing data
# (i.e. from a previous run), which may be confusing; but with how
# memory_profiler treats streams, there's no simple way to prevent
# appending data for per-element without re-implementing parts of
# memory_profiler (maybe someday?) @lynn
fmode = "w" if get_maximum else "a"
with smart_open(self.output_file, fmode=fmode) as f:
self._stream = f
self._run_pipeline()
if get_maximum:
memory_profiler.show_results(profiler, stream=self._stream)
def inner(*args, **kws):
ret = profiled_func(*args, **kws)
show_results(mprofiler)
return ret
def _profile(*args, **kwargs):
prof = LineProfiler() # backend=backend)
val = prof(f)(*args, **kwargs)
show_results(prof) # , stream=stream, precision=precision)
return val
def prof(self):
if not self.m is None:
return memory_profiler.show_results(self.m)
else:
return None
def report():
print("Show results!")
show_results(profile)
def mprun(self, parameter_s='', cell=None):
""" Execute a statement under the line-by-line memory profiler from the
memory_profiler module.
Usage, in line mode:
%mprun -f func1 -f func2 <statement>
Usage, in cell mode:
%%mprun -f func1 -f func2 [statement]
code...
code...
In cell mode, the additional code lines are appended to the (possibly
empty) statement in the first line. Cell mode allows you to easily
profile multiline blocks without having to put them in a separate
function.
The given statement (which doesn't require quote marks) is run via the
LineProfiler. Profiling is enabled for the functions specified by the -f
options. The statistics will be shown side-by-side with the code through
the pager once the statement has completed.
Options:
-f <function>: LineProfiler only profiles functions and methods it is told
to profile. This option tells the profiler about these functions. Multiple
-f options may be used. The argument may be any expression that gives
a Python function or method object. However, one must be careful to avoid
spaces that may confuse the option parser. Additionally, functions defined
in the interpreter at the In[] prompt or via %run currently cannot be
displayed. Write these functions out to a separate file and import them.
One or more -f options are required to get any useful results.
-T <filename>: dump the text-formatted statistics with the code
side-by-side out to a text file.
-r: return the LineProfiler object after it has completed profiling.
-c: If present, add the memory usage of any children process to the report.
"""
from io import StringIO
from memory_profiler import show_results, LineProfiler
# Local imports to avoid hard dependency.
from distutils.version import LooseVersion
import IPython
ipython_version = LooseVersion(IPython.__version__)
if ipython_version < '0.11':
from IPython.genutils import page
from IPython.ipstruct import Struct
from IPython.ipapi import UsageError
else:
from IPython.core.page import page
from IPython.utils.ipstruct import Struct
from IPython.core.error import UsageError
# Escape quote markers.
opts_def = Struct(T=[''], f=[])
parameter_s = parameter_s.replace('"', r'\"').replace("'", r"\'")
opts, arg_str = self.parse_options(parameter_s, 'rf:T:c',
list_all=True)
opts.merge(opts_def)
global_ns = self.shell.user_global_ns
local_ns = self.shell.user_ns
if cell is not None:
arg_str += '\n' + cell
# Get the requested functions.
funcs = []
for name in opts.f:
try:
funcs.append(eval(name, global_ns, local_ns))
except Exception as e:
raise UsageError('Could not find function %r.\n%s: %s' % (name,
e.__class__.__name__,
e))
include_children = 'c' in opts
profile = LineProfiler(include_children=include_children)
for func in funcs:
profile(func)
# Add the profiler to the builtins for @profile.
if 'profile' in builtins.__dict__:
had_profile = True
old_profile = builtins.__dict__['profile']
else:
had_profile = False
old_profile = None
builtins.__dict__['profile'] = profile
try:
profile.runctx(arg_str, global_ns, local_ns)
message = ''
except SystemExit:
message = "*** SystemExit exception caught in code being profiled."
except KeyboardInterrupt:
message = ("*** KeyboardInterrupt exception caught in code being "
"profiled.")
finally:
if had_profile:
builtins.__dict__['profile'] = old_profile
# Trap text output.
stdout_trap = StringIO()
show_results(profile, stdout_trap)
output = stdout_trap.getvalue()
output = output.rstrip()
if ipython_version < '0.11':
page(output, screen_lines=self.shell.rc.screen_length)
else:
page(output)
print(message, )
text_file = opts.T[0]
if text_file:
with open(text_file, 'w') as pfile:
pfile.write(output)
print('\n*** Profile printout saved to text file %s. %s' % (
text_file,
message))
return_value = None
if 'r' in opts:
return_value = profile
return return_value
def wrapper(*args, **kwargs):
prof = LineProfiler()
val = prof(func)(*args, **kwargs)
show_results(prof)
return val
def mprun(self, parameter_s='', cell=None):
""" Execute a statement under the line-by-line memory profiler from the
memory_profiler module.
Usage, in line mode:
%mprun -f func1 -f func2 <statement>
Usage, in cell mode:
%%mprun -f func1 -f func2 [statement]
code...
code...
In cell mode, the additional code lines are appended to the (possibly
empty) statement in the first line. Cell mode allows you to easily
profile multiline blocks without having to put them in a separate
function.
The given statement (which doesn't require quote marks) is run via the
LineProfiler. Profiling is enabled for the functions specified by the -f
options. The statistics will be shown side-by-side with the code through
the pager once the statement has completed.
Options:
-f <function>: LineProfiler only profiles functions and methods it is told
to profile. This option tells the profiler about these functions. Multiple
-f options may be used. The argument may be any expression that gives
a Python function or method object. However, one must be careful to avoid
spaces that may confuse the option parser. Additionally, functions defined
in the interpreter at the In[] prompt or via %run currently cannot be
displayed. Write these functions out to a separate file and import them.
One or more -f options are required to get any useful results.
-T <filename>: dump the text-formatted statistics with the code
side-by-side out to a text file.
-r: return the LineProfiler object after it has completed profiling.
-c: If present, add the memory usage of any children process to the report.
"""
from io import StringIO
from memory_profiler import show_results, LineProfiler
# Local imports to avoid hard dependency.
from distutils.version import LooseVersion
import IPython
ipython_version = LooseVersion(IPython.__version__)
if ipython_version < '0.11':
from IPython.genutils import page
from IPython.ipstruct import Struct
from IPython.ipapi import UsageError
else:
from IPython.core.page import page
from IPython.utils.ipstruct import Struct
from IPython.core.error import UsageError
# Escape quote markers.
opts_def = Struct(T=[''], f=[])
parameter_s = parameter_s.replace('"', r'\"').replace("'", r"\'")
opts, arg_str = self.parse_options(parameter_s, 'rf:T:c', list_all=True)
opts.merge(opts_def)
global_ns = self.shell.user_global_ns
local_ns = self.shell.user_ns
if cell is not None:
arg_str += '\n' + cell
# Get the requested functions.
funcs = []
for name in opts.f:
try:
funcs.append(eval(name, global_ns, local_ns))
except Exception as e:
raise UsageError('Could not find function %r.\n%s: %s' % (name,
e.__class__.__name__, e))
include_children = 'c' in opts
profile = LineProfiler(include_children=include_children)
for func in funcs:
profile(func)
# Add the profiler to the builtins for @profile.
if 'profile' in builtins.__dict__:
had_profile = True
old_profile = builtins.__dict__['profile']
else:
had_profile = False
old_profile = None
builtins.__dict__['profile'] = profile
try:
profile.runctx(arg_str, global_ns, local_ns)
message = ''
except SystemExit:
message = "*** SystemExit exception caught in code being profiled."
except KeyboardInterrupt:
message = ("*** KeyboardInterrupt exception caught in code being "
"profiled.")
finally:
if had_profile:
builtins.__dict__['profile'] = old_profile
# Trap text output.
stdout_trap = StringIO()
show_results(profile, stdout_trap)
output = stdout_trap.getvalue()
output = output.rstrip()
if ipython_version < '0.11':
page(output, screen_lines=self.shell.rc.screen_length)
else:
page(output)
print(message,)
text_file = opts.T[0]
if text_file:
with open(text_file, 'w') as pfile:
pfile.write(output)
print('\n*** Profile printout saved to text file %s. %s' % (text_file,
message))
return_value = None
if 'r' in opts:
return_value = profile
return return_value
if len(args.program) == 0:
print("A program to run must be provided. Use -h for help")
sys.exit(1)
target = args.program[0]
script_args = args.program[1:]
_backend = choose_backend(args.backend)
if args.timestamp:
prof = TimeStamper(_backend)
else:
prof = LineProfiler(max_mem=args.max_mem, backend=_backend)
try:
if args.program[0].endswith('.py'):
script_filename = _find_script(args.program[0])
exec_with_profiler(script_filename, prof, args.backend,
script_args)
else:
run_module_with_profiler(target, prof, args.backend, script_args)
finally:
if args.out_filename is not None:
out_file = open(args.out_filename, "a")
else:
out_file = sys.stdout
if args.timestamp:
prof.show_results(stream=out_file)
else:
show_results(prof, precision=args.precision, stream=out_file)
def record_postexecute(self, execution_time=None):
"""
Function used to record runtime information after the task we want to track is comleted, e.g.
the `execute_analysis(...)` function of a standard analysis.
The function may be overwritten in child classes to add recording of
additional runtime information.
When overwriting the function we should call super(...,self).runinfo_record_postexecute(execution_time)
in the custom version to ensure that the execution and end_time are properly
recorded.
:param execution_time: The total time it took to execute the analysis. May be None, in which
case the function will attempt to compute the execution time based on the start_time
(if available) and the the current time.
:param comm: Used for logging only. The MPI communicator to be used. Default value is None,
in which case MPI.COMM_WORLD is used.
"""
log_helper.debug(__name__,
'Recording post-execution runtime data',
root=self.mpi_root,
comm=self.mpi_comm)
# Finalize recording of post execution provenance
self['end_time'] = unicode(datetime.datetime.now())
if execution_time is not None:
self['execution_time'] = unicode(execution_time)
elif 'start_time' in self:
start_time = run_info_dict.string_to_time(self['start_time'])
stop_time = run_info_dict.string_to_time(self['end_time'])
self['execution_time'] = unicode(
stop_time - start_time
) # TODO: This only gives execution time in full seconds right now
else:
self['execution_time'] = None
# Attempt to record psutil data
try:
import psutil
process = psutil.Process()
self['memory_info_after'] = unicode(process.memory_info())
except ImportError:
log_helper.warning(
__name__,
'psutil not installed. Recording of part of runtime information not possible',
root=self.mpi_root,
comm=self.mpi_comm)
except:
warnings.warn(
"Recording of psutil-based runtime information failed: " +
str(sys.exc_info()))
# Record the time and use profiling data if possible
if self.__time_and_use_profiler is not None:
self.__time_and_use_profiler.disable()
self.__time_and_use_profiler.create_stats()
self['profile'] = unicode(self.__time_and_use_profiler.stats)
# Save the summary statistics for the profiling data
stats_io = StringIO.StringIO()
profiler_stats = pstats.Stats(
self.__time_and_use_profiler,
stream=stats_io).sort_stats('cumulative')
profiler_stats.print_stats()
self['profile_stats'] = stats_io.getvalue()
# Record the memory profiling data if possible
if self.__memory_profiler is not None and self.get_profile_memory():
log_helper.debug(__name__,
'Recording memory profiling data',
root=self.mpi_root,
comm=self.mpi_comm)
mem_stats_io = StringIO.StringIO()
memory_profiler.show_results(self.__memory_profiler,
stream=mem_stats_io)
self['profile_mem'] = unicode(self.__memory_profiler.code_map)
self['profile_mem_stats'] = mem_stats_io.getvalue()
def wrapper(*args, **kwargs):
prof = memory_profiler.LineProfiler()
val = prof(f)(*args, **kwargs)
memory_profiler.show_results(prof, stream=sys.stdout)
return val
@profile
def test_1():
# .. will be called twice ..
c = {}
for i in range(1000):
c[i] = 2
from memory_profiler import LineProfiler as MemeryProfiler, show_results
mp = MemeryProfiler(enable_global=True)
@mp
def test_inner_func():
c = dict((i, i*2) for i in xrange(10000))
def inner_func():
inner_c = [1] * 1024 * 1024 * 5
return inner_c
del c
inner_func()
import os
dirs = list(os.walk("."))
if __name__ == '__main__':
test_1()
test_1()
test_inner_func()
show_results(mp)
"""
Line-by-line profiling of RAM use of Python scripts in
:mod:`mandelbrot.implementations` using :mod:`memory_profiler`.
"""
import memory_profiler
import mandelbrot
if __name__ == "__main__":
args = mandelbrot.parsed_mandelbrot_args()
for impl in mandelbrot.all_implementations():
print(f"About to profile {impl.id_}")
profile = memory_profiler.LineProfiler()
profile.add_function(impl.callable)
cmd = (
f"{impl.fully_qualified_name}(grid_side_size={args.grid_side_size}, "
f"max_iter={args.max_iter})")
global_ns = globals()
local_ns = locals()
profile.runctx(cmd, global_ns, local_ns)
memory_profiler.show_results(profile)
sample = TiffFile(sample_path)
curr_dir = os.path.dirname(__file__)
fname = os.path.join(
curr_dir, os.path.join(sample.fpath, sample.fname))
arr = sample.asarray()
peaks = detect_peaks(arr,
shape_label=('t', 'z', 'x', 'y'),
verbose=True,
show_progress=False,
parallel=True,
**detection_params)
# del sample
# sample = None
gc.get_referrers(arr)
del arr
gc.collect()
# arr = None
# gc.collect()
data_path = "/media/thor/data/ldtracker/"
data_path = "/home/hadim/Insync/Documents/phd/data/ldtracker"
fpath = os.path.join(data_path, "..", "test.tif")
test(fpath)
show_results(prof)
help="path to a file where results will be written")
parser.add_option("--timestamp", dest="timestamp", default=False,
action="store_true",
help="""print timestamp instead of memory measurement for
decorated functions""")
if not sys.argv[1:]:
parser.print_help()
sys.exit(2)
(options, args) = parser.parse_args()
sys.argv[:] = args # Remove every memory_profiler arguments
if options.timestamp:
prof = TimeStamper()
else:
prof = LineProfiler(max_mem=options.max_mem)
script_filename = _find_script(args[0])
try:
exec_with_profiler(script_filename, prof)
finally:
if options.out_filename is not None:
out_file = open(options.out_filename, "a")
else:
out_file = sys.stdout
if options.timestamp:
prof.show_results(stream=out_file)
else:
show_results(prof, precision=options.precision, stream=out_file)
args = parser.parse_args()
if len(args.program) == 0:
print("A program to run must be provided. Use -h for help")
sys.exit(1)
target = args.program[0]
script_args = args.program[1:]
_backend = choose_backend(args.backend)
if args.timestamp:
prof = TimeStamper(_backend)
else:
prof = LineProfiler(max_mem=args.max_mem, backend=_backend)
try:
if args.program[0].endswith('.py'):
script_filename = _find_script(args.program[0])
exec_with_profiler(script_filename, prof, args.backend, script_args)
else:
run_module_with_profiler(target, prof, args.backend, script_args)
finally:
if args.out_filename is not None:
out_file = open(args.out_filename, "a")
else:
out_file = sys.stdout
if args.timestamp:
prof.show_results(stream=out_file)
else:
show_results(prof, precision=args.precision, stream=out_file)
def wrapper(*args, **kwargs):
prof = LineProfiler()
val = prof(func)(*args, **kwargs)
show_results(prof, stream=stream, precision=precision)
return val
def prof(self): if not self.m is None: return memory_profiler.show_results(self.m) else: return None
default='psutil',
help='backend using for getting memory info '
'(one of the {tracemalloc, psutil, posix})')
if not sys.argv[1:]:
parser.print_help()
sys.exit(2)
(options, args) = parser.parse_args()
sys.argv[:] = args # Remove every memory_profiler arguments
script_filename = _find_script(args[0])
_backend = choose_backend(options.backend)
if options.timestamp:
prof = TimeStamper(_backend)
else:
prof = LineProfiler(max_mem=options.max_mem, backend=_backend)
try:
exec_with_profiler(script_filename, prof, options.backend)
finally:
if options.out_filename is not None:
out_file = open(options.out_filename, "a")
else:
out_file = sys.stdout
if options.timestamp:
prof.show_results(stream=out_file)
else:
show_results(prof, precision=options.precision, stream=out_file)
def wrapper(*args, **kwargs):
prof = cls(backend="psutil")
try:
yield from prof(func)(*args, **kwargs)
finally:
mp.show_results(prof, stream=stream)
def wrapper(*args, **kwargs):
prof = LineProfiler(backend=backend)
val = prof(func)(*args, **kwargs)
show_results(prof, stream=stream, precision=precision)
return val
def _mem_profile(fn):
prof = mp.LineProfiler(backend="psutil")
prof(fn)()
mp.show_results(prof)