def enable_mem_profiler(fun): """ Enable memory profiler if specified in arguments. :param fun: function to wrap :return: LineProfiler instance """ global parsed_args mem = None if parsed_args.memory: mem = LineProfiler() mem.add_function(fun) mem.enable() return mem
def run_memory_profiler(self, func, kwargs, clean_result=False): self.helper_funcs = {} func_module = inspect.getmodule(func) funcs = inspect.getmembers(func_module, inspect.isfunction) profiler = LineProfiler() wrapper = profiler(func) for func_name, func_instance in funcs: if func_name == func.__name__: continue profiler.code_map.add(func_instance.__code__) wrapper(**kwargs) kwargs_size = self.get_kwargs_size(kwargs) func_usage, total_usage = self.extract_result_from_profiler(profiler) if clean_result: self.clean_result(profiler) show_results(profiler, precision=4) self.print_summary(kwargs_size=kwargs_size, func_usage=func_usage, total_usage=total_usage)
def wrapper(*args, **kwargs): prof = LineProfiler(backend=backend) start = time.time() val = prof(func)(*args, **kwargs) end = time.time() global all_d all_d = end - start show_results(prof, stream=stream, precision=precision) return val
def wrapper(*args, **kwargs): prof = LineProfiler() val = prof(func)(*args, **kwargs) logger.info( 'Please wait... Line graph will be ready in few seconds.') job = threading.Thread(target=determine_memory_info, args=(prof, ), kwargs={'precision': precision}) job.start() return val
def test_normal_incr(self): def normal_incr(): use_some_memory = [1] * (10**6) profiler = LineProfiler() wrapped = profiler(normal_incr) wrapped() show_results(profiler) results = list(list(profiler.code_map.values())[0].values())[-1]
def profiled_fn(*args, **kwargs): ts = time.time() prof = LineProfiler() val = prof(func)(*args, **kwargs) fpath = ''.join( [profile_data_dir, func.__name__, str(ts), ".mprofile"]) astream = io.open(fpath, 'w') show_results(prof, astream, precision=3) astream.flush() return val
def test_loop_decr(self): def loop_with_del(): for i in range(3): b = [2] * (2 * 10**7) a = 1 profiler = LineProfiler() wrapped = profiler(loop_with_del) wrapped() show_results(profiler) results = list(list(profiler.code_map.values())[0].values())[-1]
def test_loop_incr(self): def loop_incr(): a = [] for i in range(3): b = [2] * (2 * 10**7) a.append(b) profiler = LineProfiler() wrapped = profiler(loop_incr) wrapped() show_results(profiler) results = list(list(profiler.code_map.values())[0].values())[-1]
def test_normal_incr(self): def normal_incr(): use_some_memory = [1] * (10**6) profiler = LineProfiler() wrapped = profiler(normal_incr) wrapped() show_results(profiler) results = list(list(profiler.code_map.values())[0].values())[-1] self.assertGreater(results[0], 0) self.assertGreater(results[1], results[0]) self.assertEqual(results[2], 1)
def measure_memory(*args, **kwargs): gc.collect() result = {} profiler = LineProfiler(backend="psutil") func = profiler(_measure_memory_inner) func(*args, **kwargs) for (filename, lines) in profiler.code_map.items(): all_lines = linecache.getlines(filename) last_mem = None for (lineno, mem) in lines: if mem is None: result[all_lines[lineno - 1].strip()[2:]] = last_mem[1] last_mem = mem return result
def test_loop_count(self): def some_loop(): for i in range(12): # line -2 a = 1 # line -1 profiler = LineProfiler() wrapped = profiler(some_loop) wrapped() show_results(profiler) for_line = list(list(profiler.code_map.values())[0].values())[-2] looped_instruction = list( list(profiler.code_map.values())[0].values())[-1] self.assertEqual(for_line[2], 13) self.assertEqual(looped_instruction[2], 12)
def test_decr(self): def del_stuff(): b = [2] * (2 * 10**7) del b profiler = LineProfiler() wrapped = profiler(del_stuff) wrapped() show_results(profiler) b_line = list(list(profiler.code_map.values())[0].values())[-2] del_line = list(list(profiler.code_map.values())[0].values())[-1] self.assertGreater(0, del_line[0]) self.assertGreater(del_line[1], 0) self.assertAlmostEqual(-del_line[0], b_line[0], delta=1)
def test_loop_incr(self): def loop_incr(): a = [] b = [2] * (2 * 10**7) # line -4 for i in range(3): c = [2] * (2 * 10**7) # line -2 a.append(c) profiler = LineProfiler() wrapped = profiler(loop_incr) wrapped() show_results(profiler) b_line = list(list(profiler.code_map.values())[0].values())[-4] c_line = list(list(profiler.code_map.values())[0].values())[-2] self.assertAlmostEqual(b_line[2] * 3, c_line[2], delta=1) self.assertEqual(c_line[2], 3)
default='psutil', help='backend using for getting memory info ' '(one of the {tracemalloc, psutil, posix})') if not sys.argv[1:]: parser.print_help() sys.exit(2) (options, args) = parser.parse_args() sys.argv[:] = args # Remove every memory_profiler arguments script_filename = _find_script(args[0]) _backend = choose_backend(options.backend) if options.timestamp: prof = TimeStamper(_backend) else: prof = LineProfiler(max_mem=options.max_mem, backend=_backend) try: exec_with_profiler(script_filename, prof, options.backend) finally: if options.out_filename is not None: out_file = open(options.out_filename, "a") else: out_file = sys.stdout if options.timestamp: prof.show_results(stream=out_file) else: show_results(prof, precision=options.precision, stream=out_file)
nargs=REMAINDER, help= 'python script or module followed by command line arguements to run') args = parser.parse_args() if len(args.program) == 0: print("A program to run must be provided. Use -h for help") sys.exit(1) target = args.program[0] script_args = args.program[1:] _backend = choose_backend(args.backend) if args.timestamp: prof = TimeStamper(_backend) else: prof = LineProfiler(max_mem=args.max_mem, backend=_backend) try: if args.program[0].endswith('.py'): script_filename = _find_script(args.program[0]) exec_with_profiler(script_filename, prof, args.backend, script_args) else: run_module_with_profiler(target, prof, args.backend, script_args) finally: if args.out_filename is not None: out_file = open(args.out_filename, "a") else: out_file = sys.stdout if args.timestamp:
dest="timestamp", default=False, action="store_true", help="""print timestamp instead of memory measurement for decorated functions""") if not sys.argv[1:]: parser.print_help() sys.exit(2) (options, args) = parser.parse_args() sys.argv[:] = args # Remove every memory_profiler arguments if options.timestamp: prof = TimeStamper() else: prof = LineProfiler(max_mem=options.max_mem) script_filename = _find_script(args[0]) try: exec_with_profiler(script_filename, prof) finally: if options.out_filename is not None: out_file = open(options.out_filename, "a") else: out_file = sys.stdout if options.timestamp: prof.show_results(stream=out_file) else: show_results(prof, precision=options.precision, stream=out_file)
def mprun(self, parameter_s='', cell=None): """ Execute a statement under the line-by-line memory profiler from the memory_profiler module. Usage, in line mode: %mprun -f func1 -f func2 <statement> Usage, in cell mode: %%mprun -f func1 -f func2 [statement] code... code... In cell mode, the additional code lines are appended to the (possibly empty) statement in the first line. Cell mode allows you to easily profile multiline blocks without having to put them in a separate function. The given statement (which doesn't require quote marks) is run via the LineProfiler. Profiling is enabled for the functions specified by the -f options. The statistics will be shown side-by-side with the code through the pager once the statement has completed. Options: -f <function>: LineProfiler only profiles functions and methods it is told to profile. This option tells the profiler about these functions. Multiple -f options may be used. The argument may be any expression that gives a Python function or method object. However, one must be careful to avoid spaces that may confuse the option parser. Additionally, functions defined in the interpreter at the In[] prompt or via %run currently cannot be displayed. Write these functions out to a separate file and import them. One or more -f options are required to get any useful results. -T <filename>: dump the text-formatted statistics with the code side-by-side out to a text file. -r: return the LineProfiler object after it has completed profiling. -c: If present, add the memory usage of any children process to the report. """ from io import StringIO from memory_profiler import show_results, LineProfiler # Local imports to avoid hard dependency. from distutils.version import LooseVersion import IPython ipython_version = LooseVersion(IPython.__version__) if ipython_version < '0.11': from IPython.genutils import page from IPython.ipstruct import Struct from IPython.ipapi import UsageError else: from IPython.core.page import page from IPython.utils.ipstruct import Struct from IPython.core.error import UsageError # Escape quote markers. opts_def = Struct(T=[''], f=[]) parameter_s = parameter_s.replace('"', r'\"').replace("'", r"\'") opts, arg_str = self.parse_options(parameter_s, 'rf:T:c', list_all=True) opts.merge(opts_def) global_ns = self.shell.user_global_ns local_ns = self.shell.user_ns if cell is not None: arg_str += '\n' + cell # Get the requested functions. funcs = [] for name in opts.f: try: funcs.append(eval(name, global_ns, local_ns)) except Exception as e: raise UsageError('Could not find function %r.\n%s: %s' % (name, e.__class__.__name__, e)) include_children = 'c' in opts profile = LineProfiler(include_children=include_children) for func in funcs: profile(func) # Add the profiler to the builtins for @profile. if 'profile' in builtins.__dict__: had_profile = True old_profile = builtins.__dict__['profile'] else: had_profile = False old_profile = None builtins.__dict__['profile'] = profile try: profile.runctx(arg_str, global_ns, local_ns) message = '' except SystemExit: message = "*** SystemExit exception caught in code being profiled." except KeyboardInterrupt: message = ("*** KeyboardInterrupt exception caught in code being " "profiled.") finally: if had_profile: builtins.__dict__['profile'] = old_profile # Trap text output. stdout_trap = StringIO() show_results(profile, stdout_trap) output = stdout_trap.getvalue() output = output.rstrip() if ipython_version < '0.11': page(output, screen_lines=self.shell.rc.screen_length) else: page(output) print(message,) text_file = opts.T[0] if text_file: with open(text_file, 'w') as pfile: pfile.write(output) print('\n*** Profile printout saved to text file %s. %s' % (text_file, message)) return_value = None if 'r' in opts: return_value = profile return return_value
def mprun(self, parameter_s='', cell=None): """ Execute a statement under the line-by-line memory profiler from the memory_profiler module. Usage, in line mode: %mprun -f func1 -f func2 <statement> Usage, in cell mode: %%mprun -f func1 -f func2 [statement] code... code... In cell mode, the additional code lines are appended to the (possibly empty) statement in the first line. Cell mode allows you to easily profile multiline blocks without having to put them in a separate function. The given statement (which doesn't require quote marks) is run via the LineProfiler. Profiling is enabled for the functions specified by the -f options. The statistics will be shown side-by-side with the code through the pager once the statement has completed. Options: -f <function>: LineProfiler only profiles functions and methods it is told to profile. This option tells the profiler about these functions. Multiple -f options may be used. The argument may be any expression that gives a Python function or method object. However, one must be careful to avoid spaces that may confuse the option parser. Additionally, functions defined in the interpreter at the In[] prompt or via %run currently cannot be displayed. Write these functions out to a separate file and import them. One or more -f options are required to get any useful results. -T <filename>: dump the text-formatted statistics with the code side-by-side out to a text file. -r: return the LineProfiler object after it has completed profiling. -c: If present, add the memory usage of any children process to the report. """ from io import StringIO from memory_profiler import show_results, LineProfiler # Local imports to avoid hard dependency. from distutils.version import LooseVersion import IPython ipython_version = LooseVersion(IPython.__version__) if ipython_version < '0.11': from IPython.genutils import page from IPython.ipstruct import Struct from IPython.ipapi import UsageError else: from IPython.core.page import page from IPython.utils.ipstruct import Struct from IPython.core.error import UsageError # Escape quote markers. opts_def = Struct(T=[''], f=[]) parameter_s = parameter_s.replace('"', r'\"').replace("'", r"\'") opts, arg_str = self.parse_options(parameter_s, 'rf:T:c', list_all=True) opts.merge(opts_def) global_ns = self.shell.user_global_ns local_ns = self.shell.user_ns if cell is not None: arg_str += '\n' + cell # Get the requested functions. funcs = [] for name in opts.f: try: funcs.append(eval(name, global_ns, local_ns)) except Exception as e: raise UsageError('Could not find function %r.\n%s: %s' % (name, e.__class__.__name__, e)) include_children = 'c' in opts profile = LineProfiler(include_children=include_children) for func in funcs: profile(func) # Add the profiler to the builtins for @profile. if 'profile' in builtins.__dict__: had_profile = True old_profile = builtins.__dict__['profile'] else: had_profile = False old_profile = None builtins.__dict__['profile'] = profile try: profile.runctx(arg_str, global_ns, local_ns) message = '' except SystemExit: message = "*** SystemExit exception caught in code being profiled." except KeyboardInterrupt: message = ("*** KeyboardInterrupt exception caught in code being " "profiled.") finally: if had_profile: builtins.__dict__['profile'] = old_profile # Trap text output. stdout_trap = StringIO() show_results(profile, stdout_trap) output = stdout_trap.getvalue() output = output.rstrip() if ipython_version < '0.11': page(output, screen_lines=self.shell.rc.screen_length) else: page(output) print(message, ) text_file = opts.T[0] if text_file: with open(text_file, 'w') as pfile: pfile.write(output) print('\n*** Profile printout saved to text file %s. %s' % ( text_file, message)) return_value = None if 'r' in opts: return_value = profile return return_value
def wrapper(*args, **kwargs): prof = LineProfiler(backend=backend) val = prof(func)(*args, **kwargs) show_results(prof, stream=stream, precision=precision) return val
def _profile(*args, **kwargs): prof = LineProfiler() # backend=backend) val = prof(f)(*args, **kwargs) show_results(prof) # , stream=stream, precision=precision) return val