def test_stracker_create_summary(self): """Test that a summary is created correctly. This can only be done heuristically, e.g that most recent objects are included. Also check that summaries managed by the tracker are excluded if ignore_self is enabled. """ # at the beginning, there should not be an indicator object listed tmp_tracker = tracker.SummaryTracker() sn = tmp_tracker.create_summary() self.assertEqual(self._contains_indicator(sn), None) # now an indicator object should be listed o = self._get_indicator() sn = tmp_tracker.create_summary() self.assertEqual(self._contains_indicator(sn), 1) # with ignore_self enabled a second summary should not list the first # summary sn = tmp_tracker.create_summary() sn2 = tmp_tracker.create_summary() tmp = summary._sweep(summary.get_diff(sn, sn2)) self.assertEqual(len(tmp), 0) # but with ignore_self turned off, there should be some difference tmp_tracker = tracker.SummaryTracker(ignore_self=False) sn = tmp_tracker.create_summary() tmp_tracker.new_obj = self._get_indicator() sn2 = tmp_tracker.create_summary() tmp = summary._sweep(summary.get_diff(sn, sn2)) self.failIfEqual(len(tmp), 0)
def test_summary_diff(self): """Test summary diff. """ # base cases res = summary.get_diff([], []) self.assertEqual(res, []) res = summary.get_diff([], [[str(str), 3, 3*_getsizeof('a')]]) self.assertEqual(res, [[str(str), 3, 3*_getsizeof('a')]]) res = summary.get_diff([[str(str), 3, 3*_getsizeof('a')]], []) self.assertEqual(res, [[str(str), -3, -3*_getsizeof('a')]]) #interesting case left = [[str(str), 3, 3*_getsizeof('a')],\ [str(int), 2, 2*_getsizeof(1)],\ [str(list), 1, _getsizeof([])],\ [str(dict), 1, _getsizeof({})]] right = [[str(str), 2, 2*_getsizeof('a')],\ [str(int), 3, 3*_getsizeof(1)],\ [str(list), 1, _getsizeof([1,2,3])],\ [str(dict), 1, _getsizeof({})], [str(tuple), 1, _getsizeof((1,2))]] expected = [[str(str), -1, -1*_getsizeof('a')],\ [str(int), 1, +1*_getsizeof(1)],\ [str(list), 0, _getsizeof([1,2,3]) - _getsizeof([])],\ [str(dict), 0, 0], [str(tuple), 1, _getsizeof((1,2))]] res = summary.get_diff(left, right) for row_e in res: self.assertTrue(row_e in expected)
def snapshot_diff(cur_items, snapshot_file): """ Attempts to compare like PIDs. If like PIDS can't be found it will just compare the first PID listed to the first PID in the file. Any unmatched or non-first PIDs will be ignored because we don't know what to compare them to. """ try: prev_items = list(frontend_utils.get_pages(snapshot_file)) except pickle.UnpicklingError as e: frontend_utils.echo_error( f"Error unpickling the data from {snapshot_file}: {e}") return None differences = [] for cur_item in cur_items: for prev_item in prev_items: if cur_item.pid == prev_item.pid: diff = summary.get_diff(cur_item.data, prev_item.data) differences.append( RetrievedObjects( pid=cur_item.pid, title=f"Snapshot Differences for {cur_item.pid}", data=diff, )) if not differences: diff = summary.get_diff(cur_items[0].data, prev_items[0].data) differences.append( RetrievedObjects(pid=0, title=f"Snapshot Differences", data=diff)) return differences
def func5(): """ Модуль pympler muppy.get_objects() - метод считывает все существующтие обьекты в Python вызывая этот метод 2 раза в разный промежуток времени, мы можем выявить какие обьекты были созданы с первого среза. summary.get_diff() - выявляет какие обьекты как разницу между двумя срезами. summary.print_() - красиво выводит на экран обьекты и память ими занимаемую. """ from pympler import asizeof from pympler import muppy from pympler import summary print('\nИспользование muppy :') alll_obj_1 = muppy.get_objects() data = list(range(1000)) alll_obj_2 = muppy.get_objects() sum1 = summary.summarize(alll_obj_1) sum2 = summary.summarize(alll_obj_2) summary.print_(summary.get_diff(sum1, sum2))
def file_test(rows=500000, cols=50): "File test" print("Creating file with {} rows and {} columns".format(rows, cols)) file = create_file(rows, cols) print("Size of the file: {:.2f} MiB".format(getsize(file) / (1024 * 1024))) print("Reading file") sum1 = summarize(get_objects()) las = read(file) sum2 = summarize(get_objects()) diff = get_diff(sum1, sum2) print_(diff) for curve in las.curves: print("Name: {}, Min: {:.2f}, Mean: {:.2f}, Max: {:.2f}" .format(curve.mnemonic, nanmin(curve.data), nanmean(curve.data), nanmax(curve.data))) del las las = read(file) del las las = read(file) del las las = read(file) del las print("Happy end")
def process_response(self, request, response): req = request.META['PATH_INFO'] if req.find('static') == -1 and req.find('media') == -1: print req self.end_objects = muppy.get_objects() sum_start = summary.summarize(self.start_objects) sum_end = summary.summarize(self.end_objects) diff = summary.get_diff(sum_start, sum_end) summary.print_(diff) #print '~~~~~~~~~' #cb = refbrowser.ConsoleBrowser(response, maxdepth=2, \ #str_func=output_function) #cb.print_tree() print '~~~~~~~~~' a = asizeof(response) print 'Total size of response object in kB: %s' % \ str(a / 1024.0) print '~~~~~~~~~' a = asizeof(self.end_objects) print 'Total size of end_objects in MB: %s' % \ str(a / 1048576.0) b = asizeof(self.start_objects) print 'Total size of start_objects in MB: %s' % \ str(b / 1048576.0) print '~~~~~~~~~' return response
def _get_usage(function, *args): """Get the usage of a function call. This function is to be used only internally. The 'real' get_usage function is a wrapper around _get_usage, but the workload is done here. """ res = [] # init before calling (s_before, s_after) = _get_summaries(function, *args) # ignore all objects used for the measurement ignore = [] if s_before != s_after: ignore.append(s_before) for row in s_before: # ignore refs from summary and frame (loop) if len(gc.get_referrers(row)) == 2: ignore.append(row) for item in row: # ignore refs from summary and frame (loop) if len(gc.get_referrers(item)) == 2: ignore.append(item) for o in ignore: s_after = summary._subtract(s_after, o) res = summary.get_diff(s_before, s_after) return summary._sweep(res)
def on_epoch_end(self, epoch, log={}): x = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss web_browser_debug = True print(x) if x > 40000: if web_browser_debug: if epoch == 0: start_in_background() tr = tracker.SummaryTracker() tr.print_diff() else: global memlist all_objects = muppy.get_objects(include_frames=True) # print(len(all_objects)) sum1 = summary.summarize(all_objects) memlist.append(sum1) summary.print_(sum1) if len(memlist) > 1: # compare with last - prints the difference per epoch diff = summary.get_diff(memlist[-2], memlist[-1]) summary.print_(diff) my_types = muppy.filter(all_objects, Type=types.ClassType) for t in my_types: print(t)
def handle_signal_abort(self, signum, frame): Log.warn("Someone want to kill me! But I'll not die now! Hahahaha!") s = summary.summarize(muppy.get_objects()) Log.debug("Current memory usage:") summary.print_(s) diff = summary.get_diff(self.mem_sum, s) self.mem_sum = s Log.debug("New memory usage:") summary.print_(diff)
def profile_expose_method(profiled_method_wrapper, accept, args, func, kw, exclude_from_memory_profiling): """ Targeted to profile a specific method that wraps HTTP request processing endpoints into database context. :param profiled_method_wrapper: method wrapped around profiled call to be passed in to memory profiler :param accept: param specific to profiled call :param args: args of a function that is being wrapped by a profiled method :param func: function that is being wrapped by a profiled method :param kw: kwargs of a function that is being wrapped by a profiled method :return: output of a profiled method without modification """ if not exclude_from_memory_profiling and get_memory_profile_logging_on() and \ check_memory_profile_package_wide_disable(func): controller_class = args[0].__class__.__name__ if args and len(args) > 0 else '' end_point_name_parts = [s for s in [func.__module__, controller_class, func.__name__] if s != ''] end_point_name = ".".join(end_point_name_parts) is_pympler_on = _is_pympler_profiling_value_on(end_point_name) profile_output = {'output': {}} if is_pympler_on: all_objects = muppy.get_objects() all_objects_summary_before = summary.summarize(all_objects) memory_profile = memory_usage((_profile_me, (profile_output, profiled_method_wrapper, func, accept, args, kw), {}), interval=0.1) output = profile_output['output'] if is_pympler_on: all_objects_summary_after = summary.summarize(all_objects) diff = summary.get_diff(all_objects_summary_before, all_objects_summary_after) diff_less = summary.format_(diff) diff_out = '' for s in diff_less: diff_out += s+'\n' thread_log.info("================ PYMPLER OUTPUT <{}> ==============\n{}".format(end_point_name, diff_out)) try: message = json.dumps({'log_type': 'memory_profile', 'proc_id': os.getpid(), 'name': func.__name__, 'module': func.__module__, 'mem_profile': memory_profile, 'min': min(memory_profile), 'max': max(memory_profile), 'diff': max(memory_profile) - min(memory_profile), 'leaked': memory_profile[-1] - memory_profile[0], 'args': [arg for arg in args[1:]], # exclude self 'kwargs': kw}) memory_log.info(message, extra={'controller_module': func.__module__, 'controller_class': controller_class, 'endpoint': func.__name__}) except Exception as e: thread_log.exception('Logger failed: {}'.format(e)) else: output = profiled_method_wrapper(accept, args, func, kw) return output
def test_print_diff(self): """Test summary can be printed.""" try: self._stdout = sys.stdout sys.stdout = self.DevNull() sum1 = summary.summarize(muppy.get_objects()) sum2 = summary.summarize(muppy.get_objects()) sumdiff = summary.get_diff(sum1, sum2) summary.print_(sumdiff) finally: sys.stdout = self._stdout
def log_summary(self): summary_str = '\n\n=====Total Memory Stats=====\n' current_summary = summary.summarize(muppy.get_objects()) summary_str += self.str_print_(current_summary, limit=25) if self.previous_summary is None: self.previous_summary = current_summary else: summary_str += '\n\n===== Difference since last update =====\n' diff = summary.get_diff(self.previous_summary, current_summary) summary_str += self.str_print_(diff, limit=25) self.previous_summary = current_summary self.logger.info(summary_str)
def test_print_diff(self): """Test summary can be printed.""" try: self._stdout = sys.stdout stream = StringIO() sys.stdout = stream sum1 = summary.summarize(muppy.get_objects()) sum2 = summary.summarize(muppy.get_objects()) sumdiff = summary.get_diff(sum1, sum2) summary.print_(sumdiff) self.assertIn('str', stream.getvalue()) self.assertNotIn("<class 'str", stream.getvalue()) finally: sys.stdout = self._stdout
def diff(self, summary1=None, summary2=None): """Compute diff between to summaries. If no summary is provided, the diff from the last to the current summary is used. If summary1 is provided the diff from summary1 to the current summary is used. If summary1 and summary2 are provided, the diff between these two is used. """ res = None if summary2 is None: self.s1 = self.create_summary() if summary1 is None: res = summary.get_diff(self.s0, self.s1) else: res = summary.get_diff(summary1, self.s1) self.s0 = self.s1 else: if summary1 is not None: res = summary.get_diff(summary1, summary2) else: raise ValueError("You cannot provide summary2 without summary1.") return summary._sweep(res)
def memusage_before_n_after(fun, *args, **kwargs): from pympler import muppy from pympler import summary from datetime import datetime before = summary.summarize(muppy.get_objects()) before_time = datetime.now() fun_ret = fun(*args, **kwargs) after_time = datetime.now() after = summary.summarize(muppy.get_objects()) diff = summary.get_diff(before, after) print "execution time: ", after_time - before_time summary.print_(diff) return fun_ret, diff
def diff(self, summary1=None, summary2=None): """Compute diff between to summaries. If no summary is provided, the diff from the last to the current summary is used. If summary1 is provided the diff from summary1 to the current summary is used. If summary1 and summary2 are provided, the diff between these two is used. """ res = None if summary2 is None: self.s1 = self.create_summary() if summary1 is None: res = summary.get_diff(self.s0, self.s1) else: res = summary.get_diff(summary1, self.s1) self.s0 = self.s1 else: if summary1 is not None: res = summary.get_diff(summary1, summary2) else: raise ValueError( "You cannot provide summary2 without summary1.") return summary._sweep(res)
def collection_post(self): print("REQUEST {}".format(self.request.json)) sums = [] for key in 'sum1', 'sum2': sums.append(self.request.json[key]) with transaction.manager: query = self.request.dbsession.query(ObjectSummary) query = query.filter(ObjectSummary.id.in_(sums)) query = query.order_by(ObjectSummary.created) dbsums = query.all() sums = [m.content for m in dbsums] diff = summary.get_diff(*sums) output = io.StringIO() with contextlib.redirect_stdout(output): summary.print_(diff) # import pdb ; pdb.set_trace() return dict(result='testing', output=output.getvalue())
def test_summary_diff(self): """Test summary diff. """ left = [[str(str), 3, 3*getsizeof('a')],\ [str(int), 2, 2*getsizeof(1)],\ [str(list), 1, getsizeof([])],\ [str(dict), 1, getsizeof({})]] right = [[str(str), 2, 2*getsizeof('a')],\ [str(int), 3, 3*getsizeof(1)],\ [str(list), 1, getsizeof([1,2,3])],\ [str(dict), 1, getsizeof({})], [str(tuple), 1, getsizeof((1,2))]] expected = [[str(str), -1, -1*getsizeof('a')],\ [str(int), 1, +1*getsizeof(1)],\ [str(list), 0, getsizeof([1,2,3]) - getsizeof([])],\ [str(dict), 0, 0], [str(tuple), 1, getsizeof((1,2))]] res = summary.get_diff(left, right) for row_e in res: self.assertTrue(row_e in expected)
def print_muppy_sumary(): # http://pythonhosted.org/Pympler/index.html try: from pympler import muppy, summary except ImportError: print("WARNING: pympler not installed") return # from pympler.classtracker import ClassTracker # from pympler.classtracker_stats import HtmlStats global all_objects, obj_summary, class_tracker if all_objects is None: all_objects = muppy.get_objects() obj_summary = summary.summarize(all_objects) summary.print_(obj_summary) # class_tracker = ClassTracker() # class_tracker.track_class(FICSPlayer, trace=1) # class_tracker.track_class(ICGameModel, resolution_level=2, trace=1) else: obj_summary2 = summary.summarize(muppy.get_objects()) diff = summary.get_diff(obj_summary, obj_summary2) summary.print_(diff, limit=200)
def _get_usage(function, *args): """Test if more memory is used after the function has been called. The function will be invoked twice and only the second measurement will be considered. Thus, memory used in initialisation (e.g. loading modules) will not be included in the result. The goal is to identify memory leaks caused by functions which use more and more memory. Any arguments next to the function will be passed on to the function on invocation. Note that this function is currently experimental, because it is not tested thoroughly and performs poorly. """ # The usage of a function is calculated by creating one summary of all # objects before the function is invoked and afterwards. These summaries # are compared and the diff is returned. # This function works in a 2-steps process. Before the actual function is # invoked an empty dummy function is measurement to identify the overhead # involved in the measuring process. This overhead then is subtracted from # the measurement performed on the passed function. The result reflects the # actual usage of a function call. # Also, a measurement is performed twice, allowing the adjustment to # initializing things, e.g. modules res = None def _get_summaries(function, *args): """Get a 2-tuple containing one summary from before, and one summary from after the function has been invoked. """ s_before = summary.summarize(get_objects()) function(*args) s_after = summary.summarize(get_objects()) return (s_before, s_after) def _get_usage(function, *args): """Get the usage of a function call. This function is to be used only internally. The 'real' get_usage function is a wrapper around _get_usage, but the workload is done here. """ res = [] # init before calling (s_before, s_after) = _get_summaries(function, *args) # ignore all objects used for the measurement ignore = [] if s_before != s_after: ignore.append(s_before) for row in s_before: # ignore refs from summary and frame (loop) if len(gc.get_referrers(row)) == 2: ignore.append(row) for item in row: # ignore refs from summary and frame (loop) if len(gc.get_referrers(item)) == 2: ignore.append(item) for o in ignore: s_after = summary._subtract(s_after, o) res = summary.get_diff(s_before, s_after) return summary._sweep(res) # calibrate; twice for initialization def noop(): pass offset = _get_usage(noop) offset = _get_usage(noop) # perform operation twice to handle objects possibly used in # initialisation tmp = _get_usage(function, *args) tmp = _get_usage(function, *args) tmp = summary.get_diff(offset, tmp) tmp = summary._sweep(tmp) if len(tmp) != 0: res = tmp return res
def before_after_each_function(request): global _global_collect_info try: import psutil # Don't fail if not there except ImportError: yield return current_pids = set(proc.pid for proc in psutil.process_iter()) before_curr_proc_memory_info = psutil.Process().memory_info() if _global_collect_info and DEBUG_MEMORY_INFO: try: from pympler import summary, muppy sum1 = summary.summarize(muppy.get_objects()) except: log.exception() sys.stdout.write(""" =============================================================================== Memory before: %s %s =============================================================================== """ % ( request.function, format_memory_info(psutil.virtual_memory(), before_curr_proc_memory_info), )) yield processes_info = [] for proc in psutil.process_iter(): if proc.pid not in current_pids: try: try: cmdline = proc.cmdline() except: cmdline = "<unable to get>" processes_info.append("New Process: %s(%s - %s) - %s" % ( proc.name(), proc.pid, cmdline, format_process_memory_info(proc.memory_info()), )) except (psutil.NoSuchProcess, psutil.AccessDenied): pass # The process could've died in the meanwhile after_curr_proc_memory_info = psutil.Process().memory_info() if DEBUG_MEMORY_INFO: try: if (after_curr_proc_memory_info.rss - before_curr_proc_memory_info.rss > 10 * 1000 * 1000): # 10 MB leak if _global_collect_info: sum2 = summary.summarize(muppy.get_objects()) diff = summary.get_diff(sum1, sum2) sys.stdout.write( "===============================================================================\n" ) sys.stdout.write("Leak info:\n") sys.stdout.write( "===============================================================================\n" ) summary.print_(diff) sys.stdout.write( "===============================================================================\n" ) _global_collect_info = True # We'll only really collect the info on the next test (i.e.: if at one test # we used too much memory, the next one will start collecting) else: _global_collect_info = False except: log.exception() sys.stdout.write(""" =============================================================================== Memory after: %s %s%s =============================================================================== """ % ( request.function, format_memory_info(psutil.virtual_memory(), after_curr_proc_memory_info), "" if not processes_info else "\nLeaked processes:\n" + "\n".join(processes_info), ))
def process_response(self, request, response): path = request.META['PATH_INFO'] if self.is_ignored(path): return response self.end_objects = muppy.get_objects() sum_start = summary.summarize(self.start_objects) sum_end = summary.summarize(self.end_objects) diff = summary.get_diff(sum_start, sum_end) if SHOW_ON_DJANGO_DEBUG_TOOLBAR_LOGGIN_PANEL: logger.info( '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~' '~~~~~~~~~~~~~~' ) logger.info( "Top %d memory deltas after processing URL: %s", SHOW_TOP_X_MEMORY_DELTAS, path ) logger.info("%-60s %10s %10s", "type", "# objects", "total size") print '\n\n' print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'\ '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~' print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'\ '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~' print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'\ '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~' print "Top %d memory deltas after processing URL: %s" % ( SHOW_TOP_X_MEMORY_DELTAS, path) print "%60s %10s %10s" % ("type", "# objects", "total size") for row in sorted(diff, key=lambda i: i[2], reverse=True )[:SHOW_TOP_X_MEMORY_DELTAS]: if SHOW_ON_DJANGO_DEBUG_TOOLBAR_LOGGIN_PANEL: logger.info( "type: %60s , # objects: %10d, total size: %s", *(row[0], row[1], filesizeformat(row[2])) ) print "%60s %10d %s" % (row[0], row[1], filesizeformat(row[2])) start_size = asizeof(self.start_objects) end_size = asizeof(self.end_objects) if SHOW_ON_DJANGO_DEBUG_TOOLBAR_LOGGIN_PANEL: logger.info( '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~' '~~~~~~~~~~~~~~' ) logger.info( "Processed %s: memory delta %0.1f kB (%0.1f -> %0.1fMB), " "response size: %0.1f kB", path, (end_size - start_size) / 1024.0, start_size / 1048576.0, end_size / 1048576.0, len(response.content) / 1024.0, ) print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'\ '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~' print ( "Processed %s: memory delta %0.1f kB (%0.1f -> %0.1fMB), " "response size: %0.1f kB" % ( path, (end_size - start_size) / 1024.0, start_size / 1048576.0, end_size / 1048576.0, len(response.content) / 1024.0, ) ) if SHOW_ON_DJANGO_DEBUG_TOOLBAR_LOGGIN_PANEL: logger.info( '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~' '~~~~~~~~~~~~~~' ) print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'\ '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~' print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'\ '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~' print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'\ '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~' print '\n\n' return response
def process_response(self, request, response): req = request.META['PATH_INFO'] if not self.is_ignored(req): if SHOW_ON_DJANGO_DEBUG_TOOLBAR_LOGGIN_PANEL: logger.info( '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~' '~~~~~~~~~~~~~~' ) print '\n\n' print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'\ '~~~~~~~~~~~~~~~~~~~~~~~~~~' print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'\ '~~~~~~~~~~~~~~~~~~~~~~~~~~' print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'\ '~~~~~~~~~~~~~~~~~~~~~~~~~~' if SHOW_ON_DJANGO_DEBUG_TOOLBAR_LOGGIN_PANEL: logger.info(u'REQUESTED URL: {}'.format(req)) print u'REQUESTED URL: {}'.format(req) self.end_objects = muppy.get_objects() if SHOW['request_summary']: sum_start = summary.summarize(self.start_objects) if SHOW_ON_DJANGO_DEBUG_TOOLBAR_LOGGIN_PANEL: logger.info( '~~~~~~~~~ SUMMARIZE REQUEST OBJECTS ~~~~~~~~~') for row in sorted( sum_start, key=lambda i: i[2], reverse=True)[:15]: logger.info( "type: %60s , # objects: %10d, total size: %s", *(row[0], row[1], filesizeformat(row[2])) ) print '~~~~~~~~~ SUMMARIZE REQUEST OBJECTS ~~~~~~~~~' summary.print_(sum_start) if SHOW['response_summary']: sum_end = summary.summarize(self.end_objects) if SHOW_ON_DJANGO_DEBUG_TOOLBAR_LOGGIN_PANEL: logger.info( '~~~~~~~~~ SUMMARIZE RESPONSE OBJECTS ~~~~~~~~~') for row in sorted( sum_end, key=lambda i: i[2], reverse=True)[:15]: logger.info( "type: %60s , # objects: %10d, total size: %s", *(row[0], row[1], filesizeformat(row[2])) ) print '~~~~~~~~~ SUMMARIZE RESPONSE OBJECTS ~~~~~~~~~' summary.print_(sum_end) if SHOW['compared_request_response_summaries']: diff = summary.get_diff(sum_start, sum_end) if SHOW_ON_DJANGO_DEBUG_TOOLBAR_LOGGIN_PANEL: logger.info( '~~~~~~~~~ COMPARED REQUEST & RESPONSE SUMMARIES ' '~~~~~~~~~' ) for row in sorted( diff, key=lambda i: i[2], reverse=True)[:15]: logger.info( "type: %60s , # objects: %10d, total size: %s", *(row[0], row[1], filesizeformat(row[2])) ) print \ '~~~~~~~~~ COMPARED REQUEST & RESPONSE SUMMARIES ~~~~~~~~~' summary.print_(diff) # print '~~~~~~~~~' # cb = refbrowser.ConsoleBrowser( # response, maxdepth=2, str_func=output_function) # cb.print_tree() a = asizeof(response) a_string = 'Total size of response object in kB: %s' % \ str(a/1024.0) b = asizeof(self.end_objects) b_string = 'Total size of end_objects in MB: %s' % str(b/1048576.0) c = asizeof(self.start_objects) c_string = 'Total size of start_objects in MB: %s' % \ str(c/1048576.0) if SHOW_ON_DJANGO_DEBUG_TOOLBAR_LOGGIN_PANEL: logger.info( '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~' '~~~~~~~~~~~~~~' ) logger.info(a_string) logger.info(b_string) logger.info(c_string) logger.info( '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~' '~~~~~~~~~~~~~~' ) print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'\ '~~~~~~~~~~~~~~~~~~~~~~~~~~' print a_string print b_string print c_string print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'\ '~~~~~~~~~~~~~~~~~~~~~~~~~~' print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'\ '~~~~~~~~~~~~~~~~~~~~~~~~~~' print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'\ '~~~~~~~~~~~~~~~~~~~~~~~~~~' return response
def process_response(self, request, response): path = request.META['PATH_INFO'] if self.is_ignored(path): return response self.end_objects = muppy.get_objects() sum_start = summary.summarize(self.start_objects) sum_end = summary.summarize(self.end_objects) diff = summary.get_diff(sum_start, sum_end) if SHOW_ON_DJANGO_DEBUG_TOOLBAR_LOGGIN_PANEL: logger.info( '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~' '~~~~~~~~~~~~~~') logger.info("Top %d memory deltas after processing URL: %s", SHOW_TOP_X_MEMORY_DELTAS, path) logger.info("%-60s %10s %10s", "type", "# objects", "total size") print '\n\n' print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'\ '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~' print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'\ '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~' print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'\ '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~' print "Top %d memory deltas after processing URL: %s" % ( SHOW_TOP_X_MEMORY_DELTAS, path) print "%60s %10s %10s" % ("type", "# objects", "total size") for row in sorted(diff, key=lambda i: i[2], reverse=True)[:SHOW_TOP_X_MEMORY_DELTAS]: if SHOW_ON_DJANGO_DEBUG_TOOLBAR_LOGGIN_PANEL: logger.info("type: %60s , # objects: %10d, total size: %s", *(row[0], row[1], filesizeformat(row[2]))) print "%60s %10d %s" % (row[0], row[1], filesizeformat(row[2])) start_size = asizeof(self.start_objects) end_size = asizeof(self.end_objects) if SHOW_ON_DJANGO_DEBUG_TOOLBAR_LOGGIN_PANEL: logger.info( '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~' '~~~~~~~~~~~~~~') logger.info( "Processed %s: memory delta %0.1f kB (%0.1f -> %0.1fMB), " "response size: %0.1f kB", path, (end_size - start_size) / 1024.0, start_size / 1048576.0, end_size / 1048576.0, len(response.content) / 1024.0, ) print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'\ '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~' print( "Processed %s: memory delta %0.1f kB (%0.1f -> %0.1fMB), " "response size: %0.1f kB" % ( path, (end_size - start_size) / 1024.0, start_size / 1048576.0, end_size / 1048576.0, len(response.content) / 1024.0, )) if SHOW_ON_DJANGO_DEBUG_TOOLBAR_LOGGIN_PANEL: logger.info( '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~' '~~~~~~~~~~~~~~') print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'\ '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~' print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'\ '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~' print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'\ '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~' print '\n\n' return response
def before_after_each_function(request): global _global_collect_info import psutil current_pids = set(proc.pid for proc in psutil.process_iter()) before_curr_proc_memory_info = psutil.Process().memory_info() if _global_collect_info and DEBUG_MEMORY_INFO: try: from pympler import summary, muppy sum1 = summary.summarize(muppy.get_objects()) except: import traceback traceback.print_exc() sys.stdout.write(''' =============================================================================== Memory before: %s %s =============================================================================== ''' % (request.function, format_memory_info(psutil.virtual_memory(), before_curr_proc_memory_info))) yield processes_info = [] for proc in psutil.process_iter(): if proc.pid not in current_pids: try: processes_info.append( 'New Process: %s(%s) - %s' % (proc.name(), proc.pid, format_process_memory_info(proc.memory_info()))) except psutil.NoSuchProcess: pass # The process could've died in the meanwhile after_curr_proc_memory_info = psutil.Process().memory_info() if DEBUG_MEMORY_INFO: try: if after_curr_proc_memory_info.rss - before_curr_proc_memory_info.rss > 10 * 1000 * 1000: # 10 MB leak if _global_collect_info: sum2 = summary.summarize(muppy.get_objects()) diff = summary.get_diff(sum1, sum2) sys.stdout.write( '===============================================================================\n' ) sys.stdout.write('Leak info:\n') sys.stdout.write( '===============================================================================\n' ) summary.print_(diff) sys.stdout.write( '===============================================================================\n' ) _global_collect_info = True # We'll only really collect the info on the next test (i.e.: if at one test # we used too much memory, the next one will start collecting) else: _global_collect_info = False except: import traceback traceback.print_exc() sys.stdout.write( ''' =============================================================================== Memory after: %s %s%s =============================================================================== ''' % (request.function, format_memory_info(psutil.virtual_memory(), after_curr_proc_memory_info), '' if not processes_info else '\nLeaked processes:\n' + '\n'.join(processes_info)), )
all_objects = muppy.get_objects() len(all_objects) import types my_types = muppy.filter(all_objects, Type=types.ClassType) len(my_types) for t in my_types: print t from pympler import summary sum1 = summary.summarize(all_objects) summary.print_(sum1) sum2 = summary.summarize(muppy.get_objects()) diff = summary.get_diff(sum1, sum2) summary.print_(diff) from pympler import refbrowser root = "some root object" root_ref1 = [root] root_ref2 = (root, ) def output_function(o): return str(type(o)) cb = refbrowser.ConsoleBrowser(root, maxdepth=2, str_func=output_function) #ib = refbrowser.InteractiveBrowser(root) #ib.main()
# Shallow size (например, размер листа, но не объектов, которые в нем есть) print(sys.getsizeof(1)) # 28 print(sys.getsizeof(54234.000342314000)) # 24 print(sys.getsizeof(None)) # 16 print(sys.getsizeof([])) # 72 print(sys.getsizeof([1, 2, 3, 4, ['s', 'l', ['a', 1]]])) # 112 print(sys.getsizeof({})) # 248 print(sys.getsizeof(tuple())) # 56 print() # Deep size (идет по иерархии объектов вглубь, суммирует) print(asizeof.asizeof(1)) # 32 print(asizeof.asizeof(54234.000342314000)) # 24 print(asizeof.asizeof(None)) # 16 print(asizeof.asizeof([])) # 72 print(asizeof.asizeof([1, 2, 3, 4, ['s', 'l', ['a', 1]]])) # 592 print( asizeof.asizeof( [1, 2, 3, 4, ['s', 'l', ['a', 1, [None, 1, { 'a': 1, 'b': 'a' }]]]])) # 1016 print(asizeof.asizeof({})) # 248 print(asizeof.asizeof(tuple())) # 56 print(asizeof.asizeof(set())) # 232 all_objects_2 = muppy.get_objects() sum_1 = summary.summarize(all_objects_1) sum_2 = summary.summarize(all_objects_2) summary.print_(summary.get_diff(sum_1, sum_2))
""" Python heap memory allocation analysis for scripts in :mod:`mandelbrot.implementations` using :mod:`muppy`. """ from pympler import muppy, summary import mandelbrot if __name__ == "__main__": args = mandelbrot.parsed_mandelbrot_args() for impl in mandelbrot.all_implementations(): before = summary.summarize(muppy.get_objects()) impl.callable(grid_side_size=args.grid_side_size, max_iter=args.max_iter) after = summary.summarize(muppy.get_objects()) differences = summary.get_diff(after, before) print(f"Memory use by {impl.id_}:") summary.print_(differences)
def before_after_each_function(request): global _global_collect_info import psutil current_pids = set(proc.pid for proc in psutil.process_iter()) before_curr_proc_memory_info = psutil.Process().memory_info() if _global_collect_info and DEBUG_MEMORY_INFO: try: from pympler import summary, muppy sum1 = summary.summarize(muppy.get_objects()) except: pydev_log.exception() sys.stdout.write( ''' =============================================================================== Memory before: %s %s =============================================================================== ''' % (request.function, format_memory_info(psutil.virtual_memory(), before_curr_proc_memory_info))) yield processes_info = [] for proc in psutil.process_iter(): if proc.pid not in current_pids: try: processes_info.append( 'New Process: %s(%s) - %s' % ( proc.name(), proc.pid, format_process_memory_info(proc.memory_info()) ) ) except psutil.NoSuchProcess: pass # The process could've died in the meanwhile after_curr_proc_memory_info = psutil.Process().memory_info() if DEBUG_MEMORY_INFO: try: if after_curr_proc_memory_info.rss - before_curr_proc_memory_info.rss > 10 * 1000 * 1000: # 10 MB leak if _global_collect_info: sum2 = summary.summarize(muppy.get_objects()) diff = summary.get_diff(sum1, sum2) sys.stdout.write('===============================================================================\n') sys.stdout.write('Leak info:\n') sys.stdout.write('===============================================================================\n') summary.print_(diff) sys.stdout.write('===============================================================================\n') _global_collect_info = True # We'll only really collect the info on the next test (i.e.: if at one test # we used too much memory, the next one will start collecting) else: _global_collect_info = False except: pydev_log.exception() sys.stdout.write( ''' =============================================================================== Memory after: %s %s%s =============================================================================== ''' % ( request.function, format_memory_info(psutil.virtual_memory(), after_curr_proc_memory_info), '' if not processes_info else '\nLeaked processes:\n' + '\n'.join(processes_info)), )
def profile_expose_method(profiled_method_wrapper, accept, args, func, kw, exclude_from_memory_profiling): """ Targeted to profile a specific method that wraps HTTP request processing endpoints into database context. :param profiled_method_wrapper: method wrapped around profiled call to be passed in to memory profiler :param accept: param specific to profiled call :param args: args of a function that is being wrapped by a profiled method :param func: function that is being wrapped by a profiled method :param kw: kwargs of a function that is being wrapped by a profiled method :return: output of a profiled method without modification """ if not exclude_from_memory_profiling and get_memory_profile_logging_on() and \ check_memory_profile_package_wide_disable(func): controller_class = args[0].__class__.__name__ if args and len( args) > 0 else '' end_point_name_parts = [ s for s in [func.__module__, controller_class, func.__name__] if s != '' ] end_point_name = ".".join(end_point_name_parts) is_pympler_on = _is_pympler_profiling_value_on(end_point_name) profile_output = {'output': {}} if is_pympler_on: all_objects = muppy.get_objects() all_objects_summary_before = summary.summarize(all_objects) memory_profile = memory_usage( (_profile_me, (profile_output, profiled_method_wrapper, func, accept, args, kw), {}), interval=0.1) output = profile_output['output'] if is_pympler_on: all_objects_summary_after = summary.summarize(all_objects) diff = summary.get_diff(all_objects_summary_before, all_objects_summary_after) diff_less = summary.format_(diff) diff_out = '' for s in diff_less: diff_out += s + '\n' thread_log.info( "================ PYMPLER OUTPUT <{}> ==============\n{}". format(end_point_name, diff_out)) try: message = json.dumps({ 'log_type': 'memory_profile', 'proc_id': os.getpid(), 'name': func.__name__, 'module': func.__module__, 'mem_profile': memory_profile, 'min': min(memory_profile), 'max': max(memory_profile), 'diff': max(memory_profile) - min(memory_profile), 'leaked': memory_profile[-1] - memory_profile[0], 'args': [arg for arg in args[1:]], # exclude self 'kwargs': kw }) memory_log.info(message, extra={ 'controller_module': func.__module__, 'controller_class': controller_class, 'endpoint': func.__name__ }) except Exception as e: thread_log.exception('Logger failed: {}'.format(e)) else: output = profiled_method_wrapper(accept, args, func, kw) return output
def process_response(self, request, response): req = request.META['PATH_INFO'] if not self.is_ignored(req): if SHOW_ON_DJANGO_DEBUG_TOOLBAR_LOGGIN_PANEL: logger.info( '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~' '~~~~~~~~~~~~~~') print '\n\n' print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'\ '~~~~~~~~~~~~~~~~~~~~~~~~~~' print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'\ '~~~~~~~~~~~~~~~~~~~~~~~~~~' print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'\ '~~~~~~~~~~~~~~~~~~~~~~~~~~' if SHOW_ON_DJANGO_DEBUG_TOOLBAR_LOGGIN_PANEL: logger.info(u'REQUESTED URL: {}'.format(req)) print u'REQUESTED URL: {}'.format(req) self.end_objects = muppy.get_objects() if SHOW['request_summary']: sum_start = summary.summarize(self.start_objects) if SHOW_ON_DJANGO_DEBUG_TOOLBAR_LOGGIN_PANEL: logger.info( '~~~~~~~~~ SUMMARIZE REQUEST OBJECTS ~~~~~~~~~') for row in sorted(sum_start, key=lambda i: i[2], reverse=True)[:15]: logger.info( "type: %60s , # objects: %10d, total size: %s", *(row[0], row[1], filesizeformat(row[2]))) print '~~~~~~~~~ SUMMARIZE REQUEST OBJECTS ~~~~~~~~~' summary.print_(sum_start) if SHOW['response_summary']: sum_end = summary.summarize(self.end_objects) if SHOW_ON_DJANGO_DEBUG_TOOLBAR_LOGGIN_PANEL: logger.info( '~~~~~~~~~ SUMMARIZE RESPONSE OBJECTS ~~~~~~~~~') for row in sorted(sum_end, key=lambda i: i[2], reverse=True)[:15]: logger.info( "type: %60s , # objects: %10d, total size: %s", *(row[0], row[1], filesizeformat(row[2]))) print '~~~~~~~~~ SUMMARIZE RESPONSE OBJECTS ~~~~~~~~~' summary.print_(sum_end) if SHOW['compared_request_response_summaries']: diff = summary.get_diff(sum_start, sum_end) if SHOW_ON_DJANGO_DEBUG_TOOLBAR_LOGGIN_PANEL: logger.info( '~~~~~~~~~ COMPARED REQUEST & RESPONSE SUMMARIES ' '~~~~~~~~~') for row in sorted(diff, key=lambda i: i[2], reverse=True)[:15]: logger.info( "type: %60s , # objects: %10d, total size: %s", *(row[0], row[1], filesizeformat(row[2]))) print \ '~~~~~~~~~ COMPARED REQUEST & RESPONSE SUMMARIES ~~~~~~~~~' summary.print_(diff) # print '~~~~~~~~~' # cb = refbrowser.ConsoleBrowser( # response, maxdepth=2, str_func=output_function) # cb.print_tree() a = asizeof(response) a_string = 'Total size of response object in kB: %s' % \ str(a/1024.0) b = asizeof(self.end_objects) b_string = 'Total size of end_objects in MB: %s' % str( b / 1048576.0) c = asizeof(self.start_objects) c_string = 'Total size of start_objects in MB: %s' % \ str(c/1048576.0) if SHOW_ON_DJANGO_DEBUG_TOOLBAR_LOGGIN_PANEL: logger.info( '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~' '~~~~~~~~~~~~~~') logger.info(a_string) logger.info(b_string) logger.info(c_string) logger.info( '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~' '~~~~~~~~~~~~~~') print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'\ '~~~~~~~~~~~~~~~~~~~~~~~~~~' print a_string print b_string print c_string print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'\ '~~~~~~~~~~~~~~~~~~~~~~~~~~' print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'\ '~~~~~~~~~~~~~~~~~~~~~~~~~~' print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'\ '~~~~~~~~~~~~~~~~~~~~~~~~~~' return response