def test_stracker_create_summary(self):
"""Test that a summary is created correctly.
This can only be done heuristically, e.g that most recent objects are
included.
Also check that summaries managed by the tracker are excluded if
ignore_self is enabled.
"""
# at the beginning, there should not be an indicator object listed
tmp_tracker = tracker.SummaryTracker()
sn = tmp_tracker.create_summary()
self.assertEqual(self._contains_indicator(sn), None)
# now an indicator object should be listed
o = self._get_indicator()
sn = tmp_tracker.create_summary()
self.assertEqual(self._contains_indicator(sn), 1)
# with ignore_self enabled a second summary should not list the first
# summary
sn = tmp_tracker.create_summary()
sn2 = tmp_tracker.create_summary()
tmp = summary._sweep(summary.get_diff(sn, sn2))
self.assertEqual(len(tmp), 0)
# but with ignore_self turned off, there should be some difference
tmp_tracker = tracker.SummaryTracker(ignore_self=False)
sn = tmp_tracker.create_summary()
tmp_tracker.new_obj = self._get_indicator()
sn2 = tmp_tracker.create_summary()
tmp = summary._sweep(summary.get_diff(sn, sn2))
self.failIfEqual(len(tmp), 0)
def test_stracker_create_summary(self):
"""Test that a summary is created correctly.
This can only be done heuristically, e.g that most recent objects are
included.
Also check that summaries managed by the tracker are excluded if
ignore_self is enabled.
"""
# at the beginning, there should not be an indicator object listed
tmp_tracker = tracker.SummaryTracker()
sn = tmp_tracker.create_summary()
self.assertEqual(self._contains_indicator(sn), None)
# now an indicator object should be listed
o = self._get_indicator()
sn = tmp_tracker.create_summary()
self.assertEqual(self._contains_indicator(sn), 1)
# with ignore_self enabled a second summary should not list the first
# summary
sn = tmp_tracker.create_summary()
sn2 = tmp_tracker.create_summary()
tmp = summary._sweep(summary.get_diff(sn, sn2))
self.assertEqual(len(tmp), 0)
# but with ignore_self turned off, there should be some difference
tmp_tracker = tracker.SummaryTracker(ignore_self=False)
sn = tmp_tracker.create_summary()
tmp_tracker.new_obj = self._get_indicator()
sn2 = tmp_tracker.create_summary()
tmp = summary._sweep(summary.get_diff(sn, sn2))
self.failIfEqual(len(tmp), 0)
def test_summary_diff(self):
"""Test summary diff. """
# base cases
res = summary.get_diff([], [])
self.assertEqual(res, [])
res = summary.get_diff([], [[str(str), 3, 3*_getsizeof('a')]])
self.assertEqual(res, [[str(str), 3, 3*_getsizeof('a')]])
res = summary.get_diff([[str(str), 3, 3*_getsizeof('a')]], [])
self.assertEqual(res, [[str(str), -3, -3*_getsizeof('a')]])
#interesting case
left = [[str(str), 3, 3*_getsizeof('a')],\
[str(int), 2, 2*_getsizeof(1)],\
[str(list), 1, _getsizeof([])],\
[str(dict), 1, _getsizeof({})]]
right = [[str(str), 2, 2*_getsizeof('a')],\
[str(int), 3, 3*_getsizeof(1)],\
[str(list), 1, _getsizeof([1,2,3])],\
[str(dict), 1, _getsizeof({})],
[str(tuple), 1, _getsizeof((1,2))]]
expected = [[str(str), -1, -1*_getsizeof('a')],\
[str(int), 1, +1*_getsizeof(1)],\
[str(list), 0, _getsizeof([1,2,3]) - _getsizeof([])],\
[str(dict), 0, 0],
[str(tuple), 1, _getsizeof((1,2))]]
res = summary.get_diff(left, right)
for row_e in res:
self.assertTrue(row_e in expected)
def snapshot_diff(cur_items, snapshot_file):
"""
Attempts to compare like PIDs. If like PIDS can't be found it will just compare
the first PID listed to the first PID in the file. Any unmatched or non-first
PIDs will be ignored because we don't know what to compare them to.
"""
try:
prev_items = list(frontend_utils.get_pages(snapshot_file))
except pickle.UnpicklingError as e:
frontend_utils.echo_error(
f"Error unpickling the data from {snapshot_file}: {e}")
return None
differences = []
for cur_item in cur_items:
for prev_item in prev_items:
if cur_item.pid == prev_item.pid:
diff = summary.get_diff(cur_item.data, prev_item.data)
differences.append(
RetrievedObjects(
pid=cur_item.pid,
title=f"Snapshot Differences for {cur_item.pid}",
data=diff,
))
if not differences:
diff = summary.get_diff(cur_items[0].data, prev_items[0].data)
differences.append(
RetrievedObjects(pid=0, title=f"Snapshot Differences", data=diff))
return differences
def func5():
"""
Модуль pympler
muppy.get_objects() - метод считывает все существующтие
обьекты в Python вызывая этот метод 2 раза в разный
промежуток времени, мы можем выявить какие обьекты были
созданы с первого среза.
summary.get_diff() - выявляет какие обьекты как разницу между
двумя срезами.
summary.print_() - красиво выводит на экран обьекты и память
ими занимаемую.
"""
from pympler import asizeof
from pympler import muppy
from pympler import summary
print('\nИспользование muppy :')
alll_obj_1 = muppy.get_objects()
data = list(range(1000))
alll_obj_2 = muppy.get_objects()
sum1 = summary.summarize(alll_obj_1)
sum2 = summary.summarize(alll_obj_2)
summary.print_(summary.get_diff(sum1, sum2))
def file_test(rows=500000, cols=50):
"File test"
print("Creating file with {} rows and {} columns".format(rows, cols))
file = create_file(rows, cols)
print("Size of the file: {:.2f} MiB".format(getsize(file) / (1024 * 1024)))
print("Reading file")
sum1 = summarize(get_objects())
las = read(file)
sum2 = summarize(get_objects())
diff = get_diff(sum1, sum2)
print_(diff)
for curve in las.curves:
print("Name: {}, Min: {:.2f}, Mean: {:.2f}, Max: {:.2f}"
.format(curve.mnemonic, nanmin(curve.data), nanmean(curve.data),
nanmax(curve.data)))
del las
las = read(file)
del las
las = read(file)
del las
las = read(file)
del las
print("Happy end")
def process_response(self, request, response):
req = request.META['PATH_INFO']
if req.find('static') == -1 and req.find('media') == -1:
print req
self.end_objects = muppy.get_objects()
sum_start = summary.summarize(self.start_objects)
sum_end = summary.summarize(self.end_objects)
diff = summary.get_diff(sum_start, sum_end)
summary.print_(diff)
#print '~~~~~~~~~'
#cb = refbrowser.ConsoleBrowser(response, maxdepth=2, \
#str_func=output_function)
#cb.print_tree()
print '~~~~~~~~~'
a = asizeof(response)
print 'Total size of response object in kB: %s' % \
str(a / 1024.0)
print '~~~~~~~~~'
a = asizeof(self.end_objects)
print 'Total size of end_objects in MB: %s' % \
str(a / 1048576.0)
b = asizeof(self.start_objects)
print 'Total size of start_objects in MB: %s' % \
str(b / 1048576.0)
print '~~~~~~~~~'
return response
def _get_usage(function, *args):
"""Get the usage of a function call.
This function is to be used only internally. The 'real' get_usage
function is a wrapper around _get_usage, but the workload is done
here.
"""
res = []
# init before calling
(s_before, s_after) = _get_summaries(function, *args)
# ignore all objects used for the measurement
ignore = []
if s_before != s_after:
ignore.append(s_before)
for row in s_before:
# ignore refs from summary and frame (loop)
if len(gc.get_referrers(row)) == 2:
ignore.append(row)
for item in row:
# ignore refs from summary and frame (loop)
if len(gc.get_referrers(item)) == 2:
ignore.append(item)
for o in ignore:
s_after = summary._subtract(s_after, o)
res = summary.get_diff(s_before, s_after)
return summary._sweep(res)
def _get_usage(function, *args):
"""Get the usage of a function call.
This function is to be used only internally. The 'real' get_usage
function is a wrapper around _get_usage, but the workload is done
here.
"""
res = []
# init before calling
(s_before, s_after) = _get_summaries(function, *args)
# ignore all objects used for the measurement
ignore = []
if s_before != s_after:
ignore.append(s_before)
for row in s_before:
# ignore refs from summary and frame (loop)
if len(gc.get_referrers(row)) == 2:
ignore.append(row)
for item in row:
# ignore refs from summary and frame (loop)
if len(gc.get_referrers(item)) == 2:
ignore.append(item)
for o in ignore:
s_after = summary._subtract(s_after, o)
res = summary.get_diff(s_before, s_after)
return summary._sweep(res)
def on_epoch_end(self, epoch, log={}):
x = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
web_browser_debug = True
print(x)
if x > 40000:
if web_browser_debug:
if epoch == 0:
start_in_background()
tr = tracker.SummaryTracker()
tr.print_diff()
else:
global memlist
all_objects = muppy.get_objects(include_frames=True)
# print(len(all_objects))
sum1 = summary.summarize(all_objects)
memlist.append(sum1)
summary.print_(sum1)
if len(memlist) > 1:
# compare with last - prints the difference per epoch
diff = summary.get_diff(memlist[-2], memlist[-1])
summary.print_(diff)
my_types = muppy.filter(all_objects, Type=types.ClassType)
for t in my_types:
print(t)
def handle_signal_abort(self, signum, frame):
Log.warn("Someone want to kill me! But I'll not die now! Hahahaha!")
s = summary.summarize(muppy.get_objects())
Log.debug("Current memory usage:")
summary.print_(s)
diff = summary.get_diff(self.mem_sum, s)
self.mem_sum = s
Log.debug("New memory usage:")
summary.print_(diff)
def profile_expose_method(profiled_method_wrapper, accept, args, func, kw, exclude_from_memory_profiling):
"""
Targeted to profile a specific method that wraps HTTP request processing endpoints into database context.
:param profiled_method_wrapper: method wrapped around profiled call to be passed in to memory profiler
:param accept: param specific to profiled call
:param args: args of a function that is being wrapped by a profiled method
:param func: function that is being wrapped by a profiled method
:param kw: kwargs of a function that is being wrapped by a profiled method
:return: output of a profiled method without modification
"""
if not exclude_from_memory_profiling and get_memory_profile_logging_on() and \
check_memory_profile_package_wide_disable(func):
controller_class = args[0].__class__.__name__ if args and len(args) > 0 else ''
end_point_name_parts = [s for s in [func.__module__, controller_class, func.__name__] if s != '']
end_point_name = ".".join(end_point_name_parts)
is_pympler_on = _is_pympler_profiling_value_on(end_point_name)
profile_output = {'output': {}}
if is_pympler_on:
all_objects = muppy.get_objects()
all_objects_summary_before = summary.summarize(all_objects)
memory_profile = memory_usage((_profile_me,
(profile_output, profiled_method_wrapper, func, accept, args, kw),
{}),
interval=0.1)
output = profile_output['output']
if is_pympler_on:
all_objects_summary_after = summary.summarize(all_objects)
diff = summary.get_diff(all_objects_summary_before, all_objects_summary_after)
diff_less = summary.format_(diff)
diff_out = ''
for s in diff_less:
diff_out += s+'\n'
thread_log.info("================ PYMPLER OUTPUT <{}> ==============\n{}".format(end_point_name, diff_out))
try:
message = json.dumps({'log_type': 'memory_profile',
'proc_id': os.getpid(),
'name': func.__name__,
'module': func.__module__,
'mem_profile': memory_profile,
'min': min(memory_profile),
'max': max(memory_profile),
'diff': max(memory_profile) - min(memory_profile),
'leaked': memory_profile[-1] - memory_profile[0],
'args': [arg for arg in args[1:]], # exclude self
'kwargs': kw})
memory_log.info(message,
extra={'controller_module': func.__module__,
'controller_class': controller_class,
'endpoint': func.__name__})
except Exception as e:
thread_log.exception('Logger failed: {}'.format(e))
else:
output = profiled_method_wrapper(accept, args, func, kw)
return output
def test_print_diff(self):
"""Test summary can be printed."""
try:
self._stdout = sys.stdout
sys.stdout = self.DevNull()
sum1 = summary.summarize(muppy.get_objects())
sum2 = summary.summarize(muppy.get_objects())
sumdiff = summary.get_diff(sum1, sum2)
summary.print_(sumdiff)
finally:
sys.stdout = self._stdout
def test_print_diff(self):
"""Test summary can be printed."""
try:
self._stdout = sys.stdout
sys.stdout = self.DevNull()
sum1 = summary.summarize(muppy.get_objects())
sum2 = summary.summarize(muppy.get_objects())
sumdiff = summary.get_diff(sum1, sum2)
summary.print_(sumdiff)
finally:
sys.stdout = self._stdout
def log_summary(self):
summary_str = '\n\n=====Total Memory Stats=====\n'
current_summary = summary.summarize(muppy.get_objects())
summary_str += self.str_print_(current_summary, limit=25)
if self.previous_summary is None:
self.previous_summary = current_summary
else:
summary_str += '\n\n===== Difference since last update =====\n'
diff = summary.get_diff(self.previous_summary, current_summary)
summary_str += self.str_print_(diff, limit=25)
self.previous_summary = current_summary
self.logger.info(summary_str)
def test_print_diff(self):
"""Test summary can be printed."""
try:
self._stdout = sys.stdout
stream = StringIO()
sys.stdout = stream
sum1 = summary.summarize(muppy.get_objects())
sum2 = summary.summarize(muppy.get_objects())
sumdiff = summary.get_diff(sum1, sum2)
summary.print_(sumdiff)
self.assertIn('str', stream.getvalue())
self.assertNotIn("<class 'str", stream.getvalue())
finally:
sys.stdout = self._stdout
def diff(self, summary1=None, summary2=None):
"""Compute diff between to summaries.
If no summary is provided, the diff from the last to the current
summary is used. If summary1 is provided the diff from summary1
to the current summary is used. If summary1 and summary2 are
provided, the diff between these two is used.
"""
res = None
if summary2 is None:
self.s1 = self.create_summary()
if summary1 is None:
res = summary.get_diff(self.s0, self.s1)
else:
res = summary.get_diff(summary1, self.s1)
self.s0 = self.s1
else:
if summary1 is not None:
res = summary.get_diff(summary1, summary2)
else:
raise ValueError("You cannot provide summary2 without summary1.")
return summary._sweep(res)
def memusage_before_n_after(fun, *args, **kwargs):
from pympler import muppy
from pympler import summary
from datetime import datetime
before = summary.summarize(muppy.get_objects())
before_time = datetime.now()
fun_ret = fun(*args, **kwargs)
after_time = datetime.now()
after = summary.summarize(muppy.get_objects())
diff = summary.get_diff(before, after)
print "execution time: ", after_time - before_time
summary.print_(diff)
return fun_ret, diff
def diff(self, summary1=None, summary2=None):
"""Compute diff between to summaries.
If no summary is provided, the diff from the last to the current
summary is used. If summary1 is provided the diff from summary1
to the current summary is used. If summary1 and summary2 are
provided, the diff between these two is used.
"""
res = None
if summary2 is None:
self.s1 = self.create_summary()
if summary1 is None:
res = summary.get_diff(self.s0, self.s1)
else:
res = summary.get_diff(summary1, self.s1)
self.s0 = self.s1
else:
if summary1 is not None:
res = summary.get_diff(summary1, summary2)
else:
raise ValueError(
"You cannot provide summary2 without summary1.")
return summary._sweep(res)
def collection_post(self):
print("REQUEST {}".format(self.request.json))
sums = []
for key in 'sum1', 'sum2':
sums.append(self.request.json[key])
with transaction.manager:
query = self.request.dbsession.query(ObjectSummary)
query = query.filter(ObjectSummary.id.in_(sums))
query = query.order_by(ObjectSummary.created)
dbsums = query.all()
sums = [m.content for m in dbsums]
diff = summary.get_diff(*sums)
output = io.StringIO()
with contextlib.redirect_stdout(output):
summary.print_(diff)
# import pdb ; pdb.set_trace()
return dict(result='testing', output=output.getvalue())
def test_summary_diff(self):
"""Test summary diff. """
left = [[str(str), 3, 3*getsizeof('a')],\
[str(int), 2, 2*getsizeof(1)],\
[str(list), 1, getsizeof([])],\
[str(dict), 1, getsizeof({})]]
right = [[str(str), 2, 2*getsizeof('a')],\
[str(int), 3, 3*getsizeof(1)],\
[str(list), 1, getsizeof([1,2,3])],\
[str(dict), 1, getsizeof({})],
[str(tuple), 1, getsizeof((1,2))]]
expected = [[str(str), -1, -1*getsizeof('a')],\
[str(int), 1, +1*getsizeof(1)],\
[str(list), 0, getsizeof([1,2,3]) - getsizeof([])],\
[str(dict), 0, 0],
[str(tuple), 1, getsizeof((1,2))]]
res = summary.get_diff(left, right)
for row_e in res:
self.assertTrue(row_e in expected)
def test_summary_diff(self):
"""Test summary diff. """
left = [[str(str), 3, 3*getsizeof('a')],\
[str(int), 2, 2*getsizeof(1)],\
[str(list), 1, getsizeof([])],\
[str(dict), 1, getsizeof({})]]
right = [[str(str), 2, 2*getsizeof('a')],\
[str(int), 3, 3*getsizeof(1)],\
[str(list), 1, getsizeof([1,2,3])],\
[str(dict), 1, getsizeof({})],
[str(tuple), 1, getsizeof((1,2))]]
expected = [[str(str), -1, -1*getsizeof('a')],\
[str(int), 1, +1*getsizeof(1)],\
[str(list), 0, getsizeof([1,2,3]) - getsizeof([])],\
[str(dict), 0, 0],
[str(tuple), 1, getsizeof((1,2))]]
res = summary.get_diff(left, right)
for row_e in res:
self.assertTrue(row_e in expected)
def print_muppy_sumary():
# http://pythonhosted.org/Pympler/index.html
try:
from pympler import muppy, summary
except ImportError:
print("WARNING: pympler not installed")
return
# from pympler.classtracker import ClassTracker
# from pympler.classtracker_stats import HtmlStats
global all_objects, obj_summary, class_tracker
if all_objects is None:
all_objects = muppy.get_objects()
obj_summary = summary.summarize(all_objects)
summary.print_(obj_summary)
# class_tracker = ClassTracker()
# class_tracker.track_class(FICSPlayer, trace=1)
# class_tracker.track_class(ICGameModel, resolution_level=2, trace=1)
else:
obj_summary2 = summary.summarize(muppy.get_objects())
diff = summary.get_diff(obj_summary, obj_summary2)
summary.print_(diff, limit=200)
def print_muppy_sumary():
# http://pythonhosted.org/Pympler/index.html
try:
from pympler import muppy, summary
except ImportError:
print("WARNING: pympler not installed")
return
# from pympler.classtracker import ClassTracker
# from pympler.classtracker_stats import HtmlStats
global all_objects, obj_summary, class_tracker
if all_objects is None:
all_objects = muppy.get_objects()
obj_summary = summary.summarize(all_objects)
summary.print_(obj_summary)
# class_tracker = ClassTracker()
# class_tracker.track_class(FICSPlayer, trace=1)
# class_tracker.track_class(ICGameModel, resolution_level=2, trace=1)
else:
obj_summary2 = summary.summarize(muppy.get_objects())
diff = summary.get_diff(obj_summary, obj_summary2)
summary.print_(diff, limit=200)
def _get_usage(function, *args):
"""Test if more memory is used after the function has been called.
The function will be invoked twice and only the second measurement will be
considered. Thus, memory used in initialisation (e.g. loading modules)
will not be included in the result. The goal is to identify memory leaks
caused by functions which use more and more memory.
Any arguments next to the function will be passed on to the function
on invocation.
Note that this function is currently experimental, because it is not
tested thoroughly and performs poorly.
"""
# The usage of a function is calculated by creating one summary of all
# objects before the function is invoked and afterwards. These summaries
# are compared and the diff is returned.
# This function works in a 2-steps process. Before the actual function is
# invoked an empty dummy function is measurement to identify the overhead
# involved in the measuring process. This overhead then is subtracted from
# the measurement performed on the passed function. The result reflects the
# actual usage of a function call.
# Also, a measurement is performed twice, allowing the adjustment to
# initializing things, e.g. modules
res = None
def _get_summaries(function, *args):
"""Get a 2-tuple containing one summary from before, and one summary
from after the function has been invoked.
"""
s_before = summary.summarize(get_objects())
function(*args)
s_after = summary.summarize(get_objects())
return (s_before, s_after)
def _get_usage(function, *args):
"""Get the usage of a function call.
This function is to be used only internally. The 'real' get_usage
function is a wrapper around _get_usage, but the workload is done
here.
"""
res = []
# init before calling
(s_before, s_after) = _get_summaries(function, *args)
# ignore all objects used for the measurement
ignore = []
if s_before != s_after:
ignore.append(s_before)
for row in s_before:
# ignore refs from summary and frame (loop)
if len(gc.get_referrers(row)) == 2:
ignore.append(row)
for item in row:
# ignore refs from summary and frame (loop)
if len(gc.get_referrers(item)) == 2:
ignore.append(item)
for o in ignore:
s_after = summary._subtract(s_after, o)
res = summary.get_diff(s_before, s_after)
return summary._sweep(res)
# calibrate; twice for initialization
def noop():
pass
offset = _get_usage(noop)
offset = _get_usage(noop)
# perform operation twice to handle objects possibly used in
# initialisation
tmp = _get_usage(function, *args)
tmp = _get_usage(function, *args)
tmp = summary.get_diff(offset, tmp)
tmp = summary._sweep(tmp)
if len(tmp) != 0:
res = tmp
return res
def before_after_each_function(request):
global _global_collect_info
try:
import psutil # Don't fail if not there
except ImportError:
yield
return
current_pids = set(proc.pid for proc in psutil.process_iter())
before_curr_proc_memory_info = psutil.Process().memory_info()
if _global_collect_info and DEBUG_MEMORY_INFO:
try:
from pympler import summary, muppy
sum1 = summary.summarize(muppy.get_objects())
except:
log.exception()
sys.stdout.write("""
===============================================================================
Memory before: %s
%s
===============================================================================
""" % (
request.function,
format_memory_info(psutil.virtual_memory(),
before_curr_proc_memory_info),
))
yield
processes_info = []
for proc in psutil.process_iter():
if proc.pid not in current_pids:
try:
try:
cmdline = proc.cmdline()
except:
cmdline = "<unable to get>"
processes_info.append("New Process: %s(%s - %s) - %s" % (
proc.name(),
proc.pid,
cmdline,
format_process_memory_info(proc.memory_info()),
))
except (psutil.NoSuchProcess, psutil.AccessDenied):
pass # The process could've died in the meanwhile
after_curr_proc_memory_info = psutil.Process().memory_info()
if DEBUG_MEMORY_INFO:
try:
if (after_curr_proc_memory_info.rss -
before_curr_proc_memory_info.rss > 10 * 1000 * 1000):
# 10 MB leak
if _global_collect_info:
sum2 = summary.summarize(muppy.get_objects())
diff = summary.get_diff(sum1, sum2)
sys.stdout.write(
"===============================================================================\n"
)
sys.stdout.write("Leak info:\n")
sys.stdout.write(
"===============================================================================\n"
)
summary.print_(diff)
sys.stdout.write(
"===============================================================================\n"
)
_global_collect_info = True
# We'll only really collect the info on the next test (i.e.: if at one test
# we used too much memory, the next one will start collecting)
else:
_global_collect_info = False
except:
log.exception()
sys.stdout.write("""
===============================================================================
Memory after: %s
%s%s
===============================================================================
""" % (
request.function,
format_memory_info(psutil.virtual_memory(),
after_curr_proc_memory_info),
"" if not processes_info else "\nLeaked processes:\n" +
"\n".join(processes_info),
))
def process_response(self, request, response):
path = request.META['PATH_INFO']
if self.is_ignored(path):
return response
self.end_objects = muppy.get_objects()
sum_start = summary.summarize(self.start_objects)
sum_end = summary.summarize(self.end_objects)
diff = summary.get_diff(sum_start, sum_end)
if SHOW_ON_DJANGO_DEBUG_TOOLBAR_LOGGIN_PANEL:
logger.info(
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'
'~~~~~~~~~~~~~~'
)
logger.info(
"Top %d memory deltas after processing URL: %s",
SHOW_TOP_X_MEMORY_DELTAS, path
)
logger.info("%-60s %10s %10s", "type", "# objects", "total size")
print '\n\n'
print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'\
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'
print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'\
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'
print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'\
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'
print "Top %d memory deltas after processing URL: %s" % (
SHOW_TOP_X_MEMORY_DELTAS, path)
print "%60s %10s %10s" % ("type", "# objects", "total size")
for row in sorted(diff, key=lambda i: i[2], reverse=True
)[:SHOW_TOP_X_MEMORY_DELTAS]:
if SHOW_ON_DJANGO_DEBUG_TOOLBAR_LOGGIN_PANEL:
logger.info(
"type: %60s , # objects: %10d, total size: %s",
*(row[0], row[1], filesizeformat(row[2]))
)
print "%60s %10d %s" % (row[0], row[1], filesizeformat(row[2]))
start_size = asizeof(self.start_objects)
end_size = asizeof(self.end_objects)
if SHOW_ON_DJANGO_DEBUG_TOOLBAR_LOGGIN_PANEL:
logger.info(
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'
'~~~~~~~~~~~~~~'
)
logger.info(
"Processed %s: memory delta %0.1f kB (%0.1f -> %0.1fMB), "
"response size: %0.1f kB",
path,
(end_size - start_size) / 1024.0,
start_size / 1048576.0,
end_size / 1048576.0,
len(response.content) / 1024.0,
)
print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'\
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'
print (
"Processed %s: memory delta %0.1f kB (%0.1f -> %0.1fMB), "
"response size: %0.1f kB" % (
path,
(end_size - start_size) / 1024.0,
start_size / 1048576.0,
end_size / 1048576.0,
len(response.content) / 1024.0,
)
)
if SHOW_ON_DJANGO_DEBUG_TOOLBAR_LOGGIN_PANEL:
logger.info(
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'
'~~~~~~~~~~~~~~'
)
print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'\
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'
print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'\
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'
print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'\
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'
print '\n\n'
return response
def process_response(self, request, response):
req = request.META['PATH_INFO']
if not self.is_ignored(req):
if SHOW_ON_DJANGO_DEBUG_TOOLBAR_LOGGIN_PANEL:
logger.info(
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'
'~~~~~~~~~~~~~~'
)
print '\n\n'
print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'\
'~~~~~~~~~~~~~~~~~~~~~~~~~~'
print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'\
'~~~~~~~~~~~~~~~~~~~~~~~~~~'
print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'\
'~~~~~~~~~~~~~~~~~~~~~~~~~~'
if SHOW_ON_DJANGO_DEBUG_TOOLBAR_LOGGIN_PANEL:
logger.info(u'REQUESTED URL: {}'.format(req))
print u'REQUESTED URL: {}'.format(req)
self.end_objects = muppy.get_objects()
if SHOW['request_summary']:
sum_start = summary.summarize(self.start_objects)
if SHOW_ON_DJANGO_DEBUG_TOOLBAR_LOGGIN_PANEL:
logger.info(
'~~~~~~~~~ SUMMARIZE REQUEST OBJECTS ~~~~~~~~~')
for row in sorted(
sum_start, key=lambda i: i[2], reverse=True)[:15]:
logger.info(
"type: %60s , # objects: %10d, total size: %s",
*(row[0], row[1], filesizeformat(row[2]))
)
print '~~~~~~~~~ SUMMARIZE REQUEST OBJECTS ~~~~~~~~~'
summary.print_(sum_start)
if SHOW['response_summary']:
sum_end = summary.summarize(self.end_objects)
if SHOW_ON_DJANGO_DEBUG_TOOLBAR_LOGGIN_PANEL:
logger.info(
'~~~~~~~~~ SUMMARIZE RESPONSE OBJECTS ~~~~~~~~~')
for row in sorted(
sum_end, key=lambda i: i[2], reverse=True)[:15]:
logger.info(
"type: %60s , # objects: %10d, total size: %s",
*(row[0], row[1], filesizeformat(row[2]))
)
print '~~~~~~~~~ SUMMARIZE RESPONSE OBJECTS ~~~~~~~~~'
summary.print_(sum_end)
if SHOW['compared_request_response_summaries']:
diff = summary.get_diff(sum_start, sum_end)
if SHOW_ON_DJANGO_DEBUG_TOOLBAR_LOGGIN_PANEL:
logger.info(
'~~~~~~~~~ COMPARED REQUEST & RESPONSE SUMMARIES '
'~~~~~~~~~'
)
for row in sorted(
diff, key=lambda i: i[2], reverse=True)[:15]:
logger.info(
"type: %60s , # objects: %10d, total size: %s",
*(row[0], row[1], filesizeformat(row[2]))
)
print \
'~~~~~~~~~ COMPARED REQUEST & RESPONSE SUMMARIES ~~~~~~~~~'
summary.print_(diff)
# print '~~~~~~~~~'
# cb = refbrowser.ConsoleBrowser(
# response, maxdepth=2, str_func=output_function)
# cb.print_tree()
a = asizeof(response)
a_string = 'Total size of response object in kB: %s' % \
str(a/1024.0)
b = asizeof(self.end_objects)
b_string = 'Total size of end_objects in MB: %s' % str(b/1048576.0)
c = asizeof(self.start_objects)
c_string = 'Total size of start_objects in MB: %s' % \
str(c/1048576.0)
if SHOW_ON_DJANGO_DEBUG_TOOLBAR_LOGGIN_PANEL:
logger.info(
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'
'~~~~~~~~~~~~~~'
)
logger.info(a_string)
logger.info(b_string)
logger.info(c_string)
logger.info(
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'
'~~~~~~~~~~~~~~'
)
print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'\
'~~~~~~~~~~~~~~~~~~~~~~~~~~'
print a_string
print b_string
print c_string
print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'\
'~~~~~~~~~~~~~~~~~~~~~~~~~~'
print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'\
'~~~~~~~~~~~~~~~~~~~~~~~~~~'
print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'\
'~~~~~~~~~~~~~~~~~~~~~~~~~~'
return response
def process_response(self, request, response):
path = request.META['PATH_INFO']
if self.is_ignored(path):
return response
self.end_objects = muppy.get_objects()
sum_start = summary.summarize(self.start_objects)
sum_end = summary.summarize(self.end_objects)
diff = summary.get_diff(sum_start, sum_end)
if SHOW_ON_DJANGO_DEBUG_TOOLBAR_LOGGIN_PANEL:
logger.info(
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'
'~~~~~~~~~~~~~~')
logger.info("Top %d memory deltas after processing URL: %s",
SHOW_TOP_X_MEMORY_DELTAS, path)
logger.info("%-60s %10s %10s", "type", "# objects", "total size")
print '\n\n'
print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'\
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'
print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'\
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'
print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'\
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'
print "Top %d memory deltas after processing URL: %s" % (
SHOW_TOP_X_MEMORY_DELTAS, path)
print "%60s %10s %10s" % ("type", "# objects", "total size")
for row in sorted(diff, key=lambda i: i[2],
reverse=True)[:SHOW_TOP_X_MEMORY_DELTAS]:
if SHOW_ON_DJANGO_DEBUG_TOOLBAR_LOGGIN_PANEL:
logger.info("type: %60s , # objects: %10d, total size: %s",
*(row[0], row[1], filesizeformat(row[2])))
print "%60s %10d %s" % (row[0], row[1], filesizeformat(row[2]))
start_size = asizeof(self.start_objects)
end_size = asizeof(self.end_objects)
if SHOW_ON_DJANGO_DEBUG_TOOLBAR_LOGGIN_PANEL:
logger.info(
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'
'~~~~~~~~~~~~~~')
logger.info(
"Processed %s: memory delta %0.1f kB (%0.1f -> %0.1fMB), "
"response size: %0.1f kB",
path,
(end_size - start_size) / 1024.0,
start_size / 1048576.0,
end_size / 1048576.0,
len(response.content) / 1024.0,
)
print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'\
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'
print(
"Processed %s: memory delta %0.1f kB (%0.1f -> %0.1fMB), "
"response size: %0.1f kB" % (
path,
(end_size - start_size) / 1024.0,
start_size / 1048576.0,
end_size / 1048576.0,
len(response.content) / 1024.0,
))
if SHOW_ON_DJANGO_DEBUG_TOOLBAR_LOGGIN_PANEL:
logger.info(
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'
'~~~~~~~~~~~~~~')
print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'\
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'
print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'\
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'
print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'\
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'
print '\n\n'
return response
def before_after_each_function(request):
global _global_collect_info
import psutil
current_pids = set(proc.pid for proc in psutil.process_iter())
before_curr_proc_memory_info = psutil.Process().memory_info()
if _global_collect_info and DEBUG_MEMORY_INFO:
try:
from pympler import summary, muppy
sum1 = summary.summarize(muppy.get_objects())
except:
import traceback
traceback.print_exc()
sys.stdout.write('''
===============================================================================
Memory before: %s
%s
===============================================================================
''' % (request.function,
format_memory_info(psutil.virtual_memory(),
before_curr_proc_memory_info)))
yield
processes_info = []
for proc in psutil.process_iter():
if proc.pid not in current_pids:
try:
processes_info.append(
'New Process: %s(%s) - %s' %
(proc.name(), proc.pid,
format_process_memory_info(proc.memory_info())))
except psutil.NoSuchProcess:
pass # The process could've died in the meanwhile
after_curr_proc_memory_info = psutil.Process().memory_info()
if DEBUG_MEMORY_INFO:
try:
if after_curr_proc_memory_info.rss - before_curr_proc_memory_info.rss > 10 * 1000 * 1000:
# 10 MB leak
if _global_collect_info:
sum2 = summary.summarize(muppy.get_objects())
diff = summary.get_diff(sum1, sum2)
sys.stdout.write(
'===============================================================================\n'
)
sys.stdout.write('Leak info:\n')
sys.stdout.write(
'===============================================================================\n'
)
summary.print_(diff)
sys.stdout.write(
'===============================================================================\n'
)
_global_collect_info = True
# We'll only really collect the info on the next test (i.e.: if at one test
# we used too much memory, the next one will start collecting)
else:
_global_collect_info = False
except:
import traceback
traceback.print_exc()
sys.stdout.write(
'''
===============================================================================
Memory after: %s
%s%s
===============================================================================
''' % (request.function,
format_memory_info(psutil.virtual_memory(),
after_curr_proc_memory_info),
'' if not processes_info else '\nLeaked processes:\n' +
'\n'.join(processes_info)), )
all_objects = muppy.get_objects()
len(all_objects)
import types
my_types = muppy.filter(all_objects, Type=types.ClassType)
len(my_types)
for t in my_types:
print t
from pympler import summary
sum1 = summary.summarize(all_objects)
summary.print_(sum1)
sum2 = summary.summarize(muppy.get_objects())
diff = summary.get_diff(sum1, sum2)
summary.print_(diff)
from pympler import refbrowser
root = "some root object"
root_ref1 = [root]
root_ref2 = (root, )
def output_function(o):
return str(type(o))
cb = refbrowser.ConsoleBrowser(root, maxdepth=2, str_func=output_function)
#ib = refbrowser.InteractiveBrowser(root)
#ib.main()
# Shallow size (например, размер листа, но не объектов, которые в нем есть)
print(sys.getsizeof(1)) # 28
print(sys.getsizeof(54234.000342314000)) # 24
print(sys.getsizeof(None)) # 16
print(sys.getsizeof([])) # 72
print(sys.getsizeof([1, 2, 3, 4, ['s', 'l', ['a', 1]]])) # 112
print(sys.getsizeof({})) # 248
print(sys.getsizeof(tuple())) # 56
print()
# Deep size (идет по иерархии объектов вглубь, суммирует)
print(asizeof.asizeof(1)) # 32
print(asizeof.asizeof(54234.000342314000)) # 24
print(asizeof.asizeof(None)) # 16
print(asizeof.asizeof([])) # 72
print(asizeof.asizeof([1, 2, 3, 4, ['s', 'l', ['a', 1]]])) # 592
print(
asizeof.asizeof(
[1, 2, 3, 4, ['s', 'l', ['a', 1, [None, 1, {
'a': 1,
'b': 'a'
}]]]])) # 1016
print(asizeof.asizeof({})) # 248
print(asizeof.asizeof(tuple())) # 56
print(asizeof.asizeof(set())) # 232
all_objects_2 = muppy.get_objects()
sum_1 = summary.summarize(all_objects_1)
sum_2 = summary.summarize(all_objects_2)
summary.print_(summary.get_diff(sum_1, sum_2))
"""
Python heap memory allocation analysis for scripts in
:mod:`mandelbrot.implementations` using :mod:`muppy`.
"""
from pympler import muppy, summary
import mandelbrot
if __name__ == "__main__":
args = mandelbrot.parsed_mandelbrot_args()
for impl in mandelbrot.all_implementations():
before = summary.summarize(muppy.get_objects())
impl.callable(grid_side_size=args.grid_side_size, max_iter=args.max_iter)
after = summary.summarize(muppy.get_objects())
differences = summary.get_diff(after, before)
print(f"Memory use by {impl.id_}:")
summary.print_(differences)
def before_after_each_function(request):
global _global_collect_info
import psutil
current_pids = set(proc.pid for proc in psutil.process_iter())
before_curr_proc_memory_info = psutil.Process().memory_info()
if _global_collect_info and DEBUG_MEMORY_INFO:
try:
from pympler import summary, muppy
sum1 = summary.summarize(muppy.get_objects())
except:
pydev_log.exception()
sys.stdout.write(
'''
===============================================================================
Memory before: %s
%s
===============================================================================
''' % (request.function, format_memory_info(psutil.virtual_memory(), before_curr_proc_memory_info)))
yield
processes_info = []
for proc in psutil.process_iter():
if proc.pid not in current_pids:
try:
processes_info.append(
'New Process: %s(%s) - %s' % (
proc.name(),
proc.pid,
format_process_memory_info(proc.memory_info())
)
)
except psutil.NoSuchProcess:
pass # The process could've died in the meanwhile
after_curr_proc_memory_info = psutil.Process().memory_info()
if DEBUG_MEMORY_INFO:
try:
if after_curr_proc_memory_info.rss - before_curr_proc_memory_info.rss > 10 * 1000 * 1000:
# 10 MB leak
if _global_collect_info:
sum2 = summary.summarize(muppy.get_objects())
diff = summary.get_diff(sum1, sum2)
sys.stdout.write('===============================================================================\n')
sys.stdout.write('Leak info:\n')
sys.stdout.write('===============================================================================\n')
summary.print_(diff)
sys.stdout.write('===============================================================================\n')
_global_collect_info = True
# We'll only really collect the info on the next test (i.e.: if at one test
# we used too much memory, the next one will start collecting)
else:
_global_collect_info = False
except:
pydev_log.exception()
sys.stdout.write(
'''
===============================================================================
Memory after: %s
%s%s
===============================================================================
''' % (
request.function,
format_memory_info(psutil.virtual_memory(), after_curr_proc_memory_info),
'' if not processes_info else '\nLeaked processes:\n' + '\n'.join(processes_info)),
)
def _get_usage(function, *args):
"""Test if more memory is used after the function has been called.
The function will be invoked twice and only the second measurement will be
considered. Thus, memory used in initialisation (e.g. loading modules)
will not be included in the result. The goal is to identify memory leaks
caused by functions which use more and more memory.
Any arguments next to the function will be passed on to the function
on invocation.
Note that this function is currently experimental, because it is not
tested thoroughly and performs poorly.
"""
# The usage of a function is calculated by creating one summary of all
# objects before the function is invoked and afterwards. These summaries
# are compared and the diff is returned.
# This function works in a 2-steps process. Before the actual function is
# invoked an empty dummy function is measurement to identify the overhead
# involved in the measuring process. This overhead then is subtracted from
# the measurement performed on the passed function. The result reflects the
# actual usage of a function call.
# Also, a measurement is performed twice, allowing the adjustment to
# initializing things, e.g. modules
res = None
def _get_summaries(function, *args):
"""Get a 2-tuple containing one summary from before, and one summary
from after the function has been invoked.
"""
s_before = summary.summarize(get_objects())
function(*args)
s_after = summary.summarize(get_objects())
return (s_before, s_after)
def _get_usage(function, *args):
"""Get the usage of a function call.
This function is to be used only internally. The 'real' get_usage
function is a wrapper around _get_usage, but the workload is done
here.
"""
res = []
# init before calling
(s_before, s_after) = _get_summaries(function, *args)
# ignore all objects used for the measurement
ignore = []
if s_before != s_after:
ignore.append(s_before)
for row in s_before:
# ignore refs from summary and frame (loop)
if len(gc.get_referrers(row)) == 2:
ignore.append(row)
for item in row:
# ignore refs from summary and frame (loop)
if len(gc.get_referrers(item)) == 2:
ignore.append(item)
for o in ignore:
s_after = summary._subtract(s_after, o)
res = summary.get_diff(s_before, s_after)
return summary._sweep(res)
# calibrate; twice for initialization
def noop():
pass
offset = _get_usage(noop)
offset = _get_usage(noop)
# perform operation twice to handle objects possibly used in
# initialisation
tmp = _get_usage(function, *args)
tmp = _get_usage(function, *args)
tmp = summary.get_diff(offset, tmp)
tmp = summary._sweep(tmp)
if len(tmp) != 0:
res = tmp
return res
def profile_expose_method(profiled_method_wrapper, accept, args, func, kw,
exclude_from_memory_profiling):
"""
Targeted to profile a specific method that wraps HTTP request processing endpoints into database context.
:param profiled_method_wrapper: method wrapped around profiled call to be passed in to memory profiler
:param accept: param specific to profiled call
:param args: args of a function that is being wrapped by a profiled method
:param func: function that is being wrapped by a profiled method
:param kw: kwargs of a function that is being wrapped by a profiled method
:return: output of a profiled method without modification
"""
if not exclude_from_memory_profiling and get_memory_profile_logging_on() and \
check_memory_profile_package_wide_disable(func):
controller_class = args[0].__class__.__name__ if args and len(
args) > 0 else ''
end_point_name_parts = [
s for s in [func.__module__, controller_class, func.__name__]
if s != ''
]
end_point_name = ".".join(end_point_name_parts)
is_pympler_on = _is_pympler_profiling_value_on(end_point_name)
profile_output = {'output': {}}
if is_pympler_on:
all_objects = muppy.get_objects()
all_objects_summary_before = summary.summarize(all_objects)
memory_profile = memory_usage(
(_profile_me, (profile_output, profiled_method_wrapper, func,
accept, args, kw), {}),
interval=0.1)
output = profile_output['output']
if is_pympler_on:
all_objects_summary_after = summary.summarize(all_objects)
diff = summary.get_diff(all_objects_summary_before,
all_objects_summary_after)
diff_less = summary.format_(diff)
diff_out = ''
for s in diff_less:
diff_out += s + '\n'
thread_log.info(
"================ PYMPLER OUTPUT <{}> ==============\n{}".
format(end_point_name, diff_out))
try:
message = json.dumps({
'log_type':
'memory_profile',
'proc_id':
os.getpid(),
'name':
func.__name__,
'module':
func.__module__,
'mem_profile':
memory_profile,
'min':
min(memory_profile),
'max':
max(memory_profile),
'diff':
max(memory_profile) - min(memory_profile),
'leaked':
memory_profile[-1] - memory_profile[0],
'args': [arg for arg in args[1:]], # exclude self
'kwargs':
kw
})
memory_log.info(message,
extra={
'controller_module': func.__module__,
'controller_class': controller_class,
'endpoint': func.__name__
})
except Exception as e:
thread_log.exception('Logger failed: {}'.format(e))
else:
output = profiled_method_wrapper(accept, args, func, kw)
return output
def process_response(self, request, response):
req = request.META['PATH_INFO']
if not self.is_ignored(req):
if SHOW_ON_DJANGO_DEBUG_TOOLBAR_LOGGIN_PANEL:
logger.info(
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'
'~~~~~~~~~~~~~~')
print '\n\n'
print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'\
'~~~~~~~~~~~~~~~~~~~~~~~~~~'
print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'\
'~~~~~~~~~~~~~~~~~~~~~~~~~~'
print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'\
'~~~~~~~~~~~~~~~~~~~~~~~~~~'
if SHOW_ON_DJANGO_DEBUG_TOOLBAR_LOGGIN_PANEL:
logger.info(u'REQUESTED URL: {}'.format(req))
print u'REQUESTED URL: {}'.format(req)
self.end_objects = muppy.get_objects()
if SHOW['request_summary']:
sum_start = summary.summarize(self.start_objects)
if SHOW_ON_DJANGO_DEBUG_TOOLBAR_LOGGIN_PANEL:
logger.info(
'~~~~~~~~~ SUMMARIZE REQUEST OBJECTS ~~~~~~~~~')
for row in sorted(sum_start,
key=lambda i: i[2],
reverse=True)[:15]:
logger.info(
"type: %60s , # objects: %10d, total size: %s",
*(row[0], row[1], filesizeformat(row[2])))
print '~~~~~~~~~ SUMMARIZE REQUEST OBJECTS ~~~~~~~~~'
summary.print_(sum_start)
if SHOW['response_summary']:
sum_end = summary.summarize(self.end_objects)
if SHOW_ON_DJANGO_DEBUG_TOOLBAR_LOGGIN_PANEL:
logger.info(
'~~~~~~~~~ SUMMARIZE RESPONSE OBJECTS ~~~~~~~~~')
for row in sorted(sum_end,
key=lambda i: i[2],
reverse=True)[:15]:
logger.info(
"type: %60s , # objects: %10d, total size: %s",
*(row[0], row[1], filesizeformat(row[2])))
print '~~~~~~~~~ SUMMARIZE RESPONSE OBJECTS ~~~~~~~~~'
summary.print_(sum_end)
if SHOW['compared_request_response_summaries']:
diff = summary.get_diff(sum_start, sum_end)
if SHOW_ON_DJANGO_DEBUG_TOOLBAR_LOGGIN_PANEL:
logger.info(
'~~~~~~~~~ COMPARED REQUEST & RESPONSE SUMMARIES '
'~~~~~~~~~')
for row in sorted(diff, key=lambda i: i[2],
reverse=True)[:15]:
logger.info(
"type: %60s , # objects: %10d, total size: %s",
*(row[0], row[1], filesizeformat(row[2])))
print \
'~~~~~~~~~ COMPARED REQUEST & RESPONSE SUMMARIES ~~~~~~~~~'
summary.print_(diff)
# print '~~~~~~~~~'
# cb = refbrowser.ConsoleBrowser(
# response, maxdepth=2, str_func=output_function)
# cb.print_tree()
a = asizeof(response)
a_string = 'Total size of response object in kB: %s' % \
str(a/1024.0)
b = asizeof(self.end_objects)
b_string = 'Total size of end_objects in MB: %s' % str(
b / 1048576.0)
c = asizeof(self.start_objects)
c_string = 'Total size of start_objects in MB: %s' % \
str(c/1048576.0)
if SHOW_ON_DJANGO_DEBUG_TOOLBAR_LOGGIN_PANEL:
logger.info(
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'
'~~~~~~~~~~~~~~')
logger.info(a_string)
logger.info(b_string)
logger.info(c_string)
logger.info(
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'
'~~~~~~~~~~~~~~')
print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'\
'~~~~~~~~~~~~~~~~~~~~~~~~~~'
print a_string
print b_string
print c_string
print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'\
'~~~~~~~~~~~~~~~~~~~~~~~~~~'
print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'\
'~~~~~~~~~~~~~~~~~~~~~~~~~~'
print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'\
'~~~~~~~~~~~~~~~~~~~~~~~~~~'
return response