class MemoryPanel(DebugPanel):
name = 'pympler'
has_content = True
template = 'memory_panel.html'
classes = [Context, Template]
def process_request(self, request):
self._tracker = ClassTracker()
for cls in get_models() + self.classes:
self._tracker.track_class(cls, keep=True)
self._tracker.create_snapshot('before')
self._before = ProcessMemoryInfo()
def process_response(self, request, response):
self._after = ProcessMemoryInfo()
self._tracker.create_snapshot('after')
def title(self):
return 'Memory'
def nav_title(self):
return 'Memory'
def nav_subtitle(self):
rss = self._after.rss
delta = rss - self._before.rss
delta = ('(+%s)' % pp(delta)) if delta > 0 else ''
return "RSS: %s %s" % (pp(rss), delta)
def url(self):
return ''
def get_stats(self):
pass
def content(self):
stats = self._tracker.stats
stats.annotate()
context = self.context.copy()
rows = [
('Resident set size', self._after.rss),
('Virtual size', self._after.vsz),
]
rows.extend(self._after - self._before)
rows = [(key, pp(value)) for key, value in rows]
rows.extend(self._after.os_specific)
classes = []
snapshot = stats.snapshots[-1]
for model in stats.tracked_classes:
cnt = snapshot.classes.get(model, {}).get('active', 0)
size = snapshot.classes.get(model, {}).get('sum', 0)
if cnt > 0:
classes.append((model, cnt, pp(size)))
context.update({'rows': rows, 'classes': classes})
return render_to_string(self.template, context)
class Debugger(Evenement):
def __init__(self, jeu, gestionnaire, methode=False):
print("start")
super().__init__(jeu, gestionnaire)
self._debugFait, self._methode = 0, methode
if self._methode is not False:
if self._methode == "Instance":
self._tracker = ClassTracker()
self._tracker.track_class(Evenement)
self._tracker.create_snapshot("Initialisation")
elif self._methode == "Fuites":
self._tracker = pympler.tracker.SummaryTracker()
self._tracker.print_diff()
Horloge.initialiser(id(self), 1, 3000)
def cmp(self, obj):
return str(type(obj))
def traiter(self):
if Horloge.sonner(id(self), 1) is True:
if self._methode == "Instance":
if self._debugFait < 3:
self._tracker.create_snapshot("Etape n°" + str(self._debugFait))
self._debugFait += 1
Horloge.initialiser(id(self), 1, 3000)
else:
self._tracker.stats.print_summary()
elif self._methode == "Fuites":
self._tracker.print_diff()
Horloge.initialiser(id(self), 1, 3000)
class MemoryPanel(DebugPanel):
name = 'pympler'
has_content = True
template = 'memory_panel.html'
classes = [Context, Template]
def process_request(self, request):
self._tracker = ClassTracker()
for cls in get_models() + self.classes:
self._tracker.track_class(cls)
self._tracker.create_snapshot('before')
self._before = ProcessMemoryInfo()
def process_response(self, request, response):
self._after = ProcessMemoryInfo()
self._tracker.create_snapshot('after')
def title(self):
return 'Memory'
def nav_title(self):
return 'Memory'
def nav_subtitle(self):
rss = self._after.rss
delta = rss - self._before.rss
delta = ('(+%s)' % pp(delta)) if delta > 0 else ''
return "%s %s" % (pp(rss), delta)
def url(self):
return ''
def get_stats(self):
pass
def content(self):
stats = self._tracker.stats
stats.annotate()
context = self.context.copy()
rows = [('Resident set size', self._after.rss),
('Virtual size', self._after.vsz),
]
rows.extend(self._after - self._before)
rows = [(key, pp(value)) for key, value in rows]
rows.extend(self._after.os_specific)
classes = []
snapshot = stats.snapshots[-1]
for model in stats.tracked_classes:
history = [cnt for _, cnt in stats.history[model]]
size = snapshot.classes.get(model, {}).get('sum', 0)
if cnt > 0:
classes.append((model, history, pp(size)))
context.update({'rows': rows, 'classes': classes})
return render_to_string(self.template, context)
def main():
tracker = ClassTracker()
tracker.track_class(Point)
tracker.track_class(Line)
tracker.create_snapshot()
points = [get_point(i) for i in xrange(1000000)]
lines = [get_line(i) for i in xrange(100000)]
tracker.create_snapshot()
tracker.stats.print_summary()
def main():
tracker = ClassTracker()
tracker.track_class(Point)
tracker.track_class(Line)
tracker.create_snapshot()
points = [get_point(i) for i in xrange(1000000)]
lines = [get_line(i) for i in xrange(100000)]
tracker.create_snapshot()
tracker.stats.print_summary()
def run(self):
"""
Redirect bottle logging messages so it doesn't clutter the test output
and start the web GUI.
"""
tracker = ClassTracker()
tracker.track_class(Trash)
tracked_trash = Trash()
tracker.create_snapshot()
sys.stdout = StringIO()
sys.stderr = StringIO()
start_profiler(debug=True, quiet=True, tracker=tracker)
def run(self):
"""
Redirect bottle logging messages so it doesn't clutter the test output
and start the web GUI.
"""
tracker = ClassTracker()
tracker.track_class(Trash)
tracked_trash = Trash()
tracker.create_snapshot()
sys.stdout = StringIO()
sys.stderr = StringIO()
start_profiler(debug=True, quiet=True, tracker=tracker)
def profile_memory():
from pympler.classtracker import ClassTracker
from pympler.asizeof import asizeof
from soap.common import Flyweight, _cache_map
from soap.expression import Expr
tracker = ClassTracker()
tracker.track_object(Flyweight._cache)
tracker.track_class(Expr)
yield
tracker.create_snapshot()
tracker.stats.print_summary()
print('Flyweight cache size', asizeof(Flyweight._cache))
print('Global cache size', asizeof(_cache_map))
class TrackObjectTestCase(unittest.TestCase):
def setUp(self):
self.tracker = ClassTracker()
def tearDown(self):
self.tracker.detach_all()
def test_track_object(self):
"""Test object registration.
"""
foo = Foo()
bar = Bar()
self.tracker.track_object(foo)
self.tracker.track_object(bar)
self.assert_(id(foo) in self.tracker.objects)
self.assert_(id(bar) in self.tracker.objects)
self.assert_('Foo' in self.tracker.index)
self.assert_('Bar' in self.tracker.index)
self.assertEqual(self.tracker.objects[id(foo)].ref(),foo)
self.assertEqual(self.tracker.objects[id(bar)].ref(),bar)
def test_type_errors(self):
"""Test intrackable objects.
"""
i = 42
j = 'Foobar'
k = [i,j]
l = {i: j}
self.assertRaises(TypeError, self.tracker.track_object, i)
self.assertRaises(TypeError, self.tracker.track_object, j)
self.assertRaises(TypeError, self.tracker.track_object, k)
self.assertRaises(TypeError, self.tracker.track_object, l)
self.assert_(id(i) not in self.tracker.objects)
self.assert_(id(j) not in self.tracker.objects)
self.assert_(id(k) not in self.tracker.objects)
self.assert_(id(l) not in self.tracker.objects)
def test_track_by_name(self):
"""Test registering objects by name.
"""
foo = Foo()
self.tracker.track_object(foo, name='Foobar')
self.assert_('Foobar' in self.tracker.index )
self.assertEqual(self.tracker.index['Foobar'][0].ref(),foo)
def test_keep(self):
"""Test lifetime of tracked objects.
"""
foo = Foo()
bar = Bar()
self.tracker.track_object(foo, keep=1)
self.tracker.track_object(bar)
idfoo = id(foo)
idbar = id(bar)
del foo
del bar
self.assert_(self.tracker.objects[idfoo].ref() is not None)
self.assert_(self.tracker.objects[idbar].ref() is None)
def test_mixed_tracking(self):
"""Test mixed instance and class tracking.
"""
foo = Foo()
self.tracker.track_object(foo)
self.tracker.create_snapshot()
self.tracker.track_class(Foo)
objs = []
for _ in range(10):
objs.append(Foo())
self.tracker.create_snapshot()
def test_recurse(self):
"""Test recursive sizing and saving of referents.
"""
foo = Foo()
self.tracker.track_object(foo, resolution_level=1)
self.tracker.create_snapshot()
fp = self.tracker.objects[id(foo)].snapshots[-1]
refs = fp[1].refs
dref = [r for r in refs if r.name == '__dict__']
self.assertEqual(len(dref),1)
dref = dref[0]
self.assert_(dref.size > 0, dref.size)
self.assert_(dref.flat > 0, dref.flat)
self.assertEqual(dref.refs,())
# Test track_change and more fine-grained resolution
self.tracker.track_change(foo, resolution_level=2)
self.tracker.create_snapshot()
fp = self.tracker.objects[id(foo)].snapshots[-1]
refs = fp[1].refs
dref = [r for r in refs if r.name == '__dict__']
self.assertEqual(len(dref),1)
dref = dref[0]
namerefs = [r.name for r in dref.refs]
self.assert_('[K] foo' in namerefs, namerefs)
self.assert_("[V] foo: 'foo'" in namerefs, namerefs)
class SnapshotTestCase(unittest.TestCase):
def setUp(self):
self.tracker = ClassTracker()
def tearDown(self):
self.tracker.stop_periodic_snapshots()
self.tracker.clear()
def test_timestamp(self):
"""Test timestamp of snapshots.
"""
foo = Foo()
bar = Bar()
self.tracker.track_object(foo)
self.tracker.track_object(bar)
self.tracker.create_snapshot()
self.tracker.create_snapshot()
self.tracker.create_snapshot()
refts = [fp.timestamp for fp in self.tracker.snapshots]
for to in self.tracker.objects.values():
ts = [t for (t,sz) in to.snapshots[1:]]
self.assertEqual(ts,refts)
def test_snapshot_members(self):
"""Test existence and value of snapshot members.
"""
foo = Foo()
self.tracker.track_object(foo)
self.tracker.create_snapshot()
self.tracker.create_snapshot(compute_total=True)
fp = self.tracker.snapshots[0]
fp_with_total = self.tracker.snapshots[1]
self.assert_(fp.overhead > 0, fp.overhead)
self.assert_(fp.tracked_total > 0, fp.tracked_total)
self.assertEqual(fp.asizeof_total, 0)
self.assert_(fp_with_total.asizeof_total > 0, fp_with_total.asizeof_total)
self.assert_(fp_with_total.asizeof_total >= fp_with_total.tracked_total)
if pympler.process.is_available():
procmem = fp.system_total
self.assertEqual(fp.total, procmem.vsz)
self.assert_(procmem.vsz > 0, procmem)
self.assert_(procmem.rss > 0, procmem)
self.assertTrue(procmem.vsz >= procmem.rss, procmem)
self.assert_(procmem.vsz > fp.overhead, procmem)
self.assert_(procmem.vsz > fp.tracked_total, procmem)
self.assert_(fp_with_total.system_total.vsz > fp_with_total.asizeof_total)
else:
self.assertEqual(fp_with_total.total, fp_with_total.asizeof_total)
self.assertEqual(fp.total, fp.tracked_total)
def test_desc(self):
"""Test snapshot label.
"""
self.tracker.create_snapshot()
self.tracker.create_snapshot('alpha')
self.tracker.create_snapshot(description='beta')
self.tracker.create_snapshot(42)
self.assertEqual(len(self.tracker.snapshots), 4)
self.assertEqual(self.tracker.snapshots[0].desc, '')
self.assertEqual(self.tracker.snapshots[1].desc, 'alpha')
self.assertEqual(self.tracker.snapshots[2].desc, 'beta')
self.assertEqual(self.tracker.snapshots[3].desc, '42')
snapshot = self.tracker.snapshots[0]
self.assertEqual(snapshot.label, '%.3fs' % snapshot.timestamp)
snapshot = self.tracker.snapshots[1]
self.assertEqual(snapshot.label, 'alpha (%.3fs)' % snapshot.timestamp)
snapshot = self.tracker.snapshots[3]
self.assertEqual(snapshot.label, '42 (%.3fs)' % snapshot.timestamp)
def test_background_monitoring(self):
"""Test background monitoring.
"""
self.tracker.start_periodic_snapshots(0.1)
self.assertEqual(self.tracker._periodic_thread.interval, 0.1)
self.assertEqual(self.tracker._periodic_thread.getName(), 'BackgroundMonitor')
for x in range(10): # try to interfere
self.tracker.create_snapshot(str(x))
time.sleep(0.5)
self.tracker.start_periodic_snapshots(0.2)
self.assertEqual(self.tracker._periodic_thread.interval, 0.2)
self.tracker.stop_periodic_snapshots()
self.assert_(self.tracker._periodic_thread is None)
self.assert_(len(self.tracker.snapshots) > 10)
class TrackObjectTestCase(unittest.TestCase):
def setUp(self):
self.tracker = ClassTracker()
def tearDown(self):
self.tracker.detach_all()
def test_track_object(self):
"""Test object registration.
"""
foo = Foo()
bar = Bar()
self.tracker.track_object(foo)
self.tracker.track_object(bar)
self.assertTrue(id(foo) in self.tracker.objects)
self.assertTrue(id(bar) in self.tracker.objects)
self.assertTrue('Foo' in self.tracker.index)
self.assertTrue('Bar' in self.tracker.index)
self.assertEqual(self.tracker.objects[id(foo)].ref(), foo)
self.assertEqual(self.tracker.objects[id(bar)].ref(), bar)
def test_type_errors(self):
"""Test intrackable objects.
"""
i = 42
j = 'Foobar'
k = [i, j]
l = {i: j}
self.assertRaises(TypeError, self.tracker.track_object, i)
self.assertRaises(TypeError, self.tracker.track_object, j)
self.assertRaises(TypeError, self.tracker.track_object, k)
self.assertRaises(TypeError, self.tracker.track_object, l)
self.assertTrue(id(i) not in self.tracker.objects)
self.assertTrue(id(j) not in self.tracker.objects)
self.assertTrue(id(k) not in self.tracker.objects)
self.assertTrue(id(l) not in self.tracker.objects)
def test_track_by_name(self):
"""Test registering objects by name.
"""
foo = Foo()
self.tracker.track_object(foo, name='Foobar')
self.assertTrue('Foobar' in self.tracker.index)
self.assertEqual(self.tracker.index['Foobar'][0].ref(), foo)
def test_keep(self):
"""Test lifetime of tracked objects.
"""
foo = Foo()
bar = Bar()
self.tracker.track_object(foo, keep=1)
self.tracker.track_object(bar)
idfoo = id(foo)
idbar = id(bar)
del foo
del bar
self.assertTrue(self.tracker.objects[idfoo].ref() is not None)
self.assertTrue(self.tracker.objects[idbar].ref() is None)
def test_mixed_tracking(self):
"""Test mixed instance and class tracking.
"""
foo = Foo()
self.tracker.track_object(foo)
self.tracker.create_snapshot()
self.tracker.track_class(Foo)
objs = []
for _ in range(10):
objs.append(Foo())
self.tracker.create_snapshot()
def test_recurse(self):
"""Test recursive sizing and saving of referents.
"""
foo = Foo()
self.tracker.track_object(foo, resolution_level=1)
self.tracker.create_snapshot()
fp = self.tracker.objects[id(foo)].snapshots[-1]
refs = fp[1].refs
dref = [r for r in refs if r.name == '__dict__']
self.assertEqual(len(dref), 1)
dref = dref[0]
self.assertTrue(dref.size > 0, dref.size)
self.assertTrue(dref.flat > 0, dref.flat)
self.assertEqual(dref.refs, ())
# Test track_change and more fine-grained resolution
self.tracker.track_change(foo, resolution_level=2)
self.tracker.create_snapshot()
fp = self.tracker.objects[id(foo)].snapshots[-1]
refs = fp[1].refs
dref = [r for r in refs if r.name == '__dict__']
self.assertEqual(len(dref), 1)
dref = dref[0]
namerefs = [r.name for r in dref.refs]
self.assertTrue('[K] foo' in namerefs, namerefs)
self.assertTrue("[V] foo: 'foo'" in namerefs, namerefs)
class SnapshotTestCase(unittest.TestCase):
def setUp(self):
self.tracker = ClassTracker()
def tearDown(self):
self.tracker.stop_periodic_snapshots()
self.tracker.clear()
def test_timestamp(self):
"""Test timestamp of snapshots.
"""
foo = Foo()
bar = Bar()
self.tracker.track_object(foo)
self.tracker.track_object(bar)
self.tracker.create_snapshot()
self.tracker.create_snapshot()
self.tracker.create_snapshot()
refts = [fp.timestamp for fp in self.tracker.snapshots]
for to in self.tracker.objects.values():
ts = [t for (t, sz) in to.snapshots[1:]]
self.assertEqual(ts, refts)
def test_snapshot_members(self):
"""Test existence and value of snapshot members.
"""
foo = Foo()
self.tracker.track_object(foo)
self.tracker.create_snapshot()
self.tracker.create_snapshot(compute_total=True)
fp = self.tracker.snapshots[0]
fp_with_total = self.tracker.snapshots[1]
self.assertTrue(fp.overhead > 0, fp.overhead)
self.assertTrue(fp.tracked_total > 0, fp.tracked_total)
self.assertEqual(fp.asizeof_total, 0)
self.assertTrue(fp_with_total.asizeof_total > 0,
fp_with_total.asizeof_total)
self.assertTrue(
fp_with_total.asizeof_total >= fp_with_total.tracked_total)
if pympler.process.is_available():
procmem = fp.system_total
self.assertEqual(fp.total, procmem.vsz)
self.assertTrue(procmem.vsz > 0, procmem)
self.assertTrue(procmem.rss > 0, procmem)
self.assertTrue(procmem.vsz >= procmem.rss, procmem)
self.assertTrue(procmem.vsz > fp.overhead, procmem)
self.assertTrue(procmem.vsz > fp.tracked_total, procmem)
self.assertTrue(
fp_with_total.system_total.vsz > fp_with_total.asizeof_total)
else:
self.assertEqual(fp_with_total.total, fp_with_total.asizeof_total)
self.assertEqual(fp.total, fp.tracked_total)
def test_desc(self):
"""Test snapshot label.
"""
self.tracker.create_snapshot()
self.tracker.create_snapshot('alpha')
self.tracker.create_snapshot(description='beta')
self.tracker.create_snapshot(42)
self.assertEqual(len(self.tracker.snapshots), 4)
self.assertEqual(self.tracker.snapshots[0].desc, '')
self.assertEqual(self.tracker.snapshots[1].desc, 'alpha')
self.assertEqual(self.tracker.snapshots[2].desc, 'beta')
self.assertEqual(self.tracker.snapshots[3].desc, '42')
snapshot = self.tracker.snapshots[0]
self.assertEqual(snapshot.label, '%.3fs' % snapshot.timestamp)
snapshot = self.tracker.snapshots[1]
self.assertEqual(snapshot.label, 'alpha (%.3fs)' % snapshot.timestamp)
snapshot = self.tracker.snapshots[3]
self.assertEqual(snapshot.label, '42 (%.3fs)' % snapshot.timestamp)
def test_background_monitoring(self):
"""Test background monitoring.
"""
self.tracker.start_periodic_snapshots(0.1)
self.assertEqual(self.tracker._periodic_thread.interval, 0.1)
self.assertEqual(self.tracker._periodic_thread.getName(),
'BackgroundMonitor')
for x in range(10): # try to interfere
self.tracker.create_snapshot(str(x))
time.sleep(0.5)
self.tracker.start_periodic_snapshots(0.2)
self.assertEqual(self.tracker._periodic_thread.interval, 0.2)
self.tracker.stop_periodic_snapshots()
self.assertTrue(self.tracker._periodic_thread is None)
self.assertTrue(len(self.tracker.snapshots) > 10)
self.mutable = list(range(10))
self.immutable = tuple(range(10))
def create_data(tracker, iterations=20, obj_per_iteration=100):
objects = []
for x in range(iterations):
for y in range(obj_per_iteration):
objects.append(Alpha())
objects.append(Beta())
objects.append(Gamma())
tracker.create_snapshot()
return objects
tracker = ClassTracker()
tracker.track_class(Alpha)
tracker.track_class(Beta)
tracker.track_class(Gamma, trace=True, resolution_level=2)
print ("Create data")
tracker.create_snapshot()
data = create_data(tracker)
print ("Drop data")
del data
tracker.create_snapshot()
web.start_profiler(debug=True, stats=tracker.stats)
class MemoryPanel(Panel):
name = 'pympler'
title = 'Memory'
template = 'memory_panel.html'
classes = [Context, Template]
def process_request(self, request):
self._tracker = ClassTracker()
for cls in apps.get_models() + self.classes:
self._tracker.track_class(cls)
self._tracker.create_snapshot('before')
self.record_stats({'before': ProcessMemoryInfo()})
def process_response(self, request, response):
self.record_stats({'after': ProcessMemoryInfo()})
self._tracker.create_snapshot('after')
stats = self._tracker.stats
stats.annotate()
self.record_stats({'stats': stats})
def enable_instrumentation(self):
self._tracker = ClassTracker()
for cls in apps.get_models() + self.classes:
self._tracker.track_class(cls)
def disable_instrumentation(self):
self._tracker.detach_all_classes()
def nav_subtitle(self):
context = self.get_stats()
before = context['before']
after = context['after']
rss = after.rss
delta = rss - before.rss
delta = ('(+%s)' % pp(delta)) if delta > 0 else ''
return "%s %s" % (pp(rss), delta)
@property
def content(self):
context = self.get_stats()
before = context['before']
after = context['after']
stats = context['stats']
rows = [
('Resident set size', after.rss),
('Virtual size', after.vsz),
]
rows.extend(after - before)
rows = [(key, pp(value)) for key, value in rows]
rows.extend(after.os_specific)
classes = []
snapshot = stats.snapshots[-1]
for model in stats.tracked_classes:
history = [cnt for _, cnt in stats.history[model]]
size = snapshot.classes.get(model, {}).get('sum', 0)
if history and history[-1] > 0:
classes.append((model, history, pp(size)))
context.update({'rows': rows, 'classes': classes})
return render_to_string(self.template, context)
class LogTestCase(unittest.TestCase):
def setUp(self):
self.out = StringIO()
self.tracker = ClassTracker(stream=self.out)
@property
def output(self):
"""Return output recorded in `ClassTracker` output stream."""
return self.out.getvalue()
def tearDown(self):
self.tracker.stop_periodic_snapshots()
self.tracker.clear()
def test_dump(self):
"""Test serialization of log data.
"""
foo = Foo()
foo.data = range(1000)
bar = Bar()
self.tracker.track_object(foo, resolution_level=4)
self.tracker.track_object(bar)
self.tracker.create_snapshot('Footest')
f1 = StringIO()
f2 = StringIO()
ConsoleStats(tracker=self.tracker, stream=f1).print_stats()
tmp = BytesIO()
Stats(tracker=self.tracker).dump_stats(tmp, close=False)
self.tracker.clear()
stats = ConsoleStats(stream=f2)
self.assertEqual(stats.index, None)
self.assertEqual(stats.snapshots, None)
tmp.seek(0)
stats.load_stats(tmp)
tmp.close()
self.assert_('Foo' in stats.index)
stats.print_stats()
self.assertEqual(f1.getvalue(), f2.getvalue())
# Test partial printing
stats.stream = f3 = StringIO()
stats.sort_stats()
tolen = len(stats.sorted)
stats.print_stats(clsname='Bar')
self.assertEqual(len(stats.sorted), tolen)
stats.print_summary()
clsname = f3.getvalue().split('\n')[0]
self.assertNotEqual(re.search('Bar', clsname), None, clsname)
f1.close()
f2.close()
f3.close()
def test_sort_stats(self):
"""Test sort_stats and reverse_order.
"""
self.tracker.track_class(Bar, name='Bar')
foo = Foo()
foo.data = list(range(1000))
bar1 = Bar()
bar2 = Bar()
self.tracker.track_object(foo, resolution_level=4)
self.tracker.create_snapshot()
stats = self.tracker.stats
# Test sort_stats and reverse_order
self.assertEqual(stats.sort_stats('size'), stats)
self.assertEqual(stats.sorted[0].classname, 'Foo')
stats.reverse_order()
self.assertEqual(stats.sorted[0].classname, 'Bar')
stats.sort_stats('classname', 'birth')
self.assertEqual(stats.sorted[0].classname, 'Bar')
self.assertRaises(ValueError, stats.sort_stats, 'name', 42, 'classn')
stats.sort_stats('classname')
def test_dump_load_with_filename(self):
"""Test serialization with filename.
"""
foo = Foo()
self.tracker.track_object(foo, resolution_level=2)
self.tracker.create_snapshot()
fhandle, fname = mkstemp(prefix='pympler_test_dump')
os.close(fhandle)
try:
self.tracker.stats.dump_stats(fname)
output = StringIO()
stats = ConsoleStats(filename=fname, stream=output)
stats.print_stats()
self.assertTrue('<Foo>' in output.getvalue(), output.getvalue())
# Check if a Stats loaded from a dump can be dumped again
stats.dump_stats(fname)
finally:
os.unlink(fname)
def test_tracked_classes(self):
"""Test listing tracked classes.
"""
self.tracker.track_class(Foo, name='Foo')
self.tracker.track_class(Bar, name='Bar')
foo = Foo()
self.tracker.create_snapshot()
bar = Bar()
self.tracker.create_snapshot()
foo = FooNew()
self.tracker.track_object(foo)
self.tracker.create_snapshot()
stats = self.tracker.stats
self.assertEqual(stats.tracked_classes, ['Bar', 'Foo', 'FooNew'])
stats.print_summary()
def test_print_stats(self):
"""Test printing class-filtered statistics.
"""
self.tracker.track_class(Foo, name='Foo', trace=True)
self.tracker.track_class(Bar, name='Bar')
foo = Foo()
bar = Bar()
self.tracker.create_snapshot()
stats = self.tracker.stats
stats.print_stats(clsname='Foo')
self.assertTrue('Foo' in self.output, self.output)
self.assertFalse('Bar' in self.output, self.output)
self.assertTrue('foo = Foo()' in self.output, self.output)
def test_print_stats_limit(self):
"""Test printing limited statistics.
"""
self.tracker.track_class(Foo, name='Foo')
foo = [Foo() for _ in range(10)]
self.tracker.create_snapshot()
stats = self.tracker.stats
stats.print_stats(limit=3)
self.assertEqual(self.output.count('<Foo>'), 3)
self.out.seek(0)
self.out.truncate()
stats.print_stats(limit=0.5)
self.assertEqual(self.output.count('<Foo>'), 5)
def test_snapshots(self):
"""Test multiple snapshots.
"""
self.tracker.track_class(Foo, name='Foo')
self.tracker.track_class(Bar, name='Bar')
self.tracker.track_class(FooNew, name='FooNew')
self.tracker.create_snapshot()
f1 = Foo()
self.tracker.create_snapshot()
f2 = Foo()
f3 = FooNew()
self.tracker.create_snapshot()
b = Bar()
del b
self.tracker.create_snapshot()
stats = self.tracker.stats
stats.print_stats()
stats.print_summary()
def test_merge(self):
"""Test merging of reference trees.
"""
self.tracker.track_class(FooNew, name='Foo', resolution_level=2)
f1 = FooNew()
f1.a = list(range(1000))
f2 = FooNew()
f2.a = list(range(100))
f2.b = 'This is some stupid spam.'
self.tracker.create_snapshot('Merge test')
sizer = Asizer()
sz1 = sizer.asized(f1)
sz2 = sizer.asized(f2)
stats = self.tracker.stats
for fp in stats.snapshots:
if fp.desc == 'Merge test':
stats.annotate_snapshot(fp)
self.assert_(hasattr(fp, 'classes'))
classes = fp.classes
stats.annotate_snapshot(fp)
self.assertEqual(fp.classes, classes)
self.assert_('Foo' in fp.classes, fp.classes)
self.assert_('merged' in fp.classes['Foo'])
fm = fp.classes['Foo']['merged']
self.assertEqual(fm.size, sz1.size + sz2.size, (fm.size, str(sz1), str(sz2)))
refs = {}
for ref in fm.refs:
refs[ref.name] = ref
self.assert_('__dict__' in refs.keys(), refs.keys())
refs2 = {}
for ref in refs['__dict__'].refs:
refs2[ref.name] = ref
self.assert_('[V] a' in refs2.keys(), refs2.keys())
self.assert_('[V] b' in refs2.keys(), refs2.keys())
self.assertEqual(refs2['[V] a'].size, asizeof(f1.a, f2.a))
def test_html(self):
"""Test emitting HTML statistics."""
self.tracker.track_class(Foo, name='Foo', resolution_level=2)
self.tracker.track_class(Bar, name='Bar', trace=True)
f1 = Foo()
f1.a = list(range(100000))
f2 = Foo()
f2.a = list(range(1000))
f2.b = 'This is some stupid spam.'
f1 = Bar()
self.tracker.create_snapshot('Merge test')
stats = HtmlStats(tracker=self.tracker)
try:
target = mkdtemp(prefix='pympler_test')
output = os.path.join(target, 'footest.html')
stats.create_html(output)
source = open(output).read()
# Ensure relative links are used
fname = os.path.join('footest_files', 'Foo.html')
self.assertTrue('<a href="%s">' % fname in source, (fname, source))
finally:
rmtree(target)
def test_charts(self):
"""Test emitting graphic charts."""
self.tracker.track_class(Foo, name='Foo', resolution_level=2)
f1 = Foo()
f1.a = list(range(1000))
f2 = Foo()
f2.a = list(range(100))
f2.b = 'This is some stupid spam.'
self.tracker.create_snapshot('Merge test')
from pympler import charts
try:
target = mkdtemp(prefix='pympler_test')
output = os.path.join(target, 'timespace.png')
charts.tracker_timespace(output, self.tracker.stats)
finally:
rmtree(target)
class MemoryPanel(Panel):
name = 'pympler'
title = 'Memory'
template = 'memory_panel.html'
classes = [Context, Template]
def process_request(self, request):
self._tracker = ClassTracker()
for cls in get_models() + self.classes:
self._tracker.track_class(cls)
self._tracker.create_snapshot('before')
self.record_stats({'before': ProcessMemoryInfo()})
def process_response(self, request, response):
self.record_stats({'after': ProcessMemoryInfo()})
self._tracker.create_snapshot('after')
stats = self._tracker.stats
stats.annotate()
self.record_stats({'stats': stats})
def enable_instrumentation(self):
self._tracker = ClassTracker()
for cls in get_models() + self.classes:
self._tracker.track_class(cls)
def disable_instrumentation(self):
self._tracker.detach_all_classes()
def nav_subtitle(self):
context = self.get_stats()
before = context['before']
after = context['after']
rss = after.rss
delta = rss - before.rss
delta = ('(+%s)' % pp(delta)) if delta > 0 else ''
return "%s %s" % (pp(rss), delta)
@property
def content(self):
context = self.get_stats()
before = context['before']
after = context['after']
stats = context['stats']
rows = [('Resident set size', after.rss),
('Virtual size', after.vsz),
]
rows.extend(after - before)
rows = [(key, pp(value)) for key, value in rows]
rows.extend(after.os_specific)
classes = []
snapshot = stats.snapshots[-1]
for model in stats.tracked_classes:
history = [cnt for _, cnt in stats.history[model]]
size = snapshot.classes.get(model, {}).get('sum', 0)
if cnt > 0:
classes.append((model, history, pp(size)))
context.update({'rows': rows, 'classes': classes})
return render_to_string(self.template, context)
def track_memory_wrapper(*args, **kwargs):
memory_info = {}
tracker = ClassTracker()
for cls in apps.get_models() + [Context, Template]:
# track all models from registered apps, plus some standard Django ones
tracker.track_class(cls)
try:
tracker.create_snapshot("before")
memory_info["before"] = ProcessMemoryInfo()
result = fn(*args, **kwargs)
memory_info["after"] = ProcessMemoryInfo()
tracker.create_snapshot("after")
memory_info["stats"] = tracker.stats
memory_info["stats"].annotate()
return result
finally:
# record a whole bunch of memory statistics...
resources = [
("resident set size", memory_info["after"].rss),
("virtual size", memory_info["after"].vsz),
]
resources.extend(memory_info["after"] - memory_info["before"])
resources = [(k, pp(v)) for k, v in resources]
resources.extend(memory_info["after"].os_specific)
# record each tracked class as of the final snapshot...
classes_stats = []
snapshot = memory_info["stats"].snapshots[-1]
for class_name in memory_info["stats"].tracked_classes:
# history is a list of tuples that is updated on every creation/deletions: (timestamp, n_instances)
history = [
n for _, n in memory_info["stats"].history[class_name]
]
if history:
classes_stats.append({
"name":
class_name,
"n_instances":
len(history),
"min_instances":
min(history),
"max_instances":
max(history),
"size":
pp(
snapshot.classes.get(class_name,
{}).get("sum", 0)),
})
if not path:
stream = sys.stdout
else:
stream = open(path, "w")
print("\nRESOURCES", file=stream)
for k, v in resources:
print(f"{k:<26}: {v:>10}", file=stream)
print("\nCLASSES", file=stream)
for class_stats in classes_stats:
print(
"{name}: created/deleted {n_instances} times for a min/max of {min_instances}/{max_instances} instances: {size:>10}"
.format(**class_stats),
file=stream,
)
stream.closed
tracker.detach_all_classes()
class LogTestCase(unittest.TestCase):
def setUp(self):
self.out = StringIO()
self.tracker = ClassTracker(stream=self.out)
@property
def output(self):
"""Return output recorded in `ClassTracker` output stream."""
return self.out.getvalue()
def tearDown(self):
self.tracker.stop_periodic_snapshots()
self.tracker.clear()
def test_dump(self):
"""Test serialization of log data.
"""
foo = Foo()
foo.data = range(1000)
bar = Bar()
self.tracker.track_object(foo, resolution_level=4)
self.tracker.track_object(bar)
self.tracker.create_snapshot('Footest')
f1 = StringIO()
f2 = StringIO()
ConsoleStats(tracker=self.tracker, stream=f1).print_stats()
tmp = BytesIO()
Stats(tracker=self.tracker).dump_stats(tmp, close=False)
self.tracker.clear()
stats = ConsoleStats(stream=f2)
self.assertEqual(stats.index, None)
self.assertEqual(stats.snapshots, None)
tmp.seek(0)
stats.load_stats(tmp)
tmp.close()
self.assert_('Foo' in stats.index)
stats.print_stats()
self.assertEqual(f1.getvalue(), f2.getvalue())
# Test partial printing
stats.stream = f3 = StringIO()
stats.sort_stats()
tolen = len(stats.sorted)
stats.print_stats(clsname='Bar')
self.assertEqual(len(stats.sorted), tolen)
stats.print_summary()
clsname = f3.getvalue().split('\n')[0]
self.assertNotEqual(re.search('\.Bar', clsname), None, clsname)
f1.close()
f2.close()
f3.close()
def test_sort_stats(self):
"""Test sort_stats and reverse_order.
"""
self.tracker.track_class(Bar, name='Bar')
foo = Foo()
foo.data = list(range(1000))
bar1 = Bar()
bar2 = Bar()
self.tracker.track_object(foo, resolution_level=4)
self.tracker.create_snapshot()
stats = self.tracker.stats
# Test sort_stats and reverse_order
self.assertEqual(stats.sort_stats('size'), stats)
self.assertEqual(stats.sorted[0].classname, 'Foo')
stats.reverse_order()
self.assertEqual(stats.sorted[0].classname, 'Bar')
stats.sort_stats('classname', 'birth')
self.assertEqual(stats.sorted[0].classname, 'Bar')
self.assertRaises(ValueError, stats.sort_stats, 'name', 42, 'classn')
stats.sort_stats('classname')
def test_dump_load_with_filename(self):
"""Test serialization with filename.
"""
foo = Foo()
self.tracker.track_object(foo, resolution_level=2)
self.tracker.create_snapshot()
fhandle, fname = mkstemp(prefix='pympler_test_dump')
os.close(fhandle)
try:
self.tracker.stats.dump_stats(fname)
output = StringIO()
stats = ConsoleStats(filename=fname, stream=output)
stats.print_stats()
self.assertTrue('<Foo>' in output.getvalue(), output.getvalue())
# Check if a Stats loaded from a dump can be dumped again
stats.dump_stats(fname)
finally:
os.unlink(fname)
def test_tracked_classes(self):
"""Test listing tracked classes.
"""
self.tracker.track_class(Foo, name='Foo')
self.tracker.track_class(Bar, name='Bar')
foo = Foo()
self.tracker.create_snapshot()
bar = Bar()
self.tracker.create_snapshot()
foo = FooNew()
self.tracker.track_object(foo)
self.tracker.create_snapshot()
stats = self.tracker.stats
self.assertEqual(stats.tracked_classes, ['Bar', 'Foo', 'FooNew'])
stats.print_summary()
def test_print_stats(self):
"""Test printing class-filtered statistics.
"""
self.tracker.track_class(Foo, name='Foo', trace=True)
self.tracker.track_class(Bar, name='Bar')
foo = Foo()
bar = Bar()
self.tracker.create_snapshot()
stats = self.tracker.stats
stats.print_stats(clsname='Foo')
self.assertTrue('Foo' in self.output, self.output)
self.assertFalse('Bar' in self.output, self.output)
self.assertTrue('foo = Foo()' in self.output, self.output)
def test_print_stats_limit(self):
"""Test printing limited statistics.
"""
self.tracker.track_class(Foo, name='Foo')
foo = [Foo() for _ in range(10)]
self.tracker.create_snapshot()
stats = self.tracker.stats
stats.print_stats(limit=3)
self.assertEqual(self.output.count('<Foo>'), 3)
self.out.seek(0)
self.out.truncate()
stats.print_stats(limit=0.5)
self.assertEqual(self.output.count('<Foo>'), 5)
def test_snapshots(self):
"""Test multiple snapshots.
"""
self.tracker.track_class(Foo, name='Foo')
self.tracker.track_class(Bar, name='Bar')
self.tracker.track_class(FooNew, name='FooNew')
self.tracker.create_snapshot()
f1 = Foo()
self.tracker.create_snapshot()
f2 = Foo()
f3 = FooNew()
self.tracker.create_snapshot()
b = Bar()
del b
self.tracker.create_snapshot()
stats = self.tracker.stats
stats.print_stats()
stats.print_summary()
def test_merge(self):
"""Test merging of reference trees.
"""
self.tracker.track_class(FooNew, name='Foo', resolution_level=2)
f1 = FooNew()
f1.a = list(range(1000))
f2 = FooNew()
f2.a = list(range(100))
f2.b = 'This is some stupid spam.'
self.tracker.create_snapshot('Merge test')
sizer = Asizer()
sz1 = sizer.asized(f1)
sz2 = sizer.asized(f2)
stats = self.tracker.stats
for fp in stats.snapshots:
if fp.desc == 'Merge test':
stats.annotate_snapshot(fp)
self.assert_(hasattr(fp, 'classes'))
classes = fp.classes
stats.annotate_snapshot(fp)
self.assertEqual(fp.classes, classes)
self.assert_('Foo' in fp.classes, fp.classes)
self.assert_('merged' in fp.classes['Foo'])
fm = fp.classes['Foo']['merged']
self.assertEqual(fm.size, sz1.size + sz2.size,
(fm.size, str(sz1), str(sz2)))
refs = {}
for ref in fm.refs:
refs[ref.name] = ref
self.assert_('__dict__' in refs.keys(), refs.keys())
refs2 = {}
for ref in refs['__dict__'].refs:
refs2[ref.name] = ref
self.assert_('[V] a' in refs2.keys(), refs2.keys())
self.assert_('[V] b' in refs2.keys(), refs2.keys())
self.assertEqual(refs2['[V] a'].size, asizeof(f1.a, f2.a))
def test_html(self):
"""Test emitting HTML statistics."""
self.tracker.track_class(Foo, name='Foo', resolution_level=2)
self.tracker.track_class(Bar, name='Bar', trace=True)
f1 = Foo()
f1.a = list(range(100000))
f2 = Foo()
f2.a = list(range(1000))
f2.b = 'This is some stupid spam.'
f1 = Bar()
self.tracker.create_snapshot('Merge test')
stats = HtmlStats(tracker=self.tracker)
try:
target = mkdtemp(prefix='pympler_test')
output = os.path.join(target, 'footest.html')
stats.create_html(output)
source = open(output).read()
# Ensure relative links are used
fname = os.path.join('footest_files', 'Foo.html')
self.assertTrue('<a href="%s">' % fname in source, (fname, source))
finally:
rmtree(target)
def test_charts(self):
"""Test emitting graphic charts."""
self.tracker.track_class(Foo, name='Foo', resolution_level=2)
f1 = Foo()
f1.a = list(range(1000))
f2 = Foo()
f2.a = list(range(100))
f2.b = 'This is some stupid spam.'
self.tracker.create_snapshot('Merge test')
from pympler import charts
try:
target = mkdtemp(prefix='pympler_test')
output = os.path.join(target, 'timespace.png')
charts.tracker_timespace(output, self.tracker.stats)
finally:
rmtree(target)
wplist.append(wp)
objects.append(wplist)
# tracker.create_snapshot()
return objects
tracker = ClassTracker()
tracker.track_class(Waypoint)
tracker.track_class(WaypointList)
#tracker.track_class(Gamma, trace=True, resolution_level=2)
print("Create data")
tracker.create_snapshot()
data = create_data(tracker)
print(data)
print("Drop data")
for item in data:
del item
del data
tracker.create_snapshot()
print("Create tenfold data")
tracker.create_snapshot()
data = create_data(tracker, iterations=1000, obj_per_iteration=100)
print("Drop data")
for item in data:
del item
del data
class MemoryDebugPanel(DebugPanel):
"""
Sample debug panel
"""
name = 'memory'
has_content = True
template = 'pyramid_debugtoolbar_memory.panels:templates/memory.dbtmako'
title = _('Memory')
nav_title = title
def __init__(self, request):
self._tracker = ClassTracker()
self._tracker.track_class(Request)
self._tracker.track_class(Response)
def wrap_handler(self, handler):
handler = self._wrap_profile_handler(handler)
return handler
def _wrap_profile_handler(self, handler):
def profiler_handler(req):
self._tracker.create_snapshot('before')
before = ProcessMemoryInfo()
req_before = get_memory_object(req.registry)
try:
result = handler(req)
except:
raise
finally:
after = ProcessMemoryInfo()
self._tracker.create_snapshot('after')
class_stats = self._tracker.stats
class_stats.annotate()
self.stats = dict(
before=before,
after=after,
class_stats=class_stats,
req_before=req_before,
req_after=get_memory_object(req.registry),
)
return result
return profiler_handler
def render_vars(self, req):
before = self.stats['before']
after = self.stats['after']
class_stats = self.stats['class_stats']
rows = [
('Resident set size', after.rss),
('Virtual size', after.vsz),
]
rows.extend(after - before)
rows = [(key, pp(value)) for key, value in rows]
rows.extend(after.os_specific)
classes = []
snapshot = class_stats.snapshots[-1]
for model in class_stats.tracked_classes:
history = [cnt for _, cnt in class_stats.history[model]]
size = snapshot.classes.get(model, {}).get('sum', 0)
if history and history[-1] > 0:
classes.append((model, history, pp(size)))
return {
'rows': rows,
'classes': classes,
'process_history': process_history,
'req_before': self.stats['req_before'],
'req_after': self.stats['req_after'],
}
