class MemoryPanel(DebugPanel): name = 'pympler' has_content = True template = 'memory_panel.html' classes = [Context, Template] def process_request(self, request): self._tracker = ClassTracker() for cls in get_models() + self.classes: self._tracker.track_class(cls, keep=True) self._tracker.create_snapshot('before') self._before = ProcessMemoryInfo() def process_response(self, request, response): self._after = ProcessMemoryInfo() self._tracker.create_snapshot('after') def title(self): return 'Memory' def nav_title(self): return 'Memory' def nav_subtitle(self): rss = self._after.rss delta = rss - self._before.rss delta = ('(+%s)' % pp(delta)) if delta > 0 else '' return "RSS: %s %s" % (pp(rss), delta) def url(self): return '' def get_stats(self): pass def content(self): stats = self._tracker.stats stats.annotate() context = self.context.copy() rows = [ ('Resident set size', self._after.rss), ('Virtual size', self._after.vsz), ] rows.extend(self._after - self._before) rows = [(key, pp(value)) for key, value in rows] rows.extend(self._after.os_specific) classes = [] snapshot = stats.snapshots[-1] for model in stats.tracked_classes: cnt = snapshot.classes.get(model, {}).get('active', 0) size = snapshot.classes.get(model, {}).get('sum', 0) if cnt > 0: classes.append((model, cnt, pp(size))) context.update({'rows': rows, 'classes': classes}) return render_to_string(self.template, context)
class Debugger(Evenement): def __init__(self, jeu, gestionnaire, methode=False): print("start") super().__init__(jeu, gestionnaire) self._debugFait, self._methode = 0, methode if self._methode is not False: if self._methode == "Instance": self._tracker = ClassTracker() self._tracker.track_class(Evenement) self._tracker.create_snapshot("Initialisation") elif self._methode == "Fuites": self._tracker = pympler.tracker.SummaryTracker() self._tracker.print_diff() Horloge.initialiser(id(self), 1, 3000) def cmp(self, obj): return str(type(obj)) def traiter(self): if Horloge.sonner(id(self), 1) is True: if self._methode == "Instance": if self._debugFait < 3: self._tracker.create_snapshot("Etape n°" + str(self._debugFait)) self._debugFait += 1 Horloge.initialiser(id(self), 1, 3000) else: self._tracker.stats.print_summary() elif self._methode == "Fuites": self._tracker.print_diff() Horloge.initialiser(id(self), 1, 3000)
class MemoryPanel(DebugPanel): name = 'pympler' has_content = True template = 'memory_panel.html' classes = [Context, Template] def process_request(self, request): self._tracker = ClassTracker() for cls in get_models() + self.classes: self._tracker.track_class(cls) self._tracker.create_snapshot('before') self._before = ProcessMemoryInfo() def process_response(self, request, response): self._after = ProcessMemoryInfo() self._tracker.create_snapshot('after') def title(self): return 'Memory' def nav_title(self): return 'Memory' def nav_subtitle(self): rss = self._after.rss delta = rss - self._before.rss delta = ('(+%s)' % pp(delta)) if delta > 0 else '' return "%s %s" % (pp(rss), delta) def url(self): return '' def get_stats(self): pass def content(self): stats = self._tracker.stats stats.annotate() context = self.context.copy() rows = [('Resident set size', self._after.rss), ('Virtual size', self._after.vsz), ] rows.extend(self._after - self._before) rows = [(key, pp(value)) for key, value in rows] rows.extend(self._after.os_specific) classes = [] snapshot = stats.snapshots[-1] for model in stats.tracked_classes: history = [cnt for _, cnt in stats.history[model]] size = snapshot.classes.get(model, {}).get('sum', 0) if cnt > 0: classes.append((model, history, pp(size))) context.update({'rows': rows, 'classes': classes}) return render_to_string(self.template, context)
def main(): tracker = ClassTracker() tracker.track_class(Point) tracker.track_class(Line) tracker.create_snapshot() points = [get_point(i) for i in xrange(1000000)] lines = [get_line(i) for i in xrange(100000)] tracker.create_snapshot() tracker.stats.print_summary()
def run(self): """ Redirect bottle logging messages so it doesn't clutter the test output and start the web GUI. """ tracker = ClassTracker() tracker.track_class(Trash) tracked_trash = Trash() tracker.create_snapshot() sys.stdout = StringIO() sys.stderr = StringIO() start_profiler(debug=True, quiet=True, tracker=tracker)
def profile_memory(): from pympler.classtracker import ClassTracker from pympler.asizeof import asizeof from soap.common import Flyweight, _cache_map from soap.expression import Expr tracker = ClassTracker() tracker.track_object(Flyweight._cache) tracker.track_class(Expr) yield tracker.create_snapshot() tracker.stats.print_summary() print('Flyweight cache size', asizeof(Flyweight._cache)) print('Global cache size', asizeof(_cache_map))
class TrackObjectTestCase(unittest.TestCase): def setUp(self): self.tracker = ClassTracker() def tearDown(self): self.tracker.detach_all() def test_track_object(self): """Test object registration. """ foo = Foo() bar = Bar() self.tracker.track_object(foo) self.tracker.track_object(bar) self.assert_(id(foo) in self.tracker.objects) self.assert_(id(bar) in self.tracker.objects) self.assert_('Foo' in self.tracker.index) self.assert_('Bar' in self.tracker.index) self.assertEqual(self.tracker.objects[id(foo)].ref(),foo) self.assertEqual(self.tracker.objects[id(bar)].ref(),bar) def test_type_errors(self): """Test intrackable objects. """ i = 42 j = 'Foobar' k = [i,j] l = {i: j} self.assertRaises(TypeError, self.tracker.track_object, i) self.assertRaises(TypeError, self.tracker.track_object, j) self.assertRaises(TypeError, self.tracker.track_object, k) self.assertRaises(TypeError, self.tracker.track_object, l) self.assert_(id(i) not in self.tracker.objects) self.assert_(id(j) not in self.tracker.objects) self.assert_(id(k) not in self.tracker.objects) self.assert_(id(l) not in self.tracker.objects) def test_track_by_name(self): """Test registering objects by name. """ foo = Foo() self.tracker.track_object(foo, name='Foobar') self.assert_('Foobar' in self.tracker.index ) self.assertEqual(self.tracker.index['Foobar'][0].ref(),foo) def test_keep(self): """Test lifetime of tracked objects. """ foo = Foo() bar = Bar() self.tracker.track_object(foo, keep=1) self.tracker.track_object(bar) idfoo = id(foo) idbar = id(bar) del foo del bar self.assert_(self.tracker.objects[idfoo].ref() is not None) self.assert_(self.tracker.objects[idbar].ref() is None) def test_mixed_tracking(self): """Test mixed instance and class tracking. """ foo = Foo() self.tracker.track_object(foo) self.tracker.create_snapshot() self.tracker.track_class(Foo) objs = [] for _ in range(10): objs.append(Foo()) self.tracker.create_snapshot() def test_recurse(self): """Test recursive sizing and saving of referents. """ foo = Foo() self.tracker.track_object(foo, resolution_level=1) self.tracker.create_snapshot() fp = self.tracker.objects[id(foo)].snapshots[-1] refs = fp[1].refs dref = [r for r in refs if r.name == '__dict__'] self.assertEqual(len(dref),1) dref = dref[0] self.assert_(dref.size > 0, dref.size) self.assert_(dref.flat > 0, dref.flat) self.assertEqual(dref.refs,()) # Test track_change and more fine-grained resolution self.tracker.track_change(foo, resolution_level=2) self.tracker.create_snapshot() fp = self.tracker.objects[id(foo)].snapshots[-1] refs = fp[1].refs dref = [r for r in refs if r.name == '__dict__'] self.assertEqual(len(dref),1) dref = dref[0] namerefs = [r.name for r in dref.refs] self.assert_('[K] foo' in namerefs, namerefs) self.assert_("[V] foo: 'foo'" in namerefs, namerefs)
class SnapshotTestCase(unittest.TestCase): def setUp(self): self.tracker = ClassTracker() def tearDown(self): self.tracker.stop_periodic_snapshots() self.tracker.clear() def test_timestamp(self): """Test timestamp of snapshots. """ foo = Foo() bar = Bar() self.tracker.track_object(foo) self.tracker.track_object(bar) self.tracker.create_snapshot() self.tracker.create_snapshot() self.tracker.create_snapshot() refts = [fp.timestamp for fp in self.tracker.snapshots] for to in self.tracker.objects.values(): ts = [t for (t,sz) in to.snapshots[1:]] self.assertEqual(ts,refts) def test_snapshot_members(self): """Test existence and value of snapshot members. """ foo = Foo() self.tracker.track_object(foo) self.tracker.create_snapshot() self.tracker.create_snapshot(compute_total=True) fp = self.tracker.snapshots[0] fp_with_total = self.tracker.snapshots[1] self.assert_(fp.overhead > 0, fp.overhead) self.assert_(fp.tracked_total > 0, fp.tracked_total) self.assertEqual(fp.asizeof_total, 0) self.assert_(fp_with_total.asizeof_total > 0, fp_with_total.asizeof_total) self.assert_(fp_with_total.asizeof_total >= fp_with_total.tracked_total) if pympler.process.is_available(): procmem = fp.system_total self.assertEqual(fp.total, procmem.vsz) self.assert_(procmem.vsz > 0, procmem) self.assert_(procmem.rss > 0, procmem) self.assertTrue(procmem.vsz >= procmem.rss, procmem) self.assert_(procmem.vsz > fp.overhead, procmem) self.assert_(procmem.vsz > fp.tracked_total, procmem) self.assert_(fp_with_total.system_total.vsz > fp_with_total.asizeof_total) else: self.assertEqual(fp_with_total.total, fp_with_total.asizeof_total) self.assertEqual(fp.total, fp.tracked_total) def test_desc(self): """Test snapshot label. """ self.tracker.create_snapshot() self.tracker.create_snapshot('alpha') self.tracker.create_snapshot(description='beta') self.tracker.create_snapshot(42) self.assertEqual(len(self.tracker.snapshots), 4) self.assertEqual(self.tracker.snapshots[0].desc, '') self.assertEqual(self.tracker.snapshots[1].desc, 'alpha') self.assertEqual(self.tracker.snapshots[2].desc, 'beta') self.assertEqual(self.tracker.snapshots[3].desc, '42') snapshot = self.tracker.snapshots[0] self.assertEqual(snapshot.label, '%.3fs' % snapshot.timestamp) snapshot = self.tracker.snapshots[1] self.assertEqual(snapshot.label, 'alpha (%.3fs)' % snapshot.timestamp) snapshot = self.tracker.snapshots[3] self.assertEqual(snapshot.label, '42 (%.3fs)' % snapshot.timestamp) def test_background_monitoring(self): """Test background monitoring. """ self.tracker.start_periodic_snapshots(0.1) self.assertEqual(self.tracker._periodic_thread.interval, 0.1) self.assertEqual(self.tracker._periodic_thread.getName(), 'BackgroundMonitor') for x in range(10): # try to interfere self.tracker.create_snapshot(str(x)) time.sleep(0.5) self.tracker.start_periodic_snapshots(0.2) self.assertEqual(self.tracker._periodic_thread.interval, 0.2) self.tracker.stop_periodic_snapshots() self.assert_(self.tracker._periodic_thread is None) self.assert_(len(self.tracker.snapshots) > 10)
class TrackObjectTestCase(unittest.TestCase): def setUp(self): self.tracker = ClassTracker() def tearDown(self): self.tracker.detach_all() def test_track_object(self): """Test object registration. """ foo = Foo() bar = Bar() self.tracker.track_object(foo) self.tracker.track_object(bar) self.assertTrue(id(foo) in self.tracker.objects) self.assertTrue(id(bar) in self.tracker.objects) self.assertTrue('Foo' in self.tracker.index) self.assertTrue('Bar' in self.tracker.index) self.assertEqual(self.tracker.objects[id(foo)].ref(), foo) self.assertEqual(self.tracker.objects[id(bar)].ref(), bar) def test_type_errors(self): """Test intrackable objects. """ i = 42 j = 'Foobar' k = [i, j] l = {i: j} self.assertRaises(TypeError, self.tracker.track_object, i) self.assertRaises(TypeError, self.tracker.track_object, j) self.assertRaises(TypeError, self.tracker.track_object, k) self.assertRaises(TypeError, self.tracker.track_object, l) self.assertTrue(id(i) not in self.tracker.objects) self.assertTrue(id(j) not in self.tracker.objects) self.assertTrue(id(k) not in self.tracker.objects) self.assertTrue(id(l) not in self.tracker.objects) def test_track_by_name(self): """Test registering objects by name. """ foo = Foo() self.tracker.track_object(foo, name='Foobar') self.assertTrue('Foobar' in self.tracker.index) self.assertEqual(self.tracker.index['Foobar'][0].ref(), foo) def test_keep(self): """Test lifetime of tracked objects. """ foo = Foo() bar = Bar() self.tracker.track_object(foo, keep=1) self.tracker.track_object(bar) idfoo = id(foo) idbar = id(bar) del foo del bar self.assertTrue(self.tracker.objects[idfoo].ref() is not None) self.assertTrue(self.tracker.objects[idbar].ref() is None) def test_mixed_tracking(self): """Test mixed instance and class tracking. """ foo = Foo() self.tracker.track_object(foo) self.tracker.create_snapshot() self.tracker.track_class(Foo) objs = [] for _ in range(10): objs.append(Foo()) self.tracker.create_snapshot() def test_recurse(self): """Test recursive sizing and saving of referents. """ foo = Foo() self.tracker.track_object(foo, resolution_level=1) self.tracker.create_snapshot() fp = self.tracker.objects[id(foo)].snapshots[-1] refs = fp[1].refs dref = [r for r in refs if r.name == '__dict__'] self.assertEqual(len(dref), 1) dref = dref[0] self.assertTrue(dref.size > 0, dref.size) self.assertTrue(dref.flat > 0, dref.flat) self.assertEqual(dref.refs, ()) # Test track_change and more fine-grained resolution self.tracker.track_change(foo, resolution_level=2) self.tracker.create_snapshot() fp = self.tracker.objects[id(foo)].snapshots[-1] refs = fp[1].refs dref = [r for r in refs if r.name == '__dict__'] self.assertEqual(len(dref), 1) dref = dref[0] namerefs = [r.name for r in dref.refs] self.assertTrue('[K] foo' in namerefs, namerefs) self.assertTrue("[V] foo: 'foo'" in namerefs, namerefs)
class SnapshotTestCase(unittest.TestCase): def setUp(self): self.tracker = ClassTracker() def tearDown(self): self.tracker.stop_periodic_snapshots() self.tracker.clear() def test_timestamp(self): """Test timestamp of snapshots. """ foo = Foo() bar = Bar() self.tracker.track_object(foo) self.tracker.track_object(bar) self.tracker.create_snapshot() self.tracker.create_snapshot() self.tracker.create_snapshot() refts = [fp.timestamp for fp in self.tracker.snapshots] for to in self.tracker.objects.values(): ts = [t for (t, sz) in to.snapshots[1:]] self.assertEqual(ts, refts) def test_snapshot_members(self): """Test existence and value of snapshot members. """ foo = Foo() self.tracker.track_object(foo) self.tracker.create_snapshot() self.tracker.create_snapshot(compute_total=True) fp = self.tracker.snapshots[0] fp_with_total = self.tracker.snapshots[1] self.assertTrue(fp.overhead > 0, fp.overhead) self.assertTrue(fp.tracked_total > 0, fp.tracked_total) self.assertEqual(fp.asizeof_total, 0) self.assertTrue(fp_with_total.asizeof_total > 0, fp_with_total.asizeof_total) self.assertTrue( fp_with_total.asizeof_total >= fp_with_total.tracked_total) if pympler.process.is_available(): procmem = fp.system_total self.assertEqual(fp.total, procmem.vsz) self.assertTrue(procmem.vsz > 0, procmem) self.assertTrue(procmem.rss > 0, procmem) self.assertTrue(procmem.vsz >= procmem.rss, procmem) self.assertTrue(procmem.vsz > fp.overhead, procmem) self.assertTrue(procmem.vsz > fp.tracked_total, procmem) self.assertTrue( fp_with_total.system_total.vsz > fp_with_total.asizeof_total) else: self.assertEqual(fp_with_total.total, fp_with_total.asizeof_total) self.assertEqual(fp.total, fp.tracked_total) def test_desc(self): """Test snapshot label. """ self.tracker.create_snapshot() self.tracker.create_snapshot('alpha') self.tracker.create_snapshot(description='beta') self.tracker.create_snapshot(42) self.assertEqual(len(self.tracker.snapshots), 4) self.assertEqual(self.tracker.snapshots[0].desc, '') self.assertEqual(self.tracker.snapshots[1].desc, 'alpha') self.assertEqual(self.tracker.snapshots[2].desc, 'beta') self.assertEqual(self.tracker.snapshots[3].desc, '42') snapshot = self.tracker.snapshots[0] self.assertEqual(snapshot.label, '%.3fs' % snapshot.timestamp) snapshot = self.tracker.snapshots[1] self.assertEqual(snapshot.label, 'alpha (%.3fs)' % snapshot.timestamp) snapshot = self.tracker.snapshots[3] self.assertEqual(snapshot.label, '42 (%.3fs)' % snapshot.timestamp) def test_background_monitoring(self): """Test background monitoring. """ self.tracker.start_periodic_snapshots(0.1) self.assertEqual(self.tracker._periodic_thread.interval, 0.1) self.assertEqual(self.tracker._periodic_thread.getName(), 'BackgroundMonitor') for x in range(10): # try to interfere self.tracker.create_snapshot(str(x)) time.sleep(0.5) self.tracker.start_periodic_snapshots(0.2) self.assertEqual(self.tracker._periodic_thread.interval, 0.2) self.tracker.stop_periodic_snapshots() self.assertTrue(self.tracker._periodic_thread is None) self.assertTrue(len(self.tracker.snapshots) > 10)
self.mutable = list(range(10)) self.immutable = tuple(range(10)) def create_data(tracker, iterations=20, obj_per_iteration=100): objects = [] for x in range(iterations): for y in range(obj_per_iteration): objects.append(Alpha()) objects.append(Beta()) objects.append(Gamma()) tracker.create_snapshot() return objects tracker = ClassTracker() tracker.track_class(Alpha) tracker.track_class(Beta) tracker.track_class(Gamma, trace=True, resolution_level=2) print ("Create data") tracker.create_snapshot() data = create_data(tracker) print ("Drop data") del data tracker.create_snapshot() web.start_profiler(debug=True, stats=tracker.stats)
class MemoryPanel(Panel): name = 'pympler' title = 'Memory' template = 'memory_panel.html' classes = [Context, Template] def process_request(self, request): self._tracker = ClassTracker() for cls in apps.get_models() + self.classes: self._tracker.track_class(cls) self._tracker.create_snapshot('before') self.record_stats({'before': ProcessMemoryInfo()}) def process_response(self, request, response): self.record_stats({'after': ProcessMemoryInfo()}) self._tracker.create_snapshot('after') stats = self._tracker.stats stats.annotate() self.record_stats({'stats': stats}) def enable_instrumentation(self): self._tracker = ClassTracker() for cls in apps.get_models() + self.classes: self._tracker.track_class(cls) def disable_instrumentation(self): self._tracker.detach_all_classes() def nav_subtitle(self): context = self.get_stats() before = context['before'] after = context['after'] rss = after.rss delta = rss - before.rss delta = ('(+%s)' % pp(delta)) if delta > 0 else '' return "%s %s" % (pp(rss), delta) @property def content(self): context = self.get_stats() before = context['before'] after = context['after'] stats = context['stats'] rows = [ ('Resident set size', after.rss), ('Virtual size', after.vsz), ] rows.extend(after - before) rows = [(key, pp(value)) for key, value in rows] rows.extend(after.os_specific) classes = [] snapshot = stats.snapshots[-1] for model in stats.tracked_classes: history = [cnt for _, cnt in stats.history[model]] size = snapshot.classes.get(model, {}).get('sum', 0) if history and history[-1] > 0: classes.append((model, history, pp(size))) context.update({'rows': rows, 'classes': classes}) return render_to_string(self.template, context)
class LogTestCase(unittest.TestCase): def setUp(self): self.out = StringIO() self.tracker = ClassTracker(stream=self.out) @property def output(self): """Return output recorded in `ClassTracker` output stream.""" return self.out.getvalue() def tearDown(self): self.tracker.stop_periodic_snapshots() self.tracker.clear() def test_dump(self): """Test serialization of log data. """ foo = Foo() foo.data = range(1000) bar = Bar() self.tracker.track_object(foo, resolution_level=4) self.tracker.track_object(bar) self.tracker.create_snapshot('Footest') f1 = StringIO() f2 = StringIO() ConsoleStats(tracker=self.tracker, stream=f1).print_stats() tmp = BytesIO() Stats(tracker=self.tracker).dump_stats(tmp, close=False) self.tracker.clear() stats = ConsoleStats(stream=f2) self.assertEqual(stats.index, None) self.assertEqual(stats.snapshots, None) tmp.seek(0) stats.load_stats(tmp) tmp.close() self.assert_('Foo' in stats.index) stats.print_stats() self.assertEqual(f1.getvalue(), f2.getvalue()) # Test partial printing stats.stream = f3 = StringIO() stats.sort_stats() tolen = len(stats.sorted) stats.print_stats(clsname='Bar') self.assertEqual(len(stats.sorted), tolen) stats.print_summary() clsname = f3.getvalue().split('\n')[0] self.assertNotEqual(re.search('Bar', clsname), None, clsname) f1.close() f2.close() f3.close() def test_sort_stats(self): """Test sort_stats and reverse_order. """ self.tracker.track_class(Bar, name='Bar') foo = Foo() foo.data = list(range(1000)) bar1 = Bar() bar2 = Bar() self.tracker.track_object(foo, resolution_level=4) self.tracker.create_snapshot() stats = self.tracker.stats # Test sort_stats and reverse_order self.assertEqual(stats.sort_stats('size'), stats) self.assertEqual(stats.sorted[0].classname, 'Foo') stats.reverse_order() self.assertEqual(stats.sorted[0].classname, 'Bar') stats.sort_stats('classname', 'birth') self.assertEqual(stats.sorted[0].classname, 'Bar') self.assertRaises(ValueError, stats.sort_stats, 'name', 42, 'classn') stats.sort_stats('classname') def test_dump_load_with_filename(self): """Test serialization with filename. """ foo = Foo() self.tracker.track_object(foo, resolution_level=2) self.tracker.create_snapshot() fhandle, fname = mkstemp(prefix='pympler_test_dump') os.close(fhandle) try: self.tracker.stats.dump_stats(fname) output = StringIO() stats = ConsoleStats(filename=fname, stream=output) stats.print_stats() self.assertTrue('<Foo>' in output.getvalue(), output.getvalue()) # Check if a Stats loaded from a dump can be dumped again stats.dump_stats(fname) finally: os.unlink(fname) def test_tracked_classes(self): """Test listing tracked classes. """ self.tracker.track_class(Foo, name='Foo') self.tracker.track_class(Bar, name='Bar') foo = Foo() self.tracker.create_snapshot() bar = Bar() self.tracker.create_snapshot() foo = FooNew() self.tracker.track_object(foo) self.tracker.create_snapshot() stats = self.tracker.stats self.assertEqual(stats.tracked_classes, ['Bar', 'Foo', 'FooNew']) stats.print_summary() def test_print_stats(self): """Test printing class-filtered statistics. """ self.tracker.track_class(Foo, name='Foo', trace=True) self.tracker.track_class(Bar, name='Bar') foo = Foo() bar = Bar() self.tracker.create_snapshot() stats = self.tracker.stats stats.print_stats(clsname='Foo') self.assertTrue('Foo' in self.output, self.output) self.assertFalse('Bar' in self.output, self.output) self.assertTrue('foo = Foo()' in self.output, self.output) def test_print_stats_limit(self): """Test printing limited statistics. """ self.tracker.track_class(Foo, name='Foo') foo = [Foo() for _ in range(10)] self.tracker.create_snapshot() stats = self.tracker.stats stats.print_stats(limit=3) self.assertEqual(self.output.count('<Foo>'), 3) self.out.seek(0) self.out.truncate() stats.print_stats(limit=0.5) self.assertEqual(self.output.count('<Foo>'), 5) def test_snapshots(self): """Test multiple snapshots. """ self.tracker.track_class(Foo, name='Foo') self.tracker.track_class(Bar, name='Bar') self.tracker.track_class(FooNew, name='FooNew') self.tracker.create_snapshot() f1 = Foo() self.tracker.create_snapshot() f2 = Foo() f3 = FooNew() self.tracker.create_snapshot() b = Bar() del b self.tracker.create_snapshot() stats = self.tracker.stats stats.print_stats() stats.print_summary() def test_merge(self): """Test merging of reference trees. """ self.tracker.track_class(FooNew, name='Foo', resolution_level=2) f1 = FooNew() f1.a = list(range(1000)) f2 = FooNew() f2.a = list(range(100)) f2.b = 'This is some stupid spam.' self.tracker.create_snapshot('Merge test') sizer = Asizer() sz1 = sizer.asized(f1) sz2 = sizer.asized(f2) stats = self.tracker.stats for fp in stats.snapshots: if fp.desc == 'Merge test': stats.annotate_snapshot(fp) self.assert_(hasattr(fp, 'classes')) classes = fp.classes stats.annotate_snapshot(fp) self.assertEqual(fp.classes, classes) self.assert_('Foo' in fp.classes, fp.classes) self.assert_('merged' in fp.classes['Foo']) fm = fp.classes['Foo']['merged'] self.assertEqual(fm.size, sz1.size + sz2.size, (fm.size, str(sz1), str(sz2))) refs = {} for ref in fm.refs: refs[ref.name] = ref self.assert_('__dict__' in refs.keys(), refs.keys()) refs2 = {} for ref in refs['__dict__'].refs: refs2[ref.name] = ref self.assert_('[V] a' in refs2.keys(), refs2.keys()) self.assert_('[V] b' in refs2.keys(), refs2.keys()) self.assertEqual(refs2['[V] a'].size, asizeof(f1.a, f2.a)) def test_html(self): """Test emitting HTML statistics.""" self.tracker.track_class(Foo, name='Foo', resolution_level=2) self.tracker.track_class(Bar, name='Bar', trace=True) f1 = Foo() f1.a = list(range(100000)) f2 = Foo() f2.a = list(range(1000)) f2.b = 'This is some stupid spam.' f1 = Bar() self.tracker.create_snapshot('Merge test') stats = HtmlStats(tracker=self.tracker) try: target = mkdtemp(prefix='pympler_test') output = os.path.join(target, 'footest.html') stats.create_html(output) source = open(output).read() # Ensure relative links are used fname = os.path.join('footest_files', 'Foo.html') self.assertTrue('<a href="%s">' % fname in source, (fname, source)) finally: rmtree(target) def test_charts(self): """Test emitting graphic charts.""" self.tracker.track_class(Foo, name='Foo', resolution_level=2) f1 = Foo() f1.a = list(range(1000)) f2 = Foo() f2.a = list(range(100)) f2.b = 'This is some stupid spam.' self.tracker.create_snapshot('Merge test') from pympler import charts try: target = mkdtemp(prefix='pympler_test') output = os.path.join(target, 'timespace.png') charts.tracker_timespace(output, self.tracker.stats) finally: rmtree(target)
class MemoryPanel(Panel): name = 'pympler' title = 'Memory' template = 'memory_panel.html' classes = [Context, Template] def process_request(self, request): self._tracker = ClassTracker() for cls in get_models() + self.classes: self._tracker.track_class(cls) self._tracker.create_snapshot('before') self.record_stats({'before': ProcessMemoryInfo()}) def process_response(self, request, response): self.record_stats({'after': ProcessMemoryInfo()}) self._tracker.create_snapshot('after') stats = self._tracker.stats stats.annotate() self.record_stats({'stats': stats}) def enable_instrumentation(self): self._tracker = ClassTracker() for cls in get_models() + self.classes: self._tracker.track_class(cls) def disable_instrumentation(self): self._tracker.detach_all_classes() def nav_subtitle(self): context = self.get_stats() before = context['before'] after = context['after'] rss = after.rss delta = rss - before.rss delta = ('(+%s)' % pp(delta)) if delta > 0 else '' return "%s %s" % (pp(rss), delta) @property def content(self): context = self.get_stats() before = context['before'] after = context['after'] stats = context['stats'] rows = [('Resident set size', after.rss), ('Virtual size', after.vsz), ] rows.extend(after - before) rows = [(key, pp(value)) for key, value in rows] rows.extend(after.os_specific) classes = [] snapshot = stats.snapshots[-1] for model in stats.tracked_classes: history = [cnt for _, cnt in stats.history[model]] size = snapshot.classes.get(model, {}).get('sum', 0) if cnt > 0: classes.append((model, history, pp(size))) context.update({'rows': rows, 'classes': classes}) return render_to_string(self.template, context)
def track_memory_wrapper(*args, **kwargs): memory_info = {} tracker = ClassTracker() for cls in apps.get_models() + [Context, Template]: # track all models from registered apps, plus some standard Django ones tracker.track_class(cls) try: tracker.create_snapshot("before") memory_info["before"] = ProcessMemoryInfo() result = fn(*args, **kwargs) memory_info["after"] = ProcessMemoryInfo() tracker.create_snapshot("after") memory_info["stats"] = tracker.stats memory_info["stats"].annotate() return result finally: # record a whole bunch of memory statistics... resources = [ ("resident set size", memory_info["after"].rss), ("virtual size", memory_info["after"].vsz), ] resources.extend(memory_info["after"] - memory_info["before"]) resources = [(k, pp(v)) for k, v in resources] resources.extend(memory_info["after"].os_specific) # record each tracked class as of the final snapshot... classes_stats = [] snapshot = memory_info["stats"].snapshots[-1] for class_name in memory_info["stats"].tracked_classes: # history is a list of tuples that is updated on every creation/deletions: (timestamp, n_instances) history = [ n for _, n in memory_info["stats"].history[class_name] ] if history: classes_stats.append({ "name": class_name, "n_instances": len(history), "min_instances": min(history), "max_instances": max(history), "size": pp( snapshot.classes.get(class_name, {}).get("sum", 0)), }) if not path: stream = sys.stdout else: stream = open(path, "w") print("\nRESOURCES", file=stream) for k, v in resources: print(f"{k:<26}: {v:>10}", file=stream) print("\nCLASSES", file=stream) for class_stats in classes_stats: print( "{name}: created/deleted {n_instances} times for a min/max of {min_instances}/{max_instances} instances: {size:>10}" .format(**class_stats), file=stream, ) stream.closed tracker.detach_all_classes()
class LogTestCase(unittest.TestCase): def setUp(self): self.out = StringIO() self.tracker = ClassTracker(stream=self.out) @property def output(self): """Return output recorded in `ClassTracker` output stream.""" return self.out.getvalue() def tearDown(self): self.tracker.stop_periodic_snapshots() self.tracker.clear() def test_dump(self): """Test serialization of log data. """ foo = Foo() foo.data = range(1000) bar = Bar() self.tracker.track_object(foo, resolution_level=4) self.tracker.track_object(bar) self.tracker.create_snapshot('Footest') f1 = StringIO() f2 = StringIO() ConsoleStats(tracker=self.tracker, stream=f1).print_stats() tmp = BytesIO() Stats(tracker=self.tracker).dump_stats(tmp, close=False) self.tracker.clear() stats = ConsoleStats(stream=f2) self.assertEqual(stats.index, None) self.assertEqual(stats.snapshots, None) tmp.seek(0) stats.load_stats(tmp) tmp.close() self.assert_('Foo' in stats.index) stats.print_stats() self.assertEqual(f1.getvalue(), f2.getvalue()) # Test partial printing stats.stream = f3 = StringIO() stats.sort_stats() tolen = len(stats.sorted) stats.print_stats(clsname='Bar') self.assertEqual(len(stats.sorted), tolen) stats.print_summary() clsname = f3.getvalue().split('\n')[0] self.assertNotEqual(re.search('\.Bar', clsname), None, clsname) f1.close() f2.close() f3.close() def test_sort_stats(self): """Test sort_stats and reverse_order. """ self.tracker.track_class(Bar, name='Bar') foo = Foo() foo.data = list(range(1000)) bar1 = Bar() bar2 = Bar() self.tracker.track_object(foo, resolution_level=4) self.tracker.create_snapshot() stats = self.tracker.stats # Test sort_stats and reverse_order self.assertEqual(stats.sort_stats('size'), stats) self.assertEqual(stats.sorted[0].classname, 'Foo') stats.reverse_order() self.assertEqual(stats.sorted[0].classname, 'Bar') stats.sort_stats('classname', 'birth') self.assertEqual(stats.sorted[0].classname, 'Bar') self.assertRaises(ValueError, stats.sort_stats, 'name', 42, 'classn') stats.sort_stats('classname') def test_dump_load_with_filename(self): """Test serialization with filename. """ foo = Foo() self.tracker.track_object(foo, resolution_level=2) self.tracker.create_snapshot() fhandle, fname = mkstemp(prefix='pympler_test_dump') os.close(fhandle) try: self.tracker.stats.dump_stats(fname) output = StringIO() stats = ConsoleStats(filename=fname, stream=output) stats.print_stats() self.assertTrue('<Foo>' in output.getvalue(), output.getvalue()) # Check if a Stats loaded from a dump can be dumped again stats.dump_stats(fname) finally: os.unlink(fname) def test_tracked_classes(self): """Test listing tracked classes. """ self.tracker.track_class(Foo, name='Foo') self.tracker.track_class(Bar, name='Bar') foo = Foo() self.tracker.create_snapshot() bar = Bar() self.tracker.create_snapshot() foo = FooNew() self.tracker.track_object(foo) self.tracker.create_snapshot() stats = self.tracker.stats self.assertEqual(stats.tracked_classes, ['Bar', 'Foo', 'FooNew']) stats.print_summary() def test_print_stats(self): """Test printing class-filtered statistics. """ self.tracker.track_class(Foo, name='Foo', trace=True) self.tracker.track_class(Bar, name='Bar') foo = Foo() bar = Bar() self.tracker.create_snapshot() stats = self.tracker.stats stats.print_stats(clsname='Foo') self.assertTrue('Foo' in self.output, self.output) self.assertFalse('Bar' in self.output, self.output) self.assertTrue('foo = Foo()' in self.output, self.output) def test_print_stats_limit(self): """Test printing limited statistics. """ self.tracker.track_class(Foo, name='Foo') foo = [Foo() for _ in range(10)] self.tracker.create_snapshot() stats = self.tracker.stats stats.print_stats(limit=3) self.assertEqual(self.output.count('<Foo>'), 3) self.out.seek(0) self.out.truncate() stats.print_stats(limit=0.5) self.assertEqual(self.output.count('<Foo>'), 5) def test_snapshots(self): """Test multiple snapshots. """ self.tracker.track_class(Foo, name='Foo') self.tracker.track_class(Bar, name='Bar') self.tracker.track_class(FooNew, name='FooNew') self.tracker.create_snapshot() f1 = Foo() self.tracker.create_snapshot() f2 = Foo() f3 = FooNew() self.tracker.create_snapshot() b = Bar() del b self.tracker.create_snapshot() stats = self.tracker.stats stats.print_stats() stats.print_summary() def test_merge(self): """Test merging of reference trees. """ self.tracker.track_class(FooNew, name='Foo', resolution_level=2) f1 = FooNew() f1.a = list(range(1000)) f2 = FooNew() f2.a = list(range(100)) f2.b = 'This is some stupid spam.' self.tracker.create_snapshot('Merge test') sizer = Asizer() sz1 = sizer.asized(f1) sz2 = sizer.asized(f2) stats = self.tracker.stats for fp in stats.snapshots: if fp.desc == 'Merge test': stats.annotate_snapshot(fp) self.assert_(hasattr(fp, 'classes')) classes = fp.classes stats.annotate_snapshot(fp) self.assertEqual(fp.classes, classes) self.assert_('Foo' in fp.classes, fp.classes) self.assert_('merged' in fp.classes['Foo']) fm = fp.classes['Foo']['merged'] self.assertEqual(fm.size, sz1.size + sz2.size, (fm.size, str(sz1), str(sz2))) refs = {} for ref in fm.refs: refs[ref.name] = ref self.assert_('__dict__' in refs.keys(), refs.keys()) refs2 = {} for ref in refs['__dict__'].refs: refs2[ref.name] = ref self.assert_('[V] a' in refs2.keys(), refs2.keys()) self.assert_('[V] b' in refs2.keys(), refs2.keys()) self.assertEqual(refs2['[V] a'].size, asizeof(f1.a, f2.a)) def test_html(self): """Test emitting HTML statistics.""" self.tracker.track_class(Foo, name='Foo', resolution_level=2) self.tracker.track_class(Bar, name='Bar', trace=True) f1 = Foo() f1.a = list(range(100000)) f2 = Foo() f2.a = list(range(1000)) f2.b = 'This is some stupid spam.' f1 = Bar() self.tracker.create_snapshot('Merge test') stats = HtmlStats(tracker=self.tracker) try: target = mkdtemp(prefix='pympler_test') output = os.path.join(target, 'footest.html') stats.create_html(output) source = open(output).read() # Ensure relative links are used fname = os.path.join('footest_files', 'Foo.html') self.assertTrue('<a href="%s">' % fname in source, (fname, source)) finally: rmtree(target) def test_charts(self): """Test emitting graphic charts.""" self.tracker.track_class(Foo, name='Foo', resolution_level=2) f1 = Foo() f1.a = list(range(1000)) f2 = Foo() f2.a = list(range(100)) f2.b = 'This is some stupid spam.' self.tracker.create_snapshot('Merge test') from pympler import charts try: target = mkdtemp(prefix='pympler_test') output = os.path.join(target, 'timespace.png') charts.tracker_timespace(output, self.tracker.stats) finally: rmtree(target)
wplist.append(wp) objects.append(wplist) # tracker.create_snapshot() return objects tracker = ClassTracker() tracker.track_class(Waypoint) tracker.track_class(WaypointList) #tracker.track_class(Gamma, trace=True, resolution_level=2) print("Create data") tracker.create_snapshot() data = create_data(tracker) print(data) print("Drop data") for item in data: del item del data tracker.create_snapshot() print("Create tenfold data") tracker.create_snapshot() data = create_data(tracker, iterations=1000, obj_per_iteration=100) print("Drop data") for item in data: del item del data
class MemoryDebugPanel(DebugPanel): """ Sample debug panel """ name = 'memory' has_content = True template = 'pyramid_debugtoolbar_memory.panels:templates/memory.dbtmako' title = _('Memory') nav_title = title def __init__(self, request): self._tracker = ClassTracker() self._tracker.track_class(Request) self._tracker.track_class(Response) def wrap_handler(self, handler): handler = self._wrap_profile_handler(handler) return handler def _wrap_profile_handler(self, handler): def profiler_handler(req): self._tracker.create_snapshot('before') before = ProcessMemoryInfo() req_before = get_memory_object(req.registry) try: result = handler(req) except: raise finally: after = ProcessMemoryInfo() self._tracker.create_snapshot('after') class_stats = self._tracker.stats class_stats.annotate() self.stats = dict( before=before, after=after, class_stats=class_stats, req_before=req_before, req_after=get_memory_object(req.registry), ) return result return profiler_handler def render_vars(self, req): before = self.stats['before'] after = self.stats['after'] class_stats = self.stats['class_stats'] rows = [ ('Resident set size', after.rss), ('Virtual size', after.vsz), ] rows.extend(after - before) rows = [(key, pp(value)) for key, value in rows] rows.extend(after.os_specific) classes = [] snapshot = class_stats.snapshots[-1] for model in class_stats.tracked_classes: history = [cnt for _, cnt in class_stats.history[model]] size = snapshot.classes.get(model, {}).get('sum', 0) if history and history[-1] > 0: classes.append((model, history, pp(size))) return { 'rows': rows, 'classes': classes, 'process_history': process_history, 'req_before': self.stats['req_before'], 'req_after': self.stats['req_after'], }