class Profiler(object):
''' An attempt at a unified Pympler API.
Combining muppy, tracker, etc. isn't trivial.
See Pympler's docs.
Pympler version from SVN 2011-03-04 '''
def __init__(self, output=None):
self.class_tracker = ClassTracker()
self.summary_tracker = SummaryTracker()
self.set_output(output or log.stream())
log('self.output: %r' % self.output) #DEBUG
self.output.write('RAW WRITE: self.output: %r\n' % self.output) #DEBUG
def set_output(self, output):
self.output = output
self.stats = ConsoleStats(tracker=self.class_tracker, stream=self.output)
def track_object(self, obj):
self.class_tracker.track_object(obj)
def track_class(self, cls):
self.class_tracker.track_class(cls)
def save(self, filename):
self.stats.dump_stats(filename)
def create_snapshot(self, label=None):
self.class_tracker.create_snapshot(label)
def print_html(self, data_filename, html_filename):
html_stats = HtmlStats()
html_stats.load_stats(data_filename)
html_stats.create_html(html_filename)
def print_summary(self):
self.stats.print_summary()
def print_stats(self):
self.stats.print_stats()
def reset_change_monitor(self):
''' Wait until changes stabilize '''
# !! this loops forever
log('reseting change monitor')
changes = self.summary_tracker.diff()
while changes:
log('%d objects changed' % len(changes))
# Summary.print_() is hardcoded to print to sys.stdout
with redir_stdout(log.stream()):
summary.print_(changes)
sleep(1)
changes = self.summary_tracker.diff()
log('change monitor reset')
def print_changes(self):
''' Print summary of recent differences. '''
# SummaryTracker.print_diff() is hardcoded to print to sys.stdout
with redir_stdout(log.stream()):
self.summary_tracker.print_diff()
def print_objects(self, objects, label='Objects', full=True):
''' Print a list of objects '''
if objects:
if full:
print(label, format(sum(Profiler.size(obj) for obj in objects)), file=self.output)
else:
print(label, file=self.output)
for obj in objects:
if full:
print(' ', type(obj), format(Profiler.size(obj)), file=self.output)
else:
print(' ', type(obj), file=self.output)
def custom_print_diff(self, diff, full=True):
''' Print the return value from ObjectTracker.diff() '''
def custom_print_changes(key):
changes = diff[key]
if changes:
if key == '+':
label = 'Added'
else:
label = 'Removed'
self.print_objects(changes, label=label)
custom_print_changes('+')
custom_print_changes('-')
@staticmethod
def size(obj):
return asizeof.flatsize(obj)
def __init__(self, output=None):
self.class_tracker = ClassTracker()
self.summary_tracker = SummaryTracker()
self.set_output(output or log.stream())
log('self.output: %r' % self.output) #DEBUG
self.output.write('RAW WRITE: self.output: %r\n' % self.output) #DEBUG
def setUp(self):
self.tracker = ClassTracker()
class LogTestCase(unittest.TestCase):
def setUp(self):
self.tracker = ClassTracker()
def tearDown(self):
self.tracker.stop_periodic_snapshots()
self.tracker.clear()
def test_dump(self):
"""Test serialization of log data.
"""
foo = Foo()
foo.data = range(1000)
bar = Bar()
self.tracker.track_object(foo, resolution_level=4)
self.tracker.track_object(bar)
self.tracker.create_snapshot('Footest')
f1 = StringIO()
f2 = StringIO()
ConsoleStats(tracker=self.tracker, stream=f1).print_stats()
tmp = BytesIO()
Stats(tracker=self.tracker).dump_stats(tmp, close=0)
self.tracker.clear()
stats = ConsoleStats(stream=f2)
self.assert_(stats.index is None)
self.assert_(stats.footprint is None)
tmp.seek(0)
stats.load_stats(tmp)
tmp.close()
self.assert_('Foo' in stats.index)
stats.print_stats()
self.assertEqual(f1.getvalue(), f2.getvalue())
# Test sort_stats and reverse_order
self.assertEqual(stats.sort_stats('size'), stats)
self.assertEqual(stats.sorted[0].classname, 'Foo')
stats.reverse_order()
self.assertEqual(stats.sorted[0].classname, 'Bar')
stats.sort_stats('classname', 'birth')
self.assertEqual(stats.sorted[0].classname, 'Bar')
self.assertRaises(ValueError, stats.sort_stats, 'name', 42, 'classn')
# Test partial printing
stats.stream = f3 = StringIO()
stats.sort_stats()
tolen = len(stats.sorted)
stats.print_stats(clsname='Bar',limit=0.5)
self.assertEqual(len(stats.sorted), tolen)
stats.print_summary()
clsname = f3.getvalue().split('\n')[0]
self.assertNotEqual(re.search('\.Bar', clsname), None, clsname)
self.assert_(len(f3.getvalue()) < len(f1.getvalue()))
f1.close()
f2.close()
f3.close()
def test_snapshots(self):
"""Test multiple snapshots.
"""
self.tracker.track_class(Foo, name='Foo')
self.tracker.track_class(Bar, name='Bar')
self.tracker.track_class(FooNew, name='FooNew')
self.tracker.create_snapshot()
f1 = Foo()
self.tracker.create_snapshot()
f2 = Foo()
f3 = FooNew()
self.tracker.create_snapshot()
b = Bar()
del b
self.tracker.create_snapshot()
stream = StringIO()
stats = ConsoleStats(tracker=self.tracker, stream=stream)
stats.print_stats()
stats.print_summary()
def test_merge(self):
"""Test merging of reference trees.
"""
self.tracker.track_class(FooNew, name='Foo', resolution_level=2)
f1 = FooNew()
f1.a = list(range(1000))
f2 = FooNew()
f2.a = list(range(100))
f2.b = 'This is some stupid spam.'
self.tracker.create_snapshot('Merge test')
sizer = Asizer()
sz1 = sizer.asized(f1)
sz2 = sizer.asized(f2)
stats = Stats(tracker=self.tracker)
for fp in stats.footprint:
if fp.desc == 'Merge test':
stats.annotate_snapshot(fp)
self.assert_(hasattr(fp, 'classes'))
self.assert_('Foo' in fp.classes, fp.classes)
self.assert_('merged' in fp.classes['Foo'])
fm = fp.classes['Foo']['merged']
self.assertEqual(fm.size, sz1.size + sz2.size, (fm.size, str(sz1), str(sz2)))
refs = {}
for ref in fm.refs:
refs[ref.name] = ref
self.assert_('__dict__' in refs.keys(), refs.keys())
refs2 = {}
for ref in refs['__dict__'].refs:
refs2[ref.name] = ref
self.assert_('[V] a' in refs2.keys(), refs2.keys())
self.assert_('[V] b' in refs2.keys(), refs2.keys())
self.assertEqual(refs2['[V] a'].size, asizeof(f1.a, f2.a))
def test_html(self):
"""Test emitting HTML statistics."""
self.tracker.track_class(Foo, name='Foo', resolution_level=2)
f1 = Foo()
f1.a = list(range(100000))
f2 = Foo()
f2.a = list(range(1000))
f2.b = 'This is some stupid spam.'
self.tracker.create_snapshot('Merge test')
stats = HtmlStats(tracker=self.tracker)
stats.create_html('tmp/data/footest.html')
def test_charts(self):
"""Test emitting graphic charts."""
self.tracker.track_class(Foo, name='Foo', resolution_level=2)
f1 = Foo()
f1.a = list(range(1000))
f2 = Foo()
f2.a = list(range(100))
f2.b = 'This is some stupid spam.'
self.tracker.create_snapshot('Merge test')
stats = Stats(tracker=self.tracker)
from pympler.gui import charts
charts.tracker_timespace('tmp/data/timespace.png', stats)
class TrackObjectTestCase(unittest.TestCase):
def setUp(self):
self.tracker = ClassTracker()
def tearDown(self):
self.tracker.detach_all()
def test_track_object(self):
"""Test object registration.
"""
foo = Foo()
bar = Bar()
self.tracker.track_object(foo)
self.tracker.track_object(bar)
self.assert_(id(foo) in self.tracker.objects)
self.assert_(id(bar) in self.tracker.objects)
self.assert_('Foo' in self.tracker.index)
self.assert_('Bar' in self.tracker.index)
self.assertEqual(self.tracker.objects[id(foo)].ref(),foo)
self.assertEqual(self.tracker.objects[id(bar)].ref(),bar)
def test_type_errors(self):
"""Test intrackable objects.
"""
i = 42
j = 'Foobar'
k = [i,j]
l = {i: j}
self.assertRaises(TypeError, self.tracker.track_object, i)
self.assertRaises(TypeError, self.tracker.track_object, j)
self.assertRaises(TypeError, self.tracker.track_object, k)
self.assertRaises(TypeError, self.tracker.track_object, l)
self.assert_(id(i) not in self.tracker.objects)
self.assert_(id(j) not in self.tracker.objects)
self.assert_(id(k) not in self.tracker.objects)
self.assert_(id(l) not in self.tracker.objects)
def test_track_by_name(self):
"""Test registering objects by name.
"""
foo = Foo()
self.tracker.track_object(foo, name='Foobar')
self.assert_('Foobar' in self.tracker.index )
self.assertEqual(self.tracker.index['Foobar'][0].ref(),foo)
def test_keep(self):
"""Test lifetime of tracked objects.
"""
foo = Foo()
bar = Bar()
self.tracker.track_object(foo, keep=1)
self.tracker.track_object(bar)
idfoo = id(foo)
idbar = id(bar)
del foo
del bar
self.assert_(self.tracker.objects[idfoo].ref() is not None)
self.assert_(self.tracker.objects[idbar].ref() is None)
def test_mixed_tracking(self):
"""Test mixed instance and class tracking.
"""
foo = Foo()
self.tracker.track_object(foo)
self.tracker.create_snapshot()
self.tracker.track_class(Foo)
objs = []
for _ in range(10):
objs.append(Foo())
self.tracker.create_snapshot()
def test_recurse(self):
"""Test recursive sizing and saving of referents.
"""
foo = Foo()
self.tracker.track_object(foo, resolution_level=1)
self.tracker.create_snapshot()
fp = self.tracker.objects[id(foo)].footprint[-1]
refs = fp[1].refs
dref = [r for r in refs if r.name == '__dict__']
self.assertEqual(len(dref),1)
dref = dref[0]
self.assert_(dref.size > 0, dref.size)
self.assert_(dref.flat > 0, dref.flat)
self.assertEqual(dref.refs,())
# Test track_change and more fine-grained resolution
self.tracker.track_change(foo, resolution_level=2)
self.tracker.create_snapshot()
fp = self.tracker.objects[id(foo)].footprint[-1]
refs = fp[1].refs
dref = [r for r in refs if r.name == '__dict__']
self.assertEqual(len(dref),1)
dref = dref[0]
namerefs = [r.name for r in dref.refs]
self.assert_('[K] foo' in namerefs, namerefs)
self.assert_("[V] foo: 'foo'" in namerefs, namerefs)
class TrackClassTestCase(unittest.TestCase):
def setUp(self):
self.tracker = ClassTracker()
def tearDown(self):
self.tracker.stop_periodic_snapshots()
self.tracker.clear()
def test_type_errors(self):
"""Test invalid parameters for class tracking.
"""
i = 42
j = 'Foobar'
k = [i,j]
l = {i: j}
foo = Foo()
bar = Bar()
self.assertRaises(TypeError, self.tracker.track_class, i)
self.assertRaises(TypeError, self.tracker.track_class, j)
self.assertRaises(TypeError, self.tracker.track_class, k)
self.assertRaises(TypeError, self.tracker.track_class, l)
self.assertRaises(TypeError, self.tracker.track_class, foo)
self.assertRaises(TypeError, self.tracker.track_class, bar)
self.assert_(id(i) not in self.tracker.objects)
self.assert_(id(j) not in self.tracker.objects)
self.assert_(id(k) not in self.tracker.objects)
self.assert_(id(l) not in self.tracker.objects)
def test_track_class(self):
"""Test tracking objects through classes.
"""
self.tracker.track_class(Foo)
self.tracker.track_class(Bar)
self.tracker.track_class(Empty)
self.tracker.track_class(Foo)
foo = Foo()
bar = Bar()
empty = Empty()
self.assert_(id(foo) in self.tracker.objects)
self.assert_(id(bar) in self.tracker.objects)
self.assert_(id(empty) in self.tracker.objects)
def test_track_class_new(self):
"""Test tracking new style classes.
"""
self.tracker.track_class(FooNew)
self.tracker.track_class(BarNew)
foo = FooNew()
bar = BarNew()
self.assert_(id(foo) in self.tracker.objects)
self.assert_(id(bar) in self.tracker.objects)
def test_track_by_name(self):
"""Test registering objects by name.
"""
self.tracker.track_class(Foo, name='Foobar')
foo = Foo()
self.assert_('Foobar' in self.tracker.index )
self.assertEqual(self.tracker.index['Foobar'][0].ref(),foo)
def test_keep(self):
"""Test lifetime of tracked objects.
"""
self.tracker.track_class(Foo, keep=1)
self.tracker.track_class(Bar)
foo = Foo()
bar = Bar()
idfoo = id(foo)
idbar = id(bar)
del foo
del bar
self.assert_(self.tracker.objects[idfoo].ref() is not None)
self.assert_(self.tracker.objects[idbar].ref() is None)
def test_trace(self):
"""Test instantiation tracing of tracked objects.
"""
from inspect import stack
self.tracker.track_class(Foo, trace=True)
self.tracker.track_class(BarNew, trace=True)
foo = Foo()
bar = BarNew()
idfoo = id(foo)
idbar = id(bar)
trace = []
st = stack()
try:
for fr in st:
trace.insert(0, fr[1:])
finally:
del st
self.assertEqual(self.tracker.objects[idfoo].trace[-1][3][0].strip(),"foo = Foo()")
self.assertEqual(self.tracker.objects[idfoo].trace[:-1],trace[:-1], trace)
self.assertEqual(self.tracker.objects[idbar].trace[:-1],trace[:-1], trace)
def test_detach(self):
"""Test detaching from tracked classes.
"""
self.tracker.track_class(Foo)
self.tracker.track_class(Bar)
foo = Foo()
bar = Bar()
self.assert_(id(foo) in self.tracker.objects)
self.assert_(id(bar) in self.tracker.objects)
self.tracker.detach_class(Foo)
self.tracker.detach_class(Bar)
foo2 = Foo()
bar2 = Bar()
self.assert_(id(foo2) not in self.tracker.objects)
self.assert_(id(bar2) not in self.tracker.objects)
self.assertRaises(KeyError, self.tracker.detach_class, Foo)
def test_change_name(self):
"""Test modifying name.
"""
self.tracker.track_class(Foo, name='Foobar')
self.tracker.track_class(Foo, name='Baz')
foo = Foo()
self.assert_('Foobar' not in self.tracker.index)
self.assert_('Baz' in self.tracker.index)
self.assertEqual(self.tracker.index['Baz'][0].ref(),foo)
class SnapshotTestCase(unittest.TestCase):
def setUp(self):
self.tracker = ClassTracker()
def tearDown(self):
self.tracker.stop_periodic_snapshots()
self.tracker.clear()
def test_timestamp(self):
"""Test timestamp of snapshots.
"""
foo = Foo()
bar = Bar()
self.tracker.track_object(foo)
self.tracker.track_object(bar)
self.tracker.create_snapshot()
self.tracker.create_snapshot()
self.tracker.create_snapshot()
refts = [fp.timestamp for fp in self.tracker.footprint]
for to in self.tracker.objects.values():
ts = [t for (t,sz) in to.footprint[1:]]
self.assertEqual(ts,refts)
def test_snapshot_members(self):
"""Test existence and value of snapshot members.
"""
foo = Foo()
self.tracker.track_object(foo)
self.tracker.create_snapshot()
fp = self.tracker.footprint[0]
self.assert_(fp.overhead > 0, fp.overhead)
self.assert_(fp.tracked_total > 0, fp.tracked_total)
self.assert_(fp.asizeof_total > 0, fp.asizeof_total)
self.assert_(fp.asizeof_total >= fp.tracked_total)
if pympler.process.is_available():
self.assert_(fp.system_total.vsz > 0)
self.assert_(fp.system_total.rss > 0)
self.assert_(fp.system_total.vsz >= fp.system_total.rss)
self.assert_(fp.system_total.vsz > fp.overhead)
self.assert_(fp.system_total.vsz > fp.tracked_total)
self.assert_(fp.system_total.vsz > fp.asizeof_total)
def test_desc(self):
"""Test footprint description.
"""
self.tracker.create_snapshot()
self.tracker.create_snapshot('alpha')
self.tracker.create_snapshot(description='beta')
self.tracker.create_snapshot(42)
self.assertEqual(len(self.tracker.footprint), 4)
self.assertEqual(self.tracker.footprint[0].desc, '')
self.assertEqual(self.tracker.footprint[1].desc, 'alpha')
self.assertEqual(self.tracker.footprint[2].desc, 'beta')
self.assertEqual(self.tracker.footprint[3].desc, '42')
def test_background_monitoring(self):
"""Test background monitoring.
"""
import time
self.tracker.start_periodic_snapshots(0.1)
self.assertEqual(self.tracker._periodic_thread.interval, 0.1)
self.assertEqual(self.tracker._periodic_thread.getName(), 'BackgroundMonitor')
for x in range(10): # try to interfere
self.tracker.create_snapshot(str(x))
time.sleep(0.5)
self.tracker.start_periodic_snapshots(0.2)
self.assertEqual(self.tracker._periodic_thread.interval, 0.2)
self.tracker.stop_periodic_snapshots()
self.assert_(self.tracker._periodic_thread is None)
self.assert_(len(self.tracker.footprint) > 10)
