예제 #1
0
class Profiler(object):
    ''' An attempt at a  unified Pympler API.
        Combining muppy, tracker, etc. isn't trivial.
        See Pympler's docs.
        Pympler version from SVN 2011-03-04 '''

    def __init__(self, output=None):
        self.class_tracker = ClassTracker()
        self.summary_tracker = SummaryTracker()
        self.set_output(output or log.stream())
        log('self.output: %r' % self.output) #DEBUG
        self.output.write('RAW WRITE: self.output: %r\n' % self.output) #DEBUG

    def set_output(self, output):
        self.output = output
        self.stats = ConsoleStats(tracker=self.class_tracker, stream=self.output)

    def track_object(self, obj):
        self.class_tracker.track_object(obj)

    def track_class(self, cls):
        self.class_tracker.track_class(cls)

    def save(self, filename):
        self.stats.dump_stats(filename)

    def create_snapshot(self, label=None):
        self.class_tracker.create_snapshot(label)

    def print_html(self, data_filename, html_filename):
        html_stats = HtmlStats()
        html_stats.load_stats(data_filename)
        html_stats.create_html(html_filename)

    def print_summary(self):
        self.stats.print_summary()

    def print_stats(self):
        self.stats.print_stats()

    def reset_change_monitor(self):
        ''' Wait until changes stabilize '''

        # !! this loops forever
        log('reseting change monitor')
        changes = self.summary_tracker.diff()
        while changes:
            log('%d objects changed' % len(changes))

            # Summary.print_() is hardcoded to print to sys.stdout
            with redir_stdout(log.stream()):
                summary.print_(changes)

            sleep(1)
            changes = self.summary_tracker.diff()

        log('change monitor reset')

    def print_changes(self):
        ''' Print summary of recent differences. '''

        # SummaryTracker.print_diff() is hardcoded to print to sys.stdout
        with redir_stdout(log.stream()):
            self.summary_tracker.print_diff()

    def print_objects(self, objects, label='Objects', full=True):
        ''' Print a list of objects '''

        if objects:

            if full:
                print(label, format(sum(Profiler.size(obj) for obj in objects)), file=self.output)
            else:
                print(label, file=self.output)

            for obj in objects:
                if full:
                    print('    ', type(obj), format(Profiler.size(obj)), file=self.output)
                else:
                    print('    ', type(obj), file=self.output)

    def custom_print_diff(self, diff, full=True):
        ''' Print the return value from ObjectTracker.diff() '''

        def custom_print_changes(key):

            changes = diff[key]
            if changes:

                if key == '+':
                    label = 'Added'
                else:
                    label = 'Removed'
                self.print_objects(changes, label=label)

        custom_print_changes('+')
        custom_print_changes('-')

    @staticmethod
    def size(obj):
        return asizeof.flatsize(obj)
class TrackObjectTestCase(unittest.TestCase):

    def setUp(self):
        self.tracker = ClassTracker()

    def tearDown(self):
        self.tracker.detach_all()

    def test_track_object(self):
        """Test object registration.
        """
        foo = Foo()
        bar = Bar()

        self.tracker.track_object(foo)
        self.tracker.track_object(bar)

        self.assert_(id(foo) in self.tracker.objects)
        self.assert_(id(bar) in self.tracker.objects)

        self.assert_('Foo' in self.tracker.index)
        self.assert_('Bar' in self.tracker.index)

        self.assertEqual(self.tracker.objects[id(foo)].ref(),foo)
        self.assertEqual(self.tracker.objects[id(bar)].ref(),bar)

    def test_type_errors(self):
        """Test intrackable objects.
        """
        i = 42
        j = 'Foobar'
        k = [i,j]
        l = {i: j}

        self.assertRaises(TypeError, self.tracker.track_object, i)
        self.assertRaises(TypeError, self.tracker.track_object, j)
        self.assertRaises(TypeError, self.tracker.track_object, k)
        self.assertRaises(TypeError, self.tracker.track_object, l)

        self.assert_(id(i) not in self.tracker.objects)
        self.assert_(id(j) not in self.tracker.objects)
        self.assert_(id(k) not in self.tracker.objects)
        self.assert_(id(l) not in self.tracker.objects)

    def test_track_by_name(self):
        """Test registering objects by name.
        """
        foo = Foo()

        self.tracker.track_object(foo, name='Foobar')

        self.assert_('Foobar' in self.tracker.index        )
        self.assertEqual(self.tracker.index['Foobar'][0].ref(),foo)

    def test_keep(self):
        """Test lifetime of tracked objects.
        """
        foo = Foo()
        bar = Bar()

        self.tracker.track_object(foo, keep=1)
        self.tracker.track_object(bar)
       
        idfoo = id(foo)
        idbar = id(bar)

        del foo
        del bar

        self.assert_(self.tracker.objects[idfoo].ref() is not None)
        self.assert_(self.tracker.objects[idbar].ref() is None)

    def test_mixed_tracking(self):
        """Test mixed instance and class tracking.
        """
        foo = Foo()
        self.tracker.track_object(foo)
        self.tracker.create_snapshot()
        self.tracker.track_class(Foo)
        objs = []
        for _ in range(10):
            objs.append(Foo())
        self.tracker.create_snapshot()

    def test_recurse(self):
        """Test recursive sizing and saving of referents.
        """
        foo = Foo()

        self.tracker.track_object(foo, resolution_level=1)
        self.tracker.create_snapshot()

        fp = self.tracker.objects[id(foo)].footprint[-1]
        refs = fp[1].refs
        dref = [r for r in refs if r.name == '__dict__']
        self.assertEqual(len(dref),1)
        dref = dref[0]
        self.assert_(dref.size > 0, dref.size)
        self.assert_(dref.flat > 0, dref.flat)
        self.assertEqual(dref.refs,())

        # Test track_change and more fine-grained resolution
        self.tracker.track_change(foo, resolution_level=2)
        self.tracker.create_snapshot()

        fp = self.tracker.objects[id(foo)].footprint[-1]
        refs = fp[1].refs
        dref = [r for r in refs if r.name == '__dict__']
        self.assertEqual(len(dref),1)
        dref = dref[0]
        namerefs = [r.name for r in dref.refs]
        self.assert_('[K] foo' in namerefs, namerefs)
        self.assert_("[V] foo: 'foo'" in namerefs, namerefs)
예제 #3
0
class LogTestCase(unittest.TestCase):

    def setUp(self):
        self.tracker = ClassTracker()

    def tearDown(self):
        self.tracker.stop_periodic_snapshots()
        self.tracker.clear()

    def test_dump(self):
        """Test serialization of log data.
        """
        foo = Foo()
        foo.data = range(1000)
        bar = Bar()

        self.tracker.track_object(foo, resolution_level=4)
        self.tracker.track_object(bar)

        self.tracker.create_snapshot('Footest')

        f1 = StringIO()
        f2 = StringIO()

        ConsoleStats(tracker=self.tracker, stream=f1).print_stats()

        tmp = BytesIO()
        Stats(tracker=self.tracker).dump_stats(tmp, close=0)

        self.tracker.clear()

        stats = ConsoleStats(stream=f2)
        self.assert_(stats.index is None)
        self.assert_(stats.footprint is None)
        tmp.seek(0)
        stats.load_stats(tmp)
        tmp.close()
        self.assert_('Foo' in stats.index)

        stats.print_stats()

        self.assertEqual(f1.getvalue(), f2.getvalue())

        # Test sort_stats and reverse_order
        self.assertEqual(stats.sort_stats('size'), stats)
        self.assertEqual(stats.sorted[0].classname, 'Foo')
        stats.reverse_order()
        self.assertEqual(stats.sorted[0].classname, 'Bar')
        stats.sort_stats('classname', 'birth')
        self.assertEqual(stats.sorted[0].classname, 'Bar')
        self.assertRaises(ValueError, stats.sort_stats, 'name', 42, 'classn')

        # Test partial printing
        stats.stream = f3 = StringIO()
        stats.sort_stats()
        tolen = len(stats.sorted)
        stats.print_stats(clsname='Bar',limit=0.5)
        self.assertEqual(len(stats.sorted), tolen)
        stats.print_summary()
        clsname = f3.getvalue().split('\n')[0]
        self.assertNotEqual(re.search('\.Bar', clsname), None, clsname)
        self.assert_(len(f3.getvalue()) < len(f1.getvalue()))

        f1.close()
        f2.close()
        f3.close()

    def test_snapshots(self):
        """Test multiple snapshots.
        """
        self.tracker.track_class(Foo, name='Foo')
        self.tracker.track_class(Bar, name='Bar')
        self.tracker.track_class(FooNew, name='FooNew')

        self.tracker.create_snapshot()
        f1 = Foo()
        self.tracker.create_snapshot()
        f2 = Foo()
        f3 = FooNew()
        self.tracker.create_snapshot()
        b = Bar()
        del b
        self.tracker.create_snapshot()

        stream = StringIO()
        stats = ConsoleStats(tracker=self.tracker, stream=stream)
        stats.print_stats()
        stats.print_summary()

    def test_merge(self):
        """Test merging of reference trees.
        """
        self.tracker.track_class(FooNew, name='Foo', resolution_level=2)

        f1 = FooNew()
        f1.a = list(range(1000))
        f2 = FooNew()
        f2.a = list(range(100))
        f2.b = 'This is some stupid spam.'

        self.tracker.create_snapshot('Merge test')

        sizer = Asizer()
        sz1 = sizer.asized(f1)
        sz2 = sizer.asized(f2)

        stats = Stats(tracker=self.tracker)
        for fp in stats.footprint:
            if fp.desc == 'Merge test':
                stats.annotate_snapshot(fp)
                self.assert_(hasattr(fp, 'classes'))
                self.assert_('Foo' in fp.classes, fp.classes)
                self.assert_('merged' in fp.classes['Foo'])
                fm = fp.classes['Foo']['merged']
                self.assertEqual(fm.size, sz1.size + sz2.size, (fm.size, str(sz1), str(sz2)))
                refs = {}
                for ref in fm.refs:
                    refs[ref.name] = ref
                self.assert_('__dict__' in refs.keys(), refs.keys())
                refs2 = {}
                for ref in refs['__dict__'].refs:
                    refs2[ref.name] = ref
                self.assert_('[V] a' in refs2.keys(), refs2.keys())
                self.assert_('[V] b' in refs2.keys(), refs2.keys())
                self.assertEqual(refs2['[V] a'].size, asizeof(f1.a, f2.a))

    def test_html(self):
        """Test emitting HTML statistics."""
        self.tracker.track_class(Foo, name='Foo', resolution_level=2)

        f1 = Foo()
        f1.a = list(range(100000))
        f2 = Foo()
        f2.a = list(range(1000))
        f2.b = 'This is some stupid spam.'

        self.tracker.create_snapshot('Merge test')

        stats = HtmlStats(tracker=self.tracker)
        stats.create_html('tmp/data/footest.html')

    def test_charts(self):
        """Test emitting graphic charts."""
        self.tracker.track_class(Foo, name='Foo', resolution_level=2)

        f1 = Foo()
        f1.a = list(range(1000))
        f2 = Foo()
        f2.a = list(range(100))
        f2.b = 'This is some stupid spam.'

        self.tracker.create_snapshot('Merge test')

        stats = Stats(tracker=self.tracker)
        from pympler.gui import charts
        charts.tracker_timespace('tmp/data/timespace.png', stats)
class SnapshotTestCase(unittest.TestCase):

    def setUp(self):
        self.tracker = ClassTracker()

    def tearDown(self):
        self.tracker.stop_periodic_snapshots()
        self.tracker.clear()

    def test_timestamp(self):
        """Test timestamp of snapshots.
        """
        foo = Foo()
        bar = Bar()

        self.tracker.track_object(foo)
        self.tracker.track_object(bar)

        self.tracker.create_snapshot()
        self.tracker.create_snapshot()
        self.tracker.create_snapshot()

        refts = [fp.timestamp for fp in self.tracker.footprint]
        for to in self.tracker.objects.values():
            ts = [t for (t,sz) in to.footprint[1:]]
            self.assertEqual(ts,refts)

    def test_snapshot_members(self):
        """Test existence and value of snapshot members.
        """
        foo = Foo()
        self.tracker.track_object(foo)
        self.tracker.create_snapshot()

        fp = self.tracker.footprint[0]
        self.assert_(fp.overhead > 0, fp.overhead)
        self.assert_(fp.tracked_total > 0, fp.tracked_total)
        self.assert_(fp.asizeof_total > 0, fp.asizeof_total)
        self.assert_(fp.asizeof_total >= fp.tracked_total)

        if pympler.process.is_available():
            self.assert_(fp.system_total.vsz > 0)
            self.assert_(fp.system_total.rss > 0)
            self.assert_(fp.system_total.vsz >= fp.system_total.rss)
            self.assert_(fp.system_total.vsz > fp.overhead)
            self.assert_(fp.system_total.vsz > fp.tracked_total)
            self.assert_(fp.system_total.vsz > fp.asizeof_total)

    def test_desc(self):
        """Test footprint description.
        """
        self.tracker.create_snapshot()
        self.tracker.create_snapshot('alpha')
        self.tracker.create_snapshot(description='beta')
        self.tracker.create_snapshot(42)

        self.assertEqual(len(self.tracker.footprint), 4)
        self.assertEqual(self.tracker.footprint[0].desc, '')
        self.assertEqual(self.tracker.footprint[1].desc, 'alpha')
        self.assertEqual(self.tracker.footprint[2].desc, 'beta')
        self.assertEqual(self.tracker.footprint[3].desc, '42')

    def test_background_monitoring(self):
        """Test background monitoring.
        """
        import time

        self.tracker.start_periodic_snapshots(0.1)
        self.assertEqual(self.tracker._periodic_thread.interval, 0.1)
        self.assertEqual(self.tracker._periodic_thread.getName(), 'BackgroundMonitor')
        for x in range(10): # try to interfere
            self.tracker.create_snapshot(str(x))
        time.sleep(0.5)
        self.tracker.start_periodic_snapshots(0.2)
        self.assertEqual(self.tracker._periodic_thread.interval, 0.2)
        self.tracker.stop_periodic_snapshots()
        self.assert_(self.tracker._periodic_thread is None)
        self.assert_(len(self.tracker.footprint) > 10)