Beispiel #1
0
    def test_tempfile(self):
        '''repeatedly seralize to temp file, then repeatedly deserialize from
        it, checking usage all along the way.
        '''
        if cdump is None:
            logger.warn('no C dump(), skipping test_tempfile')
            return
        with tempfile.NamedTemporaryFile() as ntf:
            # first, write a bunch to temp file
            with open(ntf.name, 'wb') as fout:
                sys.stderr.write('write {!r} {}\n'.format(ntf.name, fout))
                start_usage = resource.getrusage(resource.RUSAGE_SELF)
                usage_history = [start_usage]
                for o in _range(_TEST_OUTER):
                    for i in _range(_TEST_COUNT):
                        ob = _randob()
                        cdump(ob, fout)
                    t_usage = resource.getrusage(resource.RUSAGE_SELF)
                    usage_history.append(t_usage)
                end_usage = usage_history[-1]
                dmaxrss = end_usage.ru_maxrss - start_usage.ru_maxrss
                didrss = end_usage.ru_idrss - start_usage.ru_idrss
                dmaxrsspct = ((end_usage.ru_maxrss != 0) and (dmaxrss / end_usage.ru_maxrss)) or 0
                didrsspct = ((end_usage.ru_idrss != 0) and (didrss / end_usage.ru_idrss)) or 0

                sys.stderr.write('maxrss: {} - {}, d={} ({:.2f}%)\n'.format(start_usage.ru_maxrss, end_usage.ru_maxrss, dmaxrss, dmaxrsspct * 100.0))
                sys.stderr.write('idrss: {} - {}, d={} ({:.2f}%)\n'.format(start_usage.ru_idrss, end_usage.ru_idrss, didrss, didrsspct * 100.0))

                assert (dmaxrsspct) < 0.05, [x.ru_maxrss for x in usage_history]
                assert (didrsspct) < 0.05, [x.ru_idrss for x in usage_history]

            sys.stderr.write('{!r} is {} bytes\n'.format(ntf.name, os.path.getsize(ntf.name)))

            # now, read a bunch back from temp file.
            with open(ntf.name, 'rb') as fin:
                sys.stderr.write('read {!r} {}\n'.format(ntf.name, fin))
                start_usage = resource.getrusage(resource.RUSAGE_SELF)
                usage_history = [start_usage]
                for o in _range(_TEST_OUTER):
                    for i in _range(_TEST_COUNT):
                        dob = cload(fin)
                        # and silently drop the result. I hope the garbage collector works!
                    gc.collect()
                    t_usage = resource.getrusage(resource.RUSAGE_SELF)
                    usage_history.append(t_usage)
                end_usage = usage_history[-1]
                dmaxrss = end_usage.ru_maxrss - start_usage.ru_maxrss
                didrss = end_usage.ru_idrss - start_usage.ru_idrss
                dmaxrsspct = ((end_usage.ru_maxrss != 0) and (dmaxrss / end_usage.ru_maxrss)) or 0
                didrsspct = ((end_usage.ru_idrss != 0) and (didrss / end_usage.ru_idrss)) or 0

                sys.stderr.write('maxrss: {} - {}, d={} ({:.2f}%)\n'.format(start_usage.ru_maxrss, end_usage.ru_maxrss, dmaxrss, dmaxrsspct * 100.0))
                sys.stderr.write('idrss: {} - {}, d={} ({:.2f}%)\n'.format(start_usage.ru_idrss, end_usage.ru_idrss, didrss, didrsspct * 100.0))

                assert (dmaxrsspct) < 0.05, [x.ru_maxrss for x in usage_history]
                assert (didrsspct) < 0.05, [x.ru_idrss for x in usage_history]
Beispiel #2
0
    def test_stringio_usage(self):
        '''serialize data to StringIO, read it back'''
        if cdump is None:
            logger.warn('no C dump(), skipping test_tempfile')
            return

        # warmup the rusage, allocate everything!
        fout = StringIO()
        sys.stderr.write('write 1 to StringIO\n')
        oblist = []
        for o in _range(_TEST_OUTER):
            for i in _range(_TEST_COUNT):
                ob = _randob()
                oblist.append(ob)
                cdump(ob, fout)

        # position at start to overwrite, but leave allocated
        fout.seek(0)

        sys.stderr.write('write 2 to StringIO\n')
        start_usage = resource.getrusage(resource.RUSAGE_SELF)
        usage_history = [start_usage]
        pos = 0
        for o in _range(_TEST_OUTER):
            for i in _range(_TEST_COUNT):
                ob = oblist[pos]
                pos += 1
                cdump(ob, fout)
            gc.collect()
            t_usage = resource.getrusage(resource.RUSAGE_SELF)
            usage_history.append(t_usage)
        end_usage = usage_history[-1]
        dmaxrss = end_usage.ru_maxrss - start_usage.ru_maxrss
        didrss = end_usage.ru_idrss - start_usage.ru_idrss
        dmaxrsspct = ((end_usage.ru_maxrss != 0) and
                      (dmaxrss / end_usage.ru_maxrss)) or 0
        didrsspct = ((end_usage.ru_idrss != 0) and
                     (didrss / end_usage.ru_idrss)) or 0

        sys.stderr.write('maxrss: {} - {}, d={} ({:.2f}%)\n'.format(
            start_usage.ru_maxrss, end_usage.ru_maxrss, dmaxrss,
            dmaxrsspct * 100.0))
        sys.stderr.write('idrss: {} - {}, d={} ({:.2f}%)\n'.format(
            start_usage.ru_idrss, end_usage.ru_idrss, didrss,
            didrsspct * 100.0))

        assert (dmaxrsspct) < 0.05, [x.ru_maxrss for x in usage_history]
        assert (didrsspct) < 0.05, [x.ru_idrss for x in usage_history]

        sys.stderr.write('StringIO is {} bytes\n'.format(fout.tell()))
        fout.seek(0)

        fin = fout
        sys.stderr.write('read StringIO\n')
        start_usage = resource.getrusage(resource.RUSAGE_SELF)
        usage_history = [start_usage]
        for o in _range(_TEST_OUTER):
            for i in _range(_TEST_COUNT):
                dob = cload(fin)
                # and silently drop the result. I hope the garbage collector works!
            gc.collect()
            t_usage = resource.getrusage(resource.RUSAGE_SELF)
            usage_history.append(t_usage)
        end_usage = usage_history[-1]
        dmaxrss = end_usage.ru_maxrss - start_usage.ru_maxrss
        didrss = end_usage.ru_idrss - start_usage.ru_idrss
        dmaxrsspct = ((end_usage.ru_maxrss != 0) and
                      (dmaxrss / end_usage.ru_maxrss)) or 0
        didrsspct = ((end_usage.ru_idrss != 0) and
                     (didrss / end_usage.ru_idrss)) or 0

        sys.stderr.write('maxrss: {} - {}, d={} ({:.2f}%)\n'.format(
            start_usage.ru_maxrss, end_usage.ru_maxrss, dmaxrss,
            dmaxrsspct * 100.0))
        sys.stderr.write('idrss: {} - {}, d={} ({:.2f}%)\n'.format(
            start_usage.ru_idrss, end_usage.ru_idrss, didrss,
            didrsspct * 100.0))

        assert (dmaxrsspct) < 0.05, [x.ru_maxrss for x in usage_history]
        assert (didrsspct) < 0.05, [x.ru_idrss for x in usage_history]
Beispiel #3
0
    def test_tempfile(self):
        '''repeatedly seralize to temp file, then repeatedly deserialize from
        it, checking usage all along the way.
        '''
        if cdump is None:
            logger.warn('no C dump(), skipping test_tempfile')
            return
        with tempfile.NamedTemporaryFile() as ntf:
            # first, write a bunch to temp file
            with open(ntf.name, 'wb') as fout:
                sys.stderr.write('write {!r} {}\n'.format(ntf.name, fout))
                start_usage = resource.getrusage(resource.RUSAGE_SELF)
                usage_history = [start_usage]
                for o in _range(_TEST_OUTER):
                    for i in _range(_TEST_COUNT):
                        ob = _randob()
                        cdump(ob, fout)
                    t_usage = resource.getrusage(resource.RUSAGE_SELF)
                    usage_history.append(t_usage)
                end_usage = usage_history[-1]
                dmaxrss = end_usage.ru_maxrss - start_usage.ru_maxrss
                didrss = end_usage.ru_idrss - start_usage.ru_idrss
                dmaxrsspct = ((end_usage.ru_maxrss != 0) and
                              (dmaxrss / end_usage.ru_maxrss)) or 0
                didrsspct = ((end_usage.ru_idrss != 0) and
                             (didrss / end_usage.ru_idrss)) or 0

                sys.stderr.write('maxrss: {} - {}, d={} ({:.2f}%)\n'.format(
                    start_usage.ru_maxrss, end_usage.ru_maxrss, dmaxrss,
                    dmaxrsspct * 100.0))
                sys.stderr.write('idrss: {} - {}, d={} ({:.2f}%)\n'.format(
                    start_usage.ru_idrss, end_usage.ru_idrss, didrss,
                    didrsspct * 100.0))

                assert (dmaxrsspct) < 0.05, [
                    x.ru_maxrss for x in usage_history
                ]
                assert (didrsspct) < 0.05, [x.ru_idrss for x in usage_history]

            sys.stderr.write('{!r} is {} bytes\n'.format(
                ntf.name, os.path.getsize(ntf.name)))

            # now, read a bunch back from temp file.
            with open(ntf.name, 'rb') as fin:
                sys.stderr.write('read {!r} {}\n'.format(ntf.name, fin))
                start_usage = resource.getrusage(resource.RUSAGE_SELF)
                usage_history = [start_usage]
                for o in _range(_TEST_OUTER):
                    for i in _range(_TEST_COUNT):
                        dob = cload(fin)
                        # and silently drop the result. I hope the garbage collector works!
                    gc.collect()
                    t_usage = resource.getrusage(resource.RUSAGE_SELF)
                    usage_history.append(t_usage)
                end_usage = usage_history[-1]
                dmaxrss = end_usage.ru_maxrss - start_usage.ru_maxrss
                didrss = end_usage.ru_idrss - start_usage.ru_idrss
                dmaxrsspct = ((end_usage.ru_maxrss != 0) and
                              (dmaxrss / end_usage.ru_maxrss)) or 0
                didrsspct = ((end_usage.ru_idrss != 0) and
                             (didrss / end_usage.ru_idrss)) or 0

                sys.stderr.write('maxrss: {} - {}, d={} ({:.2f}%)\n'.format(
                    start_usage.ru_maxrss, end_usage.ru_maxrss, dmaxrss,
                    dmaxrsspct * 100.0))
                sys.stderr.write('idrss: {} - {}, d={} ({:.2f}%)\n'.format(
                    start_usage.ru_idrss, end_usage.ru_idrss, didrss,
                    didrsspct * 100.0))

                assert (dmaxrsspct) < 0.05, [
                    x.ru_maxrss for x in usage_history
                ]
                assert (didrsspct) < 0.05, [x.ru_idrss for x in usage_history]
Beispiel #4
0
    def test_stringio_usage(self):
        '''serialize data to StringIO, read it back'''
        if cdump is None:
            logger.warn('no C dump(), skipping test_tempfile')
            return

        # warmup the rusage, allocate everything!
        fout = StringIO()
        sys.stderr.write('write 1 to StringIO\n')
        oblist = []
        for o in _range(_TEST_OUTER):
            for i in _range(_TEST_COUNT):
                ob = _randob()
                oblist.append(ob)
                cdump(ob, fout)

        # position at start to overwrite, but leave allocated
        fout.seek(0)

        sys.stderr.write('write 2 to StringIO\n')
        start_usage = resource.getrusage(resource.RUSAGE_SELF)
        usage_history = [start_usage]
        pos = 0
        for o in _range(_TEST_OUTER):
            for i in _range(_TEST_COUNT):
                ob = oblist[pos]
                pos += 1
                cdump(ob, fout)
            gc.collect()
            t_usage = resource.getrusage(resource.RUSAGE_SELF)
            usage_history.append(t_usage)
        end_usage = usage_history[-1]
        dmaxrss = end_usage.ru_maxrss - start_usage.ru_maxrss
        didrss = end_usage.ru_idrss - start_usage.ru_idrss
        dmaxrsspct = ((end_usage.ru_maxrss != 0) and (dmaxrss / end_usage.ru_maxrss)) or 0
        didrsspct = ((end_usage.ru_idrss != 0) and (didrss / end_usage.ru_idrss)) or 0

        sys.stderr.write('maxrss: {} - {}, d={} ({:.2f}%)\n'.format(start_usage.ru_maxrss, end_usage.ru_maxrss, dmaxrss, dmaxrsspct * 100.0))
        sys.stderr.write('idrss: {} - {}, d={} ({:.2f}%)\n'.format(start_usage.ru_idrss, end_usage.ru_idrss, didrss, didrsspct * 100.0))

        assert (dmaxrsspct) < 0.05, [x.ru_maxrss for x in usage_history]
        assert (didrsspct) < 0.05, [x.ru_idrss for x in usage_history]

        sys.stderr.write('StringIO is {} bytes\n'.format(fout.tell()))
        fout.seek(0)

        fin = fout
        sys.stderr.write('read StringIO\n')
        start_usage = resource.getrusage(resource.RUSAGE_SELF)
        usage_history = [start_usage]
        for o in _range(_TEST_OUTER):
            for i in _range(_TEST_COUNT):
                dob = cload(fin)
                # and silently drop the result. I hope the garbage collector works!
            gc.collect()
            t_usage = resource.getrusage(resource.RUSAGE_SELF)
            usage_history.append(t_usage)
        end_usage = usage_history[-1]
        dmaxrss = end_usage.ru_maxrss - start_usage.ru_maxrss
        didrss = end_usage.ru_idrss - start_usage.ru_idrss
        dmaxrsspct = ((end_usage.ru_maxrss != 0) and (dmaxrss / end_usage.ru_maxrss)) or 0
        didrsspct = ((end_usage.ru_idrss != 0) and (didrss / end_usage.ru_idrss)) or 0

        sys.stderr.write('maxrss: {} - {}, d={} ({:.2f}%)\n'.format(start_usage.ru_maxrss, end_usage.ru_maxrss, dmaxrss, dmaxrsspct * 100.0))
        sys.stderr.write('idrss: {} - {}, d={} ({:.2f}%)\n'.format(start_usage.ru_idrss, end_usage.ru_idrss, didrss, didrsspct * 100.0))

        assert (dmaxrsspct) < 0.05, [x.ru_maxrss for x in usage_history]
        assert (didrsspct) < 0.05, [x.ru_idrss for x in usage_history]