Beispiel #1
0
class TestMaster(ScenarioFixture, TestCase):
    maxDiff = None

    def setUp(self):
        # Set our test scenario differently from the default; must be BEFORE
        # the super call.
        self.scenario_dict = dict(
            name='Master Test Scenario - ablkei',
            sizes=[
                dict(name='tiny', size_min=99, size_max=100),
                dict(name='small', size_min=1990, size_max=1990,
                     crud_profile=[71, 9, 12, 8]),
                dict(name='medium', size_min=2990, size_max=3000),
                dict(name='unused', size_min=9876543, size_max=9876543),
                dict(name='large', size_min=399000, size_max=400000,
                     crud_profile=[16, 61, 7, 16]),
                dict(name='huge', size_min=49900000, size_max=71499999)],
            initial_files=dict(
                tiny=300, small=400, medium=500, large=200, huge=70,
            ),
            operation_count=5000,
            #             C  R  U  D
            crud_profile=[5, 3, 1, 1],
            user_count=2,
        )
        super(TestMaster, self).setUp()

        self.zmq_host = 'slick.queue.com'
        self.zmq_work_port = 7482
        self.zmq_results_port = 18398
        self.work_endpoint = 'tcp://%s:%d' % (self.zmq_host,
                                              self.zmq_work_port)
        self.results_endpoint = 'tcp://%s:%d' % (self.zmq_host,
                                                 self.zmq_results_port)

        self.mock_context = flexmock()
        flexmock(zmq.Context).new_instances(self.mock_context).once

        self.mock_work_push = flexmock()
        self.mock_context.should_receive('socket').with_args(
            zmq.PUSH,
        ).and_return(self.mock_work_push).once
        self.mock_work_push.should_receive('bind').with_args(
            self.work_endpoint,
        ).once

        self.mock_results_pull = flexmock()
        self.mock_context.should_receive('socket').with_args(
            zmq.PULL,
        ).and_return(self.mock_results_pull).once
        self.mock_results_pull.should_receive('bind').with_args(
            self.results_endpoint,
        ).once

        self.master = Master(self.zmq_host, self.zmq_work_port,
                             self.zmq_results_port,
                             connect_timeout=3.14159,
                             network_timeout=2.71828)

        self.result_index = 1  # for self.gen_result()

        self.stub_results = [
            self.gen_result(
                1, ssbench.CREATE_OBJECT, 'small', 100.0, 101.0, 103.0),
            self.gen_result(
                1, ssbench.READ_OBJECT, 'tiny', 103.0, 103.1, 103.8),
            self.gen_result(
                1, ssbench.CREATE_OBJECT, 'huge', 103.8, 105.0, 106.0),
            self.gen_result(
                1, ssbench.UPDATE_OBJECT, 'large', 106.1, 106.3, 106.4),
            #
            # exceptions should be ignored
            dict(worker_id=2, type=ssbench.UPDATE_OBJECT,
                 completed_at=39293.2, exception='wacky!', traceback='ugh'),
            self.gen_result(
                2, ssbench.UPDATE_OBJECT, 'medium', 100.1, 100.9, 102.9),
            self.gen_result(
                2, ssbench.DELETE_OBJECT, 'large', 102.9, 103.0, 103.3),
            self.gen_result(
                2, ssbench.CREATE_OBJECT, 'tiny', 103.3, 103.4, 103.5),
            self.gen_result(
                2, ssbench.READ_OBJECT, 'small', 103.5, 103.7, 104.0),
            #
            self.gen_result(
                3, ssbench.READ_OBJECT, 'tiny', 100.1, 101.1, 101.9),
            # worker 3 took a while (observer lower concurrency in second 102
            self.gen_result(
                3, ssbench.DELETE_OBJECT, 'small', 103.1, 103.6, 103.9),
            self.gen_result(
                3, ssbench.READ_OBJECT, 'medium', 103.9, 104.2, 104.3),
            self.gen_result(
                3, ssbench.UPDATE_OBJECT, 'tiny', 104.3, 104.9, 104.999),
        ]

    def tearDown(self):
        super(TestMaster, self).tearDown()

    def gen_result(self, worker_id, op_type, size_str, start, first_byte,
                   last_byte):
        self.result_index += 1

        return {
            # There are other keys in a "result", but these are the only ones
            # used for the reporting.
            'worker_id': worker_id,
            'type': op_type,
            'size_str': size_str,
            'size': 989,
            'first_byte_latency': first_byte - start,
            'last_byte_latency': last_byte - start,
            'trans_id': 'txID%03d' % self.result_index,
            'completed_at': last_byte,
        }

    def test_format_bytes(self):
        input_expected = [
            (0, '  0  B'),
            (7, '  7  B'),
            (17, ' 17  B'),
            (117, '117  B'),
            (999, '999  B'),
            (1000, '  1 kB'),
            (1001, '  1 kB'),
            (1100, '  1 kB'),
            (1500, '  2 kB'),
            (1999, '  2 kB'),
            (2000, '  2 kB'),
            (7500, '  8 kB'),
            (7900, '  8 kB'),
            (9999, ' 10 kB'),
            (10000, ' 10 kB'),
            (19999, ' 20 kB'),
            (20000, ' 20 kB'),
            (84999, ' 85 kB'),
            (85000, ' 85 kB'),
            (99999, '100 kB'),
            (100000, '100 kB'),
            (100001, '100 kB'),
            (700500, '701 kB'),
            (999999, '  1 MB'),
            (1000000, '  1 MB'),
            (1000001, '  1 MB'),
            (4123456, '  4 MB'),
            (4543210, '  5 MB'),
            (99999999, '100 MB'),
            (100000000, '100 MB'),
            (9999999999, ' 10 GB'),
            (10000000000, ' 10 GB'),
            (99999999999, '100 GB'),
            (100000000000, '100 GB'),
            (234567890123, '235 GB'),
        ]
        for input_bytes, formatting in input_expected:
            self.assertEqual(
                formatting, self.master._format_bytes(input_bytes),
                '_format_bytes(%d) = %s instead of %s' % (
                    input_bytes, self.master._format_bytes(input_bytes),
                    formatting))

    def test_calculate_scenario_stats_aggregate(self):
        first_byte_latency_all = [1, 0.1, 1.2, 0.2, 0.8, 0.1, 0.1,
                                  0.2, 1, 0.5, 0.3, 0.6]
        last_byte_latency_all = [3, 0.8, 2.2, 0.3, 2.8, 0.4, 0.2,
                                 0.5, 1.8, 0.8, 0.4, 0.699]
        scen_stats = self.master.calculate_scenario_stats(self.scenario,
                                                          self.stub_results)
        self.assertDictEqual(dict(
            worker_count=3, start=100.0, stop=106.4, req_count=12,
            avg_req_per_sec=round(12 / (106.4 - 100), 6),
            first_byte_latency=dict(
                min='%6.3f' % 0.1,
                max='%7.3f' % 1.2,
                avg='%7.3f' % stats.lmean(first_byte_latency_all),
                pctile='%7.3f' % 1.2,
                std_dev='%7.3f' % stats.lsamplestdev(first_byte_latency_all),
                median='%7.3f' % stats.lmedianscore(first_byte_latency_all),
            ),
            last_byte_latency=dict(
                min='%6.3f' % 0.2,
                max='%7.3f' % 3.0,
                avg='%7.3f' % stats.lmean(last_byte_latency_all),
                pctile='%7.3f' % 3.0,
                std_dev='%7.3f' % stats.lsamplestdev(last_byte_latency_all),
                median='  0.749',  # XXX why??
                # median='%7.3f' % stats.lmedianscore(last_byte_latency_all),
            ),
            worst_first_byte_latency=(1.2, 'txID004'),
            worst_last_byte_latency=(3.0, 'txID002'),
        ), scen_stats['agg_stats'])

    def test_calculate_scenario_stats_aggregate_low_pctile(self):
        first_byte_latency_all = [1, 0.1, 1.2, 0.2, 0.8, 0.1, 0.1,
                                  0.2, 1, 0.5, 0.3, 0.6]
        last_byte_latency_all = [3, 0.8, 2.2, 0.3, 2.8, 0.4, 0.2,
                                 0.5, 1.8, 0.8, 0.4, 0.699]
        scen_stats = self.master.calculate_scenario_stats(self.scenario,
                                                          self.stub_results,
                                                          nth_pctile=20)
        self.assertDictEqual(dict(
            worker_count=3, start=100.0, stop=106.4, req_count=12,
            avg_req_per_sec=round(12 / (106.4 - 100), 6),
            first_byte_latency=dict(
                min='%6.3f' % 0.1,
                max='%7.3f' % 1.2,
                avg='%7.3f' % stats.lmean(first_byte_latency_all),
                pctile='%7.3f' % sorted(first_byte_latency_all)[2],
                std_dev='%7.3f' % stats.lsamplestdev(first_byte_latency_all),
                median='%7.3f' % stats.lmedianscore(first_byte_latency_all),
            ),
            last_byte_latency=dict(
                min='%6.3f' % 0.2,
                max='%7.3f' % 3.0,
                avg='%7.3f' % stats.lmean(last_byte_latency_all),
                pctile='%7.3f' % sorted(last_byte_latency_all)[2],
                std_dev='%7.3f' % stats.lsamplestdev(last_byte_latency_all),
                median='  0.749',  # XXX why??
                # median='%7.3f' % stats.lmedianscore(last_byte_latency_all),
            ),
            worst_first_byte_latency=(1.2, 'txID004'),
            worst_last_byte_latency=(3.0, 'txID002'),
        ), scen_stats['agg_stats'])

    def test_calculate_scenario_stats_worker1(self):
        w1_first_byte_latency = [1.0, 0.1, 1.2, 0.2]
        w1_last_byte_latency = [3.0, 0.8, 2.2, 0.3]
        scen_stats = self.master.calculate_scenario_stats(self.scenario,
                                                          self.stub_results)
        self.assertDictEqual(dict(
            start=100.0, stop=106.4, req_count=4,
            avg_req_per_sec=round(4 / (106.4 - 100), 6),
            first_byte_latency=dict(
                min='%6.3f' % min(w1_first_byte_latency),
                max='%7.3f' % max(w1_first_byte_latency),
                avg='%7.3f' % stats.lmean(w1_first_byte_latency),
                pctile='%7.3f' % 1.2,
                std_dev='%7.3f' % stats.lsamplestdev(w1_first_byte_latency),
                median='%7.3f' % stats.lmedianscore(w1_first_byte_latency),
            ),
            last_byte_latency=dict(
                min='%6.3f' % min(w1_last_byte_latency),
                max='%7.3f' % max(w1_last_byte_latency),
                avg='%7.3f' % stats.lmean(w1_last_byte_latency),
                pctile='%7.3f' % 3.0,
                std_dev='%7.3f' % stats.lsamplestdev(w1_last_byte_latency),
                median='%7.3f' % stats.lmedianscore(w1_last_byte_latency),
            ),
            worst_first_byte_latency=(float(max(w1_first_byte_latency)),
                                      'txID004'),
            worst_last_byte_latency=(
                float(max(w1_last_byte_latency)), 'txID002'),
        ), scen_stats['worker_stats'][1])

    def test_calculate_scenario_stats_worker2(self):
        w2_first_byte_latency = [0.8, 0.1, 0.1, 0.2]
        w2_last_byte_latency = [2.8, 0.4, 0.2, 0.5]
        scen_stats = self.master.calculate_scenario_stats(self.scenario,
                                                          self.stub_results)
        self.assertDictEqual(dict(
            start=100.1, stop=104.0, req_count=4,
            avg_req_per_sec=round(4 / (104.0 - 100.1), 6),
            first_byte_latency=dict(
                min='%6.3f' % min(w2_first_byte_latency),
                max='%7.3f' % max(w2_first_byte_latency),
                avg='%7.3f' % stats.lmean(w2_first_byte_latency),
                pctile='%7.3f' % 0.8,
                std_dev='%7.3f' % stats.lsamplestdev(w2_first_byte_latency),
                median='%7.3f' % stats.lmedianscore(w2_first_byte_latency),
            ),
            last_byte_latency=dict(
                min='%6.3f' % min(w2_last_byte_latency),
                max='%7.3f' % max(w2_last_byte_latency),
                avg='%7.3f' % stats.lmean(w2_last_byte_latency),
                pctile='%7.3f' % 2.8,
                std_dev='%7.3f' % stats.lsamplestdev(w2_last_byte_latency),
                median='%7.3f' % stats.lmedianscore(w2_last_byte_latency),
            ),
            worst_first_byte_latency=(float(max(w2_first_byte_latency)),
                                      'txID006'),
            worst_last_byte_latency=(
                float(max(w2_last_byte_latency)), 'txID006'),
        ), scen_stats['worker_stats'][2])

    def test_calculate_scenario_stats_worker3(self):
        w3_first_byte_latency = [1, 0.5, 0.3, 0.6]
        w3_last_byte_latency = [1.8, 0.8, 0.4, 0.699]
        scen_stats = self.master.calculate_scenario_stats(self.scenario,
                                                          self.stub_results)
        self.assertDictEqual(dict(
            start=100.1, stop=104.999, req_count=4,
            avg_req_per_sec=round(4 / (104.999 - 100.1), 6),
            first_byte_latency=dict(
                min='%6.3f' % min(w3_first_byte_latency),
                max='%7.3f' % max(w3_first_byte_latency),
                avg='%7.3f' % stats.lmean(w3_first_byte_latency),
                pctile='%7.3f' % 1.0,
                std_dev='%7.3f' % stats.lsamplestdev(w3_first_byte_latency),
                median='%7.3f' % stats.lmedianscore(w3_first_byte_latency),
            ),
            last_byte_latency=dict(
                min='%6.3f' % min(w3_last_byte_latency),
                max='%7.3f' % max(w3_last_byte_latency),
                avg='%7.3f' % stats.lmean(w3_last_byte_latency),
                pctile='%7.3f' % 1.8,
                std_dev='%7.3f' % stats.lsamplestdev(w3_last_byte_latency),
                median='%7.3f' % stats.lmedianscore(w3_last_byte_latency),
            ),
            worst_first_byte_latency=(
                float(max(w3_first_byte_latency)), 'txID010'),
            worst_last_byte_latency=(
                float(max(w3_last_byte_latency)), 'txID010'),
        ), scen_stats['worker_stats'][3])

    def test_calculate_scenario_stats_create(self):
        # Stats for Create
        c_first_byte_latency = [1, 1.2, 0.1]
        c_last_byte_latency = [3.0, 2.2, 0.2]
        scen_stats = self.master.calculate_scenario_stats(self.scenario,
                                                          self.stub_results)
        self.assertDictEqual(dict(
            start=100.0, stop=106.0, req_count=3,
            avg_req_per_sec=round(3 / (106 - 100.0), 6),
            first_byte_latency=dict(
                min='%6.3f' % min(c_first_byte_latency),
                max='%7.3f' % max(c_first_byte_latency),
                pctile='%7.3f' % max(c_first_byte_latency),
                avg='%7.3f' % stats.lmean(c_first_byte_latency),
                std_dev='%7.3f' % stats.lsamplestdev(c_first_byte_latency),
                median='%7.3f' % stats.lmedianscore(c_first_byte_latency),
            ),
            last_byte_latency=dict(
                min='%6.3f' % min(c_last_byte_latency),
                max='%7.3f' % max(c_last_byte_latency),
                pctile='%7.3f' % max(c_last_byte_latency),
                avg='%7.3f' % stats.lmean(c_last_byte_latency),
                std_dev='%7.3f' % stats.lsamplestdev(c_last_byte_latency),
                median='%7.3f' % stats.lmedianscore(c_last_byte_latency),
            ),
            worst_first_byte_latency=(max(c_first_byte_latency), 'txID004'),
            worst_last_byte_latency=(max(c_last_byte_latency), 'txID002'),
            size_stats=OrderedDict([
                ('tiny', {'avg_req_per_sec': 5.0,
                          'first_byte_latency': {'avg': '%7.3f' % 0.1,
                                                 'max': '%7.3f' % 0.1,
                                                 'pctile': '%7.3f' % 0.1,
                                                 'median': '%7.3f' % 0.1,
                                                 'min': '%6.3f' % 0.1,
                                                 'std_dev': '%7.3f' % 0.0},
                          'last_byte_latency': {'avg': '%7.3f' % 0.2,
                                                'max': '%7.3f' % 0.2,
                                                'pctile': '%7.3f' % 0.2,
                                                'median': '%7.3f' % 0.2,
                                                'min': '%6.3f' % 0.2,
                                                'std_dev': '%7.3f' % 0.0},
                          'worst_first_byte_latency': (0.1, 'txID008'),
                          'worst_last_byte_latency': (0.2, 'txID008'),
                          'req_count': 1,
                          'start': 103.3,
                          'stop': 103.5}),
                ('small', {'avg_req_per_sec': 0.333333,
                           'first_byte_latency': {'avg': '%7.3f' % 1.0,
                                                  'max': '%7.3f' % 1.0,
                                                  'pctile': '%7.3f' % 1.0,
                                                  'median': '%7.3f' % 1.0,
                                                  'min': '%6.3f' % 1.0,
                                                  'std_dev': '%7.3f' % 0.0},
                           'last_byte_latency': {'avg': '%7.3f' % 3.0,
                                                 'max': '%7.3f' % 3.0,
                                                 'pctile': '%7.3f' % 3.0,
                                                 'median': '%7.3f' % 3.0,
                                                 'min': '%6.3f' % 3.0,
                                                 'std_dev': '%7.3f' % 0.0},
                           'worst_first_byte_latency': (1.0, 'txID002'),
                           'worst_last_byte_latency': (3.0, 'txID002'),
                           'req_count': 1,
                           'start': 100.0,
                           'stop': 103.0}),
                ('huge', {'avg_req_per_sec': 0.454545,
                          'first_byte_latency': {'avg': '%7.3f' % 1.2,
                                                 'max': '%7.3f' % 1.2,
                                                 'pctile': '%7.3f' % 1.2,
                                                 'median': '%7.3f' % 1.2,
                                                 'min': '%6.3f' % 1.2,
                                                 'std_dev': '%7.3f' % 0.0},
                          'last_byte_latency': {'avg': '%7.3f' % 2.2,
                                                'max': '%7.3f' % 2.2,
                                                'pctile': '%7.3f' % 2.2,
                                                'median': '%7.3f' % 2.2,
                                                'min': '%6.3f' % 2.2,
                                                'std_dev': '%7.3f' % 0.0},
                          'worst_first_byte_latency': (1.2, 'txID004'),
                          'worst_last_byte_latency': (2.2, 'txID004'),
                          'req_count': 1,
                          'start': 103.8,
                          'stop': 106.0})]),
        ), scen_stats['op_stats'][ssbench.CREATE_OBJECT])

    def test_calculate_scenario_stats_read(self):
        # Stats for Read
        r_first_byte_latency = [0.1, 0.2, 1.0, 0.3]
        r_last_byte_latency = [0.8, 0.5, 1.8, 0.4]
        scen_stats = self.master.calculate_scenario_stats(self.scenario,
                                                          self.stub_results)
        self.assertDictEqual(dict(
            start=100.1, stop=104.3, req_count=4,
            avg_req_per_sec=round(4 / (104.3 - 100.1), 6),
            first_byte_latency=dict(
                min='%6.3f' % min(r_first_byte_latency),
                max='%7.3f' % max(r_first_byte_latency),
                pctile='%7.3f' % max(r_first_byte_latency),
                avg='%7.3f' % stats.lmean(r_first_byte_latency),
                std_dev='%7.3f' % stats.lsamplestdev(r_first_byte_latency),
                median='%7.3f' % stats.lmedianscore(r_first_byte_latency),
            ),
            last_byte_latency=dict(
                min='%6.3f' % min(r_last_byte_latency),
                max='%7.3f' % max(r_last_byte_latency),
                pctile='%7.3f' % max(r_last_byte_latency),
                avg='%7.3f' % stats.lmean(r_last_byte_latency),
                std_dev='%7.3f' % stats.lsamplestdev(r_last_byte_latency),
                median='%7.3f' % stats.lmedianscore(r_last_byte_latency),
            ),
            worst_first_byte_latency=(max(r_first_byte_latency), 'txID010'),
            worst_last_byte_latency=(max(r_last_byte_latency), 'txID010'),
            size_stats=OrderedDict([
                ('tiny', {'avg_req_per_sec': 0.540541,
                          'first_byte_latency': {'avg': '%7.3f' % 0.55,
                                                 'max': '%7.3f' % 1.0,
                                                 'pctile': '%7.3f' % 1.0,
                                                 'median': '%7.3f' % 0.55,
                                                 'min': '%6.3f' % 0.1,
                                                 'std_dev': '%7.3f' % 0.45},
                          'last_byte_latency': {'avg': '%7.3f' % 1.3,
                                                'max': '%7.3f' % 1.8,
                                                'pctile': '%7.3f' % 1.8,
                                                'median': '%7.3f' % 1.3,
                                                'min': '%6.3f' % 0.8,
                                                'std_dev': '%7.3f' % 0.5},
                          'worst_first_byte_latency': (1.0, 'txID010'),
                          'worst_last_byte_latency': (1.8, 'txID010'),
                          'req_count': 2,
                          'start': 100.1,
                          'stop': 103.8}),
                ('small', {'avg_req_per_sec': 2.0,
                           'first_byte_latency': {'avg': '%7.3f' % 0.2,
                                                  'max': '%7.3f' % 0.2,
                                                  'pctile': '%7.3f' % 0.2,
                                                  'median': '%7.3f' % 0.2,
                                                  'min': '%6.3f' % 0.2,
                                                  'std_dev': '%7.3f' % 0.0},
                           'last_byte_latency': {'avg': '%7.3f' % 0.5,
                                                 'max': '%7.3f' % 0.5,
                                                 'pctile': '%7.3f' % 0.5,
                                                 'median': '%7.3f' % 0.5,
                                                 'min': '%6.3f' % 0.5,
                                                 'std_dev': '%7.3f' % 0.0},
                           'worst_first_byte_latency': (0.2, 'txID009'),
                           'worst_last_byte_latency': (0.5, 'txID009'),
                           'req_count': 1,
                           'start': 103.5,
                           'stop': 104.0}),
                ('medium', {'avg_req_per_sec': 2.5,
                            'first_byte_latency': {'avg': '%7.3f' % 0.3,
                                                   'max': '%7.3f' % 0.3,
                                                   'pctile': '%7.3f' % 0.3,
                                                   'median': '%7.3f' % 0.3,
                                                   'min': '%6.3f' % 0.3,
                                                   'std_dev': '%7.3f' % 0.0},
                            'last_byte_latency': {'avg': '%7.3f' % 0.4,
                                                  'max': '%7.3f' % 0.4,
                                                  'pctile': '%7.3f' % 0.4,
                                                  'median': '%7.3f' % 0.4,
                                                  'min': '%6.3f' % 0.4,
                                                  'std_dev': '%7.3f' % 0.0},
                            'worst_first_byte_latency': (0.3, 'txID012'),
                            'worst_last_byte_latency': (0.4, 'txID012'),
                            'req_count': 1,
                            'start': 103.9,
                            'stop': 104.3})]),
        ), scen_stats['op_stats'][ssbench.READ_OBJECT])

    def test_calculate_scenario_stats_update(self):
        u_first_byte_latency = [0.2, 0.8, 0.6]
        u_last_byte_latency = [0.3, 2.8, 0.699]
        scen_stats = self.master.calculate_scenario_stats(self.scenario,
                                                          self.stub_results)
        self.assertDictEqual(dict(
            start=100.1, stop=106.4, req_count=3,
            avg_req_per_sec=round(3 / (106.4 - 100.1), 6),
            first_byte_latency=dict(
                min='%6.3f' % min(u_first_byte_latency),
                max='%7.3f' % max(u_first_byte_latency),
                avg='%7.3f' % stats.lmean(u_first_byte_latency),
                pctile='%7.3f' % 0.8,
                std_dev='%7.3f' % stats.lsamplestdev(u_first_byte_latency),
                median='%7.3f' % stats.lmedianscore(u_first_byte_latency),
            ),
            worst_first_byte_latency=(max(u_first_byte_latency), 'txID006'),
            last_byte_latency=dict(
                min='%6.3f' % min(u_last_byte_latency),
                max='%7.3f' % max(u_last_byte_latency),
                avg='%7.3f' % stats.lmean(u_last_byte_latency),
                pctile='%7.3f' % 2.8,
                std_dev='%7.3f' % stats.lsamplestdev(u_last_byte_latency),
                median='%7.3f' % stats.lmedianscore(u_last_byte_latency),
            ),
            worst_last_byte_latency=(max(u_last_byte_latency), 'txID006'),
            size_stats=OrderedDict([
                ('tiny', {'avg_req_per_sec': 1.430615,
                          'first_byte_latency': {'avg': '%7.3f' % 0.6,
                                                 'pctile': '%7.3f' % 0.6,
                                                 'max': '%7.3f' % 0.6,
                                                 'median': '%7.3f' % 0.6,
                                                 'min': '%6.3f' % 0.6,
                                                 'std_dev': '%7.3f' % 0.0},
                          'worst_first_byte_latency': (0.6, 'txID013'),
                          'last_byte_latency': {'avg': '%7.3f' % 0.699,
                                                'pctile': '%7.3f' % 0.699,
                                                'max': '%7.3f' % 0.699,
                                                'median': '%7.3f' % 0.699,
                                                'min': '%6.3f' % 0.699,
                                                'std_dev': '%7.3f' % 0.0},
                          'worst_last_byte_latency': (0.699, 'txID013'),
                          'req_count': 1,
                          'start': 104.3,
                          'stop': 104.999}),
                ('medium', {'avg_req_per_sec': 0.357143,
                            'first_byte_latency': {'avg': '%7.3f' % 0.8,
                                                   'pctile': '%7.3f' % 0.8,
                                                   'max': '%7.3f' % 0.8,
                                                   'median': '%7.3f' % 0.8,
                                                   'min': '%6.3f' % 0.8,
                                                   'std_dev': '%7.3f' % 0.0},
                            'worst_first_byte_latency': (0.8, 'txID006'),
                            'last_byte_latency': {'avg': '%7.3f' % 2.8,
                                                  'pctile': '%7.3f' % 2.8,
                                                  'max': '%7.3f' % 2.8,
                                                  'median': '%7.3f' % 2.8,
                                                  'min': '%6.3f' % 2.8,
                                                  'std_dev': '%7.3f' % 0.0},
                            'worst_last_byte_latency': (2.8, 'txID006'),
                            'req_count': 1,
                            'start': 100.1,
                            'stop': 102.9}),
                ('large', {'avg_req_per_sec': 3.333333,
                           'first_byte_latency': {'avg': '%7.3f' % 0.2,
                                                  'pctile': '%7.3f' % 0.2,
                                                  'max': '%7.3f' % 0.2,
                                                  'median': '%7.3f' % 0.2,
                                                  'min': '%6.3f' % 0.2,
                                                  'std_dev': '%7.3f' % 0.0},
                           'worst_first_byte_latency': (0.2, 'txID005'),
                           'last_byte_latency': {'avg': '%7.3f' % 0.3,
                                                 'pctile': '%7.3f' % 0.3,
                                                 'max': '%7.3f' % 0.3,
                                                 'median': '%7.3f' % 0.3,
                                                 'min': '%6.3f' % 0.3,
                                                 'std_dev': '%7.3f' % 0.0},
                           'worst_last_byte_latency': (0.3, 'txID005'),
                           'req_count': 1,
                           'start': 106.1,
                           'stop': 106.4})]),
        ), scen_stats['op_stats'][ssbench.UPDATE_OBJECT])

    def test_calculate_scenario_stats_delete(self):
        d_first_byte_latency = [0.1, 0.5]
        d_last_byte_latency = [0.4, 0.8]
        scen_stats = self.master.calculate_scenario_stats(self.scenario,
                                                          self.stub_results)
        self.assertDictEqual(dict(
            start=102.9, stop=103.9, req_count=2,
            avg_req_per_sec=round(2 / (103.9 - 102.9), 6),
            first_byte_latency=dict(
                min='%6.3f' % min(d_first_byte_latency),
                max='%7.3f' % max(d_first_byte_latency),
                pctile='%7.3f' % max(d_first_byte_latency),
                avg='%7.3f' % stats.lmean(d_first_byte_latency),
                std_dev='%7.3f' % stats.lsamplestdev(d_first_byte_latency),
                median='%7.3f' % stats.lmedianscore(d_first_byte_latency),
            ),
            last_byte_latency=dict(
                min='%6.3f' % min(d_last_byte_latency),
                max='%7.3f' % max(d_last_byte_latency),
                pctile='%7.3f' % max(d_last_byte_latency),
                avg='%7.3f' % stats.lmean(d_last_byte_latency),
                std_dev='%7.3f' % stats.lsamplestdev(d_last_byte_latency),
                median='%7.3f' % stats.lmedianscore(d_last_byte_latency),
            ),
            worst_first_byte_latency=(max(d_first_byte_latency), 'txID011'),
            worst_last_byte_latency=(max(d_last_byte_latency), 'txID011'),
            size_stats=OrderedDict([
                ('small', {'avg_req_per_sec': 1.25,
                           'first_byte_latency': {'avg': '%7.3f' % 0.5,
                                                  'max': '%7.3f' % 0.5,
                                                  'pctile': '%7.3f' % 0.5,
                                                  'median': '%7.3f' % 0.5,
                                                  'min': '%6.3f' % 0.5,
                                                  'std_dev': '%7.3f' % 0.0},
                           'last_byte_latency': {'avg': '%7.3f' % 0.8,
                                                 'max': '%7.3f' % 0.8,
                                                 'pctile': '%7.3f' % 0.8,
                                                 'median': '%7.3f' % 0.8,
                                                 'min': '%6.3f' % 0.8,
                                                 'std_dev': '%7.3f' % 0.0},
                           'worst_first_byte_latency': (0.5, 'txID011'),
                           'worst_last_byte_latency': (0.8, 'txID011'),
                           'req_count': 1,
                           'start': 103.1,
                           'stop': 103.9}),
                ('large', {'avg_req_per_sec': 2.5,
                           'first_byte_latency': {'avg': '%7.3f' % 0.1,
                                                  'max': '%7.3f' % 0.1,
                                                  'pctile': '%7.3f' % 0.1,
                                                  'median': '%7.3f' % 0.1,
                                                  'min': '%6.3f' % 0.1,
                                                  'std_dev': '%7.3f' % 0.0},
                           'last_byte_latency': {'avg': '%7.3f' % 0.4,
                                                 'max': '%7.3f' % 0.4,
                                                 'pctile': '%7.3f' % 0.4,
                                                 'median': '%7.3f' % 0.4,
                                                 'min': '%6.3f' % 0.4,
                                                 'std_dev': '%7.3f' % 0.0},
                           'worst_first_byte_latency': (0.1, 'txID007'),
                           'worst_last_byte_latency': (0.4, 'txID007'),
                           'req_count': 1,
                           'start': 102.9,
                           'stop': 103.3})]),
        ), scen_stats['op_stats'][ssbench.DELETE_OBJECT])

    def test_calculate_scenario_size_stats(self):
        d_first_byte_latency = [0.1, 0.5]
        d_last_byte_latency = [0.4, 0.8]
        scen_stats = self.master.calculate_scenario_stats(self.scenario,
                                                          self.stub_results)
        self.assertDictEqual(OrderedDict([
            ('tiny', {'avg_req_per_sec': 0.816493,
                      'first_byte_latency': {'avg': '%7.3f' % 0.45,
                                             'max': '%7.3f' % 1.0,
                                             'pctile': '%7.3f' % 1.0,
                                             'median': '%7.3f' % 0.35,
                                             'min': '%6.3f' % 0.1,
                                             'std_dev': '%7.3f' % 0.377492},
                      'last_byte_latency': {'avg': '%7.3f' % 0.87475,
                                            'max': '%7.3f' % 1.8,
                                            'pctile': '%7.3f' % 1.8,
                                            'median': '%7.3f' % 0.7494,
                                            'min': '%6.3f' % 0.2,
                                            'std_dev': '%7.3f' % 0.580485},
                      'worst_first_byte_latency': (1.0, 'txID010'),
                      'worst_last_byte_latency': (1.8, 'txID010'),
                      'req_count': 4,
                      'start': 100.1,
                      'stop': 104.999}),
            ('small', {'avg_req_per_sec': 0.75,
                       'first_byte_latency': {'avg': '%7.3f' % 0.566667,
                                              'max': '%7.3f' % 1.0,
                                              'pctile': '%7.3f' % 1.0,
                                              'median': '%7.3f' % 0.5,
                                              'min': '%6.3f' % 0.2,
                                              'std_dev': '%7.3f' % 0.329983},
                       'last_byte_latency': {'avg': '%7.3f' % 1.433333,
                                             'max': '%7.3f' % 3.0,
                                             'pctile': '%7.3f' % 3.0,
                                             'median': '%7.3f' % 0.8,
                                             'min': '%6.3f' % 0.5,
                                             'std_dev': '%7.3f' % 1.11455},
                       'worst_first_byte_latency': (1.0, 'txID002'),
                       'worst_last_byte_latency': (3.0, 'txID002'),
                       'req_count': 3,
                       'start': 100.0,
                       'stop': 104.0}),
            ('medium', {'avg_req_per_sec': 0.47619,
                        'first_byte_latency': {'avg': '%7.3f' % 0.55,
                                               'max': '%7.3f' % 0.8,
                                               'pctile': '%7.3f' % 0.8,
                                               'median': '%7.3f' % 0.55,
                                               'min': '%6.3f' % 0.3,
                                               'std_dev': '%7.3f' % 0.25},
                        'last_byte_latency': {'avg': '%7.3f' % 1.6,
                                              'max': '%7.3f' % 2.8,
                                              'pctile': '%7.3f' % 2.8,
                                              'median': '%7.3f' % 1.6,
                                              'min': '%6.3f' % 0.4,
                                              'std_dev': '%7.3f' % 1.2},
                        'worst_first_byte_latency': (0.8, 'txID006'),
                        'worst_last_byte_latency': (2.8, 'txID006'),
                        'req_count': 2,
                        'start': 100.1,
                        'stop': 104.3}),
            ('large', {'avg_req_per_sec': 0.571429,
                       'first_byte_latency': {'avg': '%7.3f' % 0.15,
                                              'max': '%7.3f' % 0.2,
                                              'pctile': '%7.3f' % 0.2,
                                              'median': '%7.3f' % 0.15,
                                              'min': '%6.3f' % 0.1,
                                              'std_dev': '%7.3f' % 0.05},
                       'last_byte_latency': {'avg': '%7.3f' % 0.35,
                                             'max': '%7.3f' % 0.4,
                                             'pctile': '%7.3f' % 0.4,
                                             'median': '%7.3f' % 0.35,
                                             'min': '%6.3f' % 0.3,
                                             'std_dev': '%7.3f' % 0.05},
                       'worst_first_byte_latency': (0.2, 'txID005'),
                       'worst_last_byte_latency': (0.4, 'txID007'),
                       'req_count': 2,
                       'start': 102.9,
                       'stop': 106.4}),
            ('huge', {'avg_req_per_sec': 0.454545,
                      'first_byte_latency': {'avg': '%7.3f' % 1.2,
                                             'max': '%7.3f' % 1.2,
                                             'pctile': '%7.3f' % 1.2,
                                             'median': '%7.3f' % 1.2,
                                             'min': '%6.3f' % 1.2,
                                             'std_dev': '%7.3f' % 0.0},
                      'last_byte_latency': {'avg': '%7.3f' % 2.2,
                                            'max': '%7.3f' % 2.2,
                                            'pctile': '%7.3f' % 2.2,
                                            'median': '%7.3f' % 2.2,
                                            'min': '%6.3f' % 2.2,
                                            'std_dev': '%7.3f' % 0.0},
                      'worst_first_byte_latency': (1.2, 'txID004'),
                      'worst_last_byte_latency': (2.2, 'txID004'),
                      'req_count': 1,
                      'start': 103.8,
                      'stop': 106.0})]),
            scen_stats['size_stats'])

    def test_calculate_scenario_stats_time_series(self):
        # Time series (reqs completed each second
        scen_stats = self.master.calculate_scenario_stats(self.scenario,
                                                          self.stub_results)
        self.assertDictEqual(dict(
            start=101,
            start_time=99.19999999999999,
            stop=106,
            data=[1, 1, 5, 3, 0, 2],
        ), scen_stats['time_series'])

    def test_write_rps_histogram(self):
        # Write out time series data (requests-per-second histogram) to an
        # already open CSV file
        scen_stats = self.master.calculate_scenario_stats(self.scenario,
                                                          self.stub_results)

        test_csv_file = StringIO()
        self.master.write_rps_histogram(scen_stats, test_csv_file)
        test_csv_file.seek(0)
        reader = csv.reader(test_csv_file)
        self.assertListEqual([
            ["Seconds Since Start", "Requests Completed"],
            ['1', '1'],
            ['2', '1'],
            ['3', '5'],
            ['4', '3'],
            ['5', '0'],
            ['6', '2'],
        ], list(reader))

    def test_generate_scenario_report(self):
        # Time series (reqs completed each second
        scen_stats = self.master.calculate_scenario_stats(self.scenario,
                                                          self.stub_results,
                                                          nth_pctile=50)
        self.assertListEqual(u"""
Master Test Scenario - ablkei
Worker count:   3   Concurrency:   2  Ran 1970-01-01 00:01:39 UTC to 1970-01-01 00:01:46 UTC (7s)

% Ops    C   R   U   D       Size Range       Size Name
 20%   % 50  30  10  10       99  B - 100  B  tiny
 27%   % 71   9  12   8        2 kB           small
 34%   % 50  30  10  10        3 kB -   3 kB  medium
  0%   % 50  30  10  10       10 MB           unused
 14%   % 16  61   7  16      399 kB - 400 kB  large
  5%   % 50  30  10  10       50 MB -  71 MB  huge
---------------------------------------------------------------------
         51  29  10  10      CRUD weighted average

TOTAL
       Count:    12  Average requests per second:   1.9
                            min       max      avg      std_dev  50%-ile                   Worst latency TX ID
       First-byte latency:  0.100 -   1.200    0.508  (  0.386)    0.400  (all obj sizes)  txID004
       Last-byte  latency:  0.200 -   3.000    1.158  (  0.970)    0.749  (all obj sizes)  txID002
       First-byte latency:  0.100 -   1.000    0.450  (  0.377)    0.350  (    tiny objs)  txID010
       Last-byte  latency:  0.200 -   1.800    0.875  (  0.580)    0.749  (    tiny objs)  txID010
       First-byte latency:  0.200 -   1.000    0.567  (  0.330)    0.500  (   small objs)  txID002
       Last-byte  latency:  0.500 -   3.000    1.433  (  1.115)    0.800  (   small objs)  txID002
       First-byte latency:  0.300 -   0.800    0.550  (  0.250)    0.550  (  medium objs)  txID006
       Last-byte  latency:  0.400 -   2.800    1.600  (  1.200)    1.600  (  medium objs)  txID006
       First-byte latency:  0.100 -   0.200    0.150  (  0.050)    0.150  (   large objs)  txID005
       Last-byte  latency:  0.300 -   0.400    0.350  (  0.050)    0.350  (   large objs)  txID007
       First-byte latency:  1.200 -   1.200    1.200  (  0.000)    1.200  (    huge objs)  txID004
       Last-byte  latency:  2.200 -   2.200    2.200  (  0.000)    2.200  (    huge objs)  txID004

CREATE
       Count:     3  Average requests per second:   0.5
                            min       max      avg      std_dev  50%-ile                   Worst latency TX ID
       First-byte latency:  0.100 -   1.200    0.767  (  0.478)    1.000  (all obj sizes)  txID004
       Last-byte  latency:  0.200 -   3.000    1.800  (  1.178)    2.200  (all obj sizes)  txID002
       First-byte latency:  0.100 -   0.100    0.100  (  0.000)    0.100  (    tiny objs)  txID008
       Last-byte  latency:  0.200 -   0.200    0.200  (  0.000)    0.200  (    tiny objs)  txID008
       First-byte latency:  1.000 -   1.000    1.000  (  0.000)    1.000  (   small objs)  txID002
       Last-byte  latency:  3.000 -   3.000    3.000  (  0.000)    3.000  (   small objs)  txID002
       First-byte latency:  1.200 -   1.200    1.200  (  0.000)    1.200  (    huge objs)  txID004
       Last-byte  latency:  2.200 -   2.200    2.200  (  0.000)    2.200  (    huge objs)  txID004

READ
       Count:     4  Average requests per second:   1.0
                            min       max      avg      std_dev  50%-ile                   Worst latency TX ID
       First-byte latency:  0.100 -   1.000    0.400  (  0.354)    0.250  (all obj sizes)  txID010
       Last-byte  latency:  0.400 -   1.800    0.875  (  0.554)    0.650  (all obj sizes)  txID010
       First-byte latency:  0.100 -   1.000    0.550  (  0.450)    0.550  (    tiny objs)  txID010
       Last-byte  latency:  0.800 -   1.800    1.300  (  0.500)    1.300  (    tiny objs)  txID010
       First-byte latency:  0.200 -   0.200    0.200  (  0.000)    0.200  (   small objs)  txID009
       Last-byte  latency:  0.500 -   0.500    0.500  (  0.000)    0.500  (   small objs)  txID009
       First-byte latency:  0.300 -   0.300    0.300  (  0.000)    0.300  (  medium objs)  txID012
       Last-byte  latency:  0.400 -   0.400    0.400  (  0.000)    0.400  (  medium objs)  txID012

UPDATE
       Count:     3  Average requests per second:   0.5
                            min       max      avg      std_dev  50%-ile                   Worst latency TX ID
       First-byte latency:  0.200 -   0.800    0.533  (  0.249)    0.600  (all obj sizes)  txID006
       Last-byte  latency:  0.300 -   2.800    1.266  (  1.097)    0.699  (all obj sizes)  txID006
       First-byte latency:  0.600 -   0.600    0.600  (  0.000)    0.600  (    tiny objs)  txID013
       Last-byte  latency:  0.699 -   0.699    0.699  (  0.000)    0.699  (    tiny objs)  txID013
       First-byte latency:  0.800 -   0.800    0.800  (  0.000)    0.800  (  medium objs)  txID006
       Last-byte  latency:  2.800 -   2.800    2.800  (  0.000)    2.800  (  medium objs)  txID006
       First-byte latency:  0.200 -   0.200    0.200  (  0.000)    0.200  (   large objs)  txID005
       Last-byte  latency:  0.300 -   0.300    0.300  (  0.000)    0.300  (   large objs)  txID005

DELETE
       Count:     2  Average requests per second:   2.0
                            min       max      avg      std_dev  50%-ile                   Worst latency TX ID
       First-byte latency:  0.100 -   0.500    0.300  (  0.200)    0.300  (all obj sizes)  txID011
       Last-byte  latency:  0.400 -   0.800    0.600  (  0.200)    0.600  (all obj sizes)  txID011
       First-byte latency:  0.500 -   0.500    0.500  (  0.000)    0.500  (   small objs)  txID011
       Last-byte  latency:  0.800 -   0.800    0.800  (  0.000)    0.800  (   small objs)  txID011
       First-byte latency:  0.100 -   0.100    0.100  (  0.000)    0.100  (   large objs)  txID007
       Last-byte  latency:  0.400 -   0.400    0.400  (  0.000)    0.400  (   large objs)  txID007

""".split('\n'), self.master.generate_scenario_report(self.scenario, scen_stats).split('\n'))
Beispiel #2
0
class TestMaster(ScenarioFixture, TestCase):
    maxDiff = None

    def setUp(self):
        # Set our test scenario differently from the default; must be BEFORE
        # the super call.
        self.scenario_dict = dict(
            name='Master Test Scenario - ablkei',
            sizes=[
                dict(name='tiny', size_min=99, size_max=100),
                dict(name='small', size_min=199, size_max=200),
                dict(name='medium', size_min=299, size_max=300),
                dict(name='large', size_min=399, size_max=400),
                dict(name='huge', size_min=499, size_max=500)],
            initial_files=dict(
                tiny=300, small=300, medium=300, large=100, huge=70,
            ),
            operation_count=5000,
            #             C  R  U  D
            crud_profile=[5, 3, 1, 1],
            user_count=2,
        )
        super(TestMaster, self).setUp()

        self.stub_queue = flexmock()
        self.stub_queue.should_receive('watch').with_args(ssbench.STATS_TUBE).once
        self.stub_queue.should_receive('ignore').with_args(ssbench.DEFAULT_TUBE).once
        self.master = Master(self.stub_queue)

        self.result_index = 1  # for self.gen_result()

        self.stub_results = [
            self.gen_result(1, ssbench.CREATE_OBJECT, 'small', 100.0, 101.0, 103.0),
            self.gen_result(1, ssbench.READ_OBJECT, 'tiny', 103.0, 103.1, 103.8),
            self.gen_result(1, ssbench.CREATE_OBJECT, 'huge', 103.8, 105.0, 106.0),
            self.gen_result(1, ssbench.UPDATE_OBJECT, 'large', 106.1, 106.3, 106.4),
            #
            # exceptions should be ignored
            dict(worker_id=2, type=ssbench.UPDATE_OBJECT, completed_at=39293.2, exception='wacky!'),
            self.gen_result(2, ssbench.UPDATE_OBJECT, 'medium', 100.1, 100.9, 102.9),
            self.gen_result(2, ssbench.DELETE_OBJECT, 'large', 102.9, 103.0, 103.3),
            self.gen_result(2, ssbench.CREATE_OBJECT, 'tiny', 103.3, 103.4, 103.5),
            self.gen_result(2, ssbench.READ_OBJECT, 'small', 103.5, 103.7, 104.0),
            #
            self.gen_result(3, ssbench.READ_OBJECT, 'tiny', 100.1, 101.1, 101.9),
            # worker 3 took a while (observer lower concurrency in second 102
            self.gen_result(3, ssbench.DELETE_OBJECT, 'small', 103.1, 103.6, 103.9),
            self.gen_result(3, ssbench.READ_OBJECT, 'medium', 103.9, 104.2, 104.3),
            self.gen_result(3, ssbench.UPDATE_OBJECT, 'tiny', 104.3, 104.9, 104.999),
        ]

    def tearDown(self):
        super(TestMaster, self).tearDown()

    def gen_result(self, worker_id, op_type, size_str, start, first_byte,
                   last_byte):
        self.result_index += 1

        return {
            # There are other keys in a "result", but these are the only ones
            # used for the reporting.
            'worker_id': worker_id,
            'type': op_type,
            'size_str': size_str,
            'size': 989,
            'first_byte_latency': first_byte - start,
            'last_byte_latency': last_byte - start,
            'completed_at': last_byte,
        }

    def test_calculate_scenario_stats_aggregate(self):
        first_byte_latency_all = [1, 0.1, 1.2, 0.2, 0.8, 0.1, 0.1, 0.2, 1, 0.5, 0.3, 0.6]
        last_byte_latency_all =  [3, 0.8, 2.2, 0.3, 2.8, 0.4, 0.2, 0.5, 1.8, 0.8, 0.4, 0.699]
        scen_stats = self.master.calculate_scenario_stats(self.scenario,
                                                          self.stub_results)
        self.assertDictEqual(dict(
            worker_count=3, start=100.0, stop=106.4, req_count=12,
            avg_req_per_sec=round(12 / (106.4 - 100), 6),
            first_byte_latency=dict(
                min='%6.3f' % 0.1,
                max='%7.3f' % 1.2,
                avg='%7.3f' % stats.lmean(first_byte_latency_all),
                std_dev='%7.3f' % stats.lsamplestdev(first_byte_latency_all),
                median='%7.3f' % stats.lmedianscore(first_byte_latency_all),
            ),
            last_byte_latency=dict(
                min='%6.3f' % 0.2,
                max='%7.3f' % 3.0,
                avg='%7.3f' % stats.lmean(last_byte_latency_all),
                std_dev='%7.3f' % stats.lsamplestdev(last_byte_latency_all),
                median='  0.749',  # XXX why??
                #median='%7.3f' % stats.lmedianscore(last_byte_latency_all),
            ),
        ), scen_stats['agg_stats'])

    def test_calculate_scenario_stats_worker1(self):
        w1_first_byte_latency = [1.0, 0.1, 1.2, 0.2]
        w1_last_byte_latency = [3.0, 0.8, 2.2, 0.3]
        scen_stats = self.master.calculate_scenario_stats(self.scenario,
                                                          self.stub_results)
        self.assertDictEqual(dict(
            start=100.0, stop=106.4, req_count=4,
            avg_req_per_sec=round(4 / (106.4 - 100), 6),
            first_byte_latency=dict(
                min='%6.3f' % min(w1_first_byte_latency),
                max='%7.3f' % max(w1_first_byte_latency),
                avg='%7.3f' % stats.lmean(w1_first_byte_latency),
                std_dev='%7.3f' % stats.lsamplestdev(w1_first_byte_latency),
                median='%7.3f' % stats.lmedianscore(w1_first_byte_latency),
            ),
            last_byte_latency=dict(
                min='%6.3f' % min(w1_last_byte_latency),
                max='%7.3f' % max(w1_last_byte_latency),
                avg='%7.3f' % stats.lmean(w1_last_byte_latency),
                std_dev='%7.3f' % stats.lsamplestdev(w1_last_byte_latency),
                median='%7.3f' % stats.lmedianscore(w1_last_byte_latency),
            ),
        ), scen_stats['worker_stats'][1])

    def test_calculate_scenario_stats_worker2(self):
        w2_first_byte_latency = [0.8, 0.1, 0.1, 0.2]
        w2_last_byte_latency = [2.8, 0.4, 0.2, 0.5]
        scen_stats = self.master.calculate_scenario_stats(self.scenario,
                                                          self.stub_results)
        self.assertDictEqual(dict(
            start=100.1, stop=104.0, req_count=4,
            avg_req_per_sec=round(4 / (104.0 - 100.1), 6),
            first_byte_latency=dict(
                min='%6.3f' % min(w2_first_byte_latency),
                max='%7.3f' % max(w2_first_byte_latency),
                avg='%7.3f' % stats.lmean(w2_first_byte_latency),
                std_dev='%7.3f' % stats.lsamplestdev(w2_first_byte_latency),
                median='%7.3f' % stats.lmedianscore(w2_first_byte_latency),
            ),
            last_byte_latency=dict(
                min='%6.3f' % min(w2_last_byte_latency),
                max='%7.3f' % max(w2_last_byte_latency),
                avg='%7.3f' % stats.lmean(w2_last_byte_latency),
                std_dev='%7.3f' % stats.lsamplestdev(w2_last_byte_latency),
                median='%7.3f' % stats.lmedianscore(w2_last_byte_latency),
            ),
        ), scen_stats['worker_stats'][2])

    def test_calculate_scenario_stats_worker3(self):
        w3_first_byte_latency = [1, 0.5, 0.3, 0.6]
        w3_last_byte_latency = [1.8, 0.8, 0.4, 0.699]
        scen_stats = self.master.calculate_scenario_stats(self.scenario,
                                                          self.stub_results)
        self.assertDictEqual(dict(
            start=100.1, stop=104.999, req_count=4,
            avg_req_per_sec=round(4 / (104.999 - 100.1), 6),
            first_byte_latency=dict(
                min='%6.3f' % min(w3_first_byte_latency),
                max='%7.3f' % max(w3_first_byte_latency),
                avg='%7.3f' % stats.lmean(w3_first_byte_latency),
                std_dev='%7.3f' % stats.lsamplestdev(w3_first_byte_latency),
                median='%7.3f' % stats.lmedianscore(w3_first_byte_latency),
            ),
            last_byte_latency=dict(
                min='%6.3f' % min(w3_last_byte_latency),
                max='%7.3f' % max(w3_last_byte_latency),
                avg='%7.3f' % stats.lmean(w3_last_byte_latency),
                std_dev='%7.3f' % stats.lsamplestdev(w3_last_byte_latency),
                median='%7.3f' % stats.lmedianscore(w3_last_byte_latency),
            ),
        ), scen_stats['worker_stats'][3])

    def test_calculate_scenario_stats_create(self):
        # Stats for Create
        c_first_byte_latency = [1, 1.2, 0.1]
        c_last_byte_latency = [3.0, 2.2, 0.2]
        scen_stats = self.master.calculate_scenario_stats(self.scenario,
                                                          self.stub_results)
        self.assertDictEqual(dict(
            start=100.0, stop=106.0, req_count=3,
            avg_req_per_sec=round(3 / (106 - 100.0), 6),
            first_byte_latency=dict(
                min='%6.3f' % min(c_first_byte_latency),
                max='%7.3f' % max(c_first_byte_latency),
                avg='%7.3f' % stats.lmean(c_first_byte_latency),
                std_dev='%7.3f' % stats.lsamplestdev(c_first_byte_latency),
                median='%7.3f' % stats.lmedianscore(c_first_byte_latency),
            ),
            last_byte_latency=dict(
                min='%6.3f' % min(c_last_byte_latency),
                max='%7.3f' % max(c_last_byte_latency),
                avg='%7.3f' % stats.lmean(c_last_byte_latency),
                std_dev='%7.3f' % stats.lsamplestdev(c_last_byte_latency),
                median='%7.3f' % stats.lmedianscore(c_last_byte_latency),
            ),
            size_stats=OrderedDict([
                ('tiny', {'avg_req_per_sec': 5.0,
                        'first_byte_latency': {'avg': '%7.3f' % 0.1,
                                               'max': '%7.3f' % 0.1,
                                               'median': '%7.3f' % 0.1,
                                               'min': '%6.3f' % 0.1,
                                               'std_dev': '%7.3f' % 0.0},
                        'last_byte_latency': {'avg': '%7.3f' % 0.2,
                                              'max': '%7.3f' % 0.2,
                                              'median': '%7.3f' % 0.2,
                                              'min': '%6.3f' % 0.2,
                                              'std_dev': '%7.3f' % 0.0},
                        'req_count': 1,
                        'start': 103.3,
                        'stop': 103.5}),
                ('small', {'avg_req_per_sec': 0.333333,
                          'first_byte_latency': {'avg': '%7.3f' % 1.0,
                                                 'max': '%7.3f' % 1.0,
                                                 'median': '%7.3f' % 1.0,
                                                 'min': '%6.3f' % 1.0,
                                                 'std_dev': '%7.3f' % 0.0},
                          'last_byte_latency': {'avg': '%7.3f' % 3.0,
                                                'max': '%7.3f' % 3.0,
                                                'median': '%7.3f' % 3.0,
                                                'min': '%6.3f' % 3.0,
                                                'std_dev': '%7.3f' % 0.0},
                          'req_count': 1,
                          'start': 100.0,
                          'stop': 103.0}),
                ('huge', {'avg_req_per_sec': 0.454545,
                             'first_byte_latency': {'avg': '%7.3f' % 1.2,
                                                    'max': '%7.3f' % 1.2,
                                                    'median': '%7.3f' % 1.2,
                                                    'min': '%6.3f' % 1.2,
                                                    'std_dev': '%7.3f' % 0.0},
                             'last_byte_latency': {'avg': '%7.3f' % 2.2,
                                                   'max': '%7.3f' % 2.2,
                                                   'median': '%7.3f' % 2.2,
                                                   'min': '%6.3f' % 2.2,
                                                   'std_dev': '%7.3f' % 0.0},
                             'req_count': 1,
                             'start': 103.8,
                             'stop': 106.0})]),
        ), scen_stats['op_stats'][ssbench.CREATE_OBJECT])

    def test_calculate_scenario_stats_read(self):
        # Stats for Read
        r_first_byte_latency = [0.1, 0.2, 1.0, 0.3]
        r_last_byte_latency = [0.8, 0.5, 1.8, 0.4]
        scen_stats = self.master.calculate_scenario_stats(self.scenario,
                                                          self.stub_results)
        self.assertDictEqual(dict(
            start=100.1, stop=104.3, req_count=4,
            avg_req_per_sec=round(4 / (104.3 - 100.1), 6),
            first_byte_latency=dict(
                min='%6.3f' % min(r_first_byte_latency),
                max='%7.3f' % max(r_first_byte_latency),
                avg='%7.3f' % stats.lmean(r_first_byte_latency),
                std_dev='%7.3f' % stats.lsamplestdev(r_first_byte_latency),
                median='%7.3f' % stats.lmedianscore(r_first_byte_latency),
            ),
            last_byte_latency=dict(
                min='%6.3f' % min(r_last_byte_latency),
                max='%7.3f' % max(r_last_byte_latency),
                avg='%7.3f' % stats.lmean(r_last_byte_latency),
                std_dev='%7.3f' % stats.lsamplestdev(r_last_byte_latency),
                median='%7.3f' % stats.lmedianscore(r_last_byte_latency),
            ),
            size_stats=OrderedDict([
                ('tiny', {'avg_req_per_sec': 0.540541,
                        'first_byte_latency': {'avg': '%7.3f' % 0.55,
                                               'max': '%7.3f' % 1.0,
                                               'median': '%7.3f' % 0.55,
                                               'min': '%6.3f' % 0.1,
                                               'std_dev': '%7.3f' % 0.45},
                        'last_byte_latency': {'avg': '%7.3f' % 1.3,
                                              'max': '%7.3f' % 1.8,
                                              'median': '%7.3f' % 1.3,
                                              'min': '%6.3f' % 0.8,
                                              'std_dev': '%7.3f' % 0.5},
                        'req_count': 2,
                        'start': 100.1,
                        'stop': 103.8}),
                ('small', {'avg_req_per_sec': 2.0,
                          'first_byte_latency': {'avg': '%7.3f' % 0.2,
                                                 'max': '%7.3f' % 0.2,
                                                 'median': '%7.3f' % 0.2,
                                                 'min': '%6.3f' % 0.2,
                                                 'std_dev': '%7.3f' % 0.0},
                          'last_byte_latency': {'avg': '%7.3f' % 0.5,
                                                'max': '%7.3f' % 0.5,
                                                'median': '%7.3f' % 0.5,
                                                'min': '%6.3f' % 0.5,
                                                'std_dev': '%7.3f' % 0.0},
                          'req_count': 1,
                          'start': 103.5,
                          'stop': 104.0}),
                ('medium', {'avg_req_per_sec': 2.5,
                          'first_byte_latency': {'avg': '%7.3f' % 0.3,
                                                 'max': '%7.3f' % 0.3,
                                                 'median': '%7.3f' % 0.3,
                                                 'min': '%6.3f' % 0.3,
                                                 'std_dev': '%7.3f' % 0.0},
                          'last_byte_latency': {'avg': '%7.3f' % 0.4,
                                                'max': '%7.3f' % 0.4,
                                                'median': '%7.3f' % 0.4,
                                                'min': '%6.3f' % 0.4,
                                                'std_dev': '%7.3f' % 0.0},
                          'req_count': 1,
                          'start': 103.9,
                          'stop': 104.3})]),
        ), scen_stats['op_stats'][ssbench.READ_OBJECT])

    def test_calculate_scenario_stats_update(self):
        u_first_byte_latency = [0.2, 0.8, 0.6]
        u_last_byte_latency = [0.3, 2.8, 0.699]
        scen_stats = self.master.calculate_scenario_stats(self.scenario,
                                                          self.stub_results)
        self.assertDictEqual(dict(
            start=100.1, stop=106.4, req_count=3,
            avg_req_per_sec=round(3 / (106.4 - 100.1), 6),
            first_byte_latency=dict(
                min='%6.3f' % min(u_first_byte_latency),
                max='%7.3f' % max(u_first_byte_latency),
                avg='%7.3f' % stats.lmean(u_first_byte_latency),
                std_dev='%7.3f' % stats.lsamplestdev(u_first_byte_latency),
                median='%7.3f' % stats.lmedianscore(u_first_byte_latency),
            ),
            last_byte_latency=dict(
                min='%6.3f' % min(u_last_byte_latency),
                max='%7.3f' % max(u_last_byte_latency),
                avg='%7.3f' % stats.lmean(u_last_byte_latency),
                std_dev='%7.3f' % stats.lsamplestdev(u_last_byte_latency),
                median='%7.3f' % stats.lmedianscore(u_last_byte_latency),
            ),
            size_stats=OrderedDict([
                ('tiny', {'avg_req_per_sec': 1.430615,
                        'first_byte_latency': {'avg': '%7.3f' % 0.6,
                                               'max': '%7.3f' % 0.6,
                                               'median': '%7.3f' % 0.6,
                                               'min': '%6.3f' % 0.6,
                                               'std_dev': '%7.3f' % 0.0},
                        'last_byte_latency': {'avg': '%7.3f' % 0.699,
                                              'max': '%7.3f' % 0.699,
                                              'median': '%7.3f' % 0.699,
                                              'min': '%6.3f' % 0.699,
                                              'std_dev': '%7.3f' % 0.0},
                        'req_count': 1,
                        'start': 104.3,
                        'stop': 104.999}),
                ('medium', {'avg_req_per_sec': 0.357143,
                          'first_byte_latency': {'avg': '%7.3f' % 0.8,
                                                 'max': '%7.3f' % 0.8,
                                                 'median': '%7.3f' % 0.8,
                                                 'min': '%6.3f' % 0.8,
                                                 'std_dev': '%7.3f' % 0.0},
                          'last_byte_latency': {'avg': '%7.3f' % 2.8,
                                                'max': '%7.3f' % 2.8,
                                                'median': '%7.3f' % 2.8,
                                                'min': '%6.3f' % 2.8,
                                                'std_dev': '%7.3f' % 0.0},
                          'req_count': 1,
                          'start': 100.1,
                          'stop': 102.9}),
                ('large', {'avg_req_per_sec': 3.333333,
                            'first_byte_latency': {'avg': '%7.3f' % 0.2,
                                                   'max': '%7.3f' % 0.2,
                                                   'median': '%7.3f' % 0.2,
                                                   'min': '%6.3f' % 0.2,
                                                   'std_dev': '%7.3f' % 0.0},
                            'last_byte_latency': {'avg': '%7.3f' % 0.3,
                                                  'max': '%7.3f' % 0.3,
                                                  'median': '%7.3f' % 0.3,
                                                  'min': '%6.3f' % 0.3,
                                                  'std_dev': '%7.3f' % 0.0},
                            'req_count': 1,
                            'start': 106.1,
                            'stop': 106.4})]),
        ), scen_stats['op_stats'][ssbench.UPDATE_OBJECT])

    def test_calculate_scenario_stats_delete(self):
        d_first_byte_latency = [0.1, 0.5]
        d_last_byte_latency = [0.4, 0.8]
        scen_stats = self.master.calculate_scenario_stats(self.scenario,
                                                          self.stub_results)
        self.assertDictEqual(dict(
            start=102.9, stop=103.9, req_count=2,
            avg_req_per_sec=round(2 / (103.9 - 102.9), 6),
            first_byte_latency=dict(
                min='%6.3f' % min(d_first_byte_latency),
                max='%7.3f' % max(d_first_byte_latency),
                avg='%7.3f' % stats.lmean(d_first_byte_latency),
                std_dev='%7.3f' % stats.lsamplestdev(d_first_byte_latency),
                median='%7.3f' % stats.lmedianscore(d_first_byte_latency),
            ),
            last_byte_latency=dict(
                min='%6.3f' % min(d_last_byte_latency),
                max='%7.3f' % max(d_last_byte_latency),
                avg='%7.3f' % stats.lmean(d_last_byte_latency),
                std_dev='%7.3f' % stats.lsamplestdev(d_last_byte_latency),
                median='%7.3f' % stats.lmedianscore(d_last_byte_latency),
            ),
            size_stats=OrderedDict([
                ('small', {'avg_req_per_sec': 1.25,
                          'first_byte_latency': {'avg': '%7.3f' % 0.5,
                                                 'max': '%7.3f' % 0.5,
                                                 'median': '%7.3f' % 0.5,
                                                 'min': '%6.3f' % 0.5,
                                                 'std_dev': '%7.3f' % 0.0},
                          'last_byte_latency': {'avg': '%7.3f' % 0.8,
                                                'max': '%7.3f' % 0.8,
                                                'median': '%7.3f' % 0.8,
                                                'min': '%6.3f' % 0.8,
                                                'std_dev': '%7.3f' % 0.0},
                          'req_count': 1,
                          'start': 103.1,
                          'stop': 103.9}),
                ('large', {'avg_req_per_sec': 2.5,
                            'first_byte_latency': {'avg': '%7.3f' % 0.1,
                                                   'max': '%7.3f' % 0.1,
                                                   'median': '%7.3f' % 0.1,
                                                   'min': '%6.3f' % 0.1,
                                                   'std_dev': '%7.3f' % 0.0},
                            'last_byte_latency': {'avg': '%7.3f' % 0.4,
                                                  'max': '%7.3f' % 0.4,
                                                  'median': '%7.3f' % 0.4,
                                                  'min': '%6.3f' % 0.4,
                                                  'std_dev': '%7.3f' % 0.0},
                            'req_count': 1,
                            'start': 102.9,
                            'stop': 103.3})]),
        ), scen_stats['op_stats'][ssbench.DELETE_OBJECT])

    def test_calculate_scenario_size_stats(self):
        d_first_byte_latency = [0.1, 0.5]
        d_last_byte_latency = [0.4, 0.8]
        scen_stats = self.master.calculate_scenario_stats(self.scenario,
                                                          self.stub_results)
        self.assertDictEqual(OrderedDict([
            ('tiny', {'avg_req_per_sec': 0.816493,
                    'first_byte_latency': {'avg': '%7.3f' % 0.45,
                                           'max': '%7.3f' % 1.0,
                                           'median': '%7.3f' % 0.35,
                                           'min': '%6.3f' % 0.1,
                                           'std_dev': '%7.3f' % 0.377492},
                    'last_byte_latency': {'avg': '%7.3f' % 0.87475,
                                          'max': '%7.3f' % 1.8,
                                          'median': '%7.3f' % 0.7494,
                                          'min': '%6.3f' % 0.2,
                                          'std_dev': '%7.3f' % 0.580485},
                    'req_count': 4,
                    'start': 100.1,
                    'stop': 104.999}),
            ('small', {'avg_req_per_sec': 0.75,
                      'first_byte_latency': {'avg': '%7.3f' % 0.566667,
                                             'max': '%7.3f' % 1.0,
                                             'median': '%7.3f' % 0.5,
                                             'min': '%6.3f' % 0.2,
                                             'std_dev': '%7.3f' % 0.329983},
                      'last_byte_latency': {'avg': '%7.3f' % 1.433333,
                                            'max': '%7.3f' % 3.0,
                                            'median': '%7.3f' % 0.8,
                                            'min': '%6.3f' % 0.5,
                                            'std_dev': '%7.3f' % 1.11455},
                      'req_count': 3,
                      'start': 100.0,
                      'stop': 104.0}),
            ('medium', {'avg_req_per_sec': 0.47619,
                      'first_byte_latency': {'avg': '%7.3f' % 0.55,
                                             'max': '%7.3f' % 0.8,
                                             'median': '%7.3f' % 0.55,
                                             'min': '%6.3f' % 0.3,
                                             'std_dev': '%7.3f' % 0.25},
                      'last_byte_latency': {'avg': '%7.3f' % 1.6,
                                            'max': '%7.3f' % 2.8,
                                            'median': '%7.3f' % 1.6,
                                            'min': '%6.3f' % 0.4,
                                            'std_dev': '%7.3f' % 1.2},
                      'req_count': 2,
                      'start': 100.1,
                      'stop': 104.3}),
            ('large', {'avg_req_per_sec': 0.571429,
                        'first_byte_latency': {'avg': '%7.3f' % 0.15,
                                               'max': '%7.3f' % 0.2,
                                               'median': '%7.3f' % 0.15,
                                               'min': '%6.3f' % 0.1,
                                               'std_dev': '%7.3f' % 0.05},
                        'last_byte_latency': {'avg': '%7.3f' % 0.35,
                                              'max': '%7.3f' % 0.4,
                                              'median': '%7.3f' % 0.35,
                                              'min': '%6.3f' % 0.3,
                                              'std_dev': '%7.3f' % 0.05},
                        'req_count': 2,
                        'start': 102.9,
                        'stop': 106.4}),
            ('huge', {'avg_req_per_sec': 0.454545,
                         'first_byte_latency': {'avg': '%7.3f' % 1.2,
                                                'max': '%7.3f' % 1.2,
                                                'median': '%7.3f' % 1.2,
                                                'min': '%6.3f' % 1.2,
                                                'std_dev': '%7.3f' % 0.0},
                         'last_byte_latency': {'avg': '%7.3f' % 2.2,
                                               'max': '%7.3f' % 2.2,
                                               'median': '%7.3f' % 2.2,
                                               'min': '%6.3f' % 2.2,
                                               'std_dev': '%7.3f' % 0.0},
                         'req_count': 1,
                         'start': 103.8,
                         'stop': 106.0})]),
            scen_stats['size_stats'])

    def test_calculate_scenario_stats_time_series(self):
        # Time series (reqs completed each second
        scen_stats = self.master.calculate_scenario_stats(self.scenario,
                                                          self.stub_results)
        self.assertDictEqual(dict(
            start=101,
            data=[1, 1, 5, 3, 0, 2],
        ), scen_stats['time_series'])

    def test_write_rps_histogram(self):
        # Write out time series data (requests-per-second histogram) to an
        # already open CSV file
        scen_stats = self.master.calculate_scenario_stats(self.scenario,
                                                          self.stub_results)

        test_csv_file = StringIO()
        self.master.write_rps_histogram(scen_stats, test_csv_file)
        test_csv_file.seek(0)
        reader = csv.reader(test_csv_file)
        self.assertListEqual([
            ["Seconds Since Start", "Requests Completed"],
            ['1', '1'],
            ['2', '1'],
            ['3', '5'],
            ['4', '3'],
            ['5', '0'],
            ['6', '2'],
        ], list(reader))


    def test_generate_scenario_report(self):
        # Time series (reqs completed each second
        scen_stats = self.master.calculate_scenario_stats(self.scenario,
                                                          self.stub_results)
        self.assertListEqual(u"""
Master Test Scenario - ablkei
  C   R   U   D     Worker count:   3   Concurrency:   2
% 50  30  10  10

TOTAL
       Count:    12  Average requests per second:   1.9
                            min       max      avg      std_dev    median
       First-byte latency:  0.100 -   1.200    0.508  (  0.386)    0.400  (all obj sizes)
       Last-byte  latency:  0.200 -   3.000    1.158  (  0.970)    0.749  (all obj sizes)
       First-byte latency:  0.100 -   1.000    0.450  (  0.377)    0.350  (tiny objs)
       Last-byte  latency:  0.200 -   1.800    0.875  (  0.580)    0.749  (tiny objs)
       First-byte latency:  0.200 -   1.000    0.567  (  0.330)    0.500  (small objs)
       Last-byte  latency:  0.500 -   3.000    1.433  (  1.115)    0.800  (small objs)
       First-byte latency:  0.300 -   0.800    0.550  (  0.250)    0.550  (medium objs)
       Last-byte  latency:  0.400 -   2.800    1.600  (  1.200)    1.600  (medium objs)
       First-byte latency:  0.100 -   0.200    0.150  (  0.050)    0.150  (large objs)
       Last-byte  latency:  0.300 -   0.400    0.350  (  0.050)    0.350  (large objs)
       First-byte latency:  1.200 -   1.200    1.200  (  0.000)    1.200  (huge objs)
       Last-byte  latency:  2.200 -   2.200    2.200  (  0.000)    2.200  (huge objs)

CREATE
       Count:     3  Average requests per second:   0.5
                            min       max      avg      std_dev    median
       First-byte latency:  0.100 -   1.200    0.767  (  0.478)    1.000  (all obj sizes)
       Last-byte  latency:  0.200 -   3.000    1.800  (  1.178)    2.200  (all obj sizes)
       First-byte latency:  0.100 -   0.100    0.100  (  0.000)    0.100  (tiny objs)
       Last-byte  latency:  0.200 -   0.200    0.200  (  0.000)    0.200  (tiny objs)
       First-byte latency:  1.000 -   1.000    1.000  (  0.000)    1.000  (small objs)
       Last-byte  latency:  3.000 -   3.000    3.000  (  0.000)    3.000  (small objs)
       First-byte latency:  1.200 -   1.200    1.200  (  0.000)    1.200  (huge objs)
       Last-byte  latency:  2.200 -   2.200    2.200  (  0.000)    2.200  (huge objs)

READ
       Count:     4  Average requests per second:   1.0
                            min       max      avg      std_dev    median
       First-byte latency:  0.100 -   1.000    0.400  (  0.354)    0.250  (all obj sizes)
       Last-byte  latency:  0.400 -   1.800    0.875  (  0.554)    0.650  (all obj sizes)
       First-byte latency:  0.100 -   1.000    0.550  (  0.450)    0.550  (tiny objs)
       Last-byte  latency:  0.800 -   1.800    1.300  (  0.500)    1.300  (tiny objs)
       First-byte latency:  0.200 -   0.200    0.200  (  0.000)    0.200  (small objs)
       Last-byte  latency:  0.500 -   0.500    0.500  (  0.000)    0.500  (small objs)
       First-byte latency:  0.300 -   0.300    0.300  (  0.000)    0.300  (medium objs)
       Last-byte  latency:  0.400 -   0.400    0.400  (  0.000)    0.400  (medium objs)

UPDATE
       Count:     3  Average requests per second:   0.5
                            min       max      avg      std_dev    median
       First-byte latency:  0.200 -   0.800    0.533  (  0.249)    0.600  (all obj sizes)
       Last-byte  latency:  0.300 -   2.800    1.266  (  1.097)    0.699  (all obj sizes)
       First-byte latency:  0.600 -   0.600    0.600  (  0.000)    0.600  (tiny objs)
       Last-byte  latency:  0.699 -   0.699    0.699  (  0.000)    0.699  (tiny objs)
       First-byte latency:  0.800 -   0.800    0.800  (  0.000)    0.800  (medium objs)
       Last-byte  latency:  2.800 -   2.800    2.800  (  0.000)    2.800  (medium objs)
       First-byte latency:  0.200 -   0.200    0.200  (  0.000)    0.200  (large objs)
       Last-byte  latency:  0.300 -   0.300    0.300  (  0.000)    0.300  (large objs)

DELETE
       Count:     2  Average requests per second:   2.0
                            min       max      avg      std_dev    median
       First-byte latency:  0.100 -   0.500    0.300  (  0.200)    0.300  (all obj sizes)
       Last-byte  latency:  0.400 -   0.800    0.600  (  0.200)    0.600  (all obj sizes)
       First-byte latency:  0.500 -   0.500    0.500  (  0.000)    0.500  (small objs)
       Last-byte  latency:  0.800 -   0.800    0.800  (  0.000)    0.800  (small objs)
       First-byte latency:  0.100 -   0.100    0.100  (  0.000)    0.100  (large objs)
       Last-byte  latency:  0.400 -   0.400    0.400  (  0.000)    0.400  (large objs)


""".split('\n'), self.master.generate_scenario_report(self.scenario, scen_stats).split('\n'))